turnstile-rb 2.0.1 → 3.0.2

data/lib/turnstile/collector/formats.rb

@@ -1,43 +1,16 @@
 require 'json'
-require_relative 'matcher'
+require_relative 'regexp_matcher'
+require_relative 'formats/json_matcher'
+require_relative 'formats/delimited_matcher'
+require_relative 'formats/custom_matcher'
 
 module Turnstile
   module Collector
     module Formats
-      MARKER_TURNSTILE = 'x-turnstile'.freeze
 
-      # Extracts from the log file of the form:
-      # {"method":"GET","path":"/api/v1/saves/4SB8U-1Am9u-4ixC5","format":"json","duration":49.01,.....}
-      def json_matcher(*_args)
-        @json_matcher ||= Matcher.new(%r{"ip_address":"\d+},
-                                      ->(line) {
-                                        begin
-                                          data = JSON.parse(line)
-                                          [
-                                            data['platform'],
-                                            data['ip_address'],
-                                            data['user_id']
-                                          ].join(':')
-                                        rescue
-                                          nil
-                                        end
-                                      })
-      end
-
-      # Expects the form of '..... x-turnstile|desktop|10.10.2.4|1234456   ....'
-      def delimited_matcher(delimiter = '|')
-        @default_matcher ||= Matcher.new(%r{#{MARKER_TURNSTILE}},
-                                         ->(line) {
-                                           marker = line.split(/ /).find { |w| w =~ /^#{MARKER_TURNSTILE}/ }
-                                           if marker
-                                             list = marker.split(delimiter)
-                                             if list && list.size == 4
-                                               return(list[1..-1].join(':'))
-                                             end
-                                           end
-                                           nil
-                                         })
-      end
+      include JsonMatcher
+      include DelimitedMatcher
+      include CustomMatcher
 
       alias default_matcher delimited_matcher
     end

data/lib/turnstile/collector/formats/custom_matcher.rb

@@ -0,0 +1,19 @@
+module Turnstile
+  module Collector
+    module Formats
+      module CustomMatcher
+
+        # Returns a custom matcher that is expected to be pre-initialized
+        # using a config file.
+        #
+        # Example of a custom matcher configuration is in the README, and inside
+        # the `example` folder.
+        #
+        def custom_matcher
+          @custom_matcher ||= Turnstile.config.custom_matcher
+        end
+      end
+    end
+  end
+end
+

data/lib/turnstile/collector/formats/delimited_matcher.rb

@@ -0,0 +1,30 @@
+module Turnstile
+  module Collector
+    module Formats
+      module DelimitedMatcher
+        class << self
+          attr_accessor :marker
+        end
+
+        self.marker = 'x-turnstile'.freeze
+
+        # Expects the form of '..... x-turnstile|desktop|10.10.2.4|1234456   ....'
+        def delimited_matcher(delimiter = '|', match_marker = ::Turnstile::Collector::Formats::DelimitedMatcher.marker)
+          @default_matcher ||= RegexpMatcher.new(%r{#{match_marker}},
+                                                 ->(line) {
+                                             marker = line.split(/ /).find { |w| w =~ /^#{match_marker}/ }
+                                             if marker
+                                               list = marker.split(delimiter)
+                                               if list && list.size == 4
+                                                 return(list[1..-1].join(':'))
+                                               end
+                                             end
+                                             nil
+                                           })
+        end
+      end
+
+    end
+  end
+end
+

data/lib/turnstile/collector/formats/json_matcher.rb

@@ -0,0 +1,30 @@
+require 'json'
+require_relative '../regexp_matcher'
+
+module Turnstile
+  module Collector
+    module Formats
+      # Extracts from the log file of the form:
+      # {"method":"GET","path":"/api/v1/saves/4SB8U-1Am9u-4ixC5","format":"json","duration":49.01,.....}
+      module JsonMatcher
+        def json_matcher(*_args)
+          @json_matcher ||= ::Turnstile::Collector::RegexpMatcher.new(%r{"ip_address":"\d+},
+                                                                      ->(line) {
+                                                                        begin
+                                                                          data = JSON.parse(line)
+                                                                          [
+                                                                              data['platform'],
+                                                                              data['ip_address'],
+                                                                              data['user_id']
+                                                                          ].join(':')
+                                                                        rescue
+                                                                          nil
+                                                                        end
+                                                                      })
+        end
+      end
+
+    end
+  end
+end
+

data/lib/turnstile/collector/log_reader.rb

@@ -1,5 +1,6 @@
 require 'file-tail'
-require 'turnstile/collector/formats'
+require_relative 'formats'
+require_relative 'actor'
 
 module Turnstile
   module Collector
@@ -7,75 +8,112 @@ module Turnstile
       include ::File::Tail
     end
 
-    class LogReader
-      class << self
-        include Formats
+    class LogReader < Actor
+      attr_accessor :file, :filename, :matcher, :should_reopen
 
-        def pipe_delimited(file, queue)
-          new(file, queue, delimited_matcher)
-        end
+      def initialize(log_file:, matcher:, **opts)
+        super(**opts)
+        self.matcher  = matcher
+        self.filename = log_file
+        self.should_reopen = false
+        open_and_watch(opts[:tail] ? opts[:tail].to_i : 0)
 
-        def comma_delimited(file, queue)
-          new(file, queue, delimited_matcher(','))
-        end
+        reader = self
+        Signal.trap('HUP') { reader.should_reopen = true }
+      end
 
-        def colon_delimited(file, queue)
-          new(file, queue, delimited_matcher(':'))
-        end
+      def reopen(a_file = nil)
+        self.should_reopen = false
+        close rescue nil
+        self.filename = a_file if a_file
+        open_and_watch(0)
+      end
 
-        def delimited(file, queue, delimiter)
-          new(file, queue, delimited_matcher(delimiter))
+      def execute
+        self.read do |token|
+          self.queue << token if token
         end
+      rescue IOError
+        open_and_watch if File.exist?(filename)
+      ensure
+        close
+      end
 
-        def json_formatted(file, queue)
-          new(file, queue, json_matcher)
+      def read(&_block)
+        file.tail do |line|
+          token = matcher.tokenize(line) if matcher
+          yield(token) if block_given? && token
+          break if stopping?
+          reopen if should_reopen?
         end
-
-        alias default pipe_delimited
       end
 
-      attr_accessor :file, :queue, :matcher
+      def close
+        (file.close rescue nil) if file
+      end
 
-      def initialize(log_file, queue, matcher)
-        self.matcher = matcher
-        self.queue   = queue
+      private
 
-        self.file = LogFile.new(log_file)
+      def open_and_watch(tail_lines = 0)
+        self.file = LogFile.new(filename)
 
         file.interval = 1
-        file.backward(0)
+        file.backward(0) if tail_lines == 0
+        file.backward(tail_lines) if tail_lines > 0
+        file.forward(0) if tail_lines == -1
       end
 
-      def run
-        Thread.new do
-          Thread.current[:name] = 'log-reader'
-          Turnstile::Logger.log "starting to tail file #{file.path}...."
-          process!
-        end
+      def should_reopen?
+        should_reopen
       end
 
-      def read(&_block)
-        file.tail do |line|
-          token = matcher.token_from(line)
-          yield(token) if block_given? && token
+      class << self
+        include Formats
+
+        def custom(file, queue, **opts)
+          new(log_file: file,
+              queue:    queue,
+              matcher:  custom_matcher,
+              **opts)
         end
-      end
 
-      def process!
-        self.read do |token|
-          queue << token if token
+        def pipe_delimited(file, queue, **opts)
+          new(log_file: file,
+              queue:    queue,
+              matcher:  delimited_matcher,
+              **opts)
         end
-      end
 
-      def close
-        (file.close if file) rescue nil
-      end
+        def comma_delimited(file, queue, **opts)
+          new(log_file: file,
+              queue:    queue,
+              matcher:  delimited_matcher(','),
+              **opts)
+        end
 
-      private
+        def colon_delimited(file, queue, **opts)
+          new(log_file: file,
+              queue:    queue,
+              matcher:  delimited_matcher(':'),
+              **opts)
+        end
+
+        def delimited(file, queue, **opts)
+          new(log_file: file,
+              queue:    queue,
+              matcher:  delimited_matcher(opts[:delimiter]),
+              **opts)
+        end
 
-      def extract(line)
+        def json_formatted(file, queue, **opts)
+          new(log_file: file,
+              queue:    queue,
+              matcher:  json_matcher,
+              **opts)
+        end
+
+        alias default pipe_delimited
       end
     end
-
   end
 end

data/lib/turnstile/collector/{matcher.rb → regexp_matcher.rb}

@@ -1,16 +1,15 @@
 module Turnstile
   module Collector
-    class Matcher < Struct.new(:regexp, :extractor)
+    class RegexpMatcher < Struct.new(:regexp, :extractor)
       # checks if the line matches +regexp+, and if yes
       # runs it through +extractor+ to grab the token
       #
       # @param [String] line read from a log file
       # @return [String] a token in the form 'platform:ip:user'
 
-      def token_from(line)
-        return nil unless matches?(line)
-        return nil unless extractor
-        extractor ? extractor[line] : nil
+      def tokenize(line)
+        return nil unless matches?(line) && extractor
+        extractor[line]
       end
 
       def matches?(line)

data/lib/turnstile/collector/session.rb

@@ -0,0 +1,7 @@
+module Turnstile
+  module Collector
+    class Session < ::Struct.new(:uid, :platform, :ip);
+    end
+  end
+end
+

data/lib/turnstile/commands.rb

@@ -0,0 +1,20 @@
+require 'active_support/inflector'
+
+require_relative 'commands/base'
+require_relative 'commands/show'
+require_relative 'commands/flushdb'
+require_relative 'commands/print_keys'
+
+module Turnstile
+  module Commands
+    class << self
+      def command(name)
+        command_candidate = "#{self.name}::#{ActiveSupport::Inflector.camelize(name)}"
+        ActiveSupport::Inflector.constantize command_candidate
+      rescue NameError
+        raise CommandNotFoundError, "Command #{name} is not found, #{command_candidate}"
+      end
+    end
+  end
+end
+

data/lib/turnstile/commands/base.rb

@@ -0,0 +1,20 @@
+require 'turnstile/dependencies'
+
+module Turnstile
+  module Commands
+
+    class Base
+      include ::Turnstile::Dependencies
+
+      attr_accessor :options, :config
+
+      def initialize(options, config = Turnstile.config)
+        self.options = options
+        self.config  = config
+      end
+
+    end
+  end
+end
+
+

data/lib/turnstile/commands/flushdb.rb

@@ -0,0 +1,21 @@
+require 'csv'
+require_relative 'base'
+
+module Turnstile
+  module Commands
+    class Flushdb < Base
+      def execute
+        flushdb
+      end
+
+      def flushdb
+        adapter.flushdb
+      end
+    end
+  end
+end
+
+
+
+
+

data/lib/turnstile/commands/print_keys.rb

@@ -0,0 +1,19 @@
+require 'csv'
+require_relative 'base'
+
+module Turnstile
+  module Commands
+    class PrintKeys < Base
+      def execute
+        adapter.all_keys.each do |key|
+          puts key
+        end
+      end
+    end
+  end
+end
+
+
+
+
+

data/lib/turnstile/commands/show.rb

@@ -0,0 +1,89 @@
+require 'csv'
+require_relative 'base'
+
+module Turnstile
+  module Commands
+    class Show < Base
+
+      def execute(format = :json, delimiter = nil)
+        STDOUT.puts render_totals(format, delimiter)
+      end
+
+      def render_totals(format, delimiter = nil)
+        unless self.respond_to?(format)
+          raise ArgumentError, "Format #{format} is not supported"
+        end
+        self.send(format, aggregate, delimiter)
+      end
+
+      def yaml(data, *)
+        build_string(data, "\n", "---\nturnstile:") { |key, value, *| yaml_row(key, value) }
+      end
+
+      # Formats supported for the output
+      # JSON
+      def json(data, *)
+        build_string(data, "\n", '{', '}') do |key, value, index:, last:, first:|
+          json_row(key, value, index: index, first: first, last: last)
+        end
+      end
+
+
+      # NAD format for Circonus
+      def nad(data, *)
+        build_string(data) { |key, value, *| nad_row(key, value) }
+      end
+
+
+      # CSV Format
+      def csv(data, delimiter = nil)
+        build_string(data) do |key, value, *|
+          string = [key, value].to_csv
+          string.gsub!(/,/, delimiter) if delimiter
+          string.strip
+        end
+      end
+
+      private
+
+      # This method is used to build a string with
+      # opening/closing parts and looping contents inside.
+      #
+      def build_string(data,
+                       joiner = "\n",
+                       prefix = nil,
+                       suffix = nil,
+                       &_block)
+        slices = []
+        slices << prefix if prefix
+        index = 0; count = data.size
+        data.each_pair do |key, value|
+          slices << yield(
+            key,
+              value,
+              index: index,
+              first: (index == 0),
+              last: (index == count - 1)
+          ).to_s
+          index += 1
+        end
+        slices << suffix if suffix
+        slices.compact.join(joiner).strip
+      end
+
+      def json_row(key, value, last: false, **)
+        %Q(  "#{key}": #{value}#{ last ? '' : ',' })
+      end
+
+
+      def nad_row(key, value)
+        %Q(turnstile:#{key}#{"\tn\t"}#{value})
+      end
+
+      def yaml_row(key, value)
+        %Q(  #{key}: #{value})
+      end
+
+    end
+  end
+end