turnstile-rb 2.0.1 → 3.0.2

data/lib/turnstile/cli/parser.rb

@@ -0,0 +1,166 @@
+require 'optparse'
+require 'colored2'
+require 'forwardable'
+
+require 'turnstile/version'
+
+module Turnstile
+  module CLI
+    class Parser
+      extend Forwardable
+      def_delegators :@system, :stdout, :stdin, :stderr
+
+      attr_accessor :options, :argv, :system
+
+      def initialize(argv, system)
+        self.system  = system
+        self.argv    = argv.dup
+        self.options = Hashie::Mash.new
+        self.argv << '-h' if argv.empty?
+      end
+
+      def parse
+        begin
+          OptionParser.new do |opts|
+            opts.banner = ' '
+
+            opts.separator 'DESCRIPTION:'.bold.magenta
+            opts.separator '   ' + ::Turnstile::GEM_DESCRIPTION.gsub(/\n/, "\n   ").strip
+            opts.separator ''
+
+            opts.separator "USAGE:\n".bold.magenta +
+                "   # Tail the log file as a proper daemon\n".bold.black +
+                "   turnstile -f <file> [ --daemon ]  [ options ]\n\n".yellow +
+
+                "   # Add a single item and exit\n".bold.black +
+                "   turnstile -a 'platform:ip:user'   [ options ]\n\n".yellow +
+
+                "   # Print the summary stats and exit\n".bold.black +
+                '   turnstile -s [ json | csv | nad ] [ options ]'.yellow
+
+            opts.separator ''
+            opts.separator 'DETAILS:'.bold.magenta
+            opts.separator '   ' + ::Turnstile::DESCRIPTION.gsub(/\n/, "\n   ").strip
+
+            opts.separator ''
+            opts.separator 'OPTIONS:'.bold.magenta
+
+
+            opts.separator "\n  Mode of Operation:".bold.green
+            opts.on('-f', '--file FILE',
+                    'Starts Turnstile in a file-tailing mode',
+                    ' ') do |file|
+              options[:file] = file
+            end
+
+            opts.on('-s', '--show [FORMAT]',
+                    'Print current stats and exit. Optional ',
+                    'format can be "json" (default), "nad",',
+                    '"yaml", or "csv"', ' ') do |v|
+              options[:show]        = true
+              options[:show_format] = (v || 'json').to_sym
+            end
+
+            opts.on('-w', '--web [PORT]',
+                    'Starts a Sinatra app on a given port',
+                    'offering /turnstile/<json|yaml> end point.',
+                    'Can be used with file tailing mode, or',
+                    'standalone. Default port is ' +
+                        ::Turnstile::DEFAULT_PORT.to_s,
+                    ' '
+                    ) do |v|
+              options[:web] = true
+              Turnstile.config.port = v.to_i if v
+            end
+
+            opts.on('-a', '--add TOKEN',
+                    'Registers an event from the token, such as ',
+                    '"ios:123.4.4.4:32442". Use -l to customize',
+                    'the delimiter', ' ') do |v|
+              options[:token] = v
+            end
+
+            opts.on('-p', '--print-keys', 'Prints all Turnstile keys in Redis', ' ') do |v|
+              options[:print_keys] = true
+            end
+
+            opts.on('--flushdb', 'Wipes Redis database, and exit', ' ') do |v|
+              options[:flushdb] = true
+            end
+
+
+            opts.separator '  Tailing log file:'.bold.green
+
+            opts.on('-d', '--daemonize', 'Daemonize to watch the logs', ' ') do |v|
+              options[:daemonize] = true
+            end
+
+            opts.on('-b', '--read-backwards [LINES]',
+                    'Like tail, read last LINES lines',
+                    'instead of tailing from the end', ' ') do |lines|
+              options[:tail] = lines ? lines.to_i : -1
+            end
+
+            opts.on('-F', '--format FORMAT',
+                    'Log file format (see above)', ' ') do |format|
+              options[:filetype] = format
+            end
+            opts.on('-l', '--delimiter CHAR',
+                    'Forces "delimited" file type, and ',
+                    'uses CHAR as the delimiter', ' ') do |v|
+              options[:delimiter] = v
+            end
+
+            opts.on('-c', '--config FILE',
+                              'Ruby config file that can define the',
+                              'custom matcher, supporting arbitrary ',
+                              'complex logs') do |file|
+              options[:config_file] = file
+            end
+
+            opts.separator "\n  Redis Server:".bold.green
+
+            opts.on('-r', '--redis-url URL', 'Redis server URL') { |host| Turnstile.config.redis_url = host }
+            opts.on('--redis-host HOST', 'Redis server host') { |host| Turnstile.config.redis_host = host }
+            opts.on('--redis-port PORT', 'Redis server port') { |port| Turnstile.config.redis_port = port }
+            opts.on('--redis-db DB', 'Redis server db') { |db| Turnstile.config.redis_db = db }
+            opts.on('--hiredis', 'Use hiredis high performance library') do |_value|
+              begin
+                require 'redis/connection/hiredis'
+              rescue LoadError => e
+                raise HiredisDriverNotFound, "Can not use hiredis driver: #{e.message}"
+              end
+              Turnstile.config.redis_use_hiredis = true
+            end
+
+            opts.separator "\n  Miscellaneous:".bold.green
+
+            opts.on('-i', '--idle-sleep SECONDS',
+                    'When no work was detected, pause the ',
+                    'threads for several seconds.') do |v|
+              Turnstile.config.flush_interval = v.to_i
+            end
+
+            opts.on('-v', '--verbose', 'Print status to stdout') { |v| options[:verbose] = true }
+            opts.on('-t', '--trace', 'Enable trace mode') do |v|
+              options[:trace]        = true
+              Turnstile.config.trace = true
+            end
+            opts.on_tail('-h', '--help', 'Show this message') do
+              puts opts
+              return
+            end
+          end.parse!(argv)
+          options
+        rescue OptionParser::MissingArgument => e
+          terr 'Invalid Usage: ' + e.message.red, stderr
+          nil
+        rescue Exception => e
+          terr e.message.bold.red, stderr
+          nil
+        end
+      end
+
+    end
+  end
+end

data/lib/turnstile/cli/runner.rb

@@ -0,0 +1,58 @@
+require 'optparse'
+require 'colored2'
+
+require_relative '../version'
+require_relative '../configuration'
+
+require_relative 'launcher'
+require_relative 'parser'
+
+module Turnstile
+  module CLI
+    class Runner
+      attr_reader :argv, :stdin, :stdout, :stderr, :kernel
+
+      # Allow everything fun to be injected from the outside while defaulting to normal implementations.
+      def initialize(argv, stdin = STDIN, stdout = STDOUT, stderr = STDERR, kernel = Kernel)
+        @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
+      end
+
+      def execute!
+        exit_code = begin
+          Colored2.disable! unless stdout.tty?
+
+          $stderr = stderr
+          $stdin  = stdin
+          $stdout = stdout
+
+          options = Parser.new(argv, self).parse
+          Configuration.from_file(options.config_file) if options && options.config_file
+          Launcher.new(options).launch if options
+
+          # Thor::Base#start does not have a return value, assume success if no exception is raised.
+          0
+        rescue StandardError => e
+          # The ruby interpreter would pipe this to STDERR and exit 1 in the case of an unhandled exception
+          if options && options[:trace]
+            b = e.backtrace
+            terr("#{b.shift}: #{e.message} (#{e.class})")
+            terr(b.map { |s| "\tfrom #{s}" }.join("\n"))
+          else
+            terr(e.message)
+          end
+          1
+        rescue SystemExit => e
+          e.status
+        ensure
+          $stderr = STDERR
+          $stdin  = STDIN
+          $stdout = STDOUT
+        end
+
+        # Proxy our exit code back to the injected kernel.
+        @kernel.exit(exit_code)
+      end
+    end
+  end
+end
+

data/lib/turnstile/collector.rb

@@ -4,5 +4,5 @@ module Turnstile
 end
 
 require_relative 'collector/log_reader'
-require_relative 'collector/updater'
-require_relative 'collector/runner'
+require_relative 'collector/flusher'
+require_relative 'collector/controller'

data/lib/turnstile/collector/actor.rb

@@ -0,0 +1,81 @@
+require_relative 'session'
+require_relative '../logger/helper'
+require_relative '../dependencies'
+require 'turnstile'
+require 'thread'
+module Turnstile
+  module Collector
+    class Actor
+      include Logger::Helper
+
+      @name = 'abstract'
+
+      class << self
+        def actor_name(value = nil)
+          @actor_name = value if value
+          @actor_name
+        end
+      end
+
+      attr_accessor :queue,
+                    :tracker,
+                    :sleep_when_idle,
+                    :options,
+                    :stopping,
+                    :thread
+
+      def initialize(queue:,
+                     tracker: nil,
+                     sleep_when_idle: Turnstile.config.flush_interval,
+                     **opts)
+
+        self.queue           = queue
+        self.sleep_when_idle = sleep_when_idle
+        self.tracker         = tracker || Turnstile::Tracker.new
+        self.options         = opts
+        self.stopping        = false
+
+        Kernel.tdb "actor initialized: #{self.to_s}" if Turnstile.config.trace
+      end
+
+      def shutdown
+        self.stopping = true
+      end
+
+      def config
+        @config ||= Turnstile.config
+      end
+
+      def stopping?
+        self.stopping
+      end
+
+      def start
+        self.thread = create_thread(self, sleep_when_idle) do |actor, sleep_period|
+          loop do
+            items_remaining = actor.execute
+            break if actor.stopping?
+            sleep(sleep_period) unless items_remaining && items_remaining > 0
+          end
+        end
+      end
+
+      def to_s
+        "<turnsile-actor##{self.class.name.gsub(/.*::/, '').downcase}: queue size: #{queue.size}, idle=#{sleep_when_idle}"
+      end
+
+      # Return nil when there is nothing else to do
+      # Return ideally a number representing number of remaining items.
+      def execute
+        raise ArgumentError, 'Abstract Method'
+      end
+
+      def create_thread(*args, &_block)
+        Thread.new(self.class.actor_name, *args) do |actor_name, *args|
+          Thread.current[:name] = actor_name
+          yield(*args) if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/turnstile/collector/controller.rb

@@ -0,0 +1,121 @@
+require 'csv'
+require 'thread'
+require 'hashie/extensions/symbolize_keys'
+require 'daemons/daemonize'
+require 'colored2'
+
+require_relative 'flusher'
+require_relative 'actor'
+
+module Turnstile
+  module Collector
+    class Controller
+      attr_accessor :options, :actors, :threads
+
+      def initialize(*args)
+        self.options = args.last.is_a?(Hash) ? args.pop : {}
+
+        options.verbose ? Turnstile::Logger.enable : Turnstile::Logger.disable
+
+        wait_for_file(file)
+
+        self.actors = [self.reader, self.flusher]
+
+        Daemonize.daemonize if options[:daemonize]
+        STDOUT.sync = true if options[:verbose]
+      end
+
+      def start
+        self.threads = actors.map(&:start)
+        threads.map(&:join)
+      end
+
+      def stop
+        actors.each(&:shutdown)
+      end
+
+      def flusher
+        @flusher ||= Flusher.new(**flusher_arguments)
+      end
+
+      def tracker
+        @tracker ||= Turnstile::Tracker.new
+      end
+
+      def queue
+        @queue ||= Queue.new
+      end
+
+      def file
+        options.file
+      end
+
+      def reader
+        opts    = reader_arguments
+        matcher = opts.delete(:matcher).to_sym
+        @reader ||= if log_reader_class.respond_to?(matcher)
+                      log_reader_class.send(matcher, file, queue, **opts)
+                    else
+                      raise ArgumentError, "Invalid matcher #{matcher}, args #{reader_args}"
+                    end
+      end
+
+      def symbolize(opts)
+        Hashie::Extensions::SymbolizeKeys.symbolize_keys(opts.to_h)
+      end
+
+      def config
+        @config ||= Turnstile.config
+      end
+
+      private
+
+      def flusher_arguments
+        symbolize(actor_argument_hash.merge(sleep_when_idle: config.flush_interval))
+      end
+
+      def reader_arguments
+        reader_args_hash   = actor_argument_hash.merge(sleep_when_idle: config.flush_interval)
+        matcher, delimiter = select_matcher
+
+        reader_args_hash.merge!(delimiter: delimiter) if delimiter
+        reader_args_hash.merge!(matcher: matcher) if matcher
+
+        symbolize(reader_args_hash)
+      end
+
+      def actor_argument_hash
+        options.merge(queue: queue, tracker: tracker)
+      end
+
+      def wait_for_file(file)
+        sleep_period = 1
+        while !::File.exist?(file)
+          terr "File #{file.bold.yellow} does not exist, waiting for it to appear..."
+          terr 'Press Ctrl-C to abort.' if sleep_period == 1
+
+          sleep sleep_period
+          sleep_period *= 1.2
+        end
+        tdb "Detected file #{file.bold.yellow} now exists, continue..."
+      end
+
+      def log_reader_class
+        Turnstile::Collector::LogReader
+      end
+
+      def select_matcher
+        matcher   = :default
+        delimiter = nil
+
+        if options[:delimiter]
+          matcher   = :delimited
+          delimiter = options[:delimiter]
+        elsif options[:filetype]
+          matcher = options[:filetype].to_sym
+        end
+        return matcher, delimiter
+      end
+    end
+  end
+end

data/lib/turnstile/collector/flusher.rb

@@ -0,0 +1,36 @@
+require_relative 'session'
+
+module Turnstile
+  module Collector
+    class Flusher < Actor
+
+      actor_name :flusher
+
+      def execute
+        flush_current_buffer unless queue.empty?
+        queue.size
+      rescue Exception => e
+        puts e.backtrace.reverse.join("\n")
+        puts e.inspect.red
+        raise e
+      end
+
+      def flush_current_buffer
+        item = queue.pop
+        return unless item
+        session = parse(item)
+        tracker.track(session.uid,
+                      session.platform,
+                      session.ip) if session.uid
+      end
+
+      def parse(token)
+        # platform, IP, user
+        a = token.split(':')
+
+        # session is backwards
+        Session.new(a[2], a[0], a[1])
+      end
+    end
+  end
+end