turnstile-rb 2.0.0

data/lib/turnstile/collector.rb

@@ -0,0 +1,8 @@
+module Turnstile
+  module Collector
+  end
+end
+
+require_relative 'collector/log_reader'
+require_relative 'collector/updater'
+require_relative 'collector/runner'

data/lib/turnstile/collector/formats.rb

@@ -0,0 +1,46 @@
+require 'json'
+require_relative 'matcher'
+
+module Turnstile
+  module Collector
+    module Formats
+      MARKER_TURNSTILE = 'x-turnstile'.freeze
+
+      # Extracts from the log file of the form:
+      # {"method":"GET","path":"/api/v1/saves/4SB8U-1Am9u-4ixC5","format":"json","duration":49.01,.....}
+      def json_matcher(*_args)
+        @json_matcher ||= Matcher.new(%r{"ip_address":"\d+},
+                                      ->(line) {
+                                        begin
+                                          data = JSON.parse(line)
+                                          [
+                                            data['platform'],
+                                            data['ip_address'],
+                                            data['user_id']
+                                          ].join(':')
+                                        rescue
+                                          nil
+                                        end
+                                      })
+      end
+
+      # Expects the form of '..... x-turnstile|desktop|10.10.2.4|1234456   ....'
+      def delimited_matcher(delimiter = '|')
+        @default_matcher ||= Matcher.new(%r{#{MARKER_TURNSTILE}},
+                                         ->(line) {
+                                           marker = line.split(/ /).find { |w| w =~ /^#{MARKER_TURNSTILE}/ }
+                                           if marker
+                                             list = marker.split(delimiter)
+                                             if list && list.size == 4
+                                               return(list[1..-1].join(':'))
+                                             end
+                                           end
+                                           nil
+                                         })
+      end
+
+      alias default_matcher delimited_matcher
+    end
+  end
+end
+

data/lib/turnstile/collector/log_reader.rb

@@ -0,0 +1,81 @@
+require 'file-tail'
+require 'turnstile/collector/formats'
+
+module Turnstile
+  module Collector
+    class LogFile < ::File
+      include ::File::Tail
+    end
+
+    class LogReader
+      class << self
+        include Formats
+
+        def pipe_delimited(file, queue)
+          new(file, queue, delimited_matcher)
+        end
+
+        def comma_delimited(file, queue)
+          new(file, queue, delimited_matcher(','))
+        end
+
+        def colon_delimited(file, queue)
+          new(file, queue, delimited_matcher(':'))
+        end
+
+        def delimited(file, queue, delimiter)
+          new(file, queue, delimited_matcher(delimiter))
+        end
+
+        def json_formatted(file, queue)
+          new(file, queue, json_matcher)
+        end
+
+        alias default pipe_delimited
+      end
+
+      attr_accessor :file, :queue, :matcher
+
+      def initialize(log_file, queue, matcher)
+        self.matcher = matcher
+        self.queue   = queue
+
+        self.file = LogFile.new(log_file)
+
+        file.interval = 1
+        file.backward(0)
+      end
+
+      def run
+        Thread.new do
+          Thread.current[:name] = 'log-reader'
+          Turnstile::Logger.log "starting to tail file #{file.path}...."
+          process!
+        end
+      end
+
+      def read(&_block)
+        file.tail do |line|
+          token = matcher.token_from(line)
+          yield(token) if block_given? && token
+        end
+      end
+
+      def process!
+        self.read do |token|
+          queue << token if token
+        end
+      end
+
+      def close
+        (file.close if file) rescue nil
+      end
+
+      private
+
+      def extract(line)
+      end
+    end
+
+  end
+end

data/lib/turnstile/collector/matcher.rb

@@ -0,0 +1,21 @@
+module Turnstile
+  module Collector
+    class Matcher < Struct.new(:regexp, :extractor)
+      # checks if the line matches +regexp+, and if yes
+      # runs it through +extractor+ to grab the token
+      #
+      # @param [String] line read from a log file
+      # @return [String] a token in the form 'platform:ip:user'
+
+      def token_from(line)
+        return nil unless matches?(line)
+        return nil unless extractor
+        extractor ? extractor[line] : nil
+      end
+
+      def matches?(line)
+        regexp && regexp.match?(line)
+      end
+    end
+  end
+end

data/lib/turnstile/collector/runner.rb

@@ -0,0 +1,71 @@
+require 'thread'
+require 'daemons/daemonize'
+
+
+module Turnstile
+  module Collector
+    class Runner
+      attr_accessor :config, :queue, :reader, :updater, :file
+
+      def initialize(*args)
+        @config = args.last.is_a?(Hash) ? args.pop : {}
+        @file   = config[:file]
+        @queue  = Queue.new
+
+        config[:debug] ? Turnstile::Logger.enable : Turnstile::Logger.disable
+
+        wait_for_file(@file)
+
+        self.reader
+        self.updater
+
+        Daemonize.daemonize if config[:daemonize]
+        STDOUT.sync = true if config[:debug]
+      end
+
+      def wait_for_file(file)
+        sleep_period = 1
+        while !File.exist?(file)
+          STDERR.puts "File #{file} does not exist, waiting for it to appear..."
+          STDERR.puts 'Press Ctrl-C to abort.' if sleep_period == 1
+
+          sleep sleep_period
+          sleep_period *= 1.2
+        end
+      end
+
+      def run
+        threads = [reader, updater].map(&:run)
+        threads.last.join
+      end
+
+      def updater
+        @updater ||= Turnstile::Collector::Updater.new(queue,
+                                                       config[:buffer_interval] || 5,
+                                                       config[:flush_interval] || 6)
+      end
+
+      def log_reader_class
+        Turnstile::Collector::LogReader
+      end
+
+      def reader
+        args    = [file, queue]
+        matcher = :default
+
+        if config[:delimiter]
+          matcher = :delimited
+          args << config[:delimiter]
+        elsif config[:filetype]
+          matcher = config[:filetype].to_sym
+        end
+
+        @reader ||= if log_reader_class.respond_to?(matcher)
+                      log_reader_class.send(matcher, *args)
+                    else
+                      raise ArgumentError, "Invalid matcher #{matcher}"
+                    end
+      end
+    end
+  end
+end

data/lib/turnstile/collector/updater.rb

@@ -0,0 +1,86 @@
+module Turnstile
+  module Collector
+    class Updater
+
+      class Session < ::Struct.new(:uid, :platform, :ip); end
+
+      attr_accessor :queue, :semaphore, :cache, :tracker, :buffer_interval, :flush_interval
+
+      def initialize(queue, buffer_interval = 5, flush_interval = 6)
+        @queue = queue
+        @semaphore = Mutex.new
+        @cache = Hash.new(0)
+        @tracker = Turnstile::Tracker.new
+        @buffer_interval = buffer_interval
+        @buffer_interval = 6 if @buffer_interval <= 0
+        @flush_interval = flush_interval
+        @flush_interval = 5 if @flush_interval <= 0
+      end
+
+      def run
+        run_queue_popper
+        run_flusher
+      end
+
+      private
+
+      def run_flusher
+        Thread.new do
+          Thread.current[:name] = 'updater:flush'
+          loop do
+            semaphore.synchronize {
+              unless cache.empty?
+                Turnstile::Logger.logging "flushing cache with [#{cache.keys.size}] keys" do
+                  cache.keys.each do |key|
+                    session = parse(key)
+                    if session.uid && !session.uid.empty?
+                      tracker.track(session.uid, session.platform, session.ip)
+                    end
+                  end
+                  reset_cache
+                end
+              else
+                Turnstile::Logger.log "nothing to flush, sleeping #{flush_interval}s.."
+              end
+            }
+            sleep flush_interval
+          end
+        end
+      end
+
+      def run_queue_popper
+        Thread.new do
+          Thread.current[:name] = 'updater:queue'
+          loop do
+            unless queue.empty?
+              Turnstile::Logger.logging "caching [#{queue.size}] keys locally" do
+                while !queue.empty?
+                  semaphore.synchronize {
+                    add(queue.pop)
+                  }
+                end
+              end
+            else
+              Turnstile::Logger.log "nothing in the queue, sleeping #{buffer_interval}s..."
+            end
+            sleep buffer_interval
+          end
+        end
+      end
+
+      def add(token)
+        cache[token] = 1
+      end
+
+      def parse(token)
+        a = token.split(':')
+        Session.new(a[2], a[0], a[1])
+      end
+
+      def reset_cache
+        @cache.clear
+        GC.start
+      end
+    end
+  end
+end

data/lib/turnstile/configuration.rb

@@ -0,0 +1,38 @@
+require 'hashie/dash'
+require 'hashie/extensions/dash/property_translation'
+
+module Turnstile
+  class RedisConfig < ::Hashie::Dash
+    include Hashie::Extensions::Dash::PropertyTranslation
+
+    property :host, default: '127.0.0.1', required: true
+    property :port, default: 6379, required: true, transform_with: ->(value) { value.to_i }
+    property :db, default: 1, required: true, transform_with: ->(value) { value.to_i }
+    property :timeout, default: 0.05, required: true, transform_with: ->(value) { value.to_f }
+
+    def configure
+      yield self if block_given?
+      self
+    end
+  end
+
+  class Configuration < ::Hashie::Dash
+    include Hashie::Extensions::Dash::PropertyTranslation
+    property :activity_interval, default: 60, required: true, transform_with: ->(value) { value.to_i }
+    property :sampling_rate, default: 100, required: true, transform_with: ->(value) { value.to_i }
+    property :redis, default: ::Turnstile::RedisConfig.new
+
+    def configure
+      yield self if block_given?
+      self
+    end
+
+    def method_missing(method, *args, &block)
+      return super unless method.to_s =~ /^redis_/
+      prop = method.to_s.gsub(/^redis_/, '').to_sym
+      if self.redis.respond_to?(prop)
+        prop.to_s.end_with?('=') ? self.redis.send(prop, *args, &block) : self.redis.send(prop)
+      end
+    end
+  end
+end

data/lib/turnstile/logger.rb

@@ -0,0 +1,47 @@
+module Turnstile
+  module Logger
+
+    class << self
+      def enable
+        class << self
+          self.send(:define_method, :log, proc { |msg| _log(msg) })
+          self.send(:define_method, :logging, proc { |msg, &block| _logging(msg, &block) })
+        end
+      end
+
+      def disable
+        class << self
+          self.send(:define_method, :log, proc { |msg|})
+          self.send(:define_method, :logging, proc { |msg, &block| block.call })
+        end
+      end
+
+      def log(msg)
+      end
+
+      def logging(msg, &block)
+        block.call
+      end
+
+      private
+
+      def _log(msg)
+        puts "#{Time.now}: #{sprintf("%-20s", Thread.current[:name])} - #{msg}"
+      end
+
+      def _logging(message, &block)
+        start = Time.now
+        returned_from_block = yield
+        elapsed_time = Time.now - start
+        if returned_from_block.is_a?(String) && returned_from_block != ""
+          message += " - #{returned_from_block}"
+        end
+        log "(#{"%9.2f" % (1000 * elapsed_time)}ms) #{message}"
+        returned_from_block
+      rescue Exception => e
+        elapsed_time = Time.now - start
+        log "(#{"%9.2f" % (1000 * elapsed_time)}ms) error: #{e.message} for #{message} "
+      end
+    end
+  end
+end

data/lib/turnstile/nad.rb

@@ -0,0 +1,16 @@
+module Turnstile
+  class Nad
+
+    def data
+      out = ""
+      aggregate.each_pair do |key, value|
+        out << %Q(turnstile:#{key}#{"\tn\t"}#{value}\n)
+      end
+      out
+    end
+
+    def aggregate
+      Turnstile::Adapter.new.aggregate
+    end
+  end
+end

data/lib/turnstile/observer.rb

@@ -0,0 +1,33 @@
+require 'hashie/mash'
+require 'hashie/extensions/mash/symbolize_keys'
+
+module Turnstile
+  class Stats < ::Hashie::Mash
+    include ::Hashie::Extensions::Mash::SymbolizeKeys
+  end
+
+  class Observer
+    def stats
+      data      = adapter.fetch
+      platforms = Hash[data.group_by { |d| d[:platform] }.map { |k, v| [k, sampler.extrapolate(v.count)] }]
+      total     = platforms.values.inject(:+) || 0
+      Stats.new({
+                  stats: {
+                    total:     total,
+                    platforms: platforms
+                  },
+                  users: data
+                })
+    end
+
+    private
+
+    def adapter
+      @adapter ||= Adapter.new
+    end
+
+    def sampler
+      @sampler ||= Sampler.new
+    end
+  end
+end