turnstile-rb 2.0.1 → 3.0.2

data/lib/turnstile/redis/spy.rb

@@ -0,0 +1,42 @@
+module Turnstile
+  module Redis
+    class Spy < Struct.new(:redis)
+      @stream        = STDOUT
+      @disable_color = false
+
+      class << self
+        # in case someone might prefer STDERR, feel free to set
+        # it in the gem configuration:
+        # Turnstile::Redis::Spy.stream = STDOUT | STDERR | etc...
+        attr_accessor :stream, :disable_color
+      end
+
+
+      def method_missing(m, *args, &block)
+        if redis.respond_to?(m)
+          t1     = Time.now
+          result = redis.send(m, *args, &block)
+          delta  = Time.now - t1
+          colors = [:blue, nil, :blue, :blue, :yellow, :cyan, nil, :blue]
+
+          components = [
+            Time.now.strftime('%H:%M:%S.%L'), ' rtt=',
+            (sprintf '%.5f', delta*1000), ' ms ',
+            (sprintf '%15s ', m.to_s.upcase),
+            (sprintf '%-40s', args.inspect.gsub(/[",\[\]]/, '')), ' ⇒ ',
+            (result.is_a?(::Redis::Future) ? '' : result.to_s)]
+
+          components.each_with_index do |component, index|
+            color     = self.class.disable_color ? nil : colors[index]
+            component = component.send(color) if color
+            self.class.stream.printf component
+          end
+          self.class.stream.puts
+          result
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/turnstile/sampler.rb

@@ -1,5 +1,10 @@
+require_relative 'dependencies'
+
 module Turnstile
   class Sampler
+
+    include Dependencies
+
     def extrapolate(n)
       (n * 100.0 / sampling_rate).to_i
     end
@@ -11,10 +16,12 @@ module Turnstile
       ((uid.hash + Time.now.day) % 100) < sampling_rate
     end
 
-    private
+    def sampling?
+      sampling_rate && sampling_rate <= 100 && sampling_rate >= 0
+    end
 
     def sampling_rate
-      Turnstile.config.sampling_rate
+      config.sampling_rate
     end
   end
 end

data/lib/turnstile/tracker.rb

@@ -1,28 +1,28 @@
+require_relative 'dependencies'
+
 module Turnstile
   class Tracker
-    def sample(uid, platform = 'unknown', ip = nil)
-      adapter.add(uid, platform, ip) if sampler.sample(uid)
+
+    include Dependencies
+
+    def track_and_sample(uid, platform = 'unknown', ip = nil)
+      track_all(uid, platform, ip) if should_track?(uid)
     end
 
-    alias track sample
+    def should_track?(uid)
+      !sampler.sampling? || sampler.sample(uid)
+    end
 
-    def add(uid, platform = 'unknown', ip = nil)
+    def track_all(uid, platform = 'unknown', ip = nil)
       adapter.add(uid, platform, ip)
     end
 
-    def add_token(token, delimiter = ':')
+    def track_token(token, delimiter = nil)
+      delimiter ||= ':'
       platform, ip, uid = token.split(delimiter)
       adapter.add(uid, platform, ip) if uid
     end
 
-    private
-
-    def adapter
-      @adapter ||= Adapter.new
-    end
-
-    def sampler
-      @sampler ||= Sampler.new
-    end
+    alias track track_and_sample
   end
 end

data/lib/turnstile/version.rb

@@ -1,21 +1,60 @@
 module Turnstile
-  VERSION     = '2.0.1'
+  VERSION     = '3.0.2'
+  DEFAULT_PORT = 9090
 
   GEM_DESCRIPTION = <<-EOF
-  Turnstile is a Redis-based library that can accurately track total number of concurrent
-  users accessing a web/API based server application. It can break it down by "platform"
-  or a device type, and returns data in JSON, CSV of NAD formats. While user tracking
-  may happen synchronously using a Rack middleware, another method is provided that is
-  based on log file analysis, and can therefore be performed outside web server process.
+Turnstile is a Redis-based library that can accurately track total number
+of concurrent users accessing a web/API based server application. It can
+break it down by "platform" or a device type, and returns data in JSON,
+CSV of NAD formats. While user tracking may happen synchronously using a
+Rack middleware, another method is provided that is based on log file
+analysis, and can therefore be performed outside web server process.
+
+
   EOF
 
   DESCRIPTION = <<-EOF
-  Turnstile can run as a daemon, in which mode it monitors a given log file. 
-  Alternatively, turnstile binary can be used to print current stats, and even
-  add new data into the registry.  
- 
-  If you are using Turnstile to tail log files, make sure you run on each app sever
-  that's generating log files.
+Turnstile can be run as a daemon, in which case it watches a given log
+file. Or, you can run turnstile to print the current aggregated stats in
+several supported formats, such as JSON.
+
+When Turnstile is used to tail the log files, ideally you should start
+turnstile daemon on each app sever that's generating log file, or be
+content with the effects of sampling.
+
+Note that the IP address is not required to track uniqueness. Only
+platform and UID are used. Also note that custom formatter can be
+specified in a config file to parse arbitrary complex log lines.
+
+For tailing a log files, Turnstile must first match a log line expected to
+contain the tokens, and then extract is using one of the matchers. You can
+specify which matcher to use depending on whether you can add Turnstile's 
+tokens to your log or not. If you can, great! If not, implement your own 
+custom matcher and great again.
+
+The following matchers are available, and can be selected with -F:
+
+  1. Format named "delimited", which expects the following token in the
+     log file:
+
+     x-turnstile:platform:ip:user-identifier
+
+     Where ':' (the delimiter) can be set via -l option, OR you can use one
+     of the following formats: "json_formatted", "pipe_formatted",
+     "comma_formatted", "colon_formatted" (the default). The match is 
+     performed on a string "x-turnstile", other log lines are skipped.
+
+  2. Format "json_delimited", which expects to find a single-line JSON
+     Hash, containing keys "platform", "ip_address", and "user_id". The match
+     is performed on any log line containing string '"ip_address"', other
+     lines are skipped.
+    
+ 3.  Format "custom" requires passing an additional flag -c/--config
+     file.rb, which will be required, and which can define a matcher and
+     assign it to the `Turnstile.config.custom_matcher` config variable.
+
+
   EOF
 
+  NS = "x-turnstile|#{VERSION.gsub(/\./,'')}".freeze
 end

data/lib/turnstile/web_app.rb

@@ -0,0 +1,29 @@
+require 'sinatra'
+require_relative 'dependencies'
+require 'json'
+module Turnstile
+  class WebApp < ::Sinatra::Base
+    SUPPORTED_FORMATS = %i(yaml json nad)
+    set :port, Turnstile.config.port
+
+    include Dependencies
+
+    set :sessions, false
+    set :port, Turnstile.config.port
+
+    get '/turnstile/:format' do
+      fmt = params['format'].to_sym
+      if SUPPORTED_FORMATS.include?(fmt)
+        status 200
+        headers 'Content-type' => fmt.to_s
+        body aggregate.send("to_#{fmt}".to_sym)
+      else
+        status 500
+        body "Error: unsupported format #{fmt}!"
+      end
+    end
+
+    # start the server if ruby file executed directly
+    run!
+  end
+end

data/spec/spec_helper.rb

@@ -9,16 +9,31 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
 
 require 'rubygems'
 require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])
-require 'simplecov'
+require 'fileutils'
+require 'rspec/its'
 
+require 'simplecov'
 SimpleCov.start
 
 require 'turnstile'
-require 'rspec/its'
+
+require_relative 'support/logging'
 
 RSpec.configure do |config|
   config.order = 'random'
+
+  config.before :all  do
+    Turnstile::RSpec::Logging.configure
+  end
+
   config.before :each do
-    Turnstile::Adapter.new.redis.flushdb
+    Turnstile::Redis::Adapter.instance
+  end
+
+  config.around :each, logging: true do |ex|
+    logging_enabled = Turnstile::Logger.enabled
+    Turnstile::Logger.enable
+    Turnstile::Logger.info_elapsed('measuring example: ') { ex.run }
+    Turnstile::Logger.enabled = logging_enabled
   end
 end

data/spec/support/logging.rb

@@ -0,0 +1,17 @@
+module Turnstile
+  module RSpec
+    module Logging
+      TEST_LOG = 'spec/log/test.log'
+
+      def self.configure(file = TEST_LOG)
+        return if Turnstile::Logger.enabled
+        Turnstile::Logger.enable
+        FileUtils.mkdir_p(File.dirname(file))
+
+        Turnstile::Logger.logger       = ::Logger.new(file)
+        Turnstile::Logger.logger.level = ::Logger::INFO
+        Turnstile::Logger.logger
+      end
+    end
+  end
+end

data/spec/turnstile/adapter_spec.rb

@@ -1,71 +1,84 @@
 require 'spec_helper'
 
-describe Turnstile::Adapter do
-
-  subject { Turnstile::Adapter.new }
-
-  let(:redis) { subject.send(:redis) }
+describe Turnstile::Redis::Adapter, logging: true do
+  subject(:adapter) { Turnstile::Redis::Adapter.instance }
 
   let(:uid) { 1238438 }
   let(:other_uid) { 1238439 }
   let(:another_uid) { 1238440 }
-
   let(:ip) { '1.2.3.4' }
   let(:another_ip) { '4.3.2.1' }
-
   let(:platform) { :ios }
   let(:another_platform) { :android }
 
-  describe '#add' do
-    it 'calls redis with the correct params' do
-      key = "t:#{uid}:#{platform}:#{ip}"
-      expect(redis).to receive(:setex).once.with(key, Turnstile.config.activity_interval, 1)
-      subject.add(uid, platform, ip)
-    end
-  end
-
-  describe '#fetch' do
-    let(:sort_lambda) { ->(a, b) { a[:uid] <=> b[:uid] }  }
-    let(:expected_hash) do
-      [
-        { uid: uid.to_s, platform: platform.to_s, ip: ip },
-        { uid: other_uid.to_s, platform: platform.to_s, ip: ip },
-        { uid: another_uid.to_s, platform: another_platform.to_s, ip: another_ip },
-      ]
-    end
+  context 'with fake redis' do
+    let(:redis) { double }
 
     before do
-      subject.add(uid, platform, ip)
-      subject.add(other_uid, platform, ip)
-      subject.add(another_uid, another_platform, another_ip)
+      allow(adapter).to receive(:with_redis).and_yield(redis)
+      expect(redis).to receive(:setex).once.with(key, Turnstile.config.activity_interval, 1)
     end
 
-    it 'pulls the platform specific stats from redis' do
-      expect(subject.fetch.sort(&sort_lambda)).to eq(expected_hash.sort(&sort_lambda))
+    describe '#add' do
+      let(:key) { adapter.compose_key(uid, platform, ip) }
+      it 'calls redis with the correct params' do
+        subject.add(uid, platform, ip)
+      end
     end
   end
 
-  describe '#aggregate' do
-    let(:expected_hash) do
-      {
-        'android' => 3,
-        'ios'     => 2,
-        'total'   => 5
-      }
-    end
+  context 'with real redis' do
+    before { adapter.flushdb }
+    describe '#fetch' do
+      let(:sort_lambda) { ->(a, b) { a[:uid] <=> b[:uid] } }
+      let(:expected_hash) do
+        [
+          { uid: uid.to_s, platform: platform.to_s, ip: ip },
+          { uid: other_uid.to_s, platform: platform.to_s, ip: ip },
+          { uid: another_uid.to_s, platform: another_platform.to_s, ip: another_ip },
+        ]
+      end
 
-    before do
-      subject.add(123, :android, ip)
-      subject.add(124, :android, ip)
-      subject.add(125, :android, ip)
+      before do
+        subject.add(uid, platform, ip)
+        subject.add(other_uid, platform, ip)
+        subject.add(another_uid, another_platform, another_ip)
+      end
 
-      subject.add(200, :ios, ip)
-      subject.add(201, :ios, ip)
+      it 'pulls the platform specific stats from redis' do
+        expect(subject.fetch.sort(&sort_lambda)).to eq(expected_hash.sort(&sort_lambda))
+      end
     end
 
-    it 'should calculated proper aggregation' do
-      expect(subject.aggregate).to eql expected_hash
+    describe '#aggregate' do
+      let(:expected_hash) do
+        {
+          'android' => 3,
+          'ios'     => 2,
+          'total'   => 5
+        }
+      end
+
+      before do
+        subject.add(123, :android, ip)
+        subject.add(124, :android, ip)
+        subject.add(125, :android, ip)
+
+        subject.add(200, :ios, ip)
+        subject.add(201, :ios, ip)
+      end
+
+      it 'should calculated proper aggregation' do
+        expect(subject.aggregate).to eql expected_hash
+      end
+
+      its(:aggregate) { should_not include({ 'total' => 0 }) }
+      its(:prefix) { should match /^x-turnstile\|\d+/ }
+
+      describe '#flushdb' do
+        before { subject.flushdb }
+        its(:aggregate) { should include({ 'total' => 0 }) }
+      end
     end
   end
-
 end

data/spec/turnstile/collector/flusher_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+require 'file/tail'
+require 'timeout'
+require 'thread'
+require 'tempfile'
+
+
+module Turnstile
+  module Collector
+    RSpec.describe Flusher do
+
+
+    end
+  end
+end
+

data/spec/turnstile/collector/log_reader_spec.rb

@@ -5,110 +5,160 @@ require 'thread'
 require 'tempfile'
 
 require 'turnstile/collector/log_reader'
+module Turnstile
+  module Collector
 
-describe Turnstile::Collector::LogReader do
-  include Timeout
+    describe LogReader do
+      include Timeout
 
-  def consume_file(reader, read_timeout)
-    hash    = {}
-    counter = 0
+      def consume_file(reader, read_timeout)
+        hash    = {}
+        counter = 0
 
-    run_reader(read_timeout) do
-      reader.read do |token|
-        counter     += 1
-        hash[token] = 1
+        run_reader(read_timeout) do
+          reader.read do |token|
+            counter     += 1
+            hash[token] = 1
+          end
+        end
+        return counter, hash
       end
-    end
-    return counter, hash
-  end
 
-  def run_reader(read_timeout, &block)
-    t_reader = Thread.new do
-      begin
-        timeout(read_timeout) do
-          block.call
+      def run_reader(read_timeout, &block)
+        t_reader = Thread.new do
+          begin
+            timeout(read_timeout) do
+              block.call
+            end
+          rescue Timeout::Error
+          end
         end
-      rescue Timeout::Error
+        t_reader.join
       end
-    end
-    t_reader.join
-  end
 
-  let(:queue) { Queue.new }
-  let(:read_timeout) { 0.1 }
-  let(:consume_file_result) { consume_file(reader, read_timeout) }
-  let(:counter) { consume_file_result.first }
-  let(:hash) { consume_file_result.last }
+      let(:queue) { Queue.new }
+      let(:read_timeout) { 0.1 }
+      let(:consume_file_result) { consume_file(reader, read_timeout) }
+      let(:counter) { consume_file_result.first }
+      let(:hash) { consume_file_result.last }
 
-  before { reader.file.backward(1000) }
+      before { reader.file.backward(1000) }
 
-  context 'json log file' do
-    let(:file) { 'spec/fixtures/sample-production.log.json' }
-    let(:reader) { Turnstile::Collector::LogReader.json_formatted(file, queue) }
+      context 'json log file' do
+        let(:file) { 'spec/fixtures/sample-production.log.json' }
+        let(:reader) { Turnstile::Collector::LogReader.json_formatted(file, queue) }
 
-    let(:expected_uniques) { 28 }
-    let(:expected_total) { 31 }
-    let(:expected_key) { 'ipad:69.61.173.104:5462583' }
+        let(:expected_uniques) { 28 }
+        let(:expected_total) { 31 }
+        let(:expected_key) { 'ipad:69.61.173.104:5462583' }
 
-    context '#read' do
-      it 'should be able to read and parse IPs from a static file' do
-        expect(counter).to eql(expected_total)
-        expect(hash.keys.size).to eql(expected_uniques)
-        expect(hash.keys).to include(expected_key)
-      end
-    end
+        context '#read' do
+          it 'should be able to read and parse IPs from a static file' do
+            expect(counter).to eql(expected_total)
+            expect(hash.keys.size).to eql(expected_uniques)
+            expect(hash.keys).to include(expected_key)
+          end
+        end
 
-    context '#process!' do
-      it 'should read values into the queue' do
-        run_reader(read_timeout) { reader.process! }
-        expect(queue.size).to eql(31)
+        context '#process!' do
+          it 'should read values into the queue' do
+            run_reader(read_timeout) { reader.execute }
+            expect(queue.size).to eql(31)
+          end
+        end
       end
-    end
-  end
 
-  context 'pipe delimited file' do
-    let(:file) { 'spec/fixtures/sample-production.log' }
-    let(:reader) { Turnstile::Collector::LogReader.pipe_delimited(file, queue) }
+      context 'pipe delimited file' do
+        let(:file) { 'spec/fixtures/sample-production.log' }
+        let(:reader) { Turnstile::Collector::LogReader.pipe_delimited(file, queue) }
 
-    let(:log_reader) { Turnstile::Collector::LogReader }
+        let(:log_reader) { Turnstile::Collector::LogReader }
 
-    let(:expected_uniques) { 2 }
-    let(:expected_total) { 4 }
-    let(:expected_key) { 'desktop:124.5.4.3:AF39945f8f87F' }
+        let(:expected_uniques) { 2 }
+        let(:expected_total) { 4 }
+        let(:expected_key) { 'desktop:124.5.4.3:AF39945f8f87F' }
 
-    context 'matcher' do
-      subject(:matcher) { log_reader.delimited_matcher }
+        context 'matcher' do
+          subject(:matcher) { log_reader.delimited_matcher }
 
-      its(:regexp) { should_not be_nil }
+          its(:regexp) { should_not be_nil }
 
-      its(:extractor) { should_not be_nil }
-      its(:extractor) { should  be_kind_of(Proc) }
+          its(:extractor) { should_not be_nil }
+          its(:extractor) { should be_kind_of(Proc) }
 
-      it 'should match lines in the file' do
-        File.open(file).each do |line|
-          expect(line).to match(matcher.regexp)
+          it 'should match lines in the file' do
+            File.open(file).each do |line|
+              expect(line).to match(matcher.regexp)
+            end
+          end
+          it 'should extract the token from file' do
+            File.open(file).each do |line|
+              expect(matcher.tokenize(line)).to_not be_nil
+            end
+          end
         end
-      end
-      it 'should extract the token from file' do
-        File.open(file).each do |line|
-          expect(matcher.token_from(line)).to_not be_nil
+
+        context '#read' do
+          it 'should be load all matching rows' do
+            expect(counter).to eql(expected_total)
+            expect(hash.keys.size).to eql(expected_uniques)
+            expect(hash.keys).to include(expected_key)
+          end
         end
-      end
-    end
 
-    context '#read' do
-      it 'should be load all matching rows' do
-        expect(counter).to eql(expected_total)
-        expect(hash.keys.size).to eql(expected_uniques)
-        expect(hash.keys).to include(expected_key)
+        context '#process!' do
+          it 'should read values into the queue' do
+            run_reader(read_timeout) { reader.execute }
+            expect(queue.size).to eql(expected_total)
+          end
+        end
       end
-    end
 
-    context '#process!' do
-      it 'should read values into the queue' do
-        run_reader(read_timeout) { reader.process! }
-        expect(queue.size).to eql(expected_total)
+      context 'custom format' do
+        before do
+          Configuration.from_file(Dir.pwd + '/example/custom_csv_matcher.rb')
+          expect(Turnstile.config.custom_matcher).to_not be_nil
+        end
+
+        it 'should set custom matcher' do
+          expect(Turnstile.config.to_h[:custom_matcher]).to_not be_nil
+        end
+
+        let(:file) { Dir.pwd + '/spec/fixtures/custom-production.log' }
+        let(:reader) { Turnstile::Collector::LogReader.custom(file, queue) }
+        let(:log_reader) { Turnstile::Collector::LogReader }
+
+        let(:expected_uniques) { 2 }
+        let(:expected_total) { 2 }
+        let(:expected_key) { 'android:47.23.6.197:BGHSsdCsX5VvsN1jLsAR' }
+
+        context 'matcher' do
+          subject(:matcher) { log_reader.custom_matcher }
+
+          it { is_expected.to_not be_nil }
+
+          it 'should match lines in the file' do
+            File.open(file).each do |line|
+              expect(matcher.matches?(line)).to be_truthy
+            end
+          end
+
+          it 'should extract the token from file' do
+            File.open(file).each do |line|
+              expect(matcher.tokenize(line)).to_not be_nil
+            end
+          end
+        end
+
+        context '#read' do
+          it 'should be load all matching rows' do
+            expect(hash.keys.size).to eql(expected_uniques)
+            expect(hash.keys).to include(expected_key)
+            expect(counter).to eql(expected_total)
+          end
+        end
       end
     end
   end
 end
+