turnstile-rb 2.0.1 → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +3 -0
  3. data/.travis.yml +2 -2
  4. data/LICENSE.txt +2 -1
  5. data/README.md +200 -11
  6. data/Rakefile +1 -0
  7. data/bin/turnstile +4 -6
  8. data/example/custom_csv_matcher.rb +22 -0
  9. data/lib/turnstile.rb +37 -8
  10. data/lib/turnstile/cli/launcher.rb +83 -0
  11. data/lib/turnstile/cli/parser.rb +166 -0
  12. data/lib/turnstile/cli/runner.rb +58 -0
  13. data/lib/turnstile/collector.rb +2 -2
  14. data/lib/turnstile/collector/actor.rb +81 -0
  15. data/lib/turnstile/collector/controller.rb +121 -0
  16. data/lib/turnstile/collector/flusher.rb +36 -0
  17. data/lib/turnstile/collector/formats.rb +7 -34
  18. data/lib/turnstile/collector/formats/custom_matcher.rb +19 -0
  19. data/lib/turnstile/collector/formats/delimited_matcher.rb +30 -0
  20. data/lib/turnstile/collector/formats/json_matcher.rb +30 -0
  21. data/lib/turnstile/collector/log_reader.rb +84 -46
  22. data/lib/turnstile/collector/{matcher.rb → regexp_matcher.rb} +4 -5
  23. data/lib/turnstile/collector/session.rb +7 -0
  24. data/lib/turnstile/commands.rb +20 -0
  25. data/lib/turnstile/commands/base.rb +20 -0
  26. data/lib/turnstile/commands/flushdb.rb +21 -0
  27. data/lib/turnstile/commands/print_keys.rb +19 -0
  28. data/lib/turnstile/commands/show.rb +89 -0
  29. data/lib/turnstile/configuration.rb +23 -0
  30. data/lib/turnstile/dependencies.rb +31 -0
  31. data/lib/turnstile/logger.rb +9 -40
  32. data/lib/turnstile/logger/helper.rb +42 -0
  33. data/lib/turnstile/logger/provider.rb +74 -0
  34. data/lib/turnstile/observer.rb +5 -9
  35. data/lib/turnstile/redis/adapter.rb +97 -0
  36. data/lib/turnstile/redis/connection.rb +116 -0
  37. data/lib/turnstile/redis/spy.rb +42 -0
  38. data/lib/turnstile/sampler.rb +9 -2
  39. data/lib/turnstile/tracker.rb +14 -14
  40. data/lib/turnstile/version.rb +51 -12
  41. data/lib/turnstile/web_app.rb +29 -0
  42. data/spec/spec_helper.rb +18 -3
  43. data/spec/support/logging.rb +17 -0
  44. data/spec/turnstile/adapter_spec.rb +59 -46
  45. data/spec/turnstile/collector/flusher_spec.rb +16 -0
  46. data/spec/turnstile/collector/log_reader_spec.rb +127 -77
  47. data/spec/turnstile/commands/show_spec.rb +40 -0
  48. data/spec/turnstile/tracker_spec.rb +21 -7
  49. data/spec/turnstile_spec.rb +3 -0
  50. data/turnstile-rb.gemspec +5 -2
  51. metadata +89 -22
  52. data/Gemfile +0 -6
  53. data/lib/turnstile/adapter.rb +0 -61
  54. data/lib/turnstile/collector/runner.rb +0 -72
  55. data/lib/turnstile/collector/updater.rb +0 -86
  56. data/lib/turnstile/parser.rb +0 -107
  57. data/lib/turnstile/runner.rb +0 -54
  58. data/lib/turnstile/summary.rb +0 -57
  59. data/spec/turnstile/summary_spec.rb +0 -41
@@ -0,0 +1,42 @@
1
+ module Turnstile
2
+ module Redis
3
+ class Spy < Struct.new(:redis)
4
+ @stream = STDOUT
5
+ @disable_color = false
6
+
7
+ class << self
8
+ # in case someone might prefer STDERR, feel free to set
9
+ # it in the gem configuration:
10
+ # Turnstile::Redis::Spy.stream = STDOUT | STDERR | etc...
11
+ attr_accessor :stream, :disable_color
12
+ end
13
+
14
+
15
+ def method_missing(m, *args, &block)
16
+ if redis.respond_to?(m)
17
+ t1 = Time.now
18
+ result = redis.send(m, *args, &block)
19
+ delta = Time.now - t1
20
+ colors = [:blue, nil, :blue, :blue, :yellow, :cyan, nil, :blue]
21
+
22
+ components = [
23
+ Time.now.strftime('%H:%M:%S.%L'), ' rtt=',
24
+ (sprintf '%.5f', delta*1000), ' ms ',
25
+ (sprintf '%15s ', m.to_s.upcase),
26
+ (sprintf '%-40s', args.inspect.gsub(/[",\[\]]/, '')), ' ⇒ ',
27
+ (result.is_a?(::Redis::Future) ? '' : result.to_s)]
28
+
29
+ components.each_with_index do |component, index|
30
+ color = self.class.disable_color ? nil : colors[index]
31
+ component = component.send(color) if color
32
+ self.class.stream.printf component
33
+ end
34
+ self.class.stream.puts
35
+ result
36
+ else
37
+ super
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -1,5 +1,10 @@
1
+ require_relative 'dependencies'
2
+
1
3
  module Turnstile
2
4
  class Sampler
5
+
6
+ include Dependencies
7
+
3
8
  def extrapolate(n)
4
9
  (n * 100.0 / sampling_rate).to_i
5
10
  end
@@ -11,10 +16,12 @@ module Turnstile
11
16
  ((uid.hash + Time.now.day) % 100) < sampling_rate
12
17
  end
13
18
 
14
- private
19
+ def sampling?
20
+ sampling_rate && sampling_rate <= 100 && sampling_rate >= 0
21
+ end
15
22
 
16
23
  def sampling_rate
17
- Turnstile.config.sampling_rate
24
+ config.sampling_rate
18
25
  end
19
26
  end
20
27
  end
@@ -1,28 +1,28 @@
1
+ require_relative 'dependencies'
2
+
1
3
  module Turnstile
2
4
  class Tracker
3
- def sample(uid, platform = 'unknown', ip = nil)
4
- adapter.add(uid, platform, ip) if sampler.sample(uid)
5
+
6
+ include Dependencies
7
+
8
+ def track_and_sample(uid, platform = 'unknown', ip = nil)
9
+ track_all(uid, platform, ip) if should_track?(uid)
5
10
  end
6
11
 
7
- alias track sample
12
+ def should_track?(uid)
13
+ !sampler.sampling? || sampler.sample(uid)
14
+ end
8
15
 
9
- def add(uid, platform = 'unknown', ip = nil)
16
+ def track_all(uid, platform = 'unknown', ip = nil)
10
17
  adapter.add(uid, platform, ip)
11
18
  end
12
19
 
13
- def add_token(token, delimiter = ':')
20
+ def track_token(token, delimiter = nil)
21
+ delimiter ||= ':'
14
22
  platform, ip, uid = token.split(delimiter)
15
23
  adapter.add(uid, platform, ip) if uid
16
24
  end
17
25
 
18
- private
19
-
20
- def adapter
21
- @adapter ||= Adapter.new
22
- end
23
-
24
- def sampler
25
- @sampler ||= Sampler.new
26
- end
26
+ alias track track_and_sample
27
27
  end
28
28
  end
@@ -1,21 +1,60 @@
1
1
  module Turnstile
2
- VERSION = '2.0.1'
2
+ VERSION = '3.0.2'
3
+ DEFAULT_PORT = 9090
3
4
 
4
5
  GEM_DESCRIPTION = <<-EOF
5
- Turnstile is a Redis-based library that can accurately track total number of concurrent
6
- users accessing a web/API based server application. It can break it down by "platform"
7
- or a device type, and returns data in JSON, CSV of NAD formats. While user tracking
8
- may happen synchronously using a Rack middleware, another method is provided that is
9
- based on log file analysis, and can therefore be performed outside web server process.
6
+ Turnstile is a Redis-based library that can accurately track total number
7
+ of concurrent users accessing a web/API based server application. It can
8
+ break it down by "platform" or a device type, and returns data in JSON,
9
+ CSV of NAD formats. While user tracking may happen synchronously using a
10
+ Rack middleware, another method is provided that is based on log file
11
+ analysis, and can therefore be performed outside web server process.
12
+
13
+
10
14
  EOF
11
15
 
12
16
  DESCRIPTION = <<-EOF
13
- Turnstile can run as a daemon, in which mode it monitors a given log file.
14
- Alternatively, turnstile binary can be used to print current stats, and even
15
- add new data into the registry.
16
-
17
- If you are using Turnstile to tail log files, make sure you run on each app sever
18
- that's generating log files.
17
+ Turnstile can be run as a daemon, in which case it watches a given log
18
+ file. Or, you can run turnstile to print the current aggregated stats in
19
+ several supported formats, such as JSON.
20
+
21
+ When Turnstile is used to tail the log files, ideally you should start
22
+ turnstile daemon on each app sever that's generating log file, or be
23
+ content with the effects of sampling.
24
+
25
+ Note that the IP address is not required to track uniqueness. Only
26
+ platform and UID are used. Also note that custom formatter can be
27
+ specified in a config file to parse arbitrary complex log lines.
28
+
29
+ For tailing a log files, Turnstile must first match a log line expected to
30
+ contain the tokens, and then extract is using one of the matchers. You can
31
+ specify which matcher to use depending on whether you can add Turnstile's
32
+ tokens to your log or not. If you can, great! If not, implement your own
33
+ custom matcher and great again.
34
+
35
+ The following matchers are available, and can be selected with -F:
36
+
37
+ 1. Format named "delimited", which expects the following token in the
38
+ log file:
39
+
40
+ x-turnstile:platform:ip:user-identifier
41
+
42
+ Where ':' (the delimiter) can be set via -l option, OR you can use one
43
+ of the following formats: "json_formatted", "pipe_formatted",
44
+ "comma_formatted", "colon_formatted" (the default). The match is
45
+ performed on a string "x-turnstile", other log lines are skipped.
46
+
47
+ 2. Format "json_delimited", which expects to find a single-line JSON
48
+ Hash, containing keys "platform", "ip_address", and "user_id". The match
49
+ is performed on any log line containing string '"ip_address"', other
50
+ lines are skipped.
51
+
52
+ 3. Format "custom" requires passing an additional flag -c/--config
53
+ file.rb, which will be required, and which can define a matcher and
54
+ assign it to the `Turnstile.config.custom_matcher` config variable.
55
+
56
+
19
57
  EOF
20
58
 
59
+ NS = "x-turnstile|#{VERSION.gsub(/\./,'')}".freeze
21
60
  end
@@ -0,0 +1,29 @@
1
+ require 'sinatra'
2
+ require_relative 'dependencies'
3
+ require 'json'
4
+ module Turnstile
5
+ class WebApp < ::Sinatra::Base
6
+ SUPPORTED_FORMATS = %i(yaml json nad)
7
+ set :port, Turnstile.config.port
8
+
9
+ include Dependencies
10
+
11
+ set :sessions, false
12
+ set :port, Turnstile.config.port
13
+
14
+ get '/turnstile/:format' do
15
+ fmt = params['format'].to_sym
16
+ if SUPPORTED_FORMATS.include?(fmt)
17
+ status 200
18
+ headers 'Content-type' => fmt.to_s
19
+ body aggregate.send("to_#{fmt}".to_sym)
20
+ else
21
+ status 500
22
+ body "Error: unsupported format #{fmt}!"
23
+ end
24
+ end
25
+
26
+ # start the server if ruby file executed directly
27
+ run!
28
+ end
29
+ end
@@ -9,16 +9,31 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
9
9
 
10
10
  require 'rubygems'
11
11
  require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])
12
- require 'simplecov'
12
+ require 'fileutils'
13
+ require 'rspec/its'
13
14
 
15
+ require 'simplecov'
14
16
  SimpleCov.start
15
17
 
16
18
  require 'turnstile'
17
- require 'rspec/its'
19
+
20
+ require_relative 'support/logging'
18
21
 
19
22
  RSpec.configure do |config|
20
23
  config.order = 'random'
24
+
25
+ config.before :all do
26
+ Turnstile::RSpec::Logging.configure
27
+ end
28
+
21
29
  config.before :each do
22
- Turnstile::Adapter.new.redis.flushdb
30
+ Turnstile::Redis::Adapter.instance
31
+ end
32
+
33
+ config.around :each, logging: true do |ex|
34
+ logging_enabled = Turnstile::Logger.enabled
35
+ Turnstile::Logger.enable
36
+ Turnstile::Logger.info_elapsed('measuring example: ') { ex.run }
37
+ Turnstile::Logger.enabled = logging_enabled
23
38
  end
24
39
  end
@@ -0,0 +1,17 @@
1
+ module Turnstile
2
+ module RSpec
3
+ module Logging
4
+ TEST_LOG = 'spec/log/test.log'
5
+
6
+ def self.configure(file = TEST_LOG)
7
+ return if Turnstile::Logger.enabled
8
+ Turnstile::Logger.enable
9
+ FileUtils.mkdir_p(File.dirname(file))
10
+
11
+ Turnstile::Logger.logger = ::Logger.new(file)
12
+ Turnstile::Logger.logger.level = ::Logger::INFO
13
+ Turnstile::Logger.logger
14
+ end
15
+ end
16
+ end
17
+ end
@@ -1,71 +1,84 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Turnstile::Adapter do
4
-
5
- subject { Turnstile::Adapter.new }
6
-
7
- let(:redis) { subject.send(:redis) }
3
+ describe Turnstile::Redis::Adapter, logging: true do
4
+ subject(:adapter) { Turnstile::Redis::Adapter.instance }
8
5
 
9
6
  let(:uid) { 1238438 }
10
7
  let(:other_uid) { 1238439 }
11
8
  let(:another_uid) { 1238440 }
12
-
13
9
  let(:ip) { '1.2.3.4' }
14
10
  let(:another_ip) { '4.3.2.1' }
15
-
16
11
  let(:platform) { :ios }
17
12
  let(:another_platform) { :android }
18
13
 
19
- describe '#add' do
20
- it 'calls redis with the correct params' do
21
- key = "t:#{uid}:#{platform}:#{ip}"
22
- expect(redis).to receive(:setex).once.with(key, Turnstile.config.activity_interval, 1)
23
- subject.add(uid, platform, ip)
24
- end
25
- end
26
-
27
- describe '#fetch' do
28
- let(:sort_lambda) { ->(a, b) { a[:uid] <=> b[:uid] } }
29
- let(:expected_hash) do
30
- [
31
- { uid: uid.to_s, platform: platform.to_s, ip: ip },
32
- { uid: other_uid.to_s, platform: platform.to_s, ip: ip },
33
- { uid: another_uid.to_s, platform: another_platform.to_s, ip: another_ip },
34
- ]
35
- end
14
+ context 'with fake redis' do
15
+ let(:redis) { double }
36
16
 
37
17
  before do
38
- subject.add(uid, platform, ip)
39
- subject.add(other_uid, platform, ip)
40
- subject.add(another_uid, another_platform, another_ip)
18
+ allow(adapter).to receive(:with_redis).and_yield(redis)
19
+ expect(redis).to receive(:setex).once.with(key, Turnstile.config.activity_interval, 1)
41
20
  end
42
21
 
43
- it 'pulls the platform specific stats from redis' do
44
- expect(subject.fetch.sort(&sort_lambda)).to eq(expected_hash.sort(&sort_lambda))
22
+ describe '#add' do
23
+ let(:key) { adapter.compose_key(uid, platform, ip) }
24
+ it 'calls redis with the correct params' do
25
+ subject.add(uid, platform, ip)
26
+ end
45
27
  end
46
28
  end
47
29
 
48
- describe '#aggregate' do
49
- let(:expected_hash) do
50
- {
51
- 'android' => 3,
52
- 'ios' => 2,
53
- 'total' => 5
54
- }
55
- end
30
+ context 'with real redis' do
31
+ before { adapter.flushdb }
32
+ describe '#fetch' do
33
+ let(:sort_lambda) { ->(a, b) { a[:uid] <=> b[:uid] } }
34
+ let(:expected_hash) do
35
+ [
36
+ { uid: uid.to_s, platform: platform.to_s, ip: ip },
37
+ { uid: other_uid.to_s, platform: platform.to_s, ip: ip },
38
+ { uid: another_uid.to_s, platform: another_platform.to_s, ip: another_ip },
39
+ ]
40
+ end
56
41
 
57
- before do
58
- subject.add(123, :android, ip)
59
- subject.add(124, :android, ip)
60
- subject.add(125, :android, ip)
42
+ before do
43
+ subject.add(uid, platform, ip)
44
+ subject.add(other_uid, platform, ip)
45
+ subject.add(another_uid, another_platform, another_ip)
46
+ end
61
47
 
62
- subject.add(200, :ios, ip)
63
- subject.add(201, :ios, ip)
48
+ it 'pulls the platform specific stats from redis' do
49
+ expect(subject.fetch.sort(&sort_lambda)).to eq(expected_hash.sort(&sort_lambda))
50
+ end
64
51
  end
65
52
 
66
- it 'should calculated proper aggregation' do
67
- expect(subject.aggregate).to eql expected_hash
53
+ describe '#aggregate' do
54
+ let(:expected_hash) do
55
+ {
56
+ 'android' => 3,
57
+ 'ios' => 2,
58
+ 'total' => 5
59
+ }
60
+ end
61
+
62
+ before do
63
+ subject.add(123, :android, ip)
64
+ subject.add(124, :android, ip)
65
+ subject.add(125, :android, ip)
66
+
67
+ subject.add(200, :ios, ip)
68
+ subject.add(201, :ios, ip)
69
+ end
70
+
71
+ it 'should calculated proper aggregation' do
72
+ expect(subject.aggregate).to eql expected_hash
73
+ end
74
+
75
+ its(:aggregate) { should_not include({ 'total' => 0 }) }
76
+ its(:prefix) { should match /^x-turnstile\|\d+/ }
77
+
78
+ describe '#flushdb' do
79
+ before { subject.flushdb }
80
+ its(:aggregate) { should include({ 'total' => 0 }) }
81
+ end
68
82
  end
69
83
  end
70
-
71
84
  end
@@ -0,0 +1,16 @@
1
+ require 'spec_helper'
2
+ require 'file/tail'
3
+ require 'timeout'
4
+ require 'thread'
5
+ require 'tempfile'
6
+
7
+
8
+ module Turnstile
9
+ module Collector
10
+ RSpec.describe Flusher do
11
+
12
+
13
+ end
14
+ end
15
+ end
16
+
@@ -5,110 +5,160 @@ require 'thread'
5
5
  require 'tempfile'
6
6
 
7
7
  require 'turnstile/collector/log_reader'
8
+ module Turnstile
9
+ module Collector
8
10
 
9
- describe Turnstile::Collector::LogReader do
10
- include Timeout
11
+ describe LogReader do
12
+ include Timeout
11
13
 
12
- def consume_file(reader, read_timeout)
13
- hash = {}
14
- counter = 0
14
+ def consume_file(reader, read_timeout)
15
+ hash = {}
16
+ counter = 0
15
17
 
16
- run_reader(read_timeout) do
17
- reader.read do |token|
18
- counter += 1
19
- hash[token] = 1
18
+ run_reader(read_timeout) do
19
+ reader.read do |token|
20
+ counter += 1
21
+ hash[token] = 1
22
+ end
23
+ end
24
+ return counter, hash
20
25
  end
21
- end
22
- return counter, hash
23
- end
24
26
 
25
- def run_reader(read_timeout, &block)
26
- t_reader = Thread.new do
27
- begin
28
- timeout(read_timeout) do
29
- block.call
27
+ def run_reader(read_timeout, &block)
28
+ t_reader = Thread.new do
29
+ begin
30
+ timeout(read_timeout) do
31
+ block.call
32
+ end
33
+ rescue Timeout::Error
34
+ end
30
35
  end
31
- rescue Timeout::Error
36
+ t_reader.join
32
37
  end
33
- end
34
- t_reader.join
35
- end
36
38
 
37
- let(:queue) { Queue.new }
38
- let(:read_timeout) { 0.1 }
39
- let(:consume_file_result) { consume_file(reader, read_timeout) }
40
- let(:counter) { consume_file_result.first }
41
- let(:hash) { consume_file_result.last }
39
+ let(:queue) { Queue.new }
40
+ let(:read_timeout) { 0.1 }
41
+ let(:consume_file_result) { consume_file(reader, read_timeout) }
42
+ let(:counter) { consume_file_result.first }
43
+ let(:hash) { consume_file_result.last }
42
44
 
43
- before { reader.file.backward(1000) }
45
+ before { reader.file.backward(1000) }
44
46
 
45
- context 'json log file' do
46
- let(:file) { 'spec/fixtures/sample-production.log.json' }
47
- let(:reader) { Turnstile::Collector::LogReader.json_formatted(file, queue) }
47
+ context 'json log file' do
48
+ let(:file) { 'spec/fixtures/sample-production.log.json' }
49
+ let(:reader) { Turnstile::Collector::LogReader.json_formatted(file, queue) }
48
50
 
49
- let(:expected_uniques) { 28 }
50
- let(:expected_total) { 31 }
51
- let(:expected_key) { 'ipad:69.61.173.104:5462583' }
51
+ let(:expected_uniques) { 28 }
52
+ let(:expected_total) { 31 }
53
+ let(:expected_key) { 'ipad:69.61.173.104:5462583' }
52
54
 
53
- context '#read' do
54
- it 'should be able to read and parse IPs from a static file' do
55
- expect(counter).to eql(expected_total)
56
- expect(hash.keys.size).to eql(expected_uniques)
57
- expect(hash.keys).to include(expected_key)
58
- end
59
- end
55
+ context '#read' do
56
+ it 'should be able to read and parse IPs from a static file' do
57
+ expect(counter).to eql(expected_total)
58
+ expect(hash.keys.size).to eql(expected_uniques)
59
+ expect(hash.keys).to include(expected_key)
60
+ end
61
+ end
60
62
 
61
- context '#process!' do
62
- it 'should read values into the queue' do
63
- run_reader(read_timeout) { reader.process! }
64
- expect(queue.size).to eql(31)
63
+ context '#process!' do
64
+ it 'should read values into the queue' do
65
+ run_reader(read_timeout) { reader.execute }
66
+ expect(queue.size).to eql(31)
67
+ end
68
+ end
65
69
  end
66
- end
67
- end
68
70
 
69
- context 'pipe delimited file' do
70
- let(:file) { 'spec/fixtures/sample-production.log' }
71
- let(:reader) { Turnstile::Collector::LogReader.pipe_delimited(file, queue) }
71
+ context 'pipe delimited file' do
72
+ let(:file) { 'spec/fixtures/sample-production.log' }
73
+ let(:reader) { Turnstile::Collector::LogReader.pipe_delimited(file, queue) }
72
74
 
73
- let(:log_reader) { Turnstile::Collector::LogReader }
75
+ let(:log_reader) { Turnstile::Collector::LogReader }
74
76
 
75
- let(:expected_uniques) { 2 }
76
- let(:expected_total) { 4 }
77
- let(:expected_key) { 'desktop:124.5.4.3:AF39945f8f87F' }
77
+ let(:expected_uniques) { 2 }
78
+ let(:expected_total) { 4 }
79
+ let(:expected_key) { 'desktop:124.5.4.3:AF39945f8f87F' }
78
80
 
79
- context 'matcher' do
80
- subject(:matcher) { log_reader.delimited_matcher }
81
+ context 'matcher' do
82
+ subject(:matcher) { log_reader.delimited_matcher }
81
83
 
82
- its(:regexp) { should_not be_nil }
84
+ its(:regexp) { should_not be_nil }
83
85
 
84
- its(:extractor) { should_not be_nil }
85
- its(:extractor) { should be_kind_of(Proc) }
86
+ its(:extractor) { should_not be_nil }
87
+ its(:extractor) { should be_kind_of(Proc) }
86
88
 
87
- it 'should match lines in the file' do
88
- File.open(file).each do |line|
89
- expect(line).to match(matcher.regexp)
89
+ it 'should match lines in the file' do
90
+ File.open(file).each do |line|
91
+ expect(line).to match(matcher.regexp)
92
+ end
93
+ end
94
+ it 'should extract the token from file' do
95
+ File.open(file).each do |line|
96
+ expect(matcher.tokenize(line)).to_not be_nil
97
+ end
98
+ end
90
99
  end
91
- end
92
- it 'should extract the token from file' do
93
- File.open(file).each do |line|
94
- expect(matcher.token_from(line)).to_not be_nil
100
+
101
+ context '#read' do
102
+ it 'should be load all matching rows' do
103
+ expect(counter).to eql(expected_total)
104
+ expect(hash.keys.size).to eql(expected_uniques)
105
+ expect(hash.keys).to include(expected_key)
106
+ end
95
107
  end
96
- end
97
- end
98
108
 
99
- context '#read' do
100
- it 'should be load all matching rows' do
101
- expect(counter).to eql(expected_total)
102
- expect(hash.keys.size).to eql(expected_uniques)
103
- expect(hash.keys).to include(expected_key)
109
+ context '#process!' do
110
+ it 'should read values into the queue' do
111
+ run_reader(read_timeout) { reader.execute }
112
+ expect(queue.size).to eql(expected_total)
113
+ end
114
+ end
104
115
  end
105
- end
106
116
 
107
- context '#process!' do
108
- it 'should read values into the queue' do
109
- run_reader(read_timeout) { reader.process! }
110
- expect(queue.size).to eql(expected_total)
117
+ context 'custom format' do
118
+ before do
119
+ Configuration.from_file(Dir.pwd + '/example/custom_csv_matcher.rb')
120
+ expect(Turnstile.config.custom_matcher).to_not be_nil
121
+ end
122
+
123
+ it 'should set custom matcher' do
124
+ expect(Turnstile.config.to_h[:custom_matcher]).to_not be_nil
125
+ end
126
+
127
+ let(:file) { Dir.pwd + '/spec/fixtures/custom-production.log' }
128
+ let(:reader) { Turnstile::Collector::LogReader.custom(file, queue) }
129
+ let(:log_reader) { Turnstile::Collector::LogReader }
130
+
131
+ let(:expected_uniques) { 2 }
132
+ let(:expected_total) { 2 }
133
+ let(:expected_key) { 'android:47.23.6.197:BGHSsdCsX5VvsN1jLsAR' }
134
+
135
+ context 'matcher' do
136
+ subject(:matcher) { log_reader.custom_matcher }
137
+
138
+ it { is_expected.to_not be_nil }
139
+
140
+ it 'should match lines in the file' do
141
+ File.open(file).each do |line|
142
+ expect(matcher.matches?(line)).to be_truthy
143
+ end
144
+ end
145
+
146
+ it 'should extract the token from file' do
147
+ File.open(file).each do |line|
148
+ expect(matcher.tokenize(line)).to_not be_nil
149
+ end
150
+ end
151
+ end
152
+
153
+ context '#read' do
154
+ it 'should be load all matching rows' do
155
+ expect(hash.keys.size).to eql(expected_uniques)
156
+ expect(hash.keys).to include(expected_key)
157
+ expect(counter).to eql(expected_total)
158
+ end
159
+ end
111
160
  end
112
161
  end
113
162
  end
114
163
  end
164
+