turnstile-rb 2.0.1 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +3 -0
- data/.travis.yml +2 -2
- data/LICENSE.txt +2 -1
- data/README.md +200 -11
- data/Rakefile +1 -0
- data/bin/turnstile +4 -6
- data/example/custom_csv_matcher.rb +22 -0
- data/lib/turnstile.rb +37 -8
- data/lib/turnstile/cli/launcher.rb +83 -0
- data/lib/turnstile/cli/parser.rb +166 -0
- data/lib/turnstile/cli/runner.rb +58 -0
- data/lib/turnstile/collector.rb +2 -2
- data/lib/turnstile/collector/actor.rb +81 -0
- data/lib/turnstile/collector/controller.rb +121 -0
- data/lib/turnstile/collector/flusher.rb +36 -0
- data/lib/turnstile/collector/formats.rb +7 -34
- data/lib/turnstile/collector/formats/custom_matcher.rb +19 -0
- data/lib/turnstile/collector/formats/delimited_matcher.rb +30 -0
- data/lib/turnstile/collector/formats/json_matcher.rb +30 -0
- data/lib/turnstile/collector/log_reader.rb +84 -46
- data/lib/turnstile/collector/{matcher.rb → regexp_matcher.rb} +4 -5
- data/lib/turnstile/collector/session.rb +7 -0
- data/lib/turnstile/commands.rb +20 -0
- data/lib/turnstile/commands/base.rb +20 -0
- data/lib/turnstile/commands/flushdb.rb +21 -0
- data/lib/turnstile/commands/print_keys.rb +19 -0
- data/lib/turnstile/commands/show.rb +89 -0
- data/lib/turnstile/configuration.rb +23 -0
- data/lib/turnstile/dependencies.rb +31 -0
- data/lib/turnstile/logger.rb +9 -40
- data/lib/turnstile/logger/helper.rb +42 -0
- data/lib/turnstile/logger/provider.rb +74 -0
- data/lib/turnstile/observer.rb +5 -9
- data/lib/turnstile/redis/adapter.rb +97 -0
- data/lib/turnstile/redis/connection.rb +116 -0
- data/lib/turnstile/redis/spy.rb +42 -0
- data/lib/turnstile/sampler.rb +9 -2
- data/lib/turnstile/tracker.rb +14 -14
- data/lib/turnstile/version.rb +51 -12
- data/lib/turnstile/web_app.rb +29 -0
- data/spec/spec_helper.rb +18 -3
- data/spec/support/logging.rb +17 -0
- data/spec/turnstile/adapter_spec.rb +59 -46
- data/spec/turnstile/collector/flusher_spec.rb +16 -0
- data/spec/turnstile/collector/log_reader_spec.rb +127 -77
- data/spec/turnstile/commands/show_spec.rb +40 -0
- data/spec/turnstile/tracker_spec.rb +21 -7
- data/spec/turnstile_spec.rb +3 -0
- data/turnstile-rb.gemspec +5 -2
- metadata +89 -22
- data/Gemfile +0 -6
- data/lib/turnstile/adapter.rb +0 -61
- data/lib/turnstile/collector/runner.rb +0 -72
- data/lib/turnstile/collector/updater.rb +0 -86
- data/lib/turnstile/parser.rb +0 -107
- data/lib/turnstile/runner.rb +0 -54
- data/lib/turnstile/summary.rb +0 -57
- data/spec/turnstile/summary_spec.rb +0 -41
@@ -0,0 +1,42 @@
|
|
1
|
+
module Turnstile
|
2
|
+
module Redis
|
3
|
+
class Spy < Struct.new(:redis)
|
4
|
+
@stream = STDOUT
|
5
|
+
@disable_color = false
|
6
|
+
|
7
|
+
class << self
|
8
|
+
# in case someone might prefer STDERR, feel free to set
|
9
|
+
# it in the gem configuration:
|
10
|
+
# Turnstile::Redis::Spy.stream = STDOUT | STDERR | etc...
|
11
|
+
attr_accessor :stream, :disable_color
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
def method_missing(m, *args, &block)
|
16
|
+
if redis.respond_to?(m)
|
17
|
+
t1 = Time.now
|
18
|
+
result = redis.send(m, *args, &block)
|
19
|
+
delta = Time.now - t1
|
20
|
+
colors = [:blue, nil, :blue, :blue, :yellow, :cyan, nil, :blue]
|
21
|
+
|
22
|
+
components = [
|
23
|
+
Time.now.strftime('%H:%M:%S.%L'), ' rtt=',
|
24
|
+
(sprintf '%.5f', delta*1000), ' ms ',
|
25
|
+
(sprintf '%15s ', m.to_s.upcase),
|
26
|
+
(sprintf '%-40s', args.inspect.gsub(/[",\[\]]/, '')), ' ⇒ ',
|
27
|
+
(result.is_a?(::Redis::Future) ? '' : result.to_s)]
|
28
|
+
|
29
|
+
components.each_with_index do |component, index|
|
30
|
+
color = self.class.disable_color ? nil : colors[index]
|
31
|
+
component = component.send(color) if color
|
32
|
+
self.class.stream.printf component
|
33
|
+
end
|
34
|
+
self.class.stream.puts
|
35
|
+
result
|
36
|
+
else
|
37
|
+
super
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/lib/turnstile/sampler.rb
CHANGED
@@ -1,5 +1,10 @@
|
|
1
|
+
require_relative 'dependencies'
|
2
|
+
|
1
3
|
module Turnstile
|
2
4
|
class Sampler
|
5
|
+
|
6
|
+
include Dependencies
|
7
|
+
|
3
8
|
def extrapolate(n)
|
4
9
|
(n * 100.0 / sampling_rate).to_i
|
5
10
|
end
|
@@ -11,10 +16,12 @@ module Turnstile
|
|
11
16
|
((uid.hash + Time.now.day) % 100) < sampling_rate
|
12
17
|
end
|
13
18
|
|
14
|
-
|
19
|
+
def sampling?
|
20
|
+
sampling_rate && sampling_rate <= 100 && sampling_rate >= 0
|
21
|
+
end
|
15
22
|
|
16
23
|
def sampling_rate
|
17
|
-
|
24
|
+
config.sampling_rate
|
18
25
|
end
|
19
26
|
end
|
20
27
|
end
|
data/lib/turnstile/tracker.rb
CHANGED
@@ -1,28 +1,28 @@
|
|
1
|
+
require_relative 'dependencies'
|
2
|
+
|
1
3
|
module Turnstile
|
2
4
|
class Tracker
|
3
|
-
|
4
|
-
|
5
|
+
|
6
|
+
include Dependencies
|
7
|
+
|
8
|
+
def track_and_sample(uid, platform = 'unknown', ip = nil)
|
9
|
+
track_all(uid, platform, ip) if should_track?(uid)
|
5
10
|
end
|
6
11
|
|
7
|
-
|
12
|
+
def should_track?(uid)
|
13
|
+
!sampler.sampling? || sampler.sample(uid)
|
14
|
+
end
|
8
15
|
|
9
|
-
def
|
16
|
+
def track_all(uid, platform = 'unknown', ip = nil)
|
10
17
|
adapter.add(uid, platform, ip)
|
11
18
|
end
|
12
19
|
|
13
|
-
def
|
20
|
+
def track_token(token, delimiter = nil)
|
21
|
+
delimiter ||= ':'
|
14
22
|
platform, ip, uid = token.split(delimiter)
|
15
23
|
adapter.add(uid, platform, ip) if uid
|
16
24
|
end
|
17
25
|
|
18
|
-
|
19
|
-
|
20
|
-
def adapter
|
21
|
-
@adapter ||= Adapter.new
|
22
|
-
end
|
23
|
-
|
24
|
-
def sampler
|
25
|
-
@sampler ||= Sampler.new
|
26
|
-
end
|
26
|
+
alias track track_and_sample
|
27
27
|
end
|
28
28
|
end
|
data/lib/turnstile/version.rb
CHANGED
@@ -1,21 +1,60 @@
|
|
1
1
|
module Turnstile
|
2
|
-
VERSION = '
|
2
|
+
VERSION = '3.0.2'
|
3
|
+
DEFAULT_PORT = 9090
|
3
4
|
|
4
5
|
GEM_DESCRIPTION = <<-EOF
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
Turnstile is a Redis-based library that can accurately track total number
|
7
|
+
of concurrent users accessing a web/API based server application. It can
|
8
|
+
break it down by "platform" or a device type, and returns data in JSON,
|
9
|
+
CSV of NAD formats. While user tracking may happen synchronously using a
|
10
|
+
Rack middleware, another method is provided that is based on log file
|
11
|
+
analysis, and can therefore be performed outside web server process.
|
12
|
+
|
13
|
+
|
10
14
|
EOF
|
11
15
|
|
12
16
|
DESCRIPTION = <<-EOF
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
17
|
+
Turnstile can be run as a daemon, in which case it watches a given log
|
18
|
+
file. Or, you can run turnstile to print the current aggregated stats in
|
19
|
+
several supported formats, such as JSON.
|
20
|
+
|
21
|
+
When Turnstile is used to tail the log files, ideally you should start
|
22
|
+
turnstile daemon on each app sever that's generating log file, or be
|
23
|
+
content with the effects of sampling.
|
24
|
+
|
25
|
+
Note that the IP address is not required to track uniqueness. Only
|
26
|
+
platform and UID are used. Also note that custom formatter can be
|
27
|
+
specified in a config file to parse arbitrary complex log lines.
|
28
|
+
|
29
|
+
For tailing a log files, Turnstile must first match a log line expected to
|
30
|
+
contain the tokens, and then extract is using one of the matchers. You can
|
31
|
+
specify which matcher to use depending on whether you can add Turnstile's
|
32
|
+
tokens to your log or not. If you can, great! If not, implement your own
|
33
|
+
custom matcher and great again.
|
34
|
+
|
35
|
+
The following matchers are available, and can be selected with -F:
|
36
|
+
|
37
|
+
1. Format named "delimited", which expects the following token in the
|
38
|
+
log file:
|
39
|
+
|
40
|
+
x-turnstile:platform:ip:user-identifier
|
41
|
+
|
42
|
+
Where ':' (the delimiter) can be set via -l option, OR you can use one
|
43
|
+
of the following formats: "json_formatted", "pipe_formatted",
|
44
|
+
"comma_formatted", "colon_formatted" (the default). The match is
|
45
|
+
performed on a string "x-turnstile", other log lines are skipped.
|
46
|
+
|
47
|
+
2. Format "json_delimited", which expects to find a single-line JSON
|
48
|
+
Hash, containing keys "platform", "ip_address", and "user_id". The match
|
49
|
+
is performed on any log line containing string '"ip_address"', other
|
50
|
+
lines are skipped.
|
51
|
+
|
52
|
+
3. Format "custom" requires passing an additional flag -c/--config
|
53
|
+
file.rb, which will be required, and which can define a matcher and
|
54
|
+
assign it to the `Turnstile.config.custom_matcher` config variable.
|
55
|
+
|
56
|
+
|
19
57
|
EOF
|
20
58
|
|
59
|
+
NS = "x-turnstile|#{VERSION.gsub(/\./,'')}".freeze
|
21
60
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'sinatra'
|
2
|
+
require_relative 'dependencies'
|
3
|
+
require 'json'
|
4
|
+
module Turnstile
|
5
|
+
class WebApp < ::Sinatra::Base
|
6
|
+
SUPPORTED_FORMATS = %i(yaml json nad)
|
7
|
+
set :port, Turnstile.config.port
|
8
|
+
|
9
|
+
include Dependencies
|
10
|
+
|
11
|
+
set :sessions, false
|
12
|
+
set :port, Turnstile.config.port
|
13
|
+
|
14
|
+
get '/turnstile/:format' do
|
15
|
+
fmt = params['format'].to_sym
|
16
|
+
if SUPPORTED_FORMATS.include?(fmt)
|
17
|
+
status 200
|
18
|
+
headers 'Content-type' => fmt.to_s
|
19
|
+
body aggregate.send("to_#{fmt}".to_sym)
|
20
|
+
else
|
21
|
+
status 500
|
22
|
+
body "Error: unsupported format #{fmt}!"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# start the server if ruby file executed directly
|
27
|
+
run!
|
28
|
+
end
|
29
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -9,16 +9,31 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
|
|
9
9
|
|
10
10
|
require 'rubygems'
|
11
11
|
require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])
|
12
|
-
require '
|
12
|
+
require 'fileutils'
|
13
|
+
require 'rspec/its'
|
13
14
|
|
15
|
+
require 'simplecov'
|
14
16
|
SimpleCov.start
|
15
17
|
|
16
18
|
require 'turnstile'
|
17
|
-
|
19
|
+
|
20
|
+
require_relative 'support/logging'
|
18
21
|
|
19
22
|
RSpec.configure do |config|
|
20
23
|
config.order = 'random'
|
24
|
+
|
25
|
+
config.before :all do
|
26
|
+
Turnstile::RSpec::Logging.configure
|
27
|
+
end
|
28
|
+
|
21
29
|
config.before :each do
|
22
|
-
Turnstile::Adapter.
|
30
|
+
Turnstile::Redis::Adapter.instance
|
31
|
+
end
|
32
|
+
|
33
|
+
config.around :each, logging: true do |ex|
|
34
|
+
logging_enabled = Turnstile::Logger.enabled
|
35
|
+
Turnstile::Logger.enable
|
36
|
+
Turnstile::Logger.info_elapsed('measuring example: ') { ex.run }
|
37
|
+
Turnstile::Logger.enabled = logging_enabled
|
23
38
|
end
|
24
39
|
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Turnstile
|
2
|
+
module RSpec
|
3
|
+
module Logging
|
4
|
+
TEST_LOG = 'spec/log/test.log'
|
5
|
+
|
6
|
+
def self.configure(file = TEST_LOG)
|
7
|
+
return if Turnstile::Logger.enabled
|
8
|
+
Turnstile::Logger.enable
|
9
|
+
FileUtils.mkdir_p(File.dirname(file))
|
10
|
+
|
11
|
+
Turnstile::Logger.logger = ::Logger.new(file)
|
12
|
+
Turnstile::Logger.logger.level = ::Logger::INFO
|
13
|
+
Turnstile::Logger.logger
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -1,71 +1,84 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe Turnstile::Adapter do
|
4
|
-
|
5
|
-
subject { Turnstile::Adapter.new }
|
6
|
-
|
7
|
-
let(:redis) { subject.send(:redis) }
|
3
|
+
describe Turnstile::Redis::Adapter, logging: true do
|
4
|
+
subject(:adapter) { Turnstile::Redis::Adapter.instance }
|
8
5
|
|
9
6
|
let(:uid) { 1238438 }
|
10
7
|
let(:other_uid) { 1238439 }
|
11
8
|
let(:another_uid) { 1238440 }
|
12
|
-
|
13
9
|
let(:ip) { '1.2.3.4' }
|
14
10
|
let(:another_ip) { '4.3.2.1' }
|
15
|
-
|
16
11
|
let(:platform) { :ios }
|
17
12
|
let(:another_platform) { :android }
|
18
13
|
|
19
|
-
|
20
|
-
|
21
|
-
key = "t:#{uid}:#{platform}:#{ip}"
|
22
|
-
expect(redis).to receive(:setex).once.with(key, Turnstile.config.activity_interval, 1)
|
23
|
-
subject.add(uid, platform, ip)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
describe '#fetch' do
|
28
|
-
let(:sort_lambda) { ->(a, b) { a[:uid] <=> b[:uid] } }
|
29
|
-
let(:expected_hash) do
|
30
|
-
[
|
31
|
-
{ uid: uid.to_s, platform: platform.to_s, ip: ip },
|
32
|
-
{ uid: other_uid.to_s, platform: platform.to_s, ip: ip },
|
33
|
-
{ uid: another_uid.to_s, platform: another_platform.to_s, ip: another_ip },
|
34
|
-
]
|
35
|
-
end
|
14
|
+
context 'with fake redis' do
|
15
|
+
let(:redis) { double }
|
36
16
|
|
37
17
|
before do
|
38
|
-
|
39
|
-
|
40
|
-
subject.add(another_uid, another_platform, another_ip)
|
18
|
+
allow(adapter).to receive(:with_redis).and_yield(redis)
|
19
|
+
expect(redis).to receive(:setex).once.with(key, Turnstile.config.activity_interval, 1)
|
41
20
|
end
|
42
21
|
|
43
|
-
|
44
|
-
|
22
|
+
describe '#add' do
|
23
|
+
let(:key) { adapter.compose_key(uid, platform, ip) }
|
24
|
+
it 'calls redis with the correct params' do
|
25
|
+
subject.add(uid, platform, ip)
|
26
|
+
end
|
45
27
|
end
|
46
28
|
end
|
47
29
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
30
|
+
context 'with real redis' do
|
31
|
+
before { adapter.flushdb }
|
32
|
+
describe '#fetch' do
|
33
|
+
let(:sort_lambda) { ->(a, b) { a[:uid] <=> b[:uid] } }
|
34
|
+
let(:expected_hash) do
|
35
|
+
[
|
36
|
+
{ uid: uid.to_s, platform: platform.to_s, ip: ip },
|
37
|
+
{ uid: other_uid.to_s, platform: platform.to_s, ip: ip },
|
38
|
+
{ uid: another_uid.to_s, platform: another_platform.to_s, ip: another_ip },
|
39
|
+
]
|
40
|
+
end
|
56
41
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
42
|
+
before do
|
43
|
+
subject.add(uid, platform, ip)
|
44
|
+
subject.add(other_uid, platform, ip)
|
45
|
+
subject.add(another_uid, another_platform, another_ip)
|
46
|
+
end
|
61
47
|
|
62
|
-
|
63
|
-
|
48
|
+
it 'pulls the platform specific stats from redis' do
|
49
|
+
expect(subject.fetch.sort(&sort_lambda)).to eq(expected_hash.sort(&sort_lambda))
|
50
|
+
end
|
64
51
|
end
|
65
52
|
|
66
|
-
|
67
|
-
|
53
|
+
describe '#aggregate' do
|
54
|
+
let(:expected_hash) do
|
55
|
+
{
|
56
|
+
'android' => 3,
|
57
|
+
'ios' => 2,
|
58
|
+
'total' => 5
|
59
|
+
}
|
60
|
+
end
|
61
|
+
|
62
|
+
before do
|
63
|
+
subject.add(123, :android, ip)
|
64
|
+
subject.add(124, :android, ip)
|
65
|
+
subject.add(125, :android, ip)
|
66
|
+
|
67
|
+
subject.add(200, :ios, ip)
|
68
|
+
subject.add(201, :ios, ip)
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'should calculated proper aggregation' do
|
72
|
+
expect(subject.aggregate).to eql expected_hash
|
73
|
+
end
|
74
|
+
|
75
|
+
its(:aggregate) { should_not include({ 'total' => 0 }) }
|
76
|
+
its(:prefix) { should match /^x-turnstile\|\d+/ }
|
77
|
+
|
78
|
+
describe '#flushdb' do
|
79
|
+
before { subject.flushdb }
|
80
|
+
its(:aggregate) { should include({ 'total' => 0 }) }
|
81
|
+
end
|
68
82
|
end
|
69
83
|
end
|
70
|
-
|
71
84
|
end
|
@@ -5,110 +5,160 @@ require 'thread'
|
|
5
5
|
require 'tempfile'
|
6
6
|
|
7
7
|
require 'turnstile/collector/log_reader'
|
8
|
+
module Turnstile
|
9
|
+
module Collector
|
8
10
|
|
9
|
-
describe
|
10
|
-
|
11
|
+
describe LogReader do
|
12
|
+
include Timeout
|
11
13
|
|
12
|
-
|
13
|
-
|
14
|
-
|
14
|
+
def consume_file(reader, read_timeout)
|
15
|
+
hash = {}
|
16
|
+
counter = 0
|
15
17
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
18
|
+
run_reader(read_timeout) do
|
19
|
+
reader.read do |token|
|
20
|
+
counter += 1
|
21
|
+
hash[token] = 1
|
22
|
+
end
|
23
|
+
end
|
24
|
+
return counter, hash
|
20
25
|
end
|
21
|
-
end
|
22
|
-
return counter, hash
|
23
|
-
end
|
24
26
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
27
|
+
def run_reader(read_timeout, &block)
|
28
|
+
t_reader = Thread.new do
|
29
|
+
begin
|
30
|
+
timeout(read_timeout) do
|
31
|
+
block.call
|
32
|
+
end
|
33
|
+
rescue Timeout::Error
|
34
|
+
end
|
30
35
|
end
|
31
|
-
|
36
|
+
t_reader.join
|
32
37
|
end
|
33
|
-
end
|
34
|
-
t_reader.join
|
35
|
-
end
|
36
38
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
39
|
+
let(:queue) { Queue.new }
|
40
|
+
let(:read_timeout) { 0.1 }
|
41
|
+
let(:consume_file_result) { consume_file(reader, read_timeout) }
|
42
|
+
let(:counter) { consume_file_result.first }
|
43
|
+
let(:hash) { consume_file_result.last }
|
42
44
|
|
43
|
-
|
45
|
+
before { reader.file.backward(1000) }
|
44
46
|
|
45
|
-
|
46
|
-
|
47
|
-
|
47
|
+
context 'json log file' do
|
48
|
+
let(:file) { 'spec/fixtures/sample-production.log.json' }
|
49
|
+
let(:reader) { Turnstile::Collector::LogReader.json_formatted(file, queue) }
|
48
50
|
|
49
|
-
|
50
|
-
|
51
|
-
|
51
|
+
let(:expected_uniques) { 28 }
|
52
|
+
let(:expected_total) { 31 }
|
53
|
+
let(:expected_key) { 'ipad:69.61.173.104:5462583' }
|
52
54
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
55
|
+
context '#read' do
|
56
|
+
it 'should be able to read and parse IPs from a static file' do
|
57
|
+
expect(counter).to eql(expected_total)
|
58
|
+
expect(hash.keys.size).to eql(expected_uniques)
|
59
|
+
expect(hash.keys).to include(expected_key)
|
60
|
+
end
|
61
|
+
end
|
60
62
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
63
|
+
context '#process!' do
|
64
|
+
it 'should read values into the queue' do
|
65
|
+
run_reader(read_timeout) { reader.execute }
|
66
|
+
expect(queue.size).to eql(31)
|
67
|
+
end
|
68
|
+
end
|
65
69
|
end
|
66
|
-
end
|
67
|
-
end
|
68
70
|
|
69
|
-
|
70
|
-
|
71
|
-
|
71
|
+
context 'pipe delimited file' do
|
72
|
+
let(:file) { 'spec/fixtures/sample-production.log' }
|
73
|
+
let(:reader) { Turnstile::Collector::LogReader.pipe_delimited(file, queue) }
|
72
74
|
|
73
|
-
|
75
|
+
let(:log_reader) { Turnstile::Collector::LogReader }
|
74
76
|
|
75
|
-
|
76
|
-
|
77
|
-
|
77
|
+
let(:expected_uniques) { 2 }
|
78
|
+
let(:expected_total) { 4 }
|
79
|
+
let(:expected_key) { 'desktop:124.5.4.3:AF39945f8f87F' }
|
78
80
|
|
79
|
-
|
80
|
-
|
81
|
+
context 'matcher' do
|
82
|
+
subject(:matcher) { log_reader.delimited_matcher }
|
81
83
|
|
82
|
-
|
84
|
+
its(:regexp) { should_not be_nil }
|
83
85
|
|
84
|
-
|
85
|
-
|
86
|
+
its(:extractor) { should_not be_nil }
|
87
|
+
its(:extractor) { should be_kind_of(Proc) }
|
86
88
|
|
87
|
-
|
88
|
-
|
89
|
-
|
89
|
+
it 'should match lines in the file' do
|
90
|
+
File.open(file).each do |line|
|
91
|
+
expect(line).to match(matcher.regexp)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
it 'should extract the token from file' do
|
95
|
+
File.open(file).each do |line|
|
96
|
+
expect(matcher.tokenize(line)).to_not be_nil
|
97
|
+
end
|
98
|
+
end
|
90
99
|
end
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
100
|
+
|
101
|
+
context '#read' do
|
102
|
+
it 'should be load all matching rows' do
|
103
|
+
expect(counter).to eql(expected_total)
|
104
|
+
expect(hash.keys.size).to eql(expected_uniques)
|
105
|
+
expect(hash.keys).to include(expected_key)
|
106
|
+
end
|
95
107
|
end
|
96
|
-
end
|
97
|
-
end
|
98
108
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
109
|
+
context '#process!' do
|
110
|
+
it 'should read values into the queue' do
|
111
|
+
run_reader(read_timeout) { reader.execute }
|
112
|
+
expect(queue.size).to eql(expected_total)
|
113
|
+
end
|
114
|
+
end
|
104
115
|
end
|
105
|
-
end
|
106
116
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
117
|
+
context 'custom format' do
|
118
|
+
before do
|
119
|
+
Configuration.from_file(Dir.pwd + '/example/custom_csv_matcher.rb')
|
120
|
+
expect(Turnstile.config.custom_matcher).to_not be_nil
|
121
|
+
end
|
122
|
+
|
123
|
+
it 'should set custom matcher' do
|
124
|
+
expect(Turnstile.config.to_h[:custom_matcher]).to_not be_nil
|
125
|
+
end
|
126
|
+
|
127
|
+
let(:file) { Dir.pwd + '/spec/fixtures/custom-production.log' }
|
128
|
+
let(:reader) { Turnstile::Collector::LogReader.custom(file, queue) }
|
129
|
+
let(:log_reader) { Turnstile::Collector::LogReader }
|
130
|
+
|
131
|
+
let(:expected_uniques) { 2 }
|
132
|
+
let(:expected_total) { 2 }
|
133
|
+
let(:expected_key) { 'android:47.23.6.197:BGHSsdCsX5VvsN1jLsAR' }
|
134
|
+
|
135
|
+
context 'matcher' do
|
136
|
+
subject(:matcher) { log_reader.custom_matcher }
|
137
|
+
|
138
|
+
it { is_expected.to_not be_nil }
|
139
|
+
|
140
|
+
it 'should match lines in the file' do
|
141
|
+
File.open(file).each do |line|
|
142
|
+
expect(matcher.matches?(line)).to be_truthy
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
it 'should extract the token from file' do
|
147
|
+
File.open(file).each do |line|
|
148
|
+
expect(matcher.tokenize(line)).to_not be_nil
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
context '#read' do
|
154
|
+
it 'should be load all matching rows' do
|
155
|
+
expect(hash.keys.size).to eql(expected_uniques)
|
156
|
+
expect(hash.keys).to include(expected_key)
|
157
|
+
expect(counter).to eql(expected_total)
|
158
|
+
end
|
159
|
+
end
|
111
160
|
end
|
112
161
|
end
|
113
162
|
end
|
114
163
|
end
|
164
|
+
|