turnstile-rb 2.0.1 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +3 -0
- data/.travis.yml +2 -2
- data/LICENSE.txt +2 -1
- data/README.md +200 -11
- data/Rakefile +1 -0
- data/bin/turnstile +4 -6
- data/example/custom_csv_matcher.rb +22 -0
- data/lib/turnstile.rb +37 -8
- data/lib/turnstile/cli/launcher.rb +83 -0
- data/lib/turnstile/cli/parser.rb +166 -0
- data/lib/turnstile/cli/runner.rb +58 -0
- data/lib/turnstile/collector.rb +2 -2
- data/lib/turnstile/collector/actor.rb +81 -0
- data/lib/turnstile/collector/controller.rb +121 -0
- data/lib/turnstile/collector/flusher.rb +36 -0
- data/lib/turnstile/collector/formats.rb +7 -34
- data/lib/turnstile/collector/formats/custom_matcher.rb +19 -0
- data/lib/turnstile/collector/formats/delimited_matcher.rb +30 -0
- data/lib/turnstile/collector/formats/json_matcher.rb +30 -0
- data/lib/turnstile/collector/log_reader.rb +84 -46
- data/lib/turnstile/collector/{matcher.rb → regexp_matcher.rb} +4 -5
- data/lib/turnstile/collector/session.rb +7 -0
- data/lib/turnstile/commands.rb +20 -0
- data/lib/turnstile/commands/base.rb +20 -0
- data/lib/turnstile/commands/flushdb.rb +21 -0
- data/lib/turnstile/commands/print_keys.rb +19 -0
- data/lib/turnstile/commands/show.rb +89 -0
- data/lib/turnstile/configuration.rb +23 -0
- data/lib/turnstile/dependencies.rb +31 -0
- data/lib/turnstile/logger.rb +9 -40
- data/lib/turnstile/logger/helper.rb +42 -0
- data/lib/turnstile/logger/provider.rb +74 -0
- data/lib/turnstile/observer.rb +5 -9
- data/lib/turnstile/redis/adapter.rb +97 -0
- data/lib/turnstile/redis/connection.rb +116 -0
- data/lib/turnstile/redis/spy.rb +42 -0
- data/lib/turnstile/sampler.rb +9 -2
- data/lib/turnstile/tracker.rb +14 -14
- data/lib/turnstile/version.rb +51 -12
- data/lib/turnstile/web_app.rb +29 -0
- data/spec/spec_helper.rb +18 -3
- data/spec/support/logging.rb +17 -0
- data/spec/turnstile/adapter_spec.rb +59 -46
- data/spec/turnstile/collector/flusher_spec.rb +16 -0
- data/spec/turnstile/collector/log_reader_spec.rb +127 -77
- data/spec/turnstile/commands/show_spec.rb +40 -0
- data/spec/turnstile/tracker_spec.rb +21 -7
- data/spec/turnstile_spec.rb +3 -0
- data/turnstile-rb.gemspec +5 -2
- metadata +89 -22
- data/Gemfile +0 -6
- data/lib/turnstile/adapter.rb +0 -61
- data/lib/turnstile/collector/runner.rb +0 -72
- data/lib/turnstile/collector/updater.rb +0 -86
- data/lib/turnstile/parser.rb +0 -107
- data/lib/turnstile/runner.rb +0 -54
- data/lib/turnstile/summary.rb +0 -57
- data/spec/turnstile/summary_spec.rb +0 -41
@@ -1,43 +1,16 @@
|
|
1
1
|
require 'json'
|
2
|
-
require_relative '
|
2
|
+
require_relative 'regexp_matcher'
|
3
|
+
require_relative 'formats/json_matcher'
|
4
|
+
require_relative 'formats/delimited_matcher'
|
5
|
+
require_relative 'formats/custom_matcher'
|
3
6
|
|
4
7
|
module Turnstile
|
5
8
|
module Collector
|
6
9
|
module Formats
|
7
|
-
MARKER_TURNSTILE = 'x-turnstile'.freeze
|
8
10
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
@json_matcher ||= Matcher.new(%r{"ip_address":"\d+},
|
13
|
-
->(line) {
|
14
|
-
begin
|
15
|
-
data = JSON.parse(line)
|
16
|
-
[
|
17
|
-
data['platform'],
|
18
|
-
data['ip_address'],
|
19
|
-
data['user_id']
|
20
|
-
].join(':')
|
21
|
-
rescue
|
22
|
-
nil
|
23
|
-
end
|
24
|
-
})
|
25
|
-
end
|
26
|
-
|
27
|
-
# Expects the form of '..... x-turnstile|desktop|10.10.2.4|1234456 ....'
|
28
|
-
def delimited_matcher(delimiter = '|')
|
29
|
-
@default_matcher ||= Matcher.new(%r{#{MARKER_TURNSTILE}},
|
30
|
-
->(line) {
|
31
|
-
marker = line.split(/ /).find { |w| w =~ /^#{MARKER_TURNSTILE}/ }
|
32
|
-
if marker
|
33
|
-
list = marker.split(delimiter)
|
34
|
-
if list && list.size == 4
|
35
|
-
return(list[1..-1].join(':'))
|
36
|
-
end
|
37
|
-
end
|
38
|
-
nil
|
39
|
-
})
|
40
|
-
end
|
11
|
+
include JsonMatcher
|
12
|
+
include DelimitedMatcher
|
13
|
+
include CustomMatcher
|
41
14
|
|
42
15
|
alias default_matcher delimited_matcher
|
43
16
|
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Turnstile
|
2
|
+
module Collector
|
3
|
+
module Formats
|
4
|
+
module CustomMatcher
|
5
|
+
|
6
|
+
# Returns a custom matcher that is expected to be pre-initialized
|
7
|
+
# using a config file.
|
8
|
+
#
|
9
|
+
# Example of a custom matcher configuration is in the README, and inside
|
10
|
+
# the `example` folder.
|
11
|
+
#
|
12
|
+
def custom_matcher
|
13
|
+
@custom_matcher ||= Turnstile.config.custom_matcher
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Turnstile
|
2
|
+
module Collector
|
3
|
+
module Formats
|
4
|
+
module DelimitedMatcher
|
5
|
+
class << self
|
6
|
+
attr_accessor :marker
|
7
|
+
end
|
8
|
+
|
9
|
+
self.marker = 'x-turnstile'.freeze
|
10
|
+
|
11
|
+
# Expects the form of '..... x-turnstile|desktop|10.10.2.4|1234456 ....'
|
12
|
+
def delimited_matcher(delimiter = '|', match_marker = ::Turnstile::Collector::Formats::DelimitedMatcher.marker)
|
13
|
+
@default_matcher ||= RegexpMatcher.new(%r{#{match_marker}},
|
14
|
+
->(line) {
|
15
|
+
marker = line.split(/ /).find { |w| w =~ /^#{match_marker}/ }
|
16
|
+
if marker
|
17
|
+
list = marker.split(delimiter)
|
18
|
+
if list && list.size == 4
|
19
|
+
return(list[1..-1].join(':'))
|
20
|
+
end
|
21
|
+
end
|
22
|
+
nil
|
23
|
+
})
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'json'
|
2
|
+
require_relative '../regexp_matcher'
|
3
|
+
|
4
|
+
module Turnstile
|
5
|
+
module Collector
|
6
|
+
module Formats
|
7
|
+
# Extracts from the log file of the form:
|
8
|
+
# {"method":"GET","path":"/api/v1/saves/4SB8U-1Am9u-4ixC5","format":"json","duration":49.01,.....}
|
9
|
+
module JsonMatcher
|
10
|
+
def json_matcher(*_args)
|
11
|
+
@json_matcher ||= ::Turnstile::Collector::RegexpMatcher.new(%r{"ip_address":"\d+},
|
12
|
+
->(line) {
|
13
|
+
begin
|
14
|
+
data = JSON.parse(line)
|
15
|
+
[
|
16
|
+
data['platform'],
|
17
|
+
data['ip_address'],
|
18
|
+
data['user_id']
|
19
|
+
].join(':')
|
20
|
+
rescue
|
21
|
+
nil
|
22
|
+
end
|
23
|
+
})
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'file-tail'
|
2
|
-
|
2
|
+
require_relative 'formats'
|
3
|
+
require_relative 'actor'
|
3
4
|
|
4
5
|
module Turnstile
|
5
6
|
module Collector
|
@@ -7,75 +8,112 @@ module Turnstile
|
|
7
8
|
include ::File::Tail
|
8
9
|
end
|
9
10
|
|
10
|
-
class LogReader
|
11
|
-
|
12
|
-
include Formats
|
11
|
+
class LogReader < Actor
|
12
|
+
attr_accessor :file, :filename, :matcher, :should_reopen
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
def initialize(log_file:, matcher:, **opts)
|
15
|
+
super(**opts)
|
16
|
+
self.matcher = matcher
|
17
|
+
self.filename = log_file
|
18
|
+
self.should_reopen = false
|
19
|
+
open_and_watch(opts[:tail] ? opts[:tail].to_i : 0)
|
17
20
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
+
reader = self
|
22
|
+
Signal.trap('HUP') { reader.should_reopen = true }
|
23
|
+
end
|
21
24
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
+
def reopen(a_file = nil)
|
26
|
+
self.should_reopen = false
|
27
|
+
close rescue nil
|
28
|
+
self.filename = a_file if a_file
|
29
|
+
open_and_watch(0)
|
30
|
+
end
|
25
31
|
|
26
|
-
|
27
|
-
|
32
|
+
def execute
|
33
|
+
self.read do |token|
|
34
|
+
self.queue << token if token
|
28
35
|
end
|
36
|
+
rescue IOError
|
37
|
+
open_and_watch if File.exist?(filename)
|
38
|
+
ensure
|
39
|
+
close
|
40
|
+
end
|
29
41
|
|
30
|
-
|
31
|
-
|
42
|
+
def read(&_block)
|
43
|
+
file.tail do |line|
|
44
|
+
token = matcher.tokenize(line) if matcher
|
45
|
+
yield(token) if block_given? && token
|
46
|
+
break if stopping?
|
47
|
+
reopen if should_reopen?
|
32
48
|
end
|
33
|
-
|
34
|
-
alias default pipe_delimited
|
35
49
|
end
|
36
50
|
|
37
|
-
|
51
|
+
def close
|
52
|
+
(file.close rescue nil) if file
|
53
|
+
end
|
38
54
|
|
39
|
-
|
40
|
-
self.matcher = matcher
|
41
|
-
self.queue = queue
|
55
|
+
private
|
42
56
|
|
43
|
-
|
57
|
+
def open_and_watch(tail_lines = 0)
|
58
|
+
self.file = LogFile.new(filename)
|
44
59
|
|
45
60
|
file.interval = 1
|
46
|
-
file.backward(0)
|
61
|
+
file.backward(0) if tail_lines == 0
|
62
|
+
file.backward(tail_lines) if tail_lines > 0
|
63
|
+
file.forward(0) if tail_lines == -1
|
47
64
|
end
|
48
65
|
|
49
|
-
def
|
50
|
-
|
51
|
-
Thread.current[:name] = 'log-reader'
|
52
|
-
Turnstile::Logger.log "starting to tail file #{file.path}...."
|
53
|
-
process!
|
54
|
-
end
|
66
|
+
def should_reopen?
|
67
|
+
should_reopen
|
55
68
|
end
|
56
69
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
70
|
+
class << self
|
71
|
+
include Formats
|
72
|
+
|
73
|
+
def custom(file, queue, **opts)
|
74
|
+
new(log_file: file,
|
75
|
+
queue: queue,
|
76
|
+
matcher: custom_matcher,
|
77
|
+
**opts)
|
61
78
|
end
|
62
|
-
end
|
63
79
|
|
64
|
-
|
65
|
-
|
66
|
-
|
80
|
+
def pipe_delimited(file, queue, **opts)
|
81
|
+
new(log_file: file,
|
82
|
+
queue: queue,
|
83
|
+
matcher: delimited_matcher,
|
84
|
+
**opts)
|
67
85
|
end
|
68
|
-
end
|
69
86
|
|
70
|
-
|
71
|
-
|
72
|
-
|
87
|
+
def comma_delimited(file, queue, **opts)
|
88
|
+
new(log_file: file,
|
89
|
+
queue: queue,
|
90
|
+
matcher: delimited_matcher(','),
|
91
|
+
**opts)
|
92
|
+
end
|
73
93
|
|
74
|
-
|
94
|
+
def colon_delimited(file, queue, **opts)
|
95
|
+
new(log_file: file,
|
96
|
+
queue: queue,
|
97
|
+
matcher: delimited_matcher(':'),
|
98
|
+
**opts)
|
99
|
+
end
|
100
|
+
|
101
|
+
def delimited(file, queue, **opts)
|
102
|
+
new(log_file: file,
|
103
|
+
queue: queue,
|
104
|
+
matcher: delimited_matcher(opts[:delimiter]),
|
105
|
+
**opts)
|
106
|
+
end
|
75
107
|
|
76
|
-
|
108
|
+
def json_formatted(file, queue, **opts)
|
109
|
+
new(log_file: file,
|
110
|
+
queue: queue,
|
111
|
+
matcher: json_matcher,
|
112
|
+
**opts)
|
113
|
+
end
|
114
|
+
|
115
|
+
alias default pipe_delimited
|
77
116
|
end
|
78
117
|
end
|
79
|
-
|
80
118
|
end
|
81
119
|
end
|
@@ -1,16 +1,15 @@
|
|
1
1
|
module Turnstile
|
2
2
|
module Collector
|
3
|
-
class
|
3
|
+
class RegexpMatcher < Struct.new(:regexp, :extractor)
|
4
4
|
# checks if the line matches +regexp+, and if yes
|
5
5
|
# runs it through +extractor+ to grab the token
|
6
6
|
#
|
7
7
|
# @param [String] line read from a log file
|
8
8
|
# @return [String] a token in the form 'platform:ip:user'
|
9
9
|
|
10
|
-
def
|
11
|
-
return nil unless matches?(line)
|
12
|
-
|
13
|
-
extractor ? extractor[line] : nil
|
10
|
+
def tokenize(line)
|
11
|
+
return nil unless matches?(line) && extractor
|
12
|
+
extractor[line]
|
14
13
|
end
|
15
14
|
|
16
15
|
def matches?(line)
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'active_support/inflector'
|
2
|
+
|
3
|
+
require_relative 'commands/base'
|
4
|
+
require_relative 'commands/show'
|
5
|
+
require_relative 'commands/flushdb'
|
6
|
+
require_relative 'commands/print_keys'
|
7
|
+
|
8
|
+
module Turnstile
|
9
|
+
module Commands
|
10
|
+
class << self
|
11
|
+
def command(name)
|
12
|
+
command_candidate = "#{self.name}::#{ActiveSupport::Inflector.camelize(name)}"
|
13
|
+
ActiveSupport::Inflector.constantize command_candidate
|
14
|
+
rescue NameError
|
15
|
+
raise CommandNotFoundError, "Command #{name} is not found, #{command_candidate}"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'turnstile/dependencies'
|
2
|
+
|
3
|
+
module Turnstile
|
4
|
+
module Commands
|
5
|
+
|
6
|
+
class Base
|
7
|
+
include ::Turnstile::Dependencies
|
8
|
+
|
9
|
+
attr_accessor :options, :config
|
10
|
+
|
11
|
+
def initialize(options, config = Turnstile.config)
|
12
|
+
self.options = options
|
13
|
+
self.config = config
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require_relative 'base'
|
3
|
+
|
4
|
+
module Turnstile
|
5
|
+
module Commands
|
6
|
+
class Show < Base
|
7
|
+
|
8
|
+
def execute(format = :json, delimiter = nil)
|
9
|
+
STDOUT.puts render_totals(format, delimiter)
|
10
|
+
end
|
11
|
+
|
12
|
+
def render_totals(format, delimiter = nil)
|
13
|
+
unless self.respond_to?(format)
|
14
|
+
raise ArgumentError, "Format #{format} is not supported"
|
15
|
+
end
|
16
|
+
self.send(format, aggregate, delimiter)
|
17
|
+
end
|
18
|
+
|
19
|
+
def yaml(data, *)
|
20
|
+
build_string(data, "\n", "---\nturnstile:") { |key, value, *| yaml_row(key, value) }
|
21
|
+
end
|
22
|
+
|
23
|
+
# Formats supported for the output
|
24
|
+
# JSON
|
25
|
+
def json(data, *)
|
26
|
+
build_string(data, "\n", '{', '}') do |key, value, index:, last:, first:|
|
27
|
+
json_row(key, value, index: index, first: first, last: last)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
# NAD format for Circonus
|
33
|
+
def nad(data, *)
|
34
|
+
build_string(data) { |key, value, *| nad_row(key, value) }
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
# CSV Format
|
39
|
+
def csv(data, delimiter = nil)
|
40
|
+
build_string(data) do |key, value, *|
|
41
|
+
string = [key, value].to_csv
|
42
|
+
string.gsub!(/,/, delimiter) if delimiter
|
43
|
+
string.strip
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
# This method is used to build a string with
|
50
|
+
# opening/closing parts and looping contents inside.
|
51
|
+
#
|
52
|
+
def build_string(data,
|
53
|
+
joiner = "\n",
|
54
|
+
prefix = nil,
|
55
|
+
suffix = nil,
|
56
|
+
&_block)
|
57
|
+
slices = []
|
58
|
+
slices << prefix if prefix
|
59
|
+
index = 0; count = data.size
|
60
|
+
data.each_pair do |key, value|
|
61
|
+
slices << yield(
|
62
|
+
key,
|
63
|
+
value,
|
64
|
+
index: index,
|
65
|
+
first: (index == 0),
|
66
|
+
last: (index == count - 1)
|
67
|
+
).to_s
|
68
|
+
index += 1
|
69
|
+
end
|
70
|
+
slices << suffix if suffix
|
71
|
+
slices.compact.join(joiner).strip
|
72
|
+
end
|
73
|
+
|
74
|
+
def json_row(key, value, last: false, **)
|
75
|
+
%Q( "#{key}": #{value}#{ last ? '' : ',' })
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
def nad_row(key, value)
|
80
|
+
%Q(turnstile:#{key}#{"\tn\t"}#{value})
|
81
|
+
end
|
82
|
+
|
83
|
+
def yaml_row(key, value)
|
84
|
+
%Q( #{key}: #{value})
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|