turnstile-rb 2.0.1 → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +3 -0
  3. data/.travis.yml +2 -2
  4. data/LICENSE.txt +2 -1
  5. data/README.md +200 -11
  6. data/Rakefile +1 -0
  7. data/bin/turnstile +4 -6
  8. data/example/custom_csv_matcher.rb +22 -0
  9. data/lib/turnstile.rb +37 -8
  10. data/lib/turnstile/cli/launcher.rb +83 -0
  11. data/lib/turnstile/cli/parser.rb +166 -0
  12. data/lib/turnstile/cli/runner.rb +58 -0
  13. data/lib/turnstile/collector.rb +2 -2
  14. data/lib/turnstile/collector/actor.rb +81 -0
  15. data/lib/turnstile/collector/controller.rb +121 -0
  16. data/lib/turnstile/collector/flusher.rb +36 -0
  17. data/lib/turnstile/collector/formats.rb +7 -34
  18. data/lib/turnstile/collector/formats/custom_matcher.rb +19 -0
  19. data/lib/turnstile/collector/formats/delimited_matcher.rb +30 -0
  20. data/lib/turnstile/collector/formats/json_matcher.rb +30 -0
  21. data/lib/turnstile/collector/log_reader.rb +84 -46
  22. data/lib/turnstile/collector/{matcher.rb → regexp_matcher.rb} +4 -5
  23. data/lib/turnstile/collector/session.rb +7 -0
  24. data/lib/turnstile/commands.rb +20 -0
  25. data/lib/turnstile/commands/base.rb +20 -0
  26. data/lib/turnstile/commands/flushdb.rb +21 -0
  27. data/lib/turnstile/commands/print_keys.rb +19 -0
  28. data/lib/turnstile/commands/show.rb +89 -0
  29. data/lib/turnstile/configuration.rb +23 -0
  30. data/lib/turnstile/dependencies.rb +31 -0
  31. data/lib/turnstile/logger.rb +9 -40
  32. data/lib/turnstile/logger/helper.rb +42 -0
  33. data/lib/turnstile/logger/provider.rb +74 -0
  34. data/lib/turnstile/observer.rb +5 -9
  35. data/lib/turnstile/redis/adapter.rb +97 -0
  36. data/lib/turnstile/redis/connection.rb +116 -0
  37. data/lib/turnstile/redis/spy.rb +42 -0
  38. data/lib/turnstile/sampler.rb +9 -2
  39. data/lib/turnstile/tracker.rb +14 -14
  40. data/lib/turnstile/version.rb +51 -12
  41. data/lib/turnstile/web_app.rb +29 -0
  42. data/spec/spec_helper.rb +18 -3
  43. data/spec/support/logging.rb +17 -0
  44. data/spec/turnstile/adapter_spec.rb +59 -46
  45. data/spec/turnstile/collector/flusher_spec.rb +16 -0
  46. data/spec/turnstile/collector/log_reader_spec.rb +127 -77
  47. data/spec/turnstile/commands/show_spec.rb +40 -0
  48. data/spec/turnstile/tracker_spec.rb +21 -7
  49. data/spec/turnstile_spec.rb +3 -0
  50. data/turnstile-rb.gemspec +5 -2
  51. metadata +89 -22
  52. data/Gemfile +0 -6
  53. data/lib/turnstile/adapter.rb +0 -61
  54. data/lib/turnstile/collector/runner.rb +0 -72
  55. data/lib/turnstile/collector/updater.rb +0 -86
  56. data/lib/turnstile/parser.rb +0 -107
  57. data/lib/turnstile/runner.rb +0 -54
  58. data/lib/turnstile/summary.rb +0 -57
  59. data/spec/turnstile/summary_spec.rb +0 -41
@@ -1,43 +1,16 @@
1
1
  require 'json'
2
- require_relative 'matcher'
2
+ require_relative 'regexp_matcher'
3
+ require_relative 'formats/json_matcher'
4
+ require_relative 'formats/delimited_matcher'
5
+ require_relative 'formats/custom_matcher'
3
6
 
4
7
  module Turnstile
5
8
  module Collector
6
9
  module Formats
7
- MARKER_TURNSTILE = 'x-turnstile'.freeze
8
10
 
9
- # Extracts from the log file of the form:
10
- # {"method":"GET","path":"/api/v1/saves/4SB8U-1Am9u-4ixC5","format":"json","duration":49.01,.....}
11
- def json_matcher(*_args)
12
- @json_matcher ||= Matcher.new(%r{"ip_address":"\d+},
13
- ->(line) {
14
- begin
15
- data = JSON.parse(line)
16
- [
17
- data['platform'],
18
- data['ip_address'],
19
- data['user_id']
20
- ].join(':')
21
- rescue
22
- nil
23
- end
24
- })
25
- end
26
-
27
- # Expects the form of '..... x-turnstile|desktop|10.10.2.4|1234456 ....'
28
- def delimited_matcher(delimiter = '|')
29
- @default_matcher ||= Matcher.new(%r{#{MARKER_TURNSTILE}},
30
- ->(line) {
31
- marker = line.split(/ /).find { |w| w =~ /^#{MARKER_TURNSTILE}/ }
32
- if marker
33
- list = marker.split(delimiter)
34
- if list && list.size == 4
35
- return(list[1..-1].join(':'))
36
- end
37
- end
38
- nil
39
- })
40
- end
11
+ include JsonMatcher
12
+ include DelimitedMatcher
13
+ include CustomMatcher
41
14
 
42
15
  alias default_matcher delimited_matcher
43
16
  end
@@ -0,0 +1,19 @@
1
+ module Turnstile
2
+ module Collector
3
+ module Formats
4
+ module CustomMatcher
5
+
6
+ # Returns a custom matcher that is expected to be pre-initialized
7
+ # using a config file.
8
+ #
9
+ # Example of a custom matcher configuration is in the README, and inside
10
+ # the `example` folder.
11
+ #
12
+ def custom_matcher
13
+ @custom_matcher ||= Turnstile.config.custom_matcher
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
19
+
@@ -0,0 +1,30 @@
1
+ module Turnstile
2
+ module Collector
3
+ module Formats
4
+ module DelimitedMatcher
5
+ class << self
6
+ attr_accessor :marker
7
+ end
8
+
9
+ self.marker = 'x-turnstile'.freeze
10
+
11
+ # Expects the form of '..... x-turnstile|desktop|10.10.2.4|1234456 ....'
12
+ def delimited_matcher(delimiter = '|', match_marker = ::Turnstile::Collector::Formats::DelimitedMatcher.marker)
13
+ @default_matcher ||= RegexpMatcher.new(%r{#{match_marker}},
14
+ ->(line) {
15
+ marker = line.split(/ /).find { |w| w =~ /^#{match_marker}/ }
16
+ if marker
17
+ list = marker.split(delimiter)
18
+ if list && list.size == 4
19
+ return(list[1..-1].join(':'))
20
+ end
21
+ end
22
+ nil
23
+ })
24
+ end
25
+ end
26
+
27
+ end
28
+ end
29
+ end
30
+
@@ -0,0 +1,30 @@
1
+ require 'json'
2
+ require_relative '../regexp_matcher'
3
+
4
+ module Turnstile
5
+ module Collector
6
+ module Formats
7
+ # Extracts from the log file of the form:
8
+ # {"method":"GET","path":"/api/v1/saves/4SB8U-1Am9u-4ixC5","format":"json","duration":49.01,.....}
9
+ module JsonMatcher
10
+ def json_matcher(*_args)
11
+ @json_matcher ||= ::Turnstile::Collector::RegexpMatcher.new(%r{"ip_address":"\d+},
12
+ ->(line) {
13
+ begin
14
+ data = JSON.parse(line)
15
+ [
16
+ data['platform'],
17
+ data['ip_address'],
18
+ data['user_id']
19
+ ].join(':')
20
+ rescue
21
+ nil
22
+ end
23
+ })
24
+ end
25
+ end
26
+
27
+ end
28
+ end
29
+ end
30
+
@@ -1,5 +1,6 @@
1
1
  require 'file-tail'
2
- require 'turnstile/collector/formats'
2
+ require_relative 'formats'
3
+ require_relative 'actor'
3
4
 
4
5
  module Turnstile
5
6
  module Collector
@@ -7,75 +8,112 @@ module Turnstile
7
8
  include ::File::Tail
8
9
  end
9
10
 
10
- class LogReader
11
- class << self
12
- include Formats
11
+ class LogReader < Actor
12
+ attr_accessor :file, :filename, :matcher, :should_reopen
13
13
 
14
- def pipe_delimited(file, queue)
15
- new(file, queue, delimited_matcher)
16
- end
14
+ def initialize(log_file:, matcher:, **opts)
15
+ super(**opts)
16
+ self.matcher = matcher
17
+ self.filename = log_file
18
+ self.should_reopen = false
19
+ open_and_watch(opts[:tail] ? opts[:tail].to_i : 0)
17
20
 
18
- def comma_delimited(file, queue)
19
- new(file, queue, delimited_matcher(','))
20
- end
21
+ reader = self
22
+ Signal.trap('HUP') { reader.should_reopen = true }
23
+ end
21
24
 
22
- def colon_delimited(file, queue)
23
- new(file, queue, delimited_matcher(':'))
24
- end
25
+ def reopen(a_file = nil)
26
+ self.should_reopen = false
27
+ close rescue nil
28
+ self.filename = a_file if a_file
29
+ open_and_watch(0)
30
+ end
25
31
 
26
- def delimited(file, queue, delimiter)
27
- new(file, queue, delimited_matcher(delimiter))
32
+ def execute
33
+ self.read do |token|
34
+ self.queue << token if token
28
35
  end
36
+ rescue IOError
37
+ open_and_watch if File.exist?(filename)
38
+ ensure
39
+ close
40
+ end
29
41
 
30
- def json_formatted(file, queue)
31
- new(file, queue, json_matcher)
42
+ def read(&_block)
43
+ file.tail do |line|
44
+ token = matcher.tokenize(line) if matcher
45
+ yield(token) if block_given? && token
46
+ break if stopping?
47
+ reopen if should_reopen?
32
48
  end
33
-
34
- alias default pipe_delimited
35
49
  end
36
50
 
37
- attr_accessor :file, :queue, :matcher
51
+ def close
52
+ (file.close rescue nil) if file
53
+ end
38
54
 
39
- def initialize(log_file, queue, matcher)
40
- self.matcher = matcher
41
- self.queue = queue
55
+ private
42
56
 
43
- self.file = LogFile.new(log_file)
57
+ def open_and_watch(tail_lines = 0)
58
+ self.file = LogFile.new(filename)
44
59
 
45
60
  file.interval = 1
46
- file.backward(0)
61
+ file.backward(0) if tail_lines == 0
62
+ file.backward(tail_lines) if tail_lines > 0
63
+ file.forward(0) if tail_lines == -1
47
64
  end
48
65
 
49
- def run
50
- Thread.new do
51
- Thread.current[:name] = 'log-reader'
52
- Turnstile::Logger.log "starting to tail file #{file.path}...."
53
- process!
54
- end
66
+ def should_reopen?
67
+ should_reopen
55
68
  end
56
69
 
57
- def read(&_block)
58
- file.tail do |line|
59
- token = matcher.token_from(line)
60
- yield(token) if block_given? && token
70
+ class << self
71
+ include Formats
72
+
73
+ def custom(file, queue, **opts)
74
+ new(log_file: file,
75
+ queue: queue,
76
+ matcher: custom_matcher,
77
+ **opts)
61
78
  end
62
- end
63
79
 
64
- def process!
65
- self.read do |token|
66
- queue << token if token
80
+ def pipe_delimited(file, queue, **opts)
81
+ new(log_file: file,
82
+ queue: queue,
83
+ matcher: delimited_matcher,
84
+ **opts)
67
85
  end
68
- end
69
86
 
70
- def close
71
- (file.close if file) rescue nil
72
- end
87
+ def comma_delimited(file, queue, **opts)
88
+ new(log_file: file,
89
+ queue: queue,
90
+ matcher: delimited_matcher(','),
91
+ **opts)
92
+ end
73
93
 
74
- private
94
+ def colon_delimited(file, queue, **opts)
95
+ new(log_file: file,
96
+ queue: queue,
97
+ matcher: delimited_matcher(':'),
98
+ **opts)
99
+ end
100
+
101
+ def delimited(file, queue, **opts)
102
+ new(log_file: file,
103
+ queue: queue,
104
+ matcher: delimited_matcher(opts[:delimiter]),
105
+ **opts)
106
+ end
75
107
 
76
- def extract(line)
108
+ def json_formatted(file, queue, **opts)
109
+ new(log_file: file,
110
+ queue: queue,
111
+ matcher: json_matcher,
112
+ **opts)
113
+ end
114
+
115
+ alias default pipe_delimited
77
116
  end
78
117
  end
79
-
80
118
  end
81
119
  end
@@ -1,16 +1,15 @@
1
1
  module Turnstile
2
2
  module Collector
3
- class Matcher < Struct.new(:regexp, :extractor)
3
+ class RegexpMatcher < Struct.new(:regexp, :extractor)
4
4
  # checks if the line matches +regexp+, and if yes
5
5
  # runs it through +extractor+ to grab the token
6
6
  #
7
7
  # @param [String] line read from a log file
8
8
  # @return [String] a token in the form 'platform:ip:user'
9
9
 
10
- def token_from(line)
11
- return nil unless matches?(line)
12
- return nil unless extractor
13
- extractor ? extractor[line] : nil
10
+ def tokenize(line)
11
+ return nil unless matches?(line) && extractor
12
+ extractor[line]
14
13
  end
15
14
 
16
15
  def matches?(line)
@@ -0,0 +1,7 @@
1
+ module Turnstile
2
+ module Collector
3
+ class Session < ::Struct.new(:uid, :platform, :ip);
4
+ end
5
+ end
6
+ end
7
+
@@ -0,0 +1,20 @@
1
+ require 'active_support/inflector'
2
+
3
+ require_relative 'commands/base'
4
+ require_relative 'commands/show'
5
+ require_relative 'commands/flushdb'
6
+ require_relative 'commands/print_keys'
7
+
8
+ module Turnstile
9
+ module Commands
10
+ class << self
11
+ def command(name)
12
+ command_candidate = "#{self.name}::#{ActiveSupport::Inflector.camelize(name)}"
13
+ ActiveSupport::Inflector.constantize command_candidate
14
+ rescue NameError
15
+ raise CommandNotFoundError, "Command #{name} is not found, #{command_candidate}"
16
+ end
17
+ end
18
+ end
19
+ end
20
+
@@ -0,0 +1,20 @@
1
+ require 'turnstile/dependencies'
2
+
3
+ module Turnstile
4
+ module Commands
5
+
6
+ class Base
7
+ include ::Turnstile::Dependencies
8
+
9
+ attr_accessor :options, :config
10
+
11
+ def initialize(options, config = Turnstile.config)
12
+ self.options = options
13
+ self.config = config
14
+ end
15
+
16
+ end
17
+ end
18
+ end
19
+
20
+
@@ -0,0 +1,21 @@
1
+ require 'csv'
2
+ require_relative 'base'
3
+
4
+ module Turnstile
5
+ module Commands
6
+ class Flushdb < Base
7
+ def execute
8
+ flushdb
9
+ end
10
+
11
+ def flushdb
12
+ adapter.flushdb
13
+ end
14
+ end
15
+ end
16
+ end
17
+
18
+
19
+
20
+
21
+
@@ -0,0 +1,19 @@
1
+ require 'csv'
2
+ require_relative 'base'
3
+
4
+ module Turnstile
5
+ module Commands
6
+ class PrintKeys < Base
7
+ def execute
8
+ adapter.all_keys.each do |key|
9
+ puts key
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
15
+
16
+
17
+
18
+
19
+
@@ -0,0 +1,89 @@
1
+ require 'csv'
2
+ require_relative 'base'
3
+
4
+ module Turnstile
5
+ module Commands
6
+ class Show < Base
7
+
8
+ def execute(format = :json, delimiter = nil)
9
+ STDOUT.puts render_totals(format, delimiter)
10
+ end
11
+
12
+ def render_totals(format, delimiter = nil)
13
+ unless self.respond_to?(format)
14
+ raise ArgumentError, "Format #{format} is not supported"
15
+ end
16
+ self.send(format, aggregate, delimiter)
17
+ end
18
+
19
+ def yaml(data, *)
20
+ build_string(data, "\n", "---\nturnstile:") { |key, value, *| yaml_row(key, value) }
21
+ end
22
+
23
+ # Formats supported for the output
24
+ # JSON
25
+ def json(data, *)
26
+ build_string(data, "\n", '{', '}') do |key, value, index:, last:, first:|
27
+ json_row(key, value, index: index, first: first, last: last)
28
+ end
29
+ end
30
+
31
+
32
+ # NAD format for Circonus
33
+ def nad(data, *)
34
+ build_string(data) { |key, value, *| nad_row(key, value) }
35
+ end
36
+
37
+
38
+ # CSV Format
39
+ def csv(data, delimiter = nil)
40
+ build_string(data) do |key, value, *|
41
+ string = [key, value].to_csv
42
+ string.gsub!(/,/, delimiter) if delimiter
43
+ string.strip
44
+ end
45
+ end
46
+
47
+ private
48
+
49
+ # This method is used to build a string with
50
+ # opening/closing parts and looping contents inside.
51
+ #
52
+ def build_string(data,
53
+ joiner = "\n",
54
+ prefix = nil,
55
+ suffix = nil,
56
+ &_block)
57
+ slices = []
58
+ slices << prefix if prefix
59
+ index = 0; count = data.size
60
+ data.each_pair do |key, value|
61
+ slices << yield(
62
+ key,
63
+ value,
64
+ index: index,
65
+ first: (index == 0),
66
+ last: (index == count - 1)
67
+ ).to_s
68
+ index += 1
69
+ end
70
+ slices << suffix if suffix
71
+ slices.compact.join(joiner).strip
72
+ end
73
+
74
+ def json_row(key, value, last: false, **)
75
+ %Q( "#{key}": #{value}#{ last ? '' : ',' })
76
+ end
77
+
78
+
79
+ def nad_row(key, value)
80
+ %Q(turnstile:#{key}#{"\tn\t"}#{value})
81
+ end
82
+
83
+ def yaml_row(key, value)
84
+ %Q( #{key}: #{value})
85
+ end
86
+
87
+ end
88
+ end
89
+ end