turnstile-rb 2.0.1 → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +3 -0
  3. data/.travis.yml +2 -2
  4. data/LICENSE.txt +2 -1
  5. data/README.md +200 -11
  6. data/Rakefile +1 -0
  7. data/bin/turnstile +4 -6
  8. data/example/custom_csv_matcher.rb +22 -0
  9. data/lib/turnstile.rb +37 -8
  10. data/lib/turnstile/cli/launcher.rb +83 -0
  11. data/lib/turnstile/cli/parser.rb +166 -0
  12. data/lib/turnstile/cli/runner.rb +58 -0
  13. data/lib/turnstile/collector.rb +2 -2
  14. data/lib/turnstile/collector/actor.rb +81 -0
  15. data/lib/turnstile/collector/controller.rb +121 -0
  16. data/lib/turnstile/collector/flusher.rb +36 -0
  17. data/lib/turnstile/collector/formats.rb +7 -34
  18. data/lib/turnstile/collector/formats/custom_matcher.rb +19 -0
  19. data/lib/turnstile/collector/formats/delimited_matcher.rb +30 -0
  20. data/lib/turnstile/collector/formats/json_matcher.rb +30 -0
  21. data/lib/turnstile/collector/log_reader.rb +84 -46
  22. data/lib/turnstile/collector/{matcher.rb → regexp_matcher.rb} +4 -5
  23. data/lib/turnstile/collector/session.rb +7 -0
  24. data/lib/turnstile/commands.rb +20 -0
  25. data/lib/turnstile/commands/base.rb +20 -0
  26. data/lib/turnstile/commands/flushdb.rb +21 -0
  27. data/lib/turnstile/commands/print_keys.rb +19 -0
  28. data/lib/turnstile/commands/show.rb +89 -0
  29. data/lib/turnstile/configuration.rb +23 -0
  30. data/lib/turnstile/dependencies.rb +31 -0
  31. data/lib/turnstile/logger.rb +9 -40
  32. data/lib/turnstile/logger/helper.rb +42 -0
  33. data/lib/turnstile/logger/provider.rb +74 -0
  34. data/lib/turnstile/observer.rb +5 -9
  35. data/lib/turnstile/redis/adapter.rb +97 -0
  36. data/lib/turnstile/redis/connection.rb +116 -0
  37. data/lib/turnstile/redis/spy.rb +42 -0
  38. data/lib/turnstile/sampler.rb +9 -2
  39. data/lib/turnstile/tracker.rb +14 -14
  40. data/lib/turnstile/version.rb +51 -12
  41. data/lib/turnstile/web_app.rb +29 -0
  42. data/spec/spec_helper.rb +18 -3
  43. data/spec/support/logging.rb +17 -0
  44. data/spec/turnstile/adapter_spec.rb +59 -46
  45. data/spec/turnstile/collector/flusher_spec.rb +16 -0
  46. data/spec/turnstile/collector/log_reader_spec.rb +127 -77
  47. data/spec/turnstile/commands/show_spec.rb +40 -0
  48. data/spec/turnstile/tracker_spec.rb +21 -7
  49. data/spec/turnstile_spec.rb +3 -0
  50. data/turnstile-rb.gemspec +5 -2
  51. metadata +89 -22
  52. data/Gemfile +0 -6
  53. data/lib/turnstile/adapter.rb +0 -61
  54. data/lib/turnstile/collector/runner.rb +0 -72
  55. data/lib/turnstile/collector/updater.rb +0 -86
  56. data/lib/turnstile/parser.rb +0 -107
  57. data/lib/turnstile/runner.rb +0 -54
  58. data/lib/turnstile/summary.rb +0 -57
  59. data/spec/turnstile/summary_spec.rb +0 -41
@@ -1,5 +1,6 @@
1
1
  require 'hashie/dash'
2
2
  require 'hashie/extensions/dash/property_translation'
3
+ require 'pp'
3
4
 
4
5
  module Turnstile
5
6
  class RedisConfig < ::Hashie::Dash
@@ -10,6 +11,9 @@ module Turnstile
10
11
  property :port, default: 6379, required: true, transform_with: ->(value) { value.to_i }
11
12
  property :db, default: 1, required: true, transform_with: ->(value) { value.to_i }
12
13
  property :timeout, default: 0.05, required: true, transform_with: ->(value) { value.to_f }
14
+ property :namespace, default: '', required: false
15
+ property :pool_size, default: 5, required: true
16
+ property :use_hiredis, default: false
13
17
 
14
18
  def configure
15
19
  yield self if block_given?
@@ -22,12 +26,31 @@ module Turnstile
22
26
  property :activity_interval, default: 60, required: true, transform_with: ->(value) { value.to_i }
23
27
  property :sampling_rate, default: 100, required: true, transform_with: ->(value) { value.to_i }
24
28
  property :redis, default: ::Turnstile::RedisConfig.new
29
+ property :custom_matcher
30
+ property :flush_interval, default: 10
31
+ property :trace
32
+ property :port, default: ::Turnstile::DEFAULT_PORT
25
33
 
26
34
  def configure
27
35
  yield self if block_given?
28
36
  self
29
37
  end
30
38
 
39
+ class << self
40
+ def from_file(file = nil)
41
+ return unless file
42
+ require(normalize(file))
43
+ rescue Exception => e
44
+ raise ConfigFileError.new("Error reading configuration from a file #{file}: #{e.message}")
45
+ end
46
+
47
+ private
48
+
49
+ def normalize(file)
50
+ file.start_with?('/') ? file : Dir.pwd + '/' + file
51
+ end
52
+ end
53
+
31
54
  def method_missing(method, *args, &block)
32
55
  return super unless method.to_s =~ /^redis_/
33
56
  prop = method.to_s.gsub(/^redis_/, '').to_sym
@@ -0,0 +1,31 @@
1
+ require_relative 'logger/helper'
2
+ module Turnstile
3
+ module Dependencies
4
+ def self.included(base)
5
+ base.include(Turnstile::Logger::Helper)
6
+ base.class_eval do
7
+
8
+ def tracker
9
+ @tracker ||= Tracker.new
10
+ end
11
+
12
+ def adapter
13
+ @adapter ||= Redis::Adapter.instance
14
+ end
15
+
16
+ def sampler
17
+ @sampler ||= Sampler.new
18
+ end
19
+
20
+ def config
21
+ @config ||= Turnstile.config
22
+ end
23
+
24
+ def aggregate
25
+ adapter.aggregate
26
+ end
27
+
28
+ end
29
+ end
30
+ end
31
+ end
@@ -1,47 +1,16 @@
1
+ require 'logger'
2
+ require_relative 'logger/helper'
3
+ require_relative 'logger/provider'
4
+
1
5
  module Turnstile
2
6
  module Logger
7
+ STDOUT.sync = true
8
+ @logger = ::Logger.new(STDOUT)
9
+ @logger.level = ::Logger::INFO
10
+ @enabled = false
3
11
 
4
12
  class << self
5
- def enable
6
- class << self
7
- self.send(:define_method, :log, proc { |msg| _log(msg) })
8
- self.send(:define_method, :logging, proc { |msg, &block| _logging(msg, &block) })
9
- end
10
- end
11
-
12
- def disable
13
- class << self
14
- self.send(:define_method, :log, proc { |msg|})
15
- self.send(:define_method, :logging, proc { |msg, &block| block.call })
16
- end
17
- end
18
-
19
- def log(msg)
20
- end
21
-
22
- def logging(msg, &block)
23
- block.call
24
- end
25
-
26
- private
27
-
28
- def _log(msg)
29
- puts "#{Time.now}: #{sprintf("%-20s", Thread.current[:name])} - #{msg}"
30
- end
31
-
32
- def _logging(message, &block)
33
- start = Time.now
34
- returned_from_block = yield
35
- elapsed_time = Time.now - start
36
- if returned_from_block.is_a?(String) && returned_from_block != ""
37
- message += " - #{returned_from_block}"
38
- end
39
- log "(#{"%9.2f" % (1000 * elapsed_time)}ms) #{message}"
40
- returned_from_block
41
- rescue Exception => e
42
- elapsed_time = Time.now - start
43
- log "(#{"%9.2f" % (1000 * elapsed_time)}ms) error: #{e.message} for #{message} "
44
- end
13
+ include Provider
45
14
  end
46
15
  end
47
16
  end
@@ -0,0 +1,42 @@
1
+ module Turnstile
2
+ module Logger
3
+ module Helper
4
+ def self.included(base)
5
+ base.extend(Helper)
6
+ end
7
+
8
+ def debug(*args, &block)
9
+ Turnstile::Logger.log(:debug, *args, &block)
10
+ end
11
+
12
+ def info(*args, &block)
13
+ Turnstile::Logger.log(:info, *args, &block)
14
+ end
15
+
16
+ def warn(*args, &block)
17
+ Turnstile::Logger.log(:warn, *args, &block)
18
+ end
19
+
20
+ def error(*args, &block)
21
+ Turnstile::Logger.log(:error, *args, &block)
22
+ end
23
+
24
+ def with_duration(level = :info, *args, &block)
25
+ Turnstile::Logger.logging(level, *args, &block)
26
+ end
27
+
28
+ def info_elapsed(*args, &block)
29
+ with_duration(:info, *args, &block)
30
+ end
31
+
32
+ def error_elapsed(*args, &block)
33
+ with_duration(:error, *args, &block)
34
+ end
35
+
36
+ alias with_logging info_elapsed
37
+ alias around_logging info_elapsed
38
+ alias log_around info_elapsed
39
+
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,74 @@
1
+ require_relative 'helper'
2
+
3
+ module Turnstile
4
+ module Logger
5
+ module Provider
6
+ attr_accessor :logger, :enabled
7
+
8
+ include Helper
9
+
10
+
11
+ def enable
12
+ self.enabled = true
13
+ class << self
14
+ self.send(:define_method, :log, proc { |level = :info, msg| _log(level, msg) })
15
+ self.send(:define_method, :logging, proc { |level = :info, msg, &block| _logging(level, msg, &block) })
16
+ end
17
+ end
18
+
19
+
20
+ def disable
21
+ self.enabled = false
22
+ class << self
23
+ self.send(:define_method, :log, proc { |_| })
24
+ self.send(:define_method, :logging, proc { |_, &block| block.call })
25
+ end
26
+ end
27
+
28
+
29
+ def log(*)
30
+ # no op
31
+ end
32
+
33
+
34
+ def logging(*)
35
+ # No op
36
+ end
37
+
38
+
39
+ private
40
+
41
+ def _log(level = :info, msg)
42
+ logger.send(level) do
43
+ "#{sprintf('%-15s', Thread.current[:name] || 'thread-main')} | #{msg}"
44
+ end
45
+ end
46
+
47
+
48
+ def _logging(level = :info, *args, &_block)
49
+ message = args.join(' ')
50
+
51
+ log(level, message) unless block_given?
52
+ return unless block_given?
53
+
54
+ start = Time.now
55
+ yield.tap do |result|
56
+ elapsed_time = Time.now - start
57
+ if result
58
+ message += " #{result.to_s}"
59
+ end
60
+ log_elapsed(level, elapsed_time, message)
61
+ end
62
+ rescue Exception => e
63
+ elapsed_time = Time.now - start
64
+ log_elapsed(level, elapsed_time, "error: #{e.message} for #{message}")
65
+ end
66
+
67
+
68
+ def log_elapsed(level = :info, elapsed_time, message)
69
+ log(level, "(#{'%7.2f' % (1000 * elapsed_time)}ms) #{message}")
70
+ end
71
+
72
+ end
73
+ end
74
+ end
@@ -1,5 +1,8 @@
1
1
  require 'hashie/mash'
2
2
  require 'hashie/extensions/mash/symbolize_keys'
3
+ require_relative 'redis/adapter'
4
+
5
+ require_relative 'dependencies'
3
6
 
4
7
  module Turnstile
5
8
  class Stats < ::Hashie::Mash
@@ -7,6 +10,8 @@ module Turnstile
7
10
  end
8
11
 
9
12
  class Observer
13
+ include Dependencies
14
+
10
15
  def stats
11
16
  data = adapter.fetch
12
17
  platforms = Hash[data.group_by { |d| d[:platform] }.map { |k, v| [k, sampler.extrapolate(v.count)] }]
@@ -20,14 +25,5 @@ module Turnstile
20
25
  })
21
26
  end
22
27
 
23
- private
24
-
25
- def adapter
26
- @adapter ||= Adapter.new
27
- end
28
-
29
- def sampler
30
- @sampler ||= Sampler.new
31
- end
32
28
  end
33
29
  end
@@ -0,0 +1,97 @@
1
+ require 'redis'
2
+ require 'timeout'
3
+ require 'singleton'
4
+ require 'connection_pool'
5
+ require_relative '../logger/helper'
6
+ require_relative 'connection'
7
+
8
+ module Turnstile
9
+ module Redis
10
+ class Adapter
11
+ SEP = ':'
12
+
13
+ include Singleton
14
+ include Logger::Helper
15
+ include Connection
16
+
17
+ attr_accessor :connection_pool, :config
18
+
19
+ # noinspection RubyResolve
20
+ def add(uid, platform, ip)
21
+ key = compose_key(uid, platform, ip)
22
+ timeout(config.redis.timeout) do
23
+ with_redis do |redis|
24
+ redis.setex(key, config.activity_interval, 1)
25
+ end
26
+ end
27
+ rescue StandardError => e
28
+ error "exception while writing to redis: #{e.inspect}"
29
+ end
30
+
31
+ def fetch
32
+ all_keys.map do |key|
33
+ fields = key.split(SEP)
34
+ {
35
+ uid: fields[1],
36
+ platform: fields[2],
37
+ ip: fields[3],
38
+ }
39
+ end
40
+ end
41
+
42
+ def all_keys
43
+ with_redis do |redis|
44
+ redis.keys("#{prefix}*")
45
+ end
46
+ end
47
+
48
+ def aggregate
49
+ all_keys.inject({}) { |hash, key| increment_platform(hash, key) }.tap do |h|
50
+ h['total'] = h.values.inject(&:+) || 0
51
+ end
52
+ end
53
+
54
+ def increment_platform(hash, key)
55
+ tuple = key.flatten if key.is_a?(Array)
56
+ tuple = key.split(SEP) if key.is_a?(String)
57
+
58
+ platform = tuple[2]
59
+
60
+ raise ArgumentError, "can't determine platform from the key #{key.inspect}" unless platform
61
+
62
+ hash[platform] ||= 0
63
+ hash[platform] += 1
64
+ hash
65
+ end
66
+
67
+ def config
68
+ Turnstile.config
69
+ end
70
+
71
+ def rconfig
72
+ config.redis
73
+ end
74
+
75
+ def compose_key(uid, platform = nil, ip = nil)
76
+ "#{prefix}#{uid}#{SEP}#{platform}#{SEP}#{ip}"
77
+ end
78
+
79
+ # def redis
80
+ # @redis ||= ConnectionPool::Wrapper.new(size: 3, timeout: 15, &redis_factory)
81
+ # end
82
+
83
+
84
+ private
85
+
86
+ def initialize(config = Turnstile.config)
87
+ self.config = config
88
+ self.connection_pool
89
+ end
90
+
91
+ def prefix
92
+ @prefix ||= "#{Turnstile::NS}|#{config.redis.namespace.gsub(/#{SEP}/, ':')}#{SEP}"
93
+ end
94
+ end
95
+ end
96
+ end
97
+
@@ -0,0 +1,116 @@
1
+ require 'timeout'
2
+ require 'colored2'
3
+ require_relative 'adapter'
4
+
5
+ module Turnstile
6
+ module Redis
7
+ module Connection
8
+ include Timeout
9
+
10
+ attr_accessor :config
11
+
12
+ def exec(redis_method, *args, &block)
13
+ send_proc = redis_method if redis_method.respond_to?(:call)
14
+ send_proc ||= ->(redis) { redis.send(redis_method, *args, &block) }
15
+ with_redis { |redis| send_proc.call(redis) }
16
+ end
17
+
18
+ %i(set get incr decr setex expire del setnx exists zadd zrange flushdb).each do |method|
19
+ define_method(method) do |*args|
20
+ self.exec method, *args
21
+ end
22
+ end
23
+
24
+ def with_redis
25
+ with_retries do
26
+ pool.with do |redis|
27
+ yield(Turnstile.debug? ? LoggingRedis.new(redis) : redis)
28
+ end
29
+ end
30
+ end
31
+
32
+ def with_pipelined
33
+ with_retries do
34
+ with_redis do |redis|
35
+ redis.pipelined do
36
+ yield(redis)
37
+ end
38
+ end
39
+ end
40
+ end
41
+
42
+ def with_multi
43
+ with_retries do
44
+ with_redis do |redis|
45
+ redis.multi do
46
+ yield(redis)
47
+ end
48
+ end
49
+ end
50
+ end
51
+
52
+ def with_retries(tries = 3)
53
+ yield(tries)
54
+ rescue Errno::EINVAL => e
55
+ on_error e
56
+ rescue ::Redis::BaseConnectionError => e
57
+ if (tries -= 1) > 0
58
+ sleep rand(0..0.01)
59
+ retry
60
+ else
61
+ on_error e
62
+ end
63
+ rescue ::Redis::CommandError => e
64
+ (e.message =~ /loading/i || e.message =~ /connection/i) ? on_error(e) : raise(e)
65
+ end
66
+
67
+ # This is how we'll be creating redis; depending on input arguments:
68
+ def redis_proc
69
+ @redis_proc ||= (config.redis.url ? redis_proc_from_url : redis_proc_from_opts)
70
+ end
71
+
72
+ # Connection pool to the Redis server
73
+ def pool
74
+ @pool ||= ::ConnectionPool.new(size: config.redis.pool_size, &redis_proc)
75
+ end
76
+
77
+ def on_error(e)
78
+ raise Error.new(e)
79
+ end
80
+
81
+ def redis_proc_from_url
82
+ @redis_proc_from_url ||= proc do
83
+ ::Redis.new(redis_opts(url: config.redis.url)).tap do |conn|
84
+ log_connection(conn)
85
+ end
86
+ end
87
+ end
88
+
89
+ def redis_opts(opts)
90
+ opts.tap do |hash|
91
+ hash[:driver] = 'hiredis' if config.redis_use_hiredis
92
+ end
93
+ end
94
+
95
+ def redis_proc_from_opts
96
+ @redis_proc_from_opts ||= proc {
97
+ ::Redis.new(redis_opts(host: config.redis.host,
98
+ port: config.redis.port,
99
+ db: config.redis.db)).tap do |conn|
100
+ log_connection(conn)
101
+ end
102
+ }
103
+ end
104
+
105
+ private
106
+
107
+ def log_connection(conn)
108
+ if config.trace
109
+ tdb "created a new redis connection: #{conn}"
110
+ tdb "driver: #{conn.instance_variable_get(:@client).driver}"
111
+ end
112
+ end
113
+ end
114
+ end
115
+ end
116
+