turnstile-rb 2.0.1 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +3 -0
- data/.travis.yml +2 -2
- data/LICENSE.txt +2 -1
- data/README.md +200 -11
- data/Rakefile +1 -0
- data/bin/turnstile +4 -6
- data/example/custom_csv_matcher.rb +22 -0
- data/lib/turnstile.rb +37 -8
- data/lib/turnstile/cli/launcher.rb +83 -0
- data/lib/turnstile/cli/parser.rb +166 -0
- data/lib/turnstile/cli/runner.rb +58 -0
- data/lib/turnstile/collector.rb +2 -2
- data/lib/turnstile/collector/actor.rb +81 -0
- data/lib/turnstile/collector/controller.rb +121 -0
- data/lib/turnstile/collector/flusher.rb +36 -0
- data/lib/turnstile/collector/formats.rb +7 -34
- data/lib/turnstile/collector/formats/custom_matcher.rb +19 -0
- data/lib/turnstile/collector/formats/delimited_matcher.rb +30 -0
- data/lib/turnstile/collector/formats/json_matcher.rb +30 -0
- data/lib/turnstile/collector/log_reader.rb +84 -46
- data/lib/turnstile/collector/{matcher.rb → regexp_matcher.rb} +4 -5
- data/lib/turnstile/collector/session.rb +7 -0
- data/lib/turnstile/commands.rb +20 -0
- data/lib/turnstile/commands/base.rb +20 -0
- data/lib/turnstile/commands/flushdb.rb +21 -0
- data/lib/turnstile/commands/print_keys.rb +19 -0
- data/lib/turnstile/commands/show.rb +89 -0
- data/lib/turnstile/configuration.rb +23 -0
- data/lib/turnstile/dependencies.rb +31 -0
- data/lib/turnstile/logger.rb +9 -40
- data/lib/turnstile/logger/helper.rb +42 -0
- data/lib/turnstile/logger/provider.rb +74 -0
- data/lib/turnstile/observer.rb +5 -9
- data/lib/turnstile/redis/adapter.rb +97 -0
- data/lib/turnstile/redis/connection.rb +116 -0
- data/lib/turnstile/redis/spy.rb +42 -0
- data/lib/turnstile/sampler.rb +9 -2
- data/lib/turnstile/tracker.rb +14 -14
- data/lib/turnstile/version.rb +51 -12
- data/lib/turnstile/web_app.rb +29 -0
- data/spec/spec_helper.rb +18 -3
- data/spec/support/logging.rb +17 -0
- data/spec/turnstile/adapter_spec.rb +59 -46
- data/spec/turnstile/collector/flusher_spec.rb +16 -0
- data/spec/turnstile/collector/log_reader_spec.rb +127 -77
- data/spec/turnstile/commands/show_spec.rb +40 -0
- data/spec/turnstile/tracker_spec.rb +21 -7
- data/spec/turnstile_spec.rb +3 -0
- data/turnstile-rb.gemspec +5 -2
- metadata +89 -22
- data/Gemfile +0 -6
- data/lib/turnstile/adapter.rb +0 -61
- data/lib/turnstile/collector/runner.rb +0 -72
- data/lib/turnstile/collector/updater.rb +0 -86
- data/lib/turnstile/parser.rb +0 -107
- data/lib/turnstile/runner.rb +0 -54
- data/lib/turnstile/summary.rb +0 -57
- data/spec/turnstile/summary_spec.rb +0 -41
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'hashie/dash'
|
2
2
|
require 'hashie/extensions/dash/property_translation'
|
3
|
+
require 'pp'
|
3
4
|
|
4
5
|
module Turnstile
|
5
6
|
class RedisConfig < ::Hashie::Dash
|
@@ -10,6 +11,9 @@ module Turnstile
|
|
10
11
|
property :port, default: 6379, required: true, transform_with: ->(value) { value.to_i }
|
11
12
|
property :db, default: 1, required: true, transform_with: ->(value) { value.to_i }
|
12
13
|
property :timeout, default: 0.05, required: true, transform_with: ->(value) { value.to_f }
|
14
|
+
property :namespace, default: '', required: false
|
15
|
+
property :pool_size, default: 5, required: true
|
16
|
+
property :use_hiredis, default: false
|
13
17
|
|
14
18
|
def configure
|
15
19
|
yield self if block_given?
|
@@ -22,12 +26,31 @@ module Turnstile
|
|
22
26
|
property :activity_interval, default: 60, required: true, transform_with: ->(value) { value.to_i }
|
23
27
|
property :sampling_rate, default: 100, required: true, transform_with: ->(value) { value.to_i }
|
24
28
|
property :redis, default: ::Turnstile::RedisConfig.new
|
29
|
+
property :custom_matcher
|
30
|
+
property :flush_interval, default: 10
|
31
|
+
property :trace
|
32
|
+
property :port, default: ::Turnstile::DEFAULT_PORT
|
25
33
|
|
26
34
|
def configure
|
27
35
|
yield self if block_given?
|
28
36
|
self
|
29
37
|
end
|
30
38
|
|
39
|
+
class << self
|
40
|
+
def from_file(file = nil)
|
41
|
+
return unless file
|
42
|
+
require(normalize(file))
|
43
|
+
rescue Exception => e
|
44
|
+
raise ConfigFileError.new("Error reading configuration from a file #{file}: #{e.message}")
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def normalize(file)
|
50
|
+
file.start_with?('/') ? file : Dir.pwd + '/' + file
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
31
54
|
def method_missing(method, *args, &block)
|
32
55
|
return super unless method.to_s =~ /^redis_/
|
33
56
|
prop = method.to_s.gsub(/^redis_/, '').to_sym
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require_relative 'logger/helper'
|
2
|
+
module Turnstile
|
3
|
+
module Dependencies
|
4
|
+
def self.included(base)
|
5
|
+
base.include(Turnstile::Logger::Helper)
|
6
|
+
base.class_eval do
|
7
|
+
|
8
|
+
def tracker
|
9
|
+
@tracker ||= Tracker.new
|
10
|
+
end
|
11
|
+
|
12
|
+
def adapter
|
13
|
+
@adapter ||= Redis::Adapter.instance
|
14
|
+
end
|
15
|
+
|
16
|
+
def sampler
|
17
|
+
@sampler ||= Sampler.new
|
18
|
+
end
|
19
|
+
|
20
|
+
def config
|
21
|
+
@config ||= Turnstile.config
|
22
|
+
end
|
23
|
+
|
24
|
+
def aggregate
|
25
|
+
adapter.aggregate
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/lib/turnstile/logger.rb
CHANGED
@@ -1,47 +1,16 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require_relative 'logger/helper'
|
3
|
+
require_relative 'logger/provider'
|
4
|
+
|
1
5
|
module Turnstile
|
2
6
|
module Logger
|
7
|
+
STDOUT.sync = true
|
8
|
+
@logger = ::Logger.new(STDOUT)
|
9
|
+
@logger.level = ::Logger::INFO
|
10
|
+
@enabled = false
|
3
11
|
|
4
12
|
class << self
|
5
|
-
|
6
|
-
class << self
|
7
|
-
self.send(:define_method, :log, proc { |msg| _log(msg) })
|
8
|
-
self.send(:define_method, :logging, proc { |msg, &block| _logging(msg, &block) })
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
def disable
|
13
|
-
class << self
|
14
|
-
self.send(:define_method, :log, proc { |msg|})
|
15
|
-
self.send(:define_method, :logging, proc { |msg, &block| block.call })
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
def log(msg)
|
20
|
-
end
|
21
|
-
|
22
|
-
def logging(msg, &block)
|
23
|
-
block.call
|
24
|
-
end
|
25
|
-
|
26
|
-
private
|
27
|
-
|
28
|
-
def _log(msg)
|
29
|
-
puts "#{Time.now}: #{sprintf("%-20s", Thread.current[:name])} - #{msg}"
|
30
|
-
end
|
31
|
-
|
32
|
-
def _logging(message, &block)
|
33
|
-
start = Time.now
|
34
|
-
returned_from_block = yield
|
35
|
-
elapsed_time = Time.now - start
|
36
|
-
if returned_from_block.is_a?(String) && returned_from_block != ""
|
37
|
-
message += " - #{returned_from_block}"
|
38
|
-
end
|
39
|
-
log "(#{"%9.2f" % (1000 * elapsed_time)}ms) #{message}"
|
40
|
-
returned_from_block
|
41
|
-
rescue Exception => e
|
42
|
-
elapsed_time = Time.now - start
|
43
|
-
log "(#{"%9.2f" % (1000 * elapsed_time)}ms) error: #{e.message} for #{message} "
|
44
|
-
end
|
13
|
+
include Provider
|
45
14
|
end
|
46
15
|
end
|
47
16
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Turnstile
|
2
|
+
module Logger
|
3
|
+
module Helper
|
4
|
+
def self.included(base)
|
5
|
+
base.extend(Helper)
|
6
|
+
end
|
7
|
+
|
8
|
+
def debug(*args, &block)
|
9
|
+
Turnstile::Logger.log(:debug, *args, &block)
|
10
|
+
end
|
11
|
+
|
12
|
+
def info(*args, &block)
|
13
|
+
Turnstile::Logger.log(:info, *args, &block)
|
14
|
+
end
|
15
|
+
|
16
|
+
def warn(*args, &block)
|
17
|
+
Turnstile::Logger.log(:warn, *args, &block)
|
18
|
+
end
|
19
|
+
|
20
|
+
def error(*args, &block)
|
21
|
+
Turnstile::Logger.log(:error, *args, &block)
|
22
|
+
end
|
23
|
+
|
24
|
+
def with_duration(level = :info, *args, &block)
|
25
|
+
Turnstile::Logger.logging(level, *args, &block)
|
26
|
+
end
|
27
|
+
|
28
|
+
def info_elapsed(*args, &block)
|
29
|
+
with_duration(:info, *args, &block)
|
30
|
+
end
|
31
|
+
|
32
|
+
def error_elapsed(*args, &block)
|
33
|
+
with_duration(:error, *args, &block)
|
34
|
+
end
|
35
|
+
|
36
|
+
alias with_logging info_elapsed
|
37
|
+
alias around_logging info_elapsed
|
38
|
+
alias log_around info_elapsed
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require_relative 'helper'
|
2
|
+
|
3
|
+
module Turnstile
|
4
|
+
module Logger
|
5
|
+
module Provider
|
6
|
+
attr_accessor :logger, :enabled
|
7
|
+
|
8
|
+
include Helper
|
9
|
+
|
10
|
+
|
11
|
+
def enable
|
12
|
+
self.enabled = true
|
13
|
+
class << self
|
14
|
+
self.send(:define_method, :log, proc { |level = :info, msg| _log(level, msg) })
|
15
|
+
self.send(:define_method, :logging, proc { |level = :info, msg, &block| _logging(level, msg, &block) })
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
def disable
|
21
|
+
self.enabled = false
|
22
|
+
class << self
|
23
|
+
self.send(:define_method, :log, proc { |_| })
|
24
|
+
self.send(:define_method, :logging, proc { |_, &block| block.call })
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
def log(*)
|
30
|
+
# no op
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
def logging(*)
|
35
|
+
# No op
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def _log(level = :info, msg)
|
42
|
+
logger.send(level) do
|
43
|
+
"#{sprintf('%-15s', Thread.current[:name] || 'thread-main')} | #{msg}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
def _logging(level = :info, *args, &_block)
|
49
|
+
message = args.join(' ')
|
50
|
+
|
51
|
+
log(level, message) unless block_given?
|
52
|
+
return unless block_given?
|
53
|
+
|
54
|
+
start = Time.now
|
55
|
+
yield.tap do |result|
|
56
|
+
elapsed_time = Time.now - start
|
57
|
+
if result
|
58
|
+
message += " #{result.to_s}"
|
59
|
+
end
|
60
|
+
log_elapsed(level, elapsed_time, message)
|
61
|
+
end
|
62
|
+
rescue Exception => e
|
63
|
+
elapsed_time = Time.now - start
|
64
|
+
log_elapsed(level, elapsed_time, "error: #{e.message} for #{message}")
|
65
|
+
end
|
66
|
+
|
67
|
+
|
68
|
+
def log_elapsed(level = :info, elapsed_time, message)
|
69
|
+
log(level, "(#{'%7.2f' % (1000 * elapsed_time)}ms) #{message}")
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
data/lib/turnstile/observer.rb
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
require 'hashie/mash'
|
2
2
|
require 'hashie/extensions/mash/symbolize_keys'
|
3
|
+
require_relative 'redis/adapter'
|
4
|
+
|
5
|
+
require_relative 'dependencies'
|
3
6
|
|
4
7
|
module Turnstile
|
5
8
|
class Stats < ::Hashie::Mash
|
@@ -7,6 +10,8 @@ module Turnstile
|
|
7
10
|
end
|
8
11
|
|
9
12
|
class Observer
|
13
|
+
include Dependencies
|
14
|
+
|
10
15
|
def stats
|
11
16
|
data = adapter.fetch
|
12
17
|
platforms = Hash[data.group_by { |d| d[:platform] }.map { |k, v| [k, sampler.extrapolate(v.count)] }]
|
@@ -20,14 +25,5 @@ module Turnstile
|
|
20
25
|
})
|
21
26
|
end
|
22
27
|
|
23
|
-
private
|
24
|
-
|
25
|
-
def adapter
|
26
|
-
@adapter ||= Adapter.new
|
27
|
-
end
|
28
|
-
|
29
|
-
def sampler
|
30
|
-
@sampler ||= Sampler.new
|
31
|
-
end
|
32
28
|
end
|
33
29
|
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
require 'redis'
|
2
|
+
require 'timeout'
|
3
|
+
require 'singleton'
|
4
|
+
require 'connection_pool'
|
5
|
+
require_relative '../logger/helper'
|
6
|
+
require_relative 'connection'
|
7
|
+
|
8
|
+
module Turnstile
|
9
|
+
module Redis
|
10
|
+
class Adapter
|
11
|
+
SEP = ':'
|
12
|
+
|
13
|
+
include Singleton
|
14
|
+
include Logger::Helper
|
15
|
+
include Connection
|
16
|
+
|
17
|
+
attr_accessor :connection_pool, :config
|
18
|
+
|
19
|
+
# noinspection RubyResolve
|
20
|
+
def add(uid, platform, ip)
|
21
|
+
key = compose_key(uid, platform, ip)
|
22
|
+
timeout(config.redis.timeout) do
|
23
|
+
with_redis do |redis|
|
24
|
+
redis.setex(key, config.activity_interval, 1)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
rescue StandardError => e
|
28
|
+
error "exception while writing to redis: #{e.inspect}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def fetch
|
32
|
+
all_keys.map do |key|
|
33
|
+
fields = key.split(SEP)
|
34
|
+
{
|
35
|
+
uid: fields[1],
|
36
|
+
platform: fields[2],
|
37
|
+
ip: fields[3],
|
38
|
+
}
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def all_keys
|
43
|
+
with_redis do |redis|
|
44
|
+
redis.keys("#{prefix}*")
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def aggregate
|
49
|
+
all_keys.inject({}) { |hash, key| increment_platform(hash, key) }.tap do |h|
|
50
|
+
h['total'] = h.values.inject(&:+) || 0
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def increment_platform(hash, key)
|
55
|
+
tuple = key.flatten if key.is_a?(Array)
|
56
|
+
tuple = key.split(SEP) if key.is_a?(String)
|
57
|
+
|
58
|
+
platform = tuple[2]
|
59
|
+
|
60
|
+
raise ArgumentError, "can't determine platform from the key #{key.inspect}" unless platform
|
61
|
+
|
62
|
+
hash[platform] ||= 0
|
63
|
+
hash[platform] += 1
|
64
|
+
hash
|
65
|
+
end
|
66
|
+
|
67
|
+
def config
|
68
|
+
Turnstile.config
|
69
|
+
end
|
70
|
+
|
71
|
+
def rconfig
|
72
|
+
config.redis
|
73
|
+
end
|
74
|
+
|
75
|
+
def compose_key(uid, platform = nil, ip = nil)
|
76
|
+
"#{prefix}#{uid}#{SEP}#{platform}#{SEP}#{ip}"
|
77
|
+
end
|
78
|
+
|
79
|
+
# def redis
|
80
|
+
# @redis ||= ConnectionPool::Wrapper.new(size: 3, timeout: 15, &redis_factory)
|
81
|
+
# end
|
82
|
+
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
def initialize(config = Turnstile.config)
|
87
|
+
self.config = config
|
88
|
+
self.connection_pool
|
89
|
+
end
|
90
|
+
|
91
|
+
def prefix
|
92
|
+
@prefix ||= "#{Turnstile::NS}|#{config.redis.namespace.gsub(/#{SEP}/, ':')}#{SEP}"
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
@@ -0,0 +1,116 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
require 'colored2'
|
3
|
+
require_relative 'adapter'
|
4
|
+
|
5
|
+
module Turnstile
|
6
|
+
module Redis
|
7
|
+
module Connection
|
8
|
+
include Timeout
|
9
|
+
|
10
|
+
attr_accessor :config
|
11
|
+
|
12
|
+
def exec(redis_method, *args, &block)
|
13
|
+
send_proc = redis_method if redis_method.respond_to?(:call)
|
14
|
+
send_proc ||= ->(redis) { redis.send(redis_method, *args, &block) }
|
15
|
+
with_redis { |redis| send_proc.call(redis) }
|
16
|
+
end
|
17
|
+
|
18
|
+
%i(set get incr decr setex expire del setnx exists zadd zrange flushdb).each do |method|
|
19
|
+
define_method(method) do |*args|
|
20
|
+
self.exec method, *args
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def with_redis
|
25
|
+
with_retries do
|
26
|
+
pool.with do |redis|
|
27
|
+
yield(Turnstile.debug? ? LoggingRedis.new(redis) : redis)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def with_pipelined
|
33
|
+
with_retries do
|
34
|
+
with_redis do |redis|
|
35
|
+
redis.pipelined do
|
36
|
+
yield(redis)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def with_multi
|
43
|
+
with_retries do
|
44
|
+
with_redis do |redis|
|
45
|
+
redis.multi do
|
46
|
+
yield(redis)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def with_retries(tries = 3)
|
53
|
+
yield(tries)
|
54
|
+
rescue Errno::EINVAL => e
|
55
|
+
on_error e
|
56
|
+
rescue ::Redis::BaseConnectionError => e
|
57
|
+
if (tries -= 1) > 0
|
58
|
+
sleep rand(0..0.01)
|
59
|
+
retry
|
60
|
+
else
|
61
|
+
on_error e
|
62
|
+
end
|
63
|
+
rescue ::Redis::CommandError => e
|
64
|
+
(e.message =~ /loading/i || e.message =~ /connection/i) ? on_error(e) : raise(e)
|
65
|
+
end
|
66
|
+
|
67
|
+
# This is how we'll be creating redis; depending on input arguments:
|
68
|
+
def redis_proc
|
69
|
+
@redis_proc ||= (config.redis.url ? redis_proc_from_url : redis_proc_from_opts)
|
70
|
+
end
|
71
|
+
|
72
|
+
# Connection pool to the Redis server
|
73
|
+
def pool
|
74
|
+
@pool ||= ::ConnectionPool.new(size: config.redis.pool_size, &redis_proc)
|
75
|
+
end
|
76
|
+
|
77
|
+
def on_error(e)
|
78
|
+
raise Error.new(e)
|
79
|
+
end
|
80
|
+
|
81
|
+
def redis_proc_from_url
|
82
|
+
@redis_proc_from_url ||= proc do
|
83
|
+
::Redis.new(redis_opts(url: config.redis.url)).tap do |conn|
|
84
|
+
log_connection(conn)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def redis_opts(opts)
|
90
|
+
opts.tap do |hash|
|
91
|
+
hash[:driver] = 'hiredis' if config.redis_use_hiredis
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def redis_proc_from_opts
|
96
|
+
@redis_proc_from_opts ||= proc {
|
97
|
+
::Redis.new(redis_opts(host: config.redis.host,
|
98
|
+
port: config.redis.port,
|
99
|
+
db: config.redis.db)).tap do |conn|
|
100
|
+
log_connection(conn)
|
101
|
+
end
|
102
|
+
}
|
103
|
+
end
|
104
|
+
|
105
|
+
private
|
106
|
+
|
107
|
+
def log_connection(conn)
|
108
|
+
if config.trace
|
109
|
+
tdb "created a new redis connection: #{conn}"
|
110
|
+
tdb "driver: #{conn.instance_variable_get(:@client).driver}"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|