event_meter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +1081 -0
- data/exe/event_meter +5 -0
- data/lib/event_meter/auto_cleanup.rb +93 -0
- data/lib/event_meter/cli.rb +124 -0
- data/lib/event_meter/configuration.rb +244 -0
- data/lib/event_meter/errors.rb +9 -0
- data/lib/event_meter/event.rb +180 -0
- data/lib/event_meter/event_payload.rb +103 -0
- data/lib/event_meter/hash_input.rb +20 -0
- data/lib/event_meter/index_key.rb +19 -0
- data/lib/event_meter/keys.rb +63 -0
- data/lib/event_meter/path_name.rb +37 -0
- data/lib/event_meter/processor.rb +305 -0
- data/lib/event_meter/rails.rb +79 -0
- data/lib/event_meter/report_definition.rb +184 -0
- data/lib/event_meter/reports.rb +143 -0
- data/lib/event_meter/rollup.rb +148 -0
- data/lib/event_meter/stores/cleanup_helpers.rb +76 -0
- data/lib/event_meter/stores/file_helpers.rb +47 -0
- data/lib/event_meter/stores/lock_refresher.rb +75 -0
- data/lib/event_meter/stores/namespace.rb +14 -0
- data/lib/event_meter/stores/redis_lock.rb +77 -0
- data/lib/event_meter/stores/rollup/active_record_postgres.rb +135 -0
- data/lib/event_meter/stores/rollup/file.rb +736 -0
- data/lib/event_meter/stores/rollup/postgres.rb +813 -0
- data/lib/event_meter/stores/rollup/redis.rb +349 -0
- data/lib/event_meter/stores/stream/file.rb +98 -0
- data/lib/event_meter/stores/stream/redis.rb +79 -0
- data/lib/event_meter/time_buckets.rb +56 -0
- data/lib/event_meter/version.rb +3 -0
- data/lib/event_meter/write_result.rb +26 -0
- data/lib/event_meter.rb +150 -0
- data/lib/generators/event_meter/install_generator.rb +57 -0
- data/lib/generators/event_meter/templates/create_event_meter_tables.rb.erb +12 -0
- data/lib/generators/event_meter/templates/event_meter.rb.erb +12 -0
- metadata +156 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
require "securerandom"
|
|
2
|
+
|
|
3
|
+
require_relative "lock_refresher"
|
|
4
|
+
|
|
5
|
+
module EventMeter
|
|
6
|
+
module Stores
|
|
7
|
+
module RedisLock
|
|
8
|
+
LOCK_REFRESH_RATIO = 2.0
|
|
9
|
+
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def with_redis_lock(key, ttl:)
|
|
13
|
+
ttl = redis_lock_ttl(ttl)
|
|
14
|
+
token = SecureRandom.hex(16)
|
|
15
|
+
acquired = lock_redis.set(key, token, nx: true, ex: ttl)
|
|
16
|
+
return false unless acquired
|
|
17
|
+
|
|
18
|
+
refresher = start_redis_lock_refresher(key, token, ttl)
|
|
19
|
+
yield
|
|
20
|
+
true
|
|
21
|
+
ensure
|
|
22
|
+
stop_redis_lock_refresher(refresher)
|
|
23
|
+
release_redis_lock(key, token) if acquired
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def release_redis_lock(key, token)
|
|
27
|
+
lock_redis.eval(<<~LUA, keys: [key], argv: [token])
|
|
28
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
29
|
+
return redis.call("del", KEYS[1])
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
return 0
|
|
33
|
+
LUA
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def refresh_redis_lock(key, token, ttl)
|
|
37
|
+
result = lock_redis.eval(<<~LUA, keys: [key], argv: [token, ttl])
|
|
38
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
39
|
+
return redis.call("expire", KEYS[1], ARGV[2])
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
return 0
|
|
43
|
+
LUA
|
|
44
|
+
|
|
45
|
+
result == true || result.to_s == "1"
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def start_redis_lock_refresher(key, token, ttl)
|
|
49
|
+
LockRefresher.new(
|
|
50
|
+
interval: ttl.to_f / LOCK_REFRESH_RATIO,
|
|
51
|
+
refresh: -> { refresh_redis_lock(key, token, ttl) },
|
|
52
|
+
failure_message: "redis lock refresh failed",
|
|
53
|
+
thread_name: "event_meter redis lock refresher"
|
|
54
|
+
).start
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def stop_redis_lock_refresher(refresher)
|
|
58
|
+
return unless refresher
|
|
59
|
+
|
|
60
|
+
refresher.stop
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def redis_lock_ttl(ttl)
|
|
64
|
+
ttl = Integer(ttl)
|
|
65
|
+
return ttl if ttl.positive?
|
|
66
|
+
|
|
67
|
+
raise ArgumentError, "lock ttl must be positive"
|
|
68
|
+
rescue ArgumentError, TypeError, RangeError
|
|
69
|
+
raise ArgumentError, "lock ttl must be positive"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def lock_redis
|
|
73
|
+
redis
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
require_relative "postgres"
|
|
2
|
+
|
|
3
|
+
module EventMeter
|
|
4
|
+
module Stores
|
|
5
|
+
module Rollup
|
|
6
|
+
class ActiveRecordPostgres < Postgres
|
|
7
|
+
class Connection
|
|
8
|
+
TRANSACTION_COMMANDS = {
|
|
9
|
+
"BEGIN" => :begin,
|
|
10
|
+
"COMMIT" => :finish,
|
|
11
|
+
"ROLLBACK" => :finish
|
|
12
|
+
}.freeze
|
|
13
|
+
|
|
14
|
+
attr_reader :connection_class
|
|
15
|
+
|
|
16
|
+
def initialize(connection_class)
|
|
17
|
+
@connection_class = connection_class
|
|
18
|
+
@transaction_connection = nil
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def exec(sql)
|
|
22
|
+
with_connection_for(sql) { |connection| connection.exec(sql) }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def exec_params(sql, params)
|
|
26
|
+
with_connection_for(sql) { |connection| connection.exec_params(sql, params) }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def with_connection_for(sql)
|
|
32
|
+
command = transaction_command(sql)
|
|
33
|
+
|
|
34
|
+
return begin_transaction { |connection| yield connection } if command == :begin
|
|
35
|
+
return finish_transaction { |connection| yield connection } if command == :finish
|
|
36
|
+
return yield active_raw_connection if transaction_open?
|
|
37
|
+
|
|
38
|
+
connection_pool.with_connection do |connection|
|
|
39
|
+
yield connection.raw_connection
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def begin_transaction
|
|
44
|
+
raise ConfigurationError, "event_meter postgres transaction is already open" if transaction_open?
|
|
45
|
+
|
|
46
|
+
@transaction_connection = connection_pool.checkout
|
|
47
|
+
yield active_raw_connection
|
|
48
|
+
rescue StandardError
|
|
49
|
+
checkin_transaction_connection
|
|
50
|
+
raise
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def finish_transaction
|
|
54
|
+
return with_connection_for_non_transaction { |connection| yield connection } unless transaction_open?
|
|
55
|
+
|
|
56
|
+
yield active_raw_connection
|
|
57
|
+
ensure
|
|
58
|
+
checkin_transaction_connection
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def with_connection_for_non_transaction
|
|
62
|
+
connection_pool.with_connection do |connection|
|
|
63
|
+
yield connection.raw_connection
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def transaction_command(sql)
|
|
68
|
+
TRANSACTION_COMMANDS[sql.to_s.strip.upcase]
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def transaction_open?
|
|
72
|
+
!@transaction_connection.nil?
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def active_raw_connection
|
|
76
|
+
@transaction_connection.raw_connection
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def checkin_transaction_connection
|
|
80
|
+
connection = @transaction_connection
|
|
81
|
+
@transaction_connection = nil
|
|
82
|
+
connection_pool.checkin(connection) if connection
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def connection_pool
|
|
86
|
+
connection_class.connection_pool
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
attr_reader :connection_class
|
|
91
|
+
|
|
92
|
+
def self.install!(connection_class: default_connection_class, table_prefix: "event_meter")
|
|
93
|
+
connection_class.connection_pool.with_connection do |connection|
|
|
94
|
+
Postgres.install!(
|
|
95
|
+
connection: connection.raw_connection,
|
|
96
|
+
table_prefix: table_prefix
|
|
97
|
+
)
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def self.default_connection_class
|
|
102
|
+
return ::ActiveRecord::Base if defined?(::ActiveRecord::Base)
|
|
103
|
+
|
|
104
|
+
raise ConfigurationError, "ActiveRecord is required for active_record_postgres rollup storage"
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def initialize(connection_class: self.class.default_connection_class, lock_connection: nil, **options)
|
|
108
|
+
@connection_class = connection_class
|
|
109
|
+
super(
|
|
110
|
+
connection: Connection.new(connection_class),
|
|
111
|
+
lock_connection: lock_connection || Connection.new(connection_class),
|
|
112
|
+
**options
|
|
113
|
+
)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def for_report(name:, version:)
|
|
117
|
+
name = name.to_s
|
|
118
|
+
version = version.to_i
|
|
119
|
+
|
|
120
|
+
self.class.new(
|
|
121
|
+
connection_class: connection_class,
|
|
122
|
+
connection_lock: connection_lock,
|
|
123
|
+
lock_connection: lock_connection,
|
|
124
|
+
lock_connection_lock: lock_connection_lock,
|
|
125
|
+
namespace: namespace,
|
|
126
|
+
table_prefix: table_prefix,
|
|
127
|
+
report_name: name,
|
|
128
|
+
version: version,
|
|
129
|
+
lock_scope: "#{Keys.event_name(name)}:#{Keys.version_key(version)}"
|
|
130
|
+
)
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|