logster 1.2.11 → 1.3.pre
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +18 -17
- data/.travis.yml +15 -16
- data/CHANGELOG.md +130 -130
- data/Gemfile +4 -4
- data/Guardfile +8 -8
- data/LICENSE.txt +22 -22
- data/README.md +99 -96
- data/Rakefile +24 -23
- data/assets/fonts/FontAwesome.otf +0 -0
- data/assets/fonts/fontawesome-webfont.eot +0 -0
- data/assets/fonts/fontawesome-webfont.svg +639 -639
- data/assets/fonts/fontawesome-webfont.ttf +0 -0
- data/assets/fonts/fontawesome-webfont.woff +0 -0
- data/assets/fonts/fontawesome-webfont.woff2 +0 -0
- data/assets/images/Icon-144_rounded.png +0 -0
- data/assets/images/Icon-144_square.png +0 -0
- data/assets/images/icon_144x144.png +0 -0
- data/assets/images/icon_64x64.png +0 -0
- data/assets/javascript/client-app.js +81 -0
- data/assets/javascript/vendor.js +5302 -0
- data/assets/stylesheets/client-app.css +1 -0
- data/assets/stylesheets/vendor.css +4 -0
- data/build_client_app.sh +12 -0
- data/client-app/.editorconfig +20 -0
- data/client-app/.ember-cli +9 -0
- data/client-app/.eslintignore +19 -0
- data/client-app/.eslintrc.js +46 -0
- data/client-app/.gitignore +23 -0
- data/client-app/.travis.yml +27 -0
- data/client-app/.watchmanconfig +3 -0
- data/client-app/README.md +57 -0
- data/client-app/app/app.js +14 -0
- data/client-app/app/components/message-info.js +18 -0
- data/client-app/app/components/message-row.js +45 -0
- data/client-app/app/components/panel-resizer.js +75 -0
- data/client-app/app/components/tab-contents.js +27 -0
- data/client-app/app/components/tab-link.js +5 -0
- data/client-app/app/components/tabbed-section.js +32 -0
- data/client-app/app/components/time-formatter.js +25 -0
- data/client-app/app/components/update-time.js +21 -0
- data/client-app/app/controllers/index.js +83 -0
- data/client-app/app/controllers/show.js +13 -0
- data/client-app/app/index.html +29 -0
- data/client-app/app/initializers/app-init.js +55 -0
- data/client-app/app/lib/preload.js +14 -0
- data/client-app/app/lib/utilities.js +140 -0
- data/client-app/app/models/message-collection.js +158 -0
- data/client-app/app/models/message.js +99 -0
- data/client-app/app/resolver.js +3 -0
- data/client-app/app/router.js +14 -0
- data/client-app/app/routes/index.js +53 -0
- data/client-app/app/routes/show.js +14 -0
- data/{assets/stylesheets → client-app/app/styles}/app.css +387 -390
- data/{assets/javascript → client-app/app}/templates/application.hbs +2 -2
- data/client-app/app/templates/components/message-info.hbs +44 -0
- data/{assets/javascript → client-app/app/templates}/components/message-row.hbs +17 -17
- data/client-app/app/templates/components/tabbed-section.hbs +10 -0
- data/client-app/app/templates/components/time-formatter.hbs +1 -0
- data/{assets/javascript → client-app/app}/templates/index.hbs +57 -57
- data/{assets/javascript → client-app/app}/templates/show.hbs +4 -4
- data/client-app/config/environment.js +51 -0
- data/client-app/config/optional-features.json +3 -0
- data/client-app/config/targets.js +18 -0
- data/client-app/ember-cli-build.js +29 -0
- data/client-app/package-lock.json +11365 -0
- data/client-app/package.json +56 -0
- data/client-app/testem.js +25 -0
- data/client-app/tests/index.html +34 -0
- data/client-app/tests/integration/components/message-info-test.js +26 -0
- data/client-app/tests/integration/components/message-row-test.js +26 -0
- data/client-app/tests/integration/components/panel-resizer-test.js +26 -0
- data/client-app/tests/integration/components/tab-contents-test.js +26 -0
- data/client-app/tests/integration/components/tab-link-test.js +26 -0
- data/client-app/tests/integration/components/tabbed-section-test.js +26 -0
- data/client-app/tests/integration/components/time-formatter-test.js +26 -0
- data/client-app/tests/integration/components/update-time-test.js +26 -0
- data/client-app/tests/test-helper.js +8 -0
- data/client-app/tests/unit/controllers/index-test.js +12 -0
- data/client-app/tests/unit/controllers/show-test.js +12 -0
- data/client-app/tests/unit/initializers/app-init-test.js +31 -0
- data/client-app/tests/unit/routes/index-test.js +11 -0
- data/client-app/tests/unit/routes/show-test.js +11 -0
- data/lib/examples/sidekiq_logster_reporter.rb +21 -21
- data/lib/logster.rb +54 -54
- data/lib/logster/base_store.rb +130 -130
- data/lib/logster/configuration.rb +25 -25
- data/lib/logster/ignore_pattern.rb +65 -65
- data/lib/logster/logger.rb +102 -101
- data/lib/logster/message.rb +227 -226
- data/lib/logster/middleware/debug_exceptions.rb +26 -26
- data/lib/logster/middleware/reporter.rb +56 -54
- data/lib/logster/middleware/viewer.rb +220 -251
- data/lib/logster/rails/railtie.rb +58 -58
- data/lib/logster/redis_store.rb +481 -477
- data/lib/logster/version.rb +3 -3
- data/lib/logster/web.rb +14 -14
- data/logster.gemspec +34 -33
- data/test/examples/test_sidekiq_reporter_example.rb +46 -46
- data/test/fake_data/Gemfile +4 -4
- data/test/fake_data/generate.rb +10 -10
- data/test/logster/middleware/test_reporter.rb +21 -21
- data/test/logster/middleware/test_viewer.rb +96 -70
- data/test/logster/test_base_store.rb +147 -147
- data/test/logster/test_ignore_pattern.rb +41 -41
- data/test/logster/test_logger.rb +74 -74
- data/test/logster/test_message.rb +34 -34
- data/test/logster/test_redis_rate_limiter.rb +230 -230
- data/test/logster/test_redis_store.rb +427 -414
- data/test/test_helper.rb +38 -37
- data/vendor/assets/javascripts/logster.js.erb +39 -39
- metadata +83 -24
- data/assets/javascript/app.js +0 -817
- data/assets/javascript/components/message-info.hbs +0 -47
- data/assets/javascript/components/panel-resizer.hbs +0 -0
- data/assets/javascript/components/tab-contents.hbs +0 -1
- data/assets/javascript/components/tab-link.hbs +0 -1
- data/assets/javascript/components/tabbed-section.hbs +0 -6
- data/assets/javascript/external/ember-template-compiler.js +0 -22346
- data/assets/javascript/external/ember.js +0 -58500
- data/assets/javascript/external/ember.min.js +0 -17
- data/assets/javascript/external/jquery.min.js +0 -5
- data/assets/javascript/external/lodash.min.js +0 -87
- data/assets/javascript/external/moment.min.js +0 -6
- data/assets/stylesheets/font-awesome.min.css +0 -4
- data/bower.json +0 -25
@@ -1,58 +1,58 @@
|
|
1
|
-
module Logster::Rails
|
2
|
-
|
3
|
-
# this magically registers logster.js in the asset pipeline
|
4
|
-
class Engine < Rails::Engine
|
5
|
-
end
|
6
|
-
|
7
|
-
def self.set_logger(config)
|
8
|
-
return unless Logster.config.environments.include?(Rails.env.to_sym)
|
9
|
-
|
10
|
-
require 'logster/middleware/debug_exceptions'
|
11
|
-
require 'logster/middleware/reporter'
|
12
|
-
|
13
|
-
store = Logster.store ||= Logster::RedisStore.new
|
14
|
-
store.level = Logger::Severity::WARN if Rails.env.production?
|
15
|
-
|
16
|
-
logger = Logster::Logger.new(store)
|
17
|
-
logger.chain(::Rails.logger)
|
18
|
-
logger.level = ::Rails.logger.level
|
19
|
-
|
20
|
-
Logster.logger = ::Rails.logger = config.logger = logger
|
21
|
-
end
|
22
|
-
|
23
|
-
|
24
|
-
def self.initialize!(app)
|
25
|
-
return unless Logster.config.environments.include?(Rails.env.to_sym)
|
26
|
-
|
27
|
-
if Logster::Logger === Rails.logger
|
28
|
-
app.middleware.insert_before ActionDispatch::ShowExceptions, Logster::Middleware::Reporter
|
29
|
-
|
30
|
-
if Rails::VERSION::MAJOR == 3
|
31
|
-
app.middleware.insert_before ActionDispatch::DebugExceptions, Logster::Middleware::DebugExceptions
|
32
|
-
else
|
33
|
-
app.middleware.insert_before ActionDispatch::DebugExceptions, Logster::Middleware::DebugExceptions, Rails.application
|
34
|
-
end
|
35
|
-
|
36
|
-
app.middleware.delete ActionDispatch::DebugExceptions
|
37
|
-
app.config.colorize_logging = false
|
38
|
-
|
39
|
-
unless Logster.config.application_version
|
40
|
-
git_version = `cd #{Rails.root} && git rev-parse --short HEAD 2> /dev/null`
|
41
|
-
if git_version.present?
|
42
|
-
Logster.config.application_version = git_version.strip
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
class Railtie < ::Rails::Railtie
|
49
|
-
|
50
|
-
config.before_initialize do
|
51
|
-
Logster::Rails.set_logger(config)
|
52
|
-
end
|
53
|
-
|
54
|
-
initializer "logster.configure_rails_initialization" do |app|
|
55
|
-
Logster::Rails.initialize!(app)
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
1
|
+
module Logster::Rails
|
2
|
+
|
3
|
+
# this magically registers logster.js in the asset pipeline
|
4
|
+
class Engine < Rails::Engine
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.set_logger(config)
|
8
|
+
return unless Logster.config.environments.include?(Rails.env.to_sym)
|
9
|
+
|
10
|
+
require 'logster/middleware/debug_exceptions'
|
11
|
+
require 'logster/middleware/reporter'
|
12
|
+
|
13
|
+
store = Logster.store ||= Logster::RedisStore.new
|
14
|
+
store.level = Logger::Severity::WARN if Rails.env.production?
|
15
|
+
|
16
|
+
logger = Logster::Logger.new(store)
|
17
|
+
logger.chain(::Rails.logger)
|
18
|
+
logger.level = ::Rails.logger.level
|
19
|
+
|
20
|
+
Logster.logger = ::Rails.logger = config.logger = logger
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
def self.initialize!(app)
|
25
|
+
return unless Logster.config.environments.include?(Rails.env.to_sym)
|
26
|
+
|
27
|
+
if Logster::Logger === Rails.logger
|
28
|
+
app.middleware.insert_before ActionDispatch::ShowExceptions, Logster::Middleware::Reporter
|
29
|
+
|
30
|
+
if Rails::VERSION::MAJOR == 3
|
31
|
+
app.middleware.insert_before ActionDispatch::DebugExceptions, Logster::Middleware::DebugExceptions
|
32
|
+
else
|
33
|
+
app.middleware.insert_before ActionDispatch::DebugExceptions, Logster::Middleware::DebugExceptions, Rails.application
|
34
|
+
end
|
35
|
+
|
36
|
+
app.middleware.delete ActionDispatch::DebugExceptions
|
37
|
+
app.config.colorize_logging = false
|
38
|
+
|
39
|
+
unless Logster.config.application_version
|
40
|
+
git_version = `cd #{Rails.root} && git rev-parse --short HEAD 2> /dev/null`
|
41
|
+
if git_version.present?
|
42
|
+
Logster.config.application_version = git_version.strip
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
class Railtie < ::Rails::Railtie
|
49
|
+
|
50
|
+
config.before_initialize do
|
51
|
+
Logster::Rails.set_logger(config)
|
52
|
+
end
|
53
|
+
|
54
|
+
initializer "logster.configure_rails_initialization" do |app|
|
55
|
+
Logster::Rails.initialize!(app)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/lib/logster/redis_store.rb
CHANGED
@@ -1,477 +1,481 @@
|
|
1
|
-
require 'json'
|
2
|
-
require 'logster/base_store'
|
3
|
-
|
4
|
-
module Logster
|
5
|
-
class RedisRateLimiter
|
6
|
-
BUCKETS = 6
|
7
|
-
PREFIX = "__LOGSTER__RATE_LIMIT".freeze
|
8
|
-
|
9
|
-
attr_reader :duration, :callback
|
10
|
-
|
11
|
-
def self.clear_all(redis, redis_prefix=nil)
|
12
|
-
prefix = key_prefix(redis_prefix)
|
13
|
-
|
14
|
-
redis.eval "
|
15
|
-
local keys = redis.call('keys', '*#{prefix}*')
|
16
|
-
if (table.getn(keys) > 0) then
|
17
|
-
redis.call('del', unpack(keys))
|
18
|
-
end
|
19
|
-
"
|
20
|
-
end
|
21
|
-
|
22
|
-
def initialize(redis, severities, limit, duration, redis_prefix = nil, callback = nil)
|
23
|
-
@severities = severities
|
24
|
-
@limit = limit
|
25
|
-
@duration = duration
|
26
|
-
@callback = callback
|
27
|
-
@redis_prefix = redis_prefix
|
28
|
-
@redis = redis
|
29
|
-
@bucket_range = @duration / BUCKETS
|
30
|
-
@mget_keys = (0..(BUCKETS - 1)).map { |i| "#{key}:#{i}" }
|
31
|
-
end
|
32
|
-
|
33
|
-
def retrieve_rate
|
34
|
-
@redis.mget(@mget_keys).reduce(0) { |sum, value| sum + value.to_i }
|
35
|
-
end
|
36
|
-
|
37
|
-
def check(severity)
|
38
|
-
return unless @severities.include?(severity)
|
39
|
-
time = Time.now.to_i
|
40
|
-
num = bucket_number(time)
|
41
|
-
redis_key = "#{key}:#{num}"
|
42
|
-
|
43
|
-
current_rate = @redis.eval <<-LUA
|
44
|
-
local bucket_number = #{num}
|
45
|
-
local bucket_count = redis.call("INCR", "#{redis_key}")
|
46
|
-
|
47
|
-
if bucket_count == 1 then
|
48
|
-
redis.call("EXPIRE", "#{redis_key}", "#{bucket_expiry(time)}")
|
49
|
-
redis.call("DEL", "#{callback_key}")
|
50
|
-
end
|
51
|
-
|
52
|
-
local function retrieve_rate ()
|
53
|
-
local sum = 0
|
54
|
-
local values = redis.call("MGET", #{mget_keys(num)})
|
55
|
-
for index, value in ipairs(values) do
|
56
|
-
if value ~= false then sum = sum + value end
|
57
|
-
end
|
58
|
-
return sum
|
59
|
-
end
|
60
|
-
|
61
|
-
return (retrieve_rate() + bucket_count)
|
62
|
-
LUA
|
63
|
-
|
64
|
-
if !@redis.get(callback_key) && (current_rate >= @limit)
|
65
|
-
@callback.call(current_rate) if @callback
|
66
|
-
@redis.set(callback_key, 1)
|
67
|
-
end
|
68
|
-
|
69
|
-
current_rate
|
70
|
-
end
|
71
|
-
|
72
|
-
def key
|
73
|
-
# "_LOGSTER_RATE_LIMIT:012:20:30"
|
74
|
-
# Triggers callback when log levels of :debug, :info and :warn occurs 20 times within 30 secs
|
75
|
-
"#{key_prefix}:#{@severities.join("")}:#{@limit}:#{@duration}"
|
76
|
-
end
|
77
|
-
|
78
|
-
def callback_key
|
79
|
-
"#{
|
80
|
-
end
|
81
|
-
|
82
|
-
private
|
83
|
-
|
84
|
-
def self.key_prefix(redis_prefix)
|
85
|
-
if redis_prefix
|
86
|
-
"#{redis_prefix.call}:#{PREFIX}"
|
87
|
-
else
|
88
|
-
PREFIX
|
89
|
-
end
|
90
|
-
|
91
|
-
end
|
92
|
-
|
93
|
-
def key_prefix
|
94
|
-
self.class.key_prefix(@redis_prefix)
|
95
|
-
end
|
96
|
-
|
97
|
-
def mget_keys(bucket_num)
|
98
|
-
keys = @mget_keys.dup
|
99
|
-
keys.delete_at(bucket_num)
|
100
|
-
keys.map { |key| "'#{key}'" }.join(', ')
|
101
|
-
end
|
102
|
-
|
103
|
-
def bucket_number(time)
|
104
|
-
(time % @duration) / @bucket_range
|
105
|
-
end
|
106
|
-
|
107
|
-
def bucket_expiry(time)
|
108
|
-
@duration - ((time % @duration) % @bucket_range)
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
|
-
class RedisStore < BaseStore
|
113
|
-
|
114
|
-
attr_accessor :redis, :max_backlog, :redis_raw_connection
|
115
|
-
attr_writer :redis_prefix
|
116
|
-
|
117
|
-
def initialize(redis = nil)
|
118
|
-
super()
|
119
|
-
@redis = redis || Redis.new
|
120
|
-
@max_backlog = 1000
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
@redis.
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
@redis.hdel(
|
147
|
-
@redis.
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
@redis.
|
159
|
-
@redis.
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
@redis
|
234
|
-
@redis.del(
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
@redis.
|
241
|
-
@redis.
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
.
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
.
|
250
|
-
.
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
#
|
262
|
-
|
263
|
-
|
264
|
-
@redis.del(
|
265
|
-
@redis.del(
|
266
|
-
@redis.del(
|
267
|
-
@redis.del(
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
@prefix_is_proc
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
@redis.hdel(
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
end
|
445
|
-
|
446
|
-
def
|
447
|
-
@
|
448
|
-
end
|
449
|
-
|
450
|
-
def
|
451
|
-
@
|
452
|
-
end
|
453
|
-
|
454
|
-
def
|
455
|
-
@
|
456
|
-
end
|
457
|
-
|
458
|
-
def
|
459
|
-
@
|
460
|
-
end
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
)
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
1
|
+
require 'json'
|
2
|
+
require 'logster/base_store'
|
3
|
+
|
4
|
+
module Logster
|
5
|
+
class RedisRateLimiter
|
6
|
+
BUCKETS = 6
|
7
|
+
PREFIX = "__LOGSTER__RATE_LIMIT".freeze
|
8
|
+
|
9
|
+
attr_reader :duration, :callback
|
10
|
+
|
11
|
+
def self.clear_all(redis, redis_prefix=nil)
|
12
|
+
prefix = key_prefix(redis_prefix)
|
13
|
+
|
14
|
+
redis.eval "
|
15
|
+
local keys = redis.call('keys', '*#{prefix}*')
|
16
|
+
if (table.getn(keys) > 0) then
|
17
|
+
redis.call('del', unpack(keys))
|
18
|
+
end
|
19
|
+
"
|
20
|
+
end
|
21
|
+
|
22
|
+
def initialize(redis, severities, limit, duration, redis_prefix = nil, callback = nil)
|
23
|
+
@severities = severities
|
24
|
+
@limit = limit
|
25
|
+
@duration = duration
|
26
|
+
@callback = callback
|
27
|
+
@redis_prefix = redis_prefix
|
28
|
+
@redis = redis
|
29
|
+
@bucket_range = @duration / BUCKETS
|
30
|
+
@mget_keys = (0..(BUCKETS - 1)).map { |i| "#{key}:#{i}" }
|
31
|
+
end
|
32
|
+
|
33
|
+
def retrieve_rate
|
34
|
+
@redis.mget(@mget_keys).reduce(0) { |sum, value| sum + value.to_i }
|
35
|
+
end
|
36
|
+
|
37
|
+
def check(severity)
|
38
|
+
return unless @severities.include?(severity)
|
39
|
+
time = Time.now.to_i
|
40
|
+
num = bucket_number(time)
|
41
|
+
redis_key = "#{key}:#{num}"
|
42
|
+
|
43
|
+
current_rate = @redis.eval <<-LUA
|
44
|
+
local bucket_number = #{num}
|
45
|
+
local bucket_count = redis.call("INCR", "#{redis_key}")
|
46
|
+
|
47
|
+
if bucket_count == 1 then
|
48
|
+
redis.call("EXPIRE", "#{redis_key}", "#{bucket_expiry(time)}")
|
49
|
+
redis.call("DEL", "#{callback_key}")
|
50
|
+
end
|
51
|
+
|
52
|
+
local function retrieve_rate ()
|
53
|
+
local sum = 0
|
54
|
+
local values = redis.call("MGET", #{mget_keys(num)})
|
55
|
+
for index, value in ipairs(values) do
|
56
|
+
if value ~= false then sum = sum + value end
|
57
|
+
end
|
58
|
+
return sum
|
59
|
+
end
|
60
|
+
|
61
|
+
return (retrieve_rate() + bucket_count)
|
62
|
+
LUA
|
63
|
+
|
64
|
+
if !@redis.get(callback_key) && (current_rate >= @limit)
|
65
|
+
@callback.call(current_rate) if @callback
|
66
|
+
@redis.set(callback_key, 1)
|
67
|
+
end
|
68
|
+
|
69
|
+
current_rate
|
70
|
+
end
|
71
|
+
|
72
|
+
def key
|
73
|
+
# "_LOGSTER_RATE_LIMIT:012:20:30"
|
74
|
+
# Triggers callback when log levels of :debug, :info and :warn occurs 20 times within 30 secs
|
75
|
+
"#{key_prefix}:#{@severities.join("")}:#{@limit}:#{@duration}"
|
76
|
+
end
|
77
|
+
|
78
|
+
def callback_key
|
79
|
+
"#{key}:callback_triggered"
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
def self.key_prefix(redis_prefix)
|
85
|
+
if redis_prefix
|
86
|
+
"#{redis_prefix.call}:#{PREFIX}"
|
87
|
+
else
|
88
|
+
PREFIX
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
def key_prefix
|
94
|
+
self.class.key_prefix(@redis_prefix)
|
95
|
+
end
|
96
|
+
|
97
|
+
def mget_keys(bucket_num)
|
98
|
+
keys = @mget_keys.dup
|
99
|
+
keys.delete_at(bucket_num)
|
100
|
+
keys.map { |key| "'#{key}'" }.join(', ')
|
101
|
+
end
|
102
|
+
|
103
|
+
def bucket_number(time)
|
104
|
+
(time % @duration) / @bucket_range
|
105
|
+
end
|
106
|
+
|
107
|
+
def bucket_expiry(time)
|
108
|
+
@duration - ((time % @duration) % @bucket_range)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
class RedisStore < BaseStore
|
113
|
+
|
114
|
+
attr_accessor :redis, :max_backlog, :redis_raw_connection
|
115
|
+
attr_writer :redis_prefix
|
116
|
+
|
117
|
+
def initialize(redis = nil)
|
118
|
+
super()
|
119
|
+
@redis = redis || Redis.new
|
120
|
+
@max_backlog = 1000
|
121
|
+
@redis_prefix = nil
|
122
|
+
@redis_raw_connection = nil
|
123
|
+
end
|
124
|
+
|
125
|
+
def save(message)
|
126
|
+
if keys=message.solved_keys
|
127
|
+
keys.each do |solved|
|
128
|
+
return true if @redis.hget(solved_key, solved)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
@redis.multi do
|
133
|
+
@redis.hset(grouping_key, message.grouping_key, message.key)
|
134
|
+
@redis.rpush(list_key, message.key)
|
135
|
+
update_message(message)
|
136
|
+
end
|
137
|
+
|
138
|
+
trim
|
139
|
+
check_rate_limits(message.severity)
|
140
|
+
|
141
|
+
true
|
142
|
+
end
|
143
|
+
|
144
|
+
def delete(msg)
|
145
|
+
@redis.multi do
|
146
|
+
@redis.hdel(hash_key, msg.key)
|
147
|
+
@redis.hdel(grouping_key, msg.grouping_key)
|
148
|
+
@redis.lrem(list_key, -1, msg.key)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def replace_and_bump(message)
|
153
|
+
# TODO make it atomic
|
154
|
+
exists = @redis.hexists(hash_key, message.key)
|
155
|
+
return false unless exists
|
156
|
+
|
157
|
+
@redis.multi do
|
158
|
+
@redis.hset(hash_key, message.key, message.to_json)
|
159
|
+
@redis.lrem(list_key, -1, message.key)
|
160
|
+
@redis.rpush(list_key, message.key)
|
161
|
+
end
|
162
|
+
|
163
|
+
check_rate_limits(message.severity)
|
164
|
+
|
165
|
+
true
|
166
|
+
end
|
167
|
+
|
168
|
+
def similar_key(message)
|
169
|
+
@redis.hget(grouping_key, message.grouping_key)
|
170
|
+
end
|
171
|
+
|
172
|
+
def count
|
173
|
+
@redis.llen(list_key)
|
174
|
+
end
|
175
|
+
|
176
|
+
def solve(message_key)
|
177
|
+
if (message = get(message_key)) && (keys = message.solved_keys)
|
178
|
+
# add a time so we can expire it
|
179
|
+
keys.each do |s_key|
|
180
|
+
@redis.hset(solved_key, s_key, Time.now.to_f.to_i)
|
181
|
+
end
|
182
|
+
end
|
183
|
+
clear_solved
|
184
|
+
end
|
185
|
+
|
186
|
+
def latest(opts={})
|
187
|
+
limit = opts[:limit] || 50
|
188
|
+
severity = opts[:severity]
|
189
|
+
before = opts[:before]
|
190
|
+
after = opts[:after]
|
191
|
+
search = opts[:search]
|
192
|
+
|
193
|
+
start, finish = find_location(before, after, limit)
|
194
|
+
|
195
|
+
return [] unless start && finish
|
196
|
+
|
197
|
+
results = []
|
198
|
+
|
199
|
+
direction = after ? 1 : -1
|
200
|
+
|
201
|
+
begin
|
202
|
+
keys = @redis.lrange(list_key, start, finish) || []
|
203
|
+
break unless keys and keys.count > 0
|
204
|
+
rows = @redis.hmget(hash_key, keys)
|
205
|
+
|
206
|
+
temp = []
|
207
|
+
|
208
|
+
rows.each do |s|
|
209
|
+
row = Message.from_json(s)
|
210
|
+
break if before && before == row.key
|
211
|
+
row = nil if severity && !severity.include?(row.severity)
|
212
|
+
|
213
|
+
row = filter_search(row, search)
|
214
|
+
temp << row if row
|
215
|
+
end
|
216
|
+
|
217
|
+
if direction == -1
|
218
|
+
results = temp + results
|
219
|
+
else
|
220
|
+
results += temp
|
221
|
+
end
|
222
|
+
|
223
|
+
start += limit * direction
|
224
|
+
finish += limit * direction
|
225
|
+
|
226
|
+
finish = -1 if finish > -1
|
227
|
+
end while rows.length > 0 && results.length < limit && start < 0
|
228
|
+
|
229
|
+
results
|
230
|
+
end
|
231
|
+
|
232
|
+
def clear
|
233
|
+
RedisRateLimiter.clear_all(@redis)
|
234
|
+
@redis.del(solved_key)
|
235
|
+
@redis.del(list_key)
|
236
|
+
keys = @redis.smembers(protected_key) || []
|
237
|
+
if keys.empty?
|
238
|
+
@redis.del(hash_key)
|
239
|
+
else
|
240
|
+
protected = @redis.mapped_hmget(hash_key, *keys)
|
241
|
+
@redis.del(hash_key)
|
242
|
+
@redis.mapped_hmset(hash_key, protected)
|
243
|
+
|
244
|
+
sorted = protected
|
245
|
+
.values
|
246
|
+
.map { |string|
|
247
|
+
Message.from_json(string) rescue nil
|
248
|
+
}
|
249
|
+
.compact
|
250
|
+
.sort
|
251
|
+
.map(&:key)
|
252
|
+
|
253
|
+
@redis.pipelined do
|
254
|
+
sorted.each do |message_key|
|
255
|
+
@redis.rpush(list_key, message_key)
|
256
|
+
end
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
# Delete everything, included protected messages
|
262
|
+
# (use in tests)
|
263
|
+
def clear_all
|
264
|
+
@redis.del(list_key)
|
265
|
+
@redis.del(protected_key)
|
266
|
+
@redis.del(hash_key)
|
267
|
+
@redis.del(grouping_key)
|
268
|
+
@redis.del(solved_key)
|
269
|
+
end
|
270
|
+
|
271
|
+
def get(message_key)
|
272
|
+
json = @redis.hget(hash_key, message_key)
|
273
|
+
return nil unless json
|
274
|
+
|
275
|
+
Message.from_json(json)
|
276
|
+
end
|
277
|
+
|
278
|
+
def protect(message_key)
|
279
|
+
if message = get(message_key)
|
280
|
+
message.protected = true
|
281
|
+
update_message(message)
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
def unprotect(message_key)
|
286
|
+
if message = get(message_key)
|
287
|
+
message.protected = false
|
288
|
+
update_message(message)
|
289
|
+
else
|
290
|
+
raise "Message already deleted"
|
291
|
+
end
|
292
|
+
end
|
293
|
+
|
294
|
+
def solved
|
295
|
+
@redis.hkeys(solved_key) || []
|
296
|
+
end
|
297
|
+
|
298
|
+
def register_rate_limit_per_minute(severities, limit, &block)
|
299
|
+
register_rate_limit(severities, limit, 60, block)
|
300
|
+
end
|
301
|
+
|
302
|
+
def register_rate_limit_per_hour(severities, limit, &block)
|
303
|
+
register_rate_limit(severities, limit, 3600, block)
|
304
|
+
end
|
305
|
+
|
306
|
+
def redis_prefix
|
307
|
+
return 'default'.freeze if !@redis_prefix
|
308
|
+
@prefix_is_proc ||= @redis_prefix.respond_to?(:call)
|
309
|
+
@prefix_is_proc ? @redis_prefix.call : @redis_prefix
|
310
|
+
end
|
311
|
+
|
312
|
+
def rate_limits
|
313
|
+
@rate_limits ||= {}
|
314
|
+
end
|
315
|
+
|
316
|
+
protected
|
317
|
+
|
318
|
+
def clear_solved(count = nil)
|
319
|
+
|
320
|
+
ignores = Set.new(@redis.hkeys(solved_key) || [])
|
321
|
+
|
322
|
+
if ignores.length > 0
|
323
|
+
start = count ? 0 - count : 0
|
324
|
+
message_keys = @redis.lrange(list_key, start, -1 ) || []
|
325
|
+
|
326
|
+
@redis.hmget(hash_key, message_keys).each do |json|
|
327
|
+
message = Message.from_json(json)
|
328
|
+
unless (ignores & (message.solved_keys || [])).empty?
|
329
|
+
delete message
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
333
|
+
end
|
334
|
+
|
335
|
+
def trim
|
336
|
+
if @redis.llen(list_key) > max_backlog
|
337
|
+
removed_keys = []
|
338
|
+
while removed_key = @redis.lpop(list_key)
|
339
|
+
unless @redis.sismember(protected_key, removed_key)
|
340
|
+
rmsg = get removed_key
|
341
|
+
@redis.hdel(hash_key, rmsg.key)
|
342
|
+
@redis.hdel(grouping_key, rmsg.grouping_key)
|
343
|
+
break
|
344
|
+
else
|
345
|
+
removed_keys << removed_key
|
346
|
+
end
|
347
|
+
end
|
348
|
+
removed_keys.reverse.each do |key|
|
349
|
+
@redis.lpush(list_key, key)
|
350
|
+
end
|
351
|
+
end
|
352
|
+
end
|
353
|
+
|
354
|
+
def update_message(message)
|
355
|
+
@redis.hset(hash_key, message.key, message.to_json)
|
356
|
+
if message.protected
|
357
|
+
@redis.sadd(protected_key, message.key)
|
358
|
+
else
|
359
|
+
@redis.srem(protected_key, message.key)
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
def find_message(list, message_key)
|
364
|
+
limit = 50
|
365
|
+
start = 0
|
366
|
+
finish = limit - 1
|
367
|
+
|
368
|
+
found = nil
|
369
|
+
while found == nil
|
370
|
+
items = @redis.lrange(list, start, finish)
|
371
|
+
|
372
|
+
break unless items && items.length > 0
|
373
|
+
|
374
|
+
found = items.index(message_key)
|
375
|
+
break if found
|
376
|
+
|
377
|
+
start += limit
|
378
|
+
finish += limit
|
379
|
+
end
|
380
|
+
|
381
|
+
found
|
382
|
+
end
|
383
|
+
|
384
|
+
def find_location(before, after, limit)
|
385
|
+
start = -limit
|
386
|
+
finish = -1
|
387
|
+
|
388
|
+
return [start,finish] unless before || after
|
389
|
+
|
390
|
+
found = nil
|
391
|
+
find = before || after
|
392
|
+
|
393
|
+
while !found
|
394
|
+
items = @redis.lrange(list_key, start, finish)
|
395
|
+
|
396
|
+
break unless items && items.length > 0
|
397
|
+
|
398
|
+
found = items.index(find)
|
399
|
+
|
400
|
+
if items.length < limit
|
401
|
+
found += limit - items.length if found
|
402
|
+
break
|
403
|
+
end
|
404
|
+
break if found
|
405
|
+
start -= limit
|
406
|
+
finish -= limit
|
407
|
+
end
|
408
|
+
|
409
|
+
if found
|
410
|
+
if before
|
411
|
+
offset = -(limit - found)
|
412
|
+
else
|
413
|
+
offset = found + 1
|
414
|
+
end
|
415
|
+
|
416
|
+
start += offset
|
417
|
+
finish += offset
|
418
|
+
|
419
|
+
finish = -1 if finish > -1
|
420
|
+
return nil if start > -1
|
421
|
+
end
|
422
|
+
|
423
|
+
[start, finish]
|
424
|
+
end
|
425
|
+
|
426
|
+
def filter_search(row, search)
|
427
|
+
return row unless row && search
|
428
|
+
|
429
|
+
if Regexp === search
|
430
|
+
row if row.message =~ search
|
431
|
+
elsif search[0] == "-"
|
432
|
+
exclude = search.sub('-', '')
|
433
|
+
row unless row.message.include?(exclude)
|
434
|
+
elsif row.message.include?(search)
|
435
|
+
row
|
436
|
+
end
|
437
|
+
|
438
|
+
end
|
439
|
+
|
440
|
+
def check_rate_limits(severity)
|
441
|
+
rate_limits_to_check = rate_limits[self.redis_prefix]
|
442
|
+
return if !rate_limits_to_check
|
443
|
+
rate_limits_to_check.each { |rate_limit| rate_limit.check(severity) }
|
444
|
+
end
|
445
|
+
|
446
|
+
def solved_key
|
447
|
+
@solved_key ||= "__LOGSTER__SOLVED_MAP"
|
448
|
+
end
|
449
|
+
|
450
|
+
def list_key
|
451
|
+
@list_key ||= "__LOGSTER__LATEST"
|
452
|
+
end
|
453
|
+
|
454
|
+
def hash_key
|
455
|
+
@hash_key ||= "__LOGSTER__MAP"
|
456
|
+
end
|
457
|
+
|
458
|
+
def protected_key
|
459
|
+
@saved_key ||= "__LOGSTER__SAVED"
|
460
|
+
end
|
461
|
+
|
462
|
+
def grouping_key
|
463
|
+
@grouping_key ||= "__LOGSTER__GMAP"
|
464
|
+
end
|
465
|
+
|
466
|
+
private
|
467
|
+
|
468
|
+
def register_rate_limit(severities, limit, duration, callback)
|
469
|
+
severities = [severities] unless severities.is_a?(Array)
|
470
|
+
redis = (@redis_raw_connection && @redis_prefix) ? @redis_raw_connection : @redis
|
471
|
+
|
472
|
+
rate_limiter = RedisRateLimiter.new(
|
473
|
+
redis, severities, limit, duration, Proc.new { redis_prefix }, callback
|
474
|
+
)
|
475
|
+
|
476
|
+
rate_limits[self.redis_prefix] ||= []
|
477
|
+
rate_limits[self.redis_prefix] << rate_limiter
|
478
|
+
rate_limiter
|
479
|
+
end
|
480
|
+
end
|
481
|
+
end
|