logster 2.1.0 → 2.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (121) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +19 -19
  3. data/.rubocop.yml +1 -1
  4. data/.travis.yml +16 -16
  5. data/CHANGELOG.md +169 -166
  6. data/Gemfile +4 -4
  7. data/Guardfile +8 -8
  8. data/LICENSE.txt +22 -22
  9. data/README.md +99 -99
  10. data/Rakefile +21 -21
  11. data/assets/fonts/FontAwesome.otf +0 -0
  12. data/assets/fonts/fontawesome-webfont.eot +0 -0
  13. data/assets/fonts/fontawesome-webfont.svg +639 -639
  14. data/assets/fonts/fontawesome-webfont.ttf +0 -0
  15. data/assets/fonts/fontawesome-webfont.woff +0 -0
  16. data/assets/fonts/fontawesome-webfont.woff2 +0 -0
  17. data/assets/images/Icon-144_rounded.png +0 -0
  18. data/assets/images/Icon-144_square.png +0 -0
  19. data/assets/images/icon_144x144.png +0 -0
  20. data/assets/images/icon_64x64.png +0 -0
  21. data/assets/javascript/client-app.js +100 -100
  22. data/assets/stylesheets/client-app.css +0 -0
  23. data/build_client_app.sh +0 -0
  24. data/client-app/.editorconfig +20 -20
  25. data/client-app/.ember-cli +9 -9
  26. data/client-app/.eslintignore +19 -19
  27. data/client-app/.eslintrc.js +46 -46
  28. data/client-app/.gitignore +23 -23
  29. data/client-app/.travis.yml +27 -27
  30. data/client-app/.watchmanconfig +3 -3
  31. data/client-app/README.md +57 -57
  32. data/client-app/app/app.js +14 -14
  33. data/client-app/app/components/actions-menu.js +37 -37
  34. data/client-app/app/components/env-tab.js +44 -44
  35. data/client-app/app/components/message-info.js +65 -65
  36. data/client-app/app/components/message-row.js +45 -45
  37. data/client-app/app/components/panel-resizer.js +74 -74
  38. data/client-app/app/components/tab-contents.js +27 -27
  39. data/client-app/app/components/tab-link.js +5 -5
  40. data/client-app/app/components/tabbed-section.js +32 -32
  41. data/client-app/app/components/time-formatter.js +25 -25
  42. data/client-app/app/components/update-time.js +21 -21
  43. data/client-app/app/controllers/index.js +105 -105
  44. data/client-app/app/controllers/show.js +13 -13
  45. data/client-app/app/index.html +29 -29
  46. data/client-app/app/initializers/app-init.js +72 -72
  47. data/client-app/app/lib/preload.js +14 -14
  48. data/client-app/app/lib/utilities.js +140 -140
  49. data/client-app/app/models/message-collection.js +178 -178
  50. data/client-app/app/models/message.js +100 -100
  51. data/client-app/app/resolver.js +3 -3
  52. data/client-app/app/router.js +14 -14
  53. data/client-app/app/routes/index.js +57 -57
  54. data/client-app/app/routes/show.js +14 -14
  55. data/client-app/app/styles/app.css +521 -521
  56. data/client-app/app/templates/application.hbs +2 -2
  57. data/client-app/app/templates/components/actions-menu.hbs +12 -12
  58. data/client-app/app/templates/components/env-tab.hbs +10 -10
  59. data/client-app/app/templates/components/message-info.hbs +41 -41
  60. data/client-app/app/templates/components/message-row.hbs +15 -15
  61. data/client-app/app/templates/components/panel-resizer.hbs +3 -3
  62. data/client-app/app/templates/components/tabbed-section.hbs +10 -10
  63. data/client-app/app/templates/components/time-formatter.hbs +1 -1
  64. data/client-app/app/templates/index.hbs +58 -58
  65. data/client-app/app/templates/show.hbs +7 -7
  66. data/client-app/config/environment.js +51 -51
  67. data/client-app/config/optional-features.json +3 -3
  68. data/client-app/config/targets.js +18 -18
  69. data/client-app/ember-cli-build.js +29 -29
  70. data/client-app/package-lock.json +11365 -11365
  71. data/client-app/package.json +56 -56
  72. data/client-app/testem.js +25 -25
  73. data/client-app/tests/index.html +34 -34
  74. data/client-app/tests/integration/components/actions-menu-test.js +26 -26
  75. data/client-app/tests/integration/components/env-tab-test.js +73 -73
  76. data/client-app/tests/integration/components/message-info-test.js +26 -26
  77. data/client-app/tests/integration/components/message-row-test.js +26 -26
  78. data/client-app/tests/integration/components/panel-resizer-test.js +26 -26
  79. data/client-app/tests/integration/components/tab-contents-test.js +26 -26
  80. data/client-app/tests/integration/components/tab-link-test.js +26 -26
  81. data/client-app/tests/integration/components/tabbed-section-test.js +26 -26
  82. data/client-app/tests/integration/components/time-formatter-test.js +26 -26
  83. data/client-app/tests/integration/components/update-time-test.js +26 -26
  84. data/client-app/tests/test-helper.js +8 -8
  85. data/client-app/tests/unit/controllers/index-test.js +12 -12
  86. data/client-app/tests/unit/controllers/show-test.js +12 -12
  87. data/client-app/tests/unit/initializers/app-init-test.js +31 -31
  88. data/client-app/tests/unit/routes/index-test.js +11 -11
  89. data/client-app/tests/unit/routes/show-test.js +11 -11
  90. data/lib/examples/sidekiq_logster_reporter.rb +21 -21
  91. data/lib/logster.rb +54 -54
  92. data/lib/logster/base_store.rb +141 -141
  93. data/lib/logster/configuration.rb +25 -25
  94. data/lib/logster/defer_logger.rb +14 -14
  95. data/lib/logster/ignore_pattern.rb +65 -65
  96. data/lib/logster/logger.rb +113 -113
  97. data/lib/logster/message.rb +212 -212
  98. data/lib/logster/middleware/debug_exceptions.rb +26 -26
  99. data/lib/logster/middleware/reporter.rb +55 -55
  100. data/lib/logster/middleware/viewer.rb +221 -221
  101. data/lib/logster/rails/railtie.rb +63 -63
  102. data/lib/logster/redis_store.rb +566 -566
  103. data/lib/logster/scheduler.rb +54 -40
  104. data/lib/logster/version.rb +3 -3
  105. data/lib/logster/web.rb +14 -14
  106. data/logster.gemspec +35 -35
  107. data/test/examples/test_sidekiq_reporter_example.rb +46 -46
  108. data/test/fake_data/Gemfile +4 -4
  109. data/test/fake_data/generate.rb +10 -10
  110. data/test/logster/middleware/test_reporter.rb +19 -19
  111. data/test/logster/middleware/test_viewer.rb +96 -96
  112. data/test/logster/test_base_store.rb +147 -147
  113. data/test/logster/test_defer_logger.rb +34 -34
  114. data/test/logster/test_ignore_pattern.rb +41 -41
  115. data/test/logster/test_logger.rb +86 -86
  116. data/test/logster/test_message.rb +119 -119
  117. data/test/logster/test_redis_rate_limiter.rb +230 -230
  118. data/test/logster/test_redis_store.rb +720 -720
  119. data/test/test_helper.rb +38 -38
  120. data/vendor/assets/javascripts/logster.js.erb +39 -39
  121. metadata +2 -2
@@ -1,63 +1,63 @@
1
- module Logster::Rails
2
-
3
- # this magically registers logster.js in the asset pipeline
4
- class Engine < Rails::Engine
5
- end
6
-
7
- def self.set_logger(config)
8
- return unless Logster.config.environments.include?(Rails.env.to_sym)
9
-
10
- require 'logster/middleware/debug_exceptions'
11
- require 'logster/middleware/reporter'
12
-
13
- store = Logster.store ||= Logster::RedisStore.new
14
- store.level = Logger::Severity::WARN if Rails.env.production?
15
-
16
- if Rails.env.development?
17
- require 'logster/defer_logger'
18
- logger = Logster::DeferLogger.new(store)
19
- else
20
- logger = Logster::Logger.new(store)
21
- end
22
-
23
- logger.chain(::Rails.logger)
24
- logger.level = ::Rails.logger.level
25
-
26
- Logster.logger = ::Rails.logger = config.logger = logger
27
- end
28
-
29
- def self.initialize!(app)
30
- return unless Logster.config.environments.include?(Rails.env.to_sym)
31
-
32
- if Logster::Logger === Rails.logger
33
- app.middleware.insert_before ActionDispatch::ShowExceptions, Logster::Middleware::Reporter
34
-
35
- if Rails::VERSION::MAJOR == 3
36
- app.middleware.insert_before ActionDispatch::DebugExceptions, Logster::Middleware::DebugExceptions
37
- else
38
- app.middleware.insert_before ActionDispatch::DebugExceptions, Logster::Middleware::DebugExceptions, Rails.application
39
- end
40
-
41
- app.middleware.delete ActionDispatch::DebugExceptions
42
- app.config.colorize_logging = false
43
-
44
- unless Logster.config.application_version
45
- git_version = `cd #{Rails.root} && git rev-parse --short HEAD 2> /dev/null`
46
- if git_version.present?
47
- Logster.config.application_version = git_version.strip
48
- end
49
- end
50
- end
51
- end
52
-
53
- class Railtie < ::Rails::Railtie
54
-
55
- config.before_initialize do
56
- Logster::Rails.set_logger(config)
57
- end
58
-
59
- initializer "logster.configure_rails_initialization" do |app|
60
- Logster::Rails.initialize!(app)
61
- end
62
- end
63
- end
1
+ module Logster::Rails
2
+
3
+ # this magically registers logster.js in the asset pipeline
4
+ class Engine < Rails::Engine
5
+ end
6
+
7
+ def self.set_logger(config)
8
+ return unless Logster.config.environments.include?(Rails.env.to_sym)
9
+
10
+ require 'logster/middleware/debug_exceptions'
11
+ require 'logster/middleware/reporter'
12
+
13
+ store = Logster.store ||= Logster::RedisStore.new
14
+ store.level = Logger::Severity::WARN if Rails.env.production?
15
+
16
+ if Rails.env.development?
17
+ require 'logster/defer_logger'
18
+ logger = Logster::DeferLogger.new(store)
19
+ else
20
+ logger = Logster::Logger.new(store)
21
+ end
22
+
23
+ logger.chain(::Rails.logger)
24
+ logger.level = ::Rails.logger.level
25
+
26
+ Logster.logger = ::Rails.logger = config.logger = logger
27
+ end
28
+
29
+ def self.initialize!(app)
30
+ return unless Logster.config.environments.include?(Rails.env.to_sym)
31
+
32
+ if Logster::Logger === Rails.logger
33
+ app.middleware.insert_before ActionDispatch::ShowExceptions, Logster::Middleware::Reporter
34
+
35
+ if Rails::VERSION::MAJOR == 3
36
+ app.middleware.insert_before ActionDispatch::DebugExceptions, Logster::Middleware::DebugExceptions
37
+ else
38
+ app.middleware.insert_before ActionDispatch::DebugExceptions, Logster::Middleware::DebugExceptions, Rails.application
39
+ end
40
+
41
+ app.middleware.delete ActionDispatch::DebugExceptions
42
+ app.config.colorize_logging = false
43
+
44
+ unless Logster.config.application_version
45
+ git_version = `cd #{Rails.root} && git rev-parse --short HEAD 2> /dev/null`
46
+ if git_version.present?
47
+ Logster.config.application_version = git_version.strip
48
+ end
49
+ end
50
+ end
51
+ end
52
+
53
+ class Railtie < ::Rails::Railtie
54
+
55
+ config.before_initialize do
56
+ Logster::Rails.set_logger(config)
57
+ end
58
+
59
+ initializer "logster.configure_rails_initialization" do |app|
60
+ Logster::Rails.initialize!(app)
61
+ end
62
+ end
63
+ end
@@ -1,566 +1,566 @@
1
- require 'json'
2
- require 'logster/base_store'
3
-
4
- module Logster
5
- class RedisRateLimiter
6
- BUCKETS = 6
7
- PREFIX = "__LOGSTER__RATE_LIMIT".freeze
8
-
9
- attr_reader :duration, :callback
10
-
11
- def self.clear_all(redis, redis_prefix = nil)
12
- prefix = key_prefix(redis_prefix)
13
-
14
- redis.eval "
15
- local keys = redis.call('keys', '*#{prefix}*')
16
- if (table.getn(keys) > 0) then
17
- redis.call('del', unpack(keys))
18
- end
19
- "
20
- end
21
-
22
- def initialize(redis, severities, limit, duration, redis_prefix = nil, callback = nil)
23
- @severities = severities
24
- @limit = limit
25
- @duration = duration
26
- @callback = callback
27
- @redis_prefix = redis_prefix
28
- @redis = redis
29
- @bucket_range = @duration / BUCKETS
30
- @mget_keys = (0..(BUCKETS - 1)).map { |i| "#{key}:#{i}" }
31
- end
32
-
33
- def retrieve_rate
34
- @redis.mget(@mget_keys).reduce(0) { |sum, value| sum + value.to_i }
35
- end
36
-
37
- def check(severity)
38
- return unless @severities.include?(severity)
39
- time = Time.now.to_i
40
- num = bucket_number(time)
41
- redis_key = "#{key}:#{num}"
42
-
43
- current_rate = @redis.eval <<-LUA
44
- local bucket_number = #{num}
45
- local bucket_count = redis.call("INCR", "#{redis_key}")
46
-
47
- if bucket_count == 1 then
48
- redis.call("EXPIRE", "#{redis_key}", "#{bucket_expiry(time)}")
49
- redis.call("DEL", "#{callback_key}")
50
- end
51
-
52
- local function retrieve_rate ()
53
- local sum = 0
54
- local values = redis.call("MGET", #{mget_keys(num)})
55
- for index, value in ipairs(values) do
56
- if value ~= false then sum = sum + value end
57
- end
58
- return sum
59
- end
60
-
61
- return (retrieve_rate() + bucket_count)
62
- LUA
63
-
64
- if !@redis.get(callback_key) && (current_rate >= @limit)
65
- @callback.call(current_rate) if @callback
66
- @redis.set(callback_key, 1)
67
- end
68
-
69
- current_rate
70
- end
71
-
72
- def key
73
- # "_LOGSTER_RATE_LIMIT:012:20:30"
74
- # Triggers callback when log levels of :debug, :info and :warn occurs 20 times within 30 secs
75
- "#{key_prefix}:#{@severities.join("")}:#{@limit}:#{@duration}"
76
- end
77
-
78
- def callback_key
79
- "#{key}:callback_triggered"
80
- end
81
-
82
- private
83
-
84
- def self.key_prefix(redis_prefix)
85
- if redis_prefix
86
- "#{redis_prefix.call}:#{PREFIX}"
87
- else
88
- PREFIX
89
- end
90
-
91
- end
92
-
93
- def key_prefix
94
- self.class.key_prefix(@redis_prefix)
95
- end
96
-
97
- def mget_keys(bucket_num)
98
- keys = @mget_keys.dup
99
- keys.delete_at(bucket_num)
100
- keys.map { |key| "'#{key}'" }.join(', ')
101
- end
102
-
103
- def bucket_number(time)
104
- (time % @duration) / @bucket_range
105
- end
106
-
107
- def bucket_expiry(time)
108
- @duration - ((time % @duration) % @bucket_range)
109
- end
110
- end
111
-
112
- class RedisStore < BaseStore
113
-
114
- attr_accessor :redis, :max_backlog, :redis_raw_connection
115
- attr_writer :redis_prefix
116
-
117
- def initialize(redis = nil)
118
- super()
119
- @redis = redis || Redis.new
120
- @max_backlog = 1000
121
- @redis_prefix = nil
122
- @redis_raw_connection = nil
123
- end
124
-
125
- def save(message)
126
- if keys = message.solved_keys
127
- keys.each do |solved|
128
- return true if @redis.hget(solved_key, solved)
129
- end
130
- end
131
-
132
- @redis.multi do
133
- @redis.hset(grouping_key, message.grouping_key, message.key)
134
- @redis.rpush(list_key, message.key)
135
- update_message(message)
136
- end
137
-
138
- trim
139
- check_rate_limits(message.severity)
140
-
141
- true
142
- end
143
-
144
- def delete(msg)
145
- @redis.multi do
146
- @redis.hdel(hash_key, msg.key)
147
- @redis.hdel(env_key, msg.key)
148
- @redis.hdel(grouping_key, msg.grouping_key)
149
- @redis.lrem(list_key, -1, msg.key)
150
- end
151
- end
152
-
153
- def replace_and_bump(message, save_env: true)
154
- # TODO make it atomic
155
- exists = @redis.hexists(hash_key, message.key)
156
- return false unless exists
157
-
158
- @redis.multi do
159
- @redis.hset(hash_key, message.key, message.to_json(exclude_env: true))
160
- @redis.hset(env_key, message.key, (message.env || {}).to_json) if save_env
161
- @redis.lrem(list_key, -1, message.key)
162
- @redis.rpush(list_key, message.key)
163
- end
164
-
165
- check_rate_limits(message.severity)
166
-
167
- true
168
- end
169
-
170
- def similar_key(message)
171
- @redis.hget(grouping_key, message.grouping_key)
172
- end
173
-
174
- def count
175
- @redis.llen(list_key)
176
- end
177
-
178
- def solve(message_key)
179
- if (message = get(message_key)) && (keys = message.solved_keys)
180
- # add a time so we can expire it
181
- keys.each do |s_key|
182
- @redis.hset(solved_key, s_key, Time.now.to_f.to_i)
183
- end
184
- end
185
- clear_solved
186
- end
187
-
188
- def latest(opts = {})
189
- limit = opts[:limit] || 50
190
- severity = opts[:severity]
191
- before = opts[:before]
192
- after = opts[:after]
193
- search = opts[:search]
194
-
195
- start, finish = find_location(before, after, limit)
196
-
197
- return [] unless start && finish
198
-
199
- results = []
200
-
201
- direction = after ? 1 : -1
202
-
203
- begin
204
- keys = @redis.lrange(list_key, start, finish) || []
205
- break unless keys && (keys.count > 0)
206
- rows = bulk_get(keys)
207
-
208
- temp = []
209
-
210
- rows.each do |row|
211
- break if before && before == row.key
212
- row = nil if severity && !severity.include?(row.severity)
213
-
214
- row = filter_search(row, search)
215
- temp << row if row
216
- end
217
-
218
- if direction == -1
219
- results = temp + results
220
- else
221
- results += temp
222
- end
223
-
224
- start += limit * direction
225
- finish += limit * direction
226
-
227
- finish = -1 if finish > -1
228
- end while rows.length > 0 && results.length < limit && start < 0
229
-
230
- results
231
- end
232
-
233
- def clear
234
- RedisRateLimiter.clear_all(@redis)
235
- @redis.del(solved_key)
236
- @redis.del(list_key)
237
- keys = @redis.smembers(protected_key) || []
238
- if keys.empty?
239
- @redis.del(hash_key)
240
- @redis.del(env_key)
241
- else
242
- protected = @redis.mapped_hmget(hash_key, *keys)
243
- protected_env = @redis.mapped_hmget(env_key, *keys)
244
- @redis.del(hash_key)
245
- @redis.del(env_key)
246
- @redis.mapped_hmset(hash_key, protected)
247
- @redis.mapped_hmset(env_key, protected_env)
248
-
249
- sorted = protected
250
- .values
251
- .map { |string|
252
- Message.from_json(string) rescue nil
253
- }
254
- .compact
255
- .sort
256
- .map(&:key)
257
-
258
- @redis.pipelined do
259
- sorted.each do |message_key|
260
- @redis.rpush(list_key, message_key)
261
- end
262
- end
263
- end
264
- end
265
-
266
- # Delete everything, included protected messages
267
- # (use in tests)
268
- def clear_all
269
- @redis.del(list_key)
270
- @redis.del(protected_key)
271
- @redis.del(hash_key)
272
- @redis.del(env_key)
273
- @redis.del(grouping_key)
274
- @redis.del(solved_key)
275
- end
276
-
277
- def get(message_key, load_env: true)
278
- json = @redis.hget(hash_key, message_key)
279
- return nil unless json
280
-
281
- message = Message.from_json(json)
282
- if load_env
283
- message.env = get_env(message_key) || {}
284
- end
285
- message
286
- end
287
-
288
- def bulk_get(message_keys)
289
- envs = @redis.hmget(env_key, message_keys)
290
- @redis.hmget(hash_key, message_keys).map!.with_index do |json, ind|
291
- message = Message.from_json(json)
292
- env = envs[ind]
293
- if !message.env || message.env.size == 0
294
- env = env && env.size > 0 ? ::JSON.parse(env) : {}
295
- message.env = env
296
- end
297
- message
298
- end
299
- end
300
-
301
- def get_env(message_key)
302
- json = @redis.hget(env_key, message_key)
303
- return if !json || json.size == 0
304
- JSON.parse(json)
305
- end
306
-
307
- def protect(message_key)
308
- if message = get(message_key)
309
- message.protected = true
310
- update_message(message)
311
- end
312
- end
313
-
314
- def unprotect(message_key)
315
- if message = get(message_key)
316
- message.protected = false
317
- update_message(message)
318
- else
319
- raise "Message already deleted"
320
- end
321
- end
322
-
323
- def solved
324
- @redis.hkeys(solved_key) || []
325
- end
326
-
327
- def register_rate_limit_per_minute(severities, limit, &block)
328
- register_rate_limit(severities, limit, 60, block)
329
- end
330
-
331
- def register_rate_limit_per_hour(severities, limit, &block)
332
- register_rate_limit(severities, limit, 3600, block)
333
- end
334
-
335
- def redis_prefix
336
- return 'default'.freeze if !@redis_prefix
337
- @prefix_is_proc ||= @redis_prefix.respond_to?(:call)
338
- @prefix_is_proc ? @redis_prefix.call : @redis_prefix
339
- end
340
-
341
- def rate_limits
342
- @rate_limits ||= {}
343
- end
344
-
345
- protected
346
-
347
- def clear_solved(count = nil)
348
-
349
- ignores = Set.new(@redis.hkeys(solved_key) || [])
350
-
351
- if ignores.length > 0
352
- start = count ? 0 - count : 0
353
- message_keys = @redis.lrange(list_key, start, -1) || []
354
-
355
- bulk_get(message_keys).each do |message|
356
- unless (ignores & (message.solved_keys || [])).empty?
357
- delete message
358
- end
359
- end
360
- end
361
- end
362
-
363
- def trim
364
- if @redis.llen(list_key) > max_backlog
365
- removed_keys = []
366
- while removed_key = @redis.lpop(list_key)
367
- unless @redis.sismember(protected_key, removed_key)
368
- rmsg = get removed_key
369
- @redis.hdel(hash_key, rmsg.key)
370
- @redis.hdel(env_key, rmsg.key)
371
- @redis.hdel(grouping_key, rmsg.grouping_key)
372
- break
373
- else
374
- removed_keys << removed_key
375
- end
376
- end
377
- removed_keys.reverse.each do |key|
378
- @redis.lpush(list_key, key)
379
- end
380
- end
381
- end
382
-
383
- def update_message(message)
384
- @redis.hset(hash_key, message.key, message.to_json(exclude_env: true))
385
- @redis.hset(env_key, message.key, (message.env || {}).to_json)
386
- if message.protected
387
- @redis.sadd(protected_key, message.key)
388
- else
389
- @redis.srem(protected_key, message.key)
390
- end
391
- end
392
-
393
- def find_message(list, message_key)
394
- limit = 50
395
- start = 0
396
- finish = limit - 1
397
-
398
- found = nil
399
- while found == nil
400
- items = @redis.lrange(list, start, finish)
401
-
402
- break unless items && items.length > 0
403
-
404
- found = items.index(message_key)
405
- break if found
406
-
407
- start += limit
408
- finish += limit
409
- end
410
-
411
- found
412
- end
413
-
414
- def find_location(before, after, limit)
415
- start = -limit
416
- finish = -1
417
-
418
- return [start, finish] unless before || after
419
-
420
- found = nil
421
- find = before || after
422
-
423
- while !found
424
- items = @redis.lrange(list_key, start, finish)
425
-
426
- break unless items && items.length > 0
427
-
428
- found = items.index(find)
429
-
430
- if items.length < limit
431
- found += limit - items.length if found
432
- break
433
- end
434
- break if found
435
- start -= limit
436
- finish -= limit
437
- end
438
-
439
- if found
440
- if before
441
- offset = -(limit - found)
442
- else
443
- offset = found + 1
444
- end
445
-
446
- start += offset
447
- finish += offset
448
-
449
- finish = -1 if finish > -1
450
- return nil if start > -1
451
- end
452
-
453
- [start, finish]
454
- end
455
-
456
- def get_search(search)
457
- exclude = false
458
- if String === search && search[0] == "-"
459
- exclude = true
460
- search = search.sub("-", "")
461
- end
462
- [search, exclude]
463
- end
464
-
465
- def filter_search(row, search)
466
- search, exclude = get_search(search)
467
- return row unless row && search
468
-
469
- if exclude
470
- row if !(row =~ search) && filter_env!(row, search, exclude)
471
- else
472
- row if row =~ search || filter_env!(row, search)
473
- end
474
- end
475
-
476
- def filter_env!(message, search, exclude = false)
477
- if Array === message.env
478
- array_env_matches?(message, search, exclude)
479
- else
480
- if exclude
481
- !env_matches?(message.env, search)
482
- else
483
- env_matches?(message.env, search)
484
- end
485
- end
486
- end
487
-
488
- def env_matches?(env, search)
489
- return false unless env && search
490
-
491
- env.values.any? do |value|
492
- if Hash === value
493
- env_matches?(value, search)
494
- else
495
- case search
496
- when Regexp
497
- value.to_s =~ search
498
- when String
499
- value.to_s =~ Regexp.new(search, Regexp::IGNORECASE)
500
- else
501
- false
502
- end
503
- end
504
- end
505
- end
506
-
507
- def array_env_matches?(message, search, exclude)
508
- matches = message.env.select do |env|
509
- if exclude
510
- !env_matches?(env, search)
511
- else
512
- env_matches?(env, search)
513
- end
514
- end
515
- return false if matches.empty?
516
- message.env = matches
517
- message.count = matches.size
518
- true
519
- end
520
-
521
- def check_rate_limits(severity)
522
- rate_limits_to_check = rate_limits[self.redis_prefix]
523
- return if !rate_limits_to_check
524
- rate_limits_to_check.each { |rate_limit| rate_limit.check(severity) }
525
- end
526
-
527
- def solved_key
528
- @solved_key ||= "__LOGSTER__SOLVED_MAP"
529
- end
530
-
531
- def list_key
532
- @list_key ||= "__LOGSTER__LATEST"
533
- end
534
-
535
- def hash_key
536
- @hash_key ||= "__LOGSTER__MAP"
537
- end
538
-
539
- def env_key
540
- @env_key ||= "__LOGSTER__ENV_MAP"
541
- end
542
-
543
- def protected_key
544
- @saved_key ||= "__LOGSTER__SAVED"
545
- end
546
-
547
- def grouping_key
548
- @grouping_key ||= "__LOGSTER__GMAP"
549
- end
550
-
551
- private
552
-
553
- def register_rate_limit(severities, limit, duration, callback)
554
- severities = [severities] unless severities.is_a?(Array)
555
- redis = (@redis_raw_connection && @redis_prefix) ? @redis_raw_connection : @redis
556
-
557
- rate_limiter = RedisRateLimiter.new(
558
- redis, severities, limit, duration, Proc.new { redis_prefix }, callback
559
- )
560
-
561
- rate_limits[self.redis_prefix] ||= []
562
- rate_limits[self.redis_prefix] << rate_limiter
563
- rate_limiter
564
- end
565
- end
566
- end
1
+ require 'json'
2
+ require 'logster/base_store'
3
+
4
+ module Logster
5
+ class RedisRateLimiter
6
+ BUCKETS = 6
7
+ PREFIX = "__LOGSTER__RATE_LIMIT".freeze
8
+
9
+ attr_reader :duration, :callback
10
+
11
+ def self.clear_all(redis, redis_prefix = nil)
12
+ prefix = key_prefix(redis_prefix)
13
+
14
+ redis.eval "
15
+ local keys = redis.call('keys', '*#{prefix}*')
16
+ if (table.getn(keys) > 0) then
17
+ redis.call('del', unpack(keys))
18
+ end
19
+ "
20
+ end
21
+
22
+ def initialize(redis, severities, limit, duration, redis_prefix = nil, callback = nil)
23
+ @severities = severities
24
+ @limit = limit
25
+ @duration = duration
26
+ @callback = callback
27
+ @redis_prefix = redis_prefix
28
+ @redis = redis
29
+ @bucket_range = @duration / BUCKETS
30
+ @mget_keys = (0..(BUCKETS - 1)).map { |i| "#{key}:#{i}" }
31
+ end
32
+
33
+ def retrieve_rate
34
+ @redis.mget(@mget_keys).reduce(0) { |sum, value| sum + value.to_i }
35
+ end
36
+
37
+ def check(severity)
38
+ return unless @severities.include?(severity)
39
+ time = Time.now.to_i
40
+ num = bucket_number(time)
41
+ redis_key = "#{key}:#{num}"
42
+
43
+ current_rate = @redis.eval <<-LUA
44
+ local bucket_number = #{num}
45
+ local bucket_count = redis.call("INCR", "#{redis_key}")
46
+
47
+ if bucket_count == 1 then
48
+ redis.call("EXPIRE", "#{redis_key}", "#{bucket_expiry(time)}")
49
+ redis.call("DEL", "#{callback_key}")
50
+ end
51
+
52
+ local function retrieve_rate ()
53
+ local sum = 0
54
+ local values = redis.call("MGET", #{mget_keys(num)})
55
+ for index, value in ipairs(values) do
56
+ if value ~= false then sum = sum + value end
57
+ end
58
+ return sum
59
+ end
60
+
61
+ return (retrieve_rate() + bucket_count)
62
+ LUA
63
+
64
+ if !@redis.get(callback_key) && (current_rate >= @limit)
65
+ @callback.call(current_rate) if @callback
66
+ @redis.set(callback_key, 1)
67
+ end
68
+
69
+ current_rate
70
+ end
71
+
72
+ def key
73
+ # "_LOGSTER_RATE_LIMIT:012:20:30"
74
+ # Triggers callback when log levels of :debug, :info and :warn occurs 20 times within 30 secs
75
+ "#{key_prefix}:#{@severities.join("")}:#{@limit}:#{@duration}"
76
+ end
77
+
78
+ def callback_key
79
+ "#{key}:callback_triggered"
80
+ end
81
+
82
+ private
83
+
84
+ def self.key_prefix(redis_prefix)
85
+ if redis_prefix
86
+ "#{redis_prefix.call}:#{PREFIX}"
87
+ else
88
+ PREFIX
89
+ end
90
+
91
+ end
92
+
93
+ def key_prefix
94
+ self.class.key_prefix(@redis_prefix)
95
+ end
96
+
97
+ def mget_keys(bucket_num)
98
+ keys = @mget_keys.dup
99
+ keys.delete_at(bucket_num)
100
+ keys.map { |key| "'#{key}'" }.join(', ')
101
+ end
102
+
103
+ def bucket_number(time)
104
+ (time % @duration) / @bucket_range
105
+ end
106
+
107
+ def bucket_expiry(time)
108
+ @duration - ((time % @duration) % @bucket_range)
109
+ end
110
+ end
111
+
112
+ class RedisStore < BaseStore
113
+
114
+ attr_accessor :redis, :max_backlog, :redis_raw_connection
115
+ attr_writer :redis_prefix
116
+
117
+ def initialize(redis = nil)
118
+ super()
119
+ @redis = redis || Redis.new
120
+ @max_backlog = 1000
121
+ @redis_prefix = nil
122
+ @redis_raw_connection = nil
123
+ end
124
+
125
+ def save(message)
126
+ if keys = message.solved_keys
127
+ keys.each do |solved|
128
+ return true if @redis.hget(solved_key, solved)
129
+ end
130
+ end
131
+
132
+ @redis.multi do
133
+ @redis.hset(grouping_key, message.grouping_key, message.key)
134
+ @redis.rpush(list_key, message.key)
135
+ update_message(message)
136
+ end
137
+
138
+ trim
139
+ check_rate_limits(message.severity)
140
+
141
+ true
142
+ end
143
+
144
+ def delete(msg)
145
+ @redis.multi do
146
+ @redis.hdel(hash_key, msg.key)
147
+ @redis.hdel(env_key, msg.key)
148
+ @redis.hdel(grouping_key, msg.grouping_key)
149
+ @redis.lrem(list_key, -1, msg.key)
150
+ end
151
+ end
152
+
153
+ def replace_and_bump(message, save_env: true)
154
+ # TODO make it atomic
155
+ exists = @redis.hexists(hash_key, message.key)
156
+ return false unless exists
157
+
158
+ @redis.multi do
159
+ @redis.hset(hash_key, message.key, message.to_json(exclude_env: true))
160
+ @redis.hset(env_key, message.key, (message.env || {}).to_json) if save_env
161
+ @redis.lrem(list_key, -1, message.key)
162
+ @redis.rpush(list_key, message.key)
163
+ end
164
+
165
+ check_rate_limits(message.severity)
166
+
167
+ true
168
+ end
169
+
170
+ def similar_key(message)
171
+ @redis.hget(grouping_key, message.grouping_key)
172
+ end
173
+
174
+ def count
175
+ @redis.llen(list_key)
176
+ end
177
+
178
+ def solve(message_key)
179
+ if (message = get(message_key)) && (keys = message.solved_keys)
180
+ # add a time so we can expire it
181
+ keys.each do |s_key|
182
+ @redis.hset(solved_key, s_key, Time.now.to_f.to_i)
183
+ end
184
+ end
185
+ clear_solved
186
+ end
187
+
188
+ def latest(opts = {})
189
+ limit = opts[:limit] || 50
190
+ severity = opts[:severity]
191
+ before = opts[:before]
192
+ after = opts[:after]
193
+ search = opts[:search]
194
+
195
+ start, finish = find_location(before, after, limit)
196
+
197
+ return [] unless start && finish
198
+
199
+ results = []
200
+
201
+ direction = after ? 1 : -1
202
+
203
+ begin
204
+ keys = @redis.lrange(list_key, start, finish) || []
205
+ break unless keys && (keys.count > 0)
206
+ rows = bulk_get(keys)
207
+
208
+ temp = []
209
+
210
+ rows.each do |row|
211
+ break if before && before == row.key
212
+ row = nil if severity && !severity.include?(row.severity)
213
+
214
+ row = filter_search(row, search)
215
+ temp << row if row
216
+ end
217
+
218
+ if direction == -1
219
+ results = temp + results
220
+ else
221
+ results += temp
222
+ end
223
+
224
+ start += limit * direction
225
+ finish += limit * direction
226
+
227
+ finish = -1 if finish > -1
228
+ end while rows.length > 0 && results.length < limit && start < 0
229
+
230
+ results
231
+ end
232
+
233
+ def clear
234
+ RedisRateLimiter.clear_all(@redis)
235
+ @redis.del(solved_key)
236
+ @redis.del(list_key)
237
+ keys = @redis.smembers(protected_key) || []
238
+ if keys.empty?
239
+ @redis.del(hash_key)
240
+ @redis.del(env_key)
241
+ else
242
+ protected = @redis.mapped_hmget(hash_key, *keys)
243
+ protected_env = @redis.mapped_hmget(env_key, *keys)
244
+ @redis.del(hash_key)
245
+ @redis.del(env_key)
246
+ @redis.mapped_hmset(hash_key, protected)
247
+ @redis.mapped_hmset(env_key, protected_env)
248
+
249
+ sorted = protected
250
+ .values
251
+ .map { |string|
252
+ Message.from_json(string) rescue nil
253
+ }
254
+ .compact
255
+ .sort
256
+ .map(&:key)
257
+
258
+ @redis.pipelined do
259
+ sorted.each do |message_key|
260
+ @redis.rpush(list_key, message_key)
261
+ end
262
+ end
263
+ end
264
+ end
265
+
266
+ # Delete everything, included protected messages
267
+ # (use in tests)
268
+ def clear_all
269
+ @redis.del(list_key)
270
+ @redis.del(protected_key)
271
+ @redis.del(hash_key)
272
+ @redis.del(env_key)
273
+ @redis.del(grouping_key)
274
+ @redis.del(solved_key)
275
+ end
276
+
277
+ def get(message_key, load_env: true)
278
+ json = @redis.hget(hash_key, message_key)
279
+ return nil unless json
280
+
281
+ message = Message.from_json(json)
282
+ if load_env
283
+ message.env = get_env(message_key) || {}
284
+ end
285
+ message
286
+ end
287
+
288
+ def bulk_get(message_keys)
289
+ envs = @redis.hmget(env_key, message_keys)
290
+ @redis.hmget(hash_key, message_keys).map!.with_index do |json, ind|
291
+ message = Message.from_json(json)
292
+ env = envs[ind]
293
+ if !message.env || message.env.size == 0
294
+ env = env && env.size > 0 ? ::JSON.parse(env) : {}
295
+ message.env = env
296
+ end
297
+ message
298
+ end
299
+ end
300
+
301
+ def get_env(message_key)
302
+ json = @redis.hget(env_key, message_key)
303
+ return if !json || json.size == 0
304
+ JSON.parse(json)
305
+ end
306
+
307
+ def protect(message_key)
308
+ if message = get(message_key)
309
+ message.protected = true
310
+ update_message(message)
311
+ end
312
+ end
313
+
314
+ def unprotect(message_key)
315
+ if message = get(message_key)
316
+ message.protected = false
317
+ update_message(message)
318
+ else
319
+ raise "Message already deleted"
320
+ end
321
+ end
322
+
323
+ def solved
324
+ @redis.hkeys(solved_key) || []
325
+ end
326
+
327
+ def register_rate_limit_per_minute(severities, limit, &block)
328
+ register_rate_limit(severities, limit, 60, block)
329
+ end
330
+
331
+ def register_rate_limit_per_hour(severities, limit, &block)
332
+ register_rate_limit(severities, limit, 3600, block)
333
+ end
334
+
335
+ def redis_prefix
336
+ return 'default'.freeze if !@redis_prefix
337
+ @prefix_is_proc ||= @redis_prefix.respond_to?(:call)
338
+ @prefix_is_proc ? @redis_prefix.call : @redis_prefix
339
+ end
340
+
341
+ def rate_limits
342
+ @rate_limits ||= {}
343
+ end
344
+
345
+ protected
346
+
347
+ def clear_solved(count = nil)
348
+
349
+ ignores = Set.new(@redis.hkeys(solved_key) || [])
350
+
351
+ if ignores.length > 0
352
+ start = count ? 0 - count : 0
353
+ message_keys = @redis.lrange(list_key, start, -1) || []
354
+
355
+ bulk_get(message_keys).each do |message|
356
+ unless (ignores & (message.solved_keys || [])).empty?
357
+ delete message
358
+ end
359
+ end
360
+ end
361
+ end
362
+
363
+ def trim
364
+ if @redis.llen(list_key) > max_backlog
365
+ removed_keys = []
366
+ while removed_key = @redis.lpop(list_key)
367
+ unless @redis.sismember(protected_key, removed_key)
368
+ rmsg = get removed_key
369
+ @redis.hdel(hash_key, rmsg.key)
370
+ @redis.hdel(env_key, rmsg.key)
371
+ @redis.hdel(grouping_key, rmsg.grouping_key)
372
+ break
373
+ else
374
+ removed_keys << removed_key
375
+ end
376
+ end
377
+ removed_keys.reverse.each do |key|
378
+ @redis.lpush(list_key, key)
379
+ end
380
+ end
381
+ end
382
+
383
+ def update_message(message)
384
+ @redis.hset(hash_key, message.key, message.to_json(exclude_env: true))
385
+ @redis.hset(env_key, message.key, (message.env || {}).to_json)
386
+ if message.protected
387
+ @redis.sadd(protected_key, message.key)
388
+ else
389
+ @redis.srem(protected_key, message.key)
390
+ end
391
+ end
392
+
393
+ def find_message(list, message_key)
394
+ limit = 50
395
+ start = 0
396
+ finish = limit - 1
397
+
398
+ found = nil
399
+ while found == nil
400
+ items = @redis.lrange(list, start, finish)
401
+
402
+ break unless items && items.length > 0
403
+
404
+ found = items.index(message_key)
405
+ break if found
406
+
407
+ start += limit
408
+ finish += limit
409
+ end
410
+
411
+ found
412
+ end
413
+
414
+ def find_location(before, after, limit)
415
+ start = -limit
416
+ finish = -1
417
+
418
+ return [start, finish] unless before || after
419
+
420
+ found = nil
421
+ find = before || after
422
+
423
+ while !found
424
+ items = @redis.lrange(list_key, start, finish)
425
+
426
+ break unless items && items.length > 0
427
+
428
+ found = items.index(find)
429
+
430
+ if items.length < limit
431
+ found += limit - items.length if found
432
+ break
433
+ end
434
+ break if found
435
+ start -= limit
436
+ finish -= limit
437
+ end
438
+
439
+ if found
440
+ if before
441
+ offset = -(limit - found)
442
+ else
443
+ offset = found + 1
444
+ end
445
+
446
+ start += offset
447
+ finish += offset
448
+
449
+ finish = -1 if finish > -1
450
+ return nil if start > -1
451
+ end
452
+
453
+ [start, finish]
454
+ end
455
+
456
+ def get_search(search)
457
+ exclude = false
458
+ if String === search && search[0] == "-"
459
+ exclude = true
460
+ search = search.sub("-", "")
461
+ end
462
+ [search, exclude]
463
+ end
464
+
465
+ def filter_search(row, search)
466
+ search, exclude = get_search(search)
467
+ return row unless row && search
468
+
469
+ if exclude
470
+ row if !(row =~ search) && filter_env!(row, search, exclude)
471
+ else
472
+ row if row =~ search || filter_env!(row, search)
473
+ end
474
+ end
475
+
476
+ def filter_env!(message, search, exclude = false)
477
+ if Array === message.env
478
+ array_env_matches?(message, search, exclude)
479
+ else
480
+ if exclude
481
+ !env_matches?(message.env, search)
482
+ else
483
+ env_matches?(message.env, search)
484
+ end
485
+ end
486
+ end
487
+
488
+ def env_matches?(env, search)
489
+ return false unless env && search
490
+
491
+ env.values.any? do |value|
492
+ if Hash === value
493
+ env_matches?(value, search)
494
+ else
495
+ case search
496
+ when Regexp
497
+ value.to_s =~ search
498
+ when String
499
+ value.to_s =~ Regexp.new(search, Regexp::IGNORECASE)
500
+ else
501
+ false
502
+ end
503
+ end
504
+ end
505
+ end
506
+
507
+ def array_env_matches?(message, search, exclude)
508
+ matches = message.env.select do |env|
509
+ if exclude
510
+ !env_matches?(env, search)
511
+ else
512
+ env_matches?(env, search)
513
+ end
514
+ end
515
+ return false if matches.empty?
516
+ message.env = matches
517
+ message.count = matches.size
518
+ true
519
+ end
520
+
521
+ def check_rate_limits(severity)
522
+ rate_limits_to_check = rate_limits[self.redis_prefix]
523
+ return if !rate_limits_to_check
524
+ rate_limits_to_check.each { |rate_limit| rate_limit.check(severity) }
525
+ end
526
+
527
+ def solved_key
528
+ @solved_key ||= "__LOGSTER__SOLVED_MAP"
529
+ end
530
+
531
+ def list_key
532
+ @list_key ||= "__LOGSTER__LATEST"
533
+ end
534
+
535
+ def hash_key
536
+ @hash_key ||= "__LOGSTER__MAP"
537
+ end
538
+
539
+ def env_key
540
+ @env_key ||= "__LOGSTER__ENV_MAP"
541
+ end
542
+
543
+ def protected_key
544
+ @saved_key ||= "__LOGSTER__SAVED"
545
+ end
546
+
547
+ def grouping_key
548
+ @grouping_key ||= "__LOGSTER__GMAP"
549
+ end
550
+
551
+ private
552
+
553
+ def register_rate_limit(severities, limit, duration, callback)
554
+ severities = [severities] unless severities.is_a?(Array)
555
+ redis = (@redis_raw_connection && @redis_prefix) ? @redis_raw_connection : @redis
556
+
557
+ rate_limiter = RedisRateLimiter.new(
558
+ redis, severities, limit, duration, Proc.new { redis_prefix }, callback
559
+ )
560
+
561
+ rate_limits[self.redis_prefix] ||= []
562
+ rate_limits[self.redis_prefix] << rate_limiter
563
+ rate_limiter
564
+ end
565
+ end
566
+ end