logster 2.4.2 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/Gemfile +2 -0
  4. data/Guardfile +2 -0
  5. data/README.md +1 -1
  6. data/Rakefile +2 -0
  7. data/assets/javascript/client-app.js +69 -53
  8. data/assets/javascript/vendor.js +580 -559
  9. data/assets/stylesheets/client-app.css +1 -1
  10. data/client-app/README.md +2 -2
  11. data/client-app/app/components/env-tab.js +16 -36
  12. data/client-app/app/components/message-row.js +1 -1
  13. data/client-app/app/components/page-nav.js +30 -0
  14. data/client-app/app/components/patterns-list.js +2 -1
  15. data/client-app/app/controllers/index.js +68 -92
  16. data/client-app/app/controllers/show.js +6 -0
  17. data/client-app/app/models/group.js +20 -0
  18. data/client-app/app/models/message-collection.js +159 -57
  19. data/client-app/app/routes/index.js +0 -2
  20. data/client-app/app/routes/settings.js +3 -1
  21. data/client-app/app/styles/app.css +17 -2
  22. data/client-app/app/templates/components/env-tab.hbs +5 -7
  23. data/client-app/app/templates/components/message-info.hbs +13 -8
  24. data/client-app/app/templates/components/page-nav.hbs +13 -0
  25. data/client-app/app/templates/components/patterns-list.hbs +6 -4
  26. data/client-app/app/templates/index.hbs +45 -11
  27. data/client-app/app/templates/settings.hbs +10 -1
  28. data/client-app/app/templates/show.hbs +2 -0
  29. data/client-app/package-lock.json +2817 -1215
  30. data/client-app/package.json +12 -12
  31. data/client-app/tests/integration/components/env-tab-test.js +29 -8
  32. data/client-app/tests/integration/components/message-info-test.js +10 -2
  33. data/lib/examples/sidekiq_logster_reporter.rb +2 -0
  34. data/lib/logster.rb +2 -2
  35. data/lib/logster/base_store.rb +40 -4
  36. data/lib/logster/cache.rb +9 -8
  37. data/lib/logster/defer_logger.rb +2 -0
  38. data/lib/logster/group.rb +124 -0
  39. data/lib/logster/grouping_pattern.rb +29 -0
  40. data/lib/logster/ignore_pattern.rb +2 -0
  41. data/lib/logster/logger.rb +2 -0
  42. data/lib/logster/message.rb +3 -1
  43. data/lib/logster/middleware/reporter.rb +2 -2
  44. data/lib/logster/middleware/viewer.rb +12 -1
  45. data/lib/logster/pattern.rb +13 -0
  46. data/lib/logster/redis_store.rb +99 -10
  47. data/lib/logster/scheduler.rb +2 -0
  48. data/lib/logster/suppression_pattern.rb +5 -2
  49. data/lib/logster/version.rb +1 -1
  50. data/lib/logster/web.rb +2 -0
  51. data/logster.gemspec +4 -1
  52. data/test/examples/test_sidekiq_reporter_example.rb +2 -0
  53. data/test/fake_data/Gemfile +2 -0
  54. data/test/fake_data/generate.rb +2 -0
  55. data/test/logster/middleware/test_viewer.rb +3 -1
  56. data/test/logster/test_base_store.rb +2 -0
  57. data/test/logster/test_cache.rb +19 -12
  58. data/test/logster/test_defer_logger.rb +2 -0
  59. data/test/logster/test_group.rb +92 -0
  60. data/test/logster/test_ignore_pattern.rb +2 -0
  61. data/test/logster/test_logger.rb +3 -1
  62. data/test/logster/test_message.rb +2 -0
  63. data/test/logster/test_pattern.rb +2 -2
  64. data/test/logster/test_redis_rate_limiter.rb +2 -0
  65. data/test/logster/test_redis_store.rb +253 -0
  66. data/test/test_helper.rb +6 -0
  67. metadata +26 -6
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'logger'
2
4
 
3
5
  module Logster
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'digest/sha1'
2
4
  require 'securerandom'
3
5
 
@@ -112,7 +114,7 @@ module Logster
112
114
 
113
115
  # in its own method so it can be overridden
114
116
  def grouping_hash
115
- return { message: self.message, severity: self.severity, backtrace: self.backtrace }
117
+ { message: self.message, severity: self.severity, backtrace: self.backtrace }
116
118
  end
117
119
 
118
120
  # todo - memoize?
@@ -25,14 +25,14 @@ module Logster
25
25
  if path == @error_path
26
26
 
27
27
  if !Logster.config.enable_js_error_reporting
28
- return [403, {}, "Access Denied"]
28
+ return [403, {}, ["Access Denied"]]
29
29
  end
30
30
 
31
31
  Logster.config.current_context.call(env) do
32
32
  if Logster.config.rate_limit_error_reporting
33
33
  req = Rack::Request.new(env)
34
34
  if Logster.store.rate_limited?(req.ip, perform: true)
35
- return [429, {}, "Rate Limited"]
35
+ return [429, {}, ["Rate Limited"]]
36
36
  end
37
37
  end
38
38
  report_js_error(env)
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'json'
2
4
 
3
5
  module Logster
@@ -122,7 +124,11 @@ module Logster
122
124
  count = ignore_count[pattern] || 0
123
125
  suppression << { value: pattern, count: count }
124
126
  end
125
- [200, { "Content-Type" => "application/json; charset=utf-8" }, [JSON.generate(suppression: suppression)]]
127
+
128
+ grouping = Logster::GroupingPattern.find_all(raw: true).map do |pattern|
129
+ { value: pattern }
130
+ end
131
+ [200, { "Content-Type" => "application/json; charset=utf-8" }, [JSON.generate(suppression: suppression, grouping: grouping)]]
126
132
  else
127
133
  [200, { "Content-Type" => "text/html; charset=utf-8" }, [body(preload_json)]]
128
134
  end
@@ -199,6 +205,9 @@ module Logster
199
205
  opts[:search] = search
200
206
  end
201
207
  search = opts[:search]
208
+ if params["known_groups"]
209
+ opts[:known_groups] = params["known_groups"]
210
+ end
202
211
  opts[:with_env] = (String === search && search.size > 0) || Regexp === search
203
212
 
204
213
  payload = {
@@ -253,6 +262,8 @@ module Logster
253
262
  case set_name
254
263
  when "suppression"
255
264
  Logster::SuppressionPattern
265
+ when "grouping"
266
+ Logster::GroupingPattern
256
267
  else
257
268
  nil
258
269
  end
@@ -1,7 +1,20 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Logster
2
4
  class Pattern
5
+ @child_classes = []
6
+
3
7
  class PatternError < StandardError; end
4
8
 
9
+ def self.inherited(subclass)
10
+ @child_classes << subclass
11
+ super
12
+ end
13
+
14
+ def self.child_classes
15
+ @child_classes
16
+ end
17
+
5
18
  def self.set_name
6
19
  raise "Please override the `set_name` method and specify and a name for this set"
7
20
  end
@@ -39,7 +39,12 @@ module Logster
39
39
  end
40
40
 
41
41
  def delete(msg)
42
+ groups = find_pattern_groups() { |pat| msg.message =~ pat }
42
43
  @redis.multi do
44
+ groups.each do |group|
45
+ group.remove_message(msg)
46
+ save_pattern_group(group) if group.changed?
47
+ end
43
48
  @redis.hdel(hash_key, msg.key)
44
49
  @redis.hdel(env_key, msg.key)
45
50
  @redis.hdel(grouping_key, msg.grouping_key)
@@ -48,7 +53,12 @@ module Logster
48
53
  end
49
54
 
50
55
  def bulk_delete(message_keys, grouping_keys)
56
+ groups = find_pattern_groups(load_messages: true)
51
57
  @redis.multi do
58
+ groups.each do |group|
59
+ group.messages = group.messages.reject { |m| message_keys.include?(m.key) }
60
+ save_pattern_group(group) if group.changed?
61
+ end
52
62
  @redis.hdel(hash_key, message_keys)
53
63
  @redis.hdel(env_key, message_keys)
54
64
  @redis.hdel(grouping_key, grouping_keys)
@@ -100,19 +110,21 @@ module Logster
100
110
  after = opts[:after]
101
111
  search = opts[:search]
102
112
  with_env = opts.key?(:with_env) ? opts[:with_env] : true
113
+ known_groups = opts[:known_groups]&.dup || []
103
114
 
104
115
  start, finish = find_location(before, after, limit)
105
116
 
106
117
  return [] unless start && finish
107
118
 
108
119
  results = []
120
+ pattern_groups = find_pattern_groups(load_messages: true)
109
121
 
110
122
  direction = after ? 1 : -1
111
123
 
112
124
  begin
113
125
  keys = @redis.lrange(list_key, start, finish) || []
114
- break unless keys && (keys.count > 0)
115
- rows = bulk_get(keys, with_env: with_env)
126
+ break if !keys || keys.count <= 0
127
+ rows = bulk_get(keys, with_env: with_env).reverse
116
128
 
117
129
  temp = []
118
130
 
@@ -121,9 +133,18 @@ module Logster
121
133
  row = nil if severity && !severity.include?(row.severity)
122
134
 
123
135
  row = filter_search(row, search)
124
- temp << row if row
136
+ if row
137
+ group = pattern_groups.find { |g| g.messages_keys.include?(row.key) }
138
+ if group && !known_groups.include?(group.key)
139
+ known_groups << group.key
140
+ temp << serialize_group(group, row.key)
141
+ elsif !group
142
+ temp << row
143
+ end
144
+ end
125
145
  end
126
146
 
147
+ temp.reverse!
127
148
  if direction == -1
128
149
  results = temp + results
129
150
  else
@@ -147,6 +168,8 @@ module Logster
147
168
  if keys.empty?
148
169
  @redis.del(hash_key)
149
170
  @redis.del(env_key)
171
+ @redis.del(pattern_groups_key)
172
+ @redis.del(grouping_key)
150
173
  else
151
174
  protected = @redis.mapped_hmget(hash_key, *keys)
152
175
  protected_env = @redis.mapped_hmget(env_key, *keys)
@@ -169,6 +192,10 @@ module Logster
169
192
  @redis.rpush(list_key, message_key)
170
193
  end
171
194
  end
195
+ find_pattern_groups(load_messages: true).each do |group|
196
+ group.messages = group.messages.select { |m| sorted.include?(m.key) }
197
+ save_pattern_group(group) if group.changed?
198
+ end
172
199
  end
173
200
  end
174
201
 
@@ -182,7 +209,8 @@ module Logster
182
209
  @redis.del(grouping_key)
183
210
  @redis.del(solved_key)
184
211
  @redis.del(ignored_logs_count_key)
185
- Logster::PATTERNS.each do |klass|
212
+ @redis.del(pattern_groups_key)
213
+ Logster::Pattern.child_classes.each do |klass|
186
214
  @redis.del(klass.set_name)
187
215
  end
188
216
  @redis.keys.each do |key|
@@ -202,13 +230,15 @@ module Logster
202
230
  message
203
231
  end
204
232
 
205
- def get_all_messages
206
- bulk_get(@redis.lrange(list_key, 0, -1))
233
+ def get_all_messages(with_env: true)
234
+ bulk_get(@redis.lrange(list_key, 0, -1), with_env: with_env)
207
235
  end
208
236
 
209
237
  def bulk_get(message_keys, with_env: true)
238
+ return [] if !message_keys || message_keys.size == 0
210
239
  envs = @redis.mapped_hmget(env_key, *message_keys) if with_env
211
- @redis.hmget(hash_key, message_keys).map! do |json|
240
+ messages = @redis.hmget(hash_key, message_keys).map! do |json|
241
+ next if !json || json.size == 0
212
242
  message = Message.from_json(json)
213
243
  if with_env
214
244
  env = envs[message.key]
@@ -219,6 +249,8 @@ module Logster
219
249
  end
220
250
  message
221
251
  end
252
+ messages.compact!
253
+ messages
222
254
  end
223
255
 
224
256
  def get_env(message_key)
@@ -301,6 +333,45 @@ module Logster
301
333
  limited
302
334
  end
303
335
 
336
+ def find_pattern_groups(load_messages: false)
337
+ patterns = @patterns_cache.fetch(Logster::GroupingPattern::CACHE_KEY) do
338
+ Logster::GroupingPattern.find_all(store: self)
339
+ end
340
+ patterns = patterns.select do |pattern|
341
+ if block_given?
342
+ yield(pattern)
343
+ else
344
+ true
345
+ end
346
+ end
347
+ return [] if patterns.size == 0
348
+ mapped = patterns.map(&:inspect)
349
+ jsons = @redis.hmget(pattern_groups_key, mapped)
350
+ jsons.map! do |json|
351
+ if json && json.size > 0
352
+ group = Logster::Group.from_json(json)
353
+ if load_messages
354
+ group.messages = bulk_get(group.messages_keys, with_env: false)
355
+ end
356
+ group
357
+ end
358
+ end
359
+ jsons.compact!
360
+ jsons
361
+ end
362
+
363
+ def save_pattern_group(group)
364
+ if group.count == 0
365
+ @redis.hdel(pattern_groups_key, group.key)
366
+ else
367
+ @redis.hset(pattern_groups_key, group.key, group.to_json)
368
+ end
369
+ end
370
+
371
+ def remove_pattern_group(pattern)
372
+ @redis.hdel(pattern_groups_key, pattern.inspect)
373
+ end
374
+
304
375
  protected
305
376
 
306
377
  def clear_solved(count = nil)
@@ -325,9 +396,7 @@ module Logster
325
396
  while removed_key = @redis.lpop(list_key)
326
397
  unless @redis.sismember(protected_key, removed_key)
327
398
  rmsg = get removed_key
328
- @redis.hdel(hash_key, rmsg.key)
329
- @redis.hdel(env_key, rmsg.key)
330
- @redis.hdel(grouping_key, rmsg.grouping_key)
399
+ delete(rmsg)
331
400
  break
332
401
  else
333
402
  removed_keys << removed_key
@@ -515,8 +584,28 @@ module Logster
515
584
  "__LOGSTER__IP_RATE_LIMIT_#{ip_address}"
516
585
  end
517
586
 
587
+ def pattern_groups_key
588
+ @pattern_groups_key ||= "__LOGSTER__PATTERN_GROUPS_KEY__MAP"
589
+ end
590
+
518
591
  private
519
592
 
593
+ def serialize_group(group, row_id)
594
+ # row_id should be the key of the most recent *message* that is
595
+ # included in the group.
596
+ # It's used by the client in the before (not after) query param
597
+ # when you hit load more and the first row is a group.
598
+ # The server uses this info (row_id) to know where it needs to
599
+ # start scanning messages when looking up older messages.
600
+ Logster::Group::GroupWeb.new(
601
+ group.key,
602
+ group.count,
603
+ group.timestamp,
604
+ group.messages,
605
+ row_id
606
+ )
607
+ end
608
+
520
609
  def apply_max_size_limit(message)
521
610
  size = message.to_json(exclude_env: true).bytesize
522
611
  env_size = message.env_json.bytesize
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Logster
2
4
  module Deferer
3
5
  attr_reader :queue, :thread
@@ -1,19 +1,22 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Logster
2
4
  class SuppressionPattern < Pattern
5
+ CACHE_KEY = :suppression
3
6
  def self.set_name
4
7
  "__LOGSTER__suppression_patterns_set".freeze
5
8
  end
6
9
 
7
10
  def save(args = {})
8
11
  super
9
- @store.clear_suppression_patterns_cache
12
+ @store.clear_patterns_cache(CACHE_KEY)
10
13
  retro_delete_messages if args[:retroactive]
11
14
  end
12
15
 
13
16
  def destroy(clear_cache: true) # arg used in tests
14
17
  super()
15
18
  @store.remove_ignore_count(self.to_s)
16
- @store.clear_suppression_patterns_cache if clear_cache
19
+ @store.clear_patterns_cache(CACHE_KEY) if clear_cache
17
20
  end
18
21
 
19
22
  private
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Logster
4
- VERSION = "2.4.2"
4
+ VERSION = "2.5.0"
5
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'logster/middleware/viewer'
2
4
 
3
5
  class Logster::Web
@@ -1,4 +1,6 @@
1
1
  # coding: utf-8
2
+ # frozen_string_literal: true
3
+
2
4
  lib = File.expand_path('../lib', __FILE__)
3
5
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
6
  require 'logster/version'
@@ -31,5 +33,6 @@ Gem::Specification.new do |spec|
31
33
  spec.add_development_dependency "guard-minitest"
32
34
  spec.add_development_dependency "timecop"
33
35
  spec.add_development_dependency "byebug"
34
- spec.add_development_dependency "rubocop", "~> 0.61.1"
36
+ spec.add_development_dependency "rubocop", "~> 0.69.0"
37
+ spec.add_development_dependency "rubocop-discourse"
35
38
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative '../test_helper'
2
4
  require 'logster/logger'
3
5
  require 'logster/redis_store'
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source 'https://rubygems.org'
2
4
 
3
5
  gem 'redis'
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'redis'
2
4
  require 'logster'
3
5
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative '../../test_helper'
2
4
  require 'rack'
3
5
  require 'logster/redis_store'
@@ -126,7 +128,7 @@ class TestViewer < Minitest::Test
126
128
  params: { pattern: "disallowedpattern" }
127
129
  )
128
130
  assert_equal(404, response.status)
129
- Logster::PATTERNS.each do |klass|
131
+ Logster::Pattern.child_classes.each do |klass|
130
132
  assert_equal(0, klass.find_all.size)
131
133
  end
132
134
  ensure
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative '../test_helper'
2
4
  require 'logster/base_store'
3
5
  require 'logster/ignore_pattern'
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative '../test_helper'
2
4
  require 'logster/cache'
3
5
 
@@ -7,32 +9,37 @@ class TestCache < Minitest::Test
7
9
  end
8
10
 
9
11
  def test_cache_works
10
- value = "I should be retured"
11
- prc = Proc.new do
12
- @cache.fetch do
12
+ prc = Proc.new do |key, value|
13
+ @cache.fetch(key) do
13
14
  value
14
15
  end
15
16
  end
16
- assert_equal(value, prc.call)
17
+ value = "I should be retured"
18
+ assert_equal(value, prc.call(:key1, value))
17
19
  cached_value = value
18
20
  value = "I shouldn't be returned"
19
- assert_equal(cached_value, prc.call)
21
+ assert_equal(cached_value, prc.call(:key1, value))
22
+ value2 = "value for key2"
23
+ assert_equal(value2, prc.call(:key2, value2))
20
24
 
21
- value = "Now I should be returned again"
25
+ value = value2 = "Now I should be returned"
22
26
  Process.stub :clock_gettime, Process.clock_gettime(Process::CLOCK_MONOTONIC) + 6 do
23
- assert_equal(value, prc.call)
27
+ assert_equal(value, prc.call(:key1, value))
28
+ assert_equal(value2, prc.call(:key2, value2))
24
29
  end
25
30
  end
26
31
 
27
32
  def test_cache_can_be_cleared
28
33
  value = "cached"
29
- prc = Proc.new do
30
- @cache.fetch { value }
34
+ prc = Proc.new do |key, val|
35
+ @cache.fetch(key) { val }
31
36
  end
32
- assert_equal(value, prc.call)
37
+ assert_equal(value, prc.call(:key1, value))
38
+ assert_equal("v2", prc.call(:key2, "v2"))
33
39
 
34
40
  value = "new value"
35
- @cache.clear
36
- assert_equal(value, prc.call)
41
+ @cache.clear(:key1)
42
+ assert_equal(value, prc.call(:key1, value))
43
+ assert_equal("v2", prc.call(:key2, "v2.2"))
37
44
  end
38
45
  end