logster 2.4.2 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile +2 -0
- data/Guardfile +2 -0
- data/README.md +1 -1
- data/Rakefile +2 -0
- data/assets/javascript/client-app.js +69 -53
- data/assets/javascript/vendor.js +580 -559
- data/assets/stylesheets/client-app.css +1 -1
- data/client-app/README.md +2 -2
- data/client-app/app/components/env-tab.js +16 -36
- data/client-app/app/components/message-row.js +1 -1
- data/client-app/app/components/page-nav.js +30 -0
- data/client-app/app/components/patterns-list.js +2 -1
- data/client-app/app/controllers/index.js +68 -92
- data/client-app/app/controllers/show.js +6 -0
- data/client-app/app/models/group.js +20 -0
- data/client-app/app/models/message-collection.js +159 -57
- data/client-app/app/routes/index.js +0 -2
- data/client-app/app/routes/settings.js +3 -1
- data/client-app/app/styles/app.css +17 -2
- data/client-app/app/templates/components/env-tab.hbs +5 -7
- data/client-app/app/templates/components/message-info.hbs +13 -8
- data/client-app/app/templates/components/page-nav.hbs +13 -0
- data/client-app/app/templates/components/patterns-list.hbs +6 -4
- data/client-app/app/templates/index.hbs +45 -11
- data/client-app/app/templates/settings.hbs +10 -1
- data/client-app/app/templates/show.hbs +2 -0
- data/client-app/package-lock.json +2817 -1215
- data/client-app/package.json +12 -12
- data/client-app/tests/integration/components/env-tab-test.js +29 -8
- data/client-app/tests/integration/components/message-info-test.js +10 -2
- data/lib/examples/sidekiq_logster_reporter.rb +2 -0
- data/lib/logster.rb +2 -2
- data/lib/logster/base_store.rb +40 -4
- data/lib/logster/cache.rb +9 -8
- data/lib/logster/defer_logger.rb +2 -0
- data/lib/logster/group.rb +124 -0
- data/lib/logster/grouping_pattern.rb +29 -0
- data/lib/logster/ignore_pattern.rb +2 -0
- data/lib/logster/logger.rb +2 -0
- data/lib/logster/message.rb +3 -1
- data/lib/logster/middleware/reporter.rb +2 -2
- data/lib/logster/middleware/viewer.rb +12 -1
- data/lib/logster/pattern.rb +13 -0
- data/lib/logster/redis_store.rb +99 -10
- data/lib/logster/scheduler.rb +2 -0
- data/lib/logster/suppression_pattern.rb +5 -2
- data/lib/logster/version.rb +1 -1
- data/lib/logster/web.rb +2 -0
- data/logster.gemspec +4 -1
- data/test/examples/test_sidekiq_reporter_example.rb +2 -0
- data/test/fake_data/Gemfile +2 -0
- data/test/fake_data/generate.rb +2 -0
- data/test/logster/middleware/test_viewer.rb +3 -1
- data/test/logster/test_base_store.rb +2 -0
- data/test/logster/test_cache.rb +19 -12
- data/test/logster/test_defer_logger.rb +2 -0
- data/test/logster/test_group.rb +92 -0
- data/test/logster/test_ignore_pattern.rb +2 -0
- data/test/logster/test_logger.rb +3 -1
- data/test/logster/test_message.rb +2 -0
- data/test/logster/test_pattern.rb +2 -2
- data/test/logster/test_redis_rate_limiter.rb +2 -0
- data/test/logster/test_redis_store.rb +253 -0
- data/test/test_helper.rb +6 -0
- metadata +26 -6
data/lib/logster/logger.rb
CHANGED
data/lib/logster/message.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'digest/sha1'
|
2
4
|
require 'securerandom'
|
3
5
|
|
@@ -112,7 +114,7 @@ module Logster
|
|
112
114
|
|
113
115
|
# in its own method so it can be overridden
|
114
116
|
def grouping_hash
|
115
|
-
|
117
|
+
{ message: self.message, severity: self.severity, backtrace: self.backtrace }
|
116
118
|
end
|
117
119
|
|
118
120
|
# todo - memoize?
|
@@ -25,14 +25,14 @@ module Logster
|
|
25
25
|
if path == @error_path
|
26
26
|
|
27
27
|
if !Logster.config.enable_js_error_reporting
|
28
|
-
return [403, {}, "Access Denied"]
|
28
|
+
return [403, {}, ["Access Denied"]]
|
29
29
|
end
|
30
30
|
|
31
31
|
Logster.config.current_context.call(env) do
|
32
32
|
if Logster.config.rate_limit_error_reporting
|
33
33
|
req = Rack::Request.new(env)
|
34
34
|
if Logster.store.rate_limited?(req.ip, perform: true)
|
35
|
-
return [429, {}, "Rate Limited"]
|
35
|
+
return [429, {}, ["Rate Limited"]]
|
36
36
|
end
|
37
37
|
end
|
38
38
|
report_js_error(env)
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'json'
|
2
4
|
|
3
5
|
module Logster
|
@@ -122,7 +124,11 @@ module Logster
|
|
122
124
|
count = ignore_count[pattern] || 0
|
123
125
|
suppression << { value: pattern, count: count }
|
124
126
|
end
|
125
|
-
|
127
|
+
|
128
|
+
grouping = Logster::GroupingPattern.find_all(raw: true).map do |pattern|
|
129
|
+
{ value: pattern }
|
130
|
+
end
|
131
|
+
[200, { "Content-Type" => "application/json; charset=utf-8" }, [JSON.generate(suppression: suppression, grouping: grouping)]]
|
126
132
|
else
|
127
133
|
[200, { "Content-Type" => "text/html; charset=utf-8" }, [body(preload_json)]]
|
128
134
|
end
|
@@ -199,6 +205,9 @@ module Logster
|
|
199
205
|
opts[:search] = search
|
200
206
|
end
|
201
207
|
search = opts[:search]
|
208
|
+
if params["known_groups"]
|
209
|
+
opts[:known_groups] = params["known_groups"]
|
210
|
+
end
|
202
211
|
opts[:with_env] = (String === search && search.size > 0) || Regexp === search
|
203
212
|
|
204
213
|
payload = {
|
@@ -253,6 +262,8 @@ module Logster
|
|
253
262
|
case set_name
|
254
263
|
when "suppression"
|
255
264
|
Logster::SuppressionPattern
|
265
|
+
when "grouping"
|
266
|
+
Logster::GroupingPattern
|
256
267
|
else
|
257
268
|
nil
|
258
269
|
end
|
data/lib/logster/pattern.rb
CHANGED
@@ -1,7 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Logster
|
2
4
|
class Pattern
|
5
|
+
@child_classes = []
|
6
|
+
|
3
7
|
class PatternError < StandardError; end
|
4
8
|
|
9
|
+
def self.inherited(subclass)
|
10
|
+
@child_classes << subclass
|
11
|
+
super
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.child_classes
|
15
|
+
@child_classes
|
16
|
+
end
|
17
|
+
|
5
18
|
def self.set_name
|
6
19
|
raise "Please override the `set_name` method and specify and a name for this set"
|
7
20
|
end
|
data/lib/logster/redis_store.rb
CHANGED
@@ -39,7 +39,12 @@ module Logster
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def delete(msg)
|
42
|
+
groups = find_pattern_groups() { |pat| msg.message =~ pat }
|
42
43
|
@redis.multi do
|
44
|
+
groups.each do |group|
|
45
|
+
group.remove_message(msg)
|
46
|
+
save_pattern_group(group) if group.changed?
|
47
|
+
end
|
43
48
|
@redis.hdel(hash_key, msg.key)
|
44
49
|
@redis.hdel(env_key, msg.key)
|
45
50
|
@redis.hdel(grouping_key, msg.grouping_key)
|
@@ -48,7 +53,12 @@ module Logster
|
|
48
53
|
end
|
49
54
|
|
50
55
|
def bulk_delete(message_keys, grouping_keys)
|
56
|
+
groups = find_pattern_groups(load_messages: true)
|
51
57
|
@redis.multi do
|
58
|
+
groups.each do |group|
|
59
|
+
group.messages = group.messages.reject { |m| message_keys.include?(m.key) }
|
60
|
+
save_pattern_group(group) if group.changed?
|
61
|
+
end
|
52
62
|
@redis.hdel(hash_key, message_keys)
|
53
63
|
@redis.hdel(env_key, message_keys)
|
54
64
|
@redis.hdel(grouping_key, grouping_keys)
|
@@ -100,19 +110,21 @@ module Logster
|
|
100
110
|
after = opts[:after]
|
101
111
|
search = opts[:search]
|
102
112
|
with_env = opts.key?(:with_env) ? opts[:with_env] : true
|
113
|
+
known_groups = opts[:known_groups]&.dup || []
|
103
114
|
|
104
115
|
start, finish = find_location(before, after, limit)
|
105
116
|
|
106
117
|
return [] unless start && finish
|
107
118
|
|
108
119
|
results = []
|
120
|
+
pattern_groups = find_pattern_groups(load_messages: true)
|
109
121
|
|
110
122
|
direction = after ? 1 : -1
|
111
123
|
|
112
124
|
begin
|
113
125
|
keys = @redis.lrange(list_key, start, finish) || []
|
114
|
-
break
|
115
|
-
rows = bulk_get(keys, with_env: with_env)
|
126
|
+
break if !keys || keys.count <= 0
|
127
|
+
rows = bulk_get(keys, with_env: with_env).reverse
|
116
128
|
|
117
129
|
temp = []
|
118
130
|
|
@@ -121,9 +133,18 @@ module Logster
|
|
121
133
|
row = nil if severity && !severity.include?(row.severity)
|
122
134
|
|
123
135
|
row = filter_search(row, search)
|
124
|
-
|
136
|
+
if row
|
137
|
+
group = pattern_groups.find { |g| g.messages_keys.include?(row.key) }
|
138
|
+
if group && !known_groups.include?(group.key)
|
139
|
+
known_groups << group.key
|
140
|
+
temp << serialize_group(group, row.key)
|
141
|
+
elsif !group
|
142
|
+
temp << row
|
143
|
+
end
|
144
|
+
end
|
125
145
|
end
|
126
146
|
|
147
|
+
temp.reverse!
|
127
148
|
if direction == -1
|
128
149
|
results = temp + results
|
129
150
|
else
|
@@ -147,6 +168,8 @@ module Logster
|
|
147
168
|
if keys.empty?
|
148
169
|
@redis.del(hash_key)
|
149
170
|
@redis.del(env_key)
|
171
|
+
@redis.del(pattern_groups_key)
|
172
|
+
@redis.del(grouping_key)
|
150
173
|
else
|
151
174
|
protected = @redis.mapped_hmget(hash_key, *keys)
|
152
175
|
protected_env = @redis.mapped_hmget(env_key, *keys)
|
@@ -169,6 +192,10 @@ module Logster
|
|
169
192
|
@redis.rpush(list_key, message_key)
|
170
193
|
end
|
171
194
|
end
|
195
|
+
find_pattern_groups(load_messages: true).each do |group|
|
196
|
+
group.messages = group.messages.select { |m| sorted.include?(m.key) }
|
197
|
+
save_pattern_group(group) if group.changed?
|
198
|
+
end
|
172
199
|
end
|
173
200
|
end
|
174
201
|
|
@@ -182,7 +209,8 @@ module Logster
|
|
182
209
|
@redis.del(grouping_key)
|
183
210
|
@redis.del(solved_key)
|
184
211
|
@redis.del(ignored_logs_count_key)
|
185
|
-
|
212
|
+
@redis.del(pattern_groups_key)
|
213
|
+
Logster::Pattern.child_classes.each do |klass|
|
186
214
|
@redis.del(klass.set_name)
|
187
215
|
end
|
188
216
|
@redis.keys.each do |key|
|
@@ -202,13 +230,15 @@ module Logster
|
|
202
230
|
message
|
203
231
|
end
|
204
232
|
|
205
|
-
def get_all_messages
|
206
|
-
bulk_get(@redis.lrange(list_key, 0, -1))
|
233
|
+
def get_all_messages(with_env: true)
|
234
|
+
bulk_get(@redis.lrange(list_key, 0, -1), with_env: with_env)
|
207
235
|
end
|
208
236
|
|
209
237
|
def bulk_get(message_keys, with_env: true)
|
238
|
+
return [] if !message_keys || message_keys.size == 0
|
210
239
|
envs = @redis.mapped_hmget(env_key, *message_keys) if with_env
|
211
|
-
@redis.hmget(hash_key, message_keys).map! do |json|
|
240
|
+
messages = @redis.hmget(hash_key, message_keys).map! do |json|
|
241
|
+
next if !json || json.size == 0
|
212
242
|
message = Message.from_json(json)
|
213
243
|
if with_env
|
214
244
|
env = envs[message.key]
|
@@ -219,6 +249,8 @@ module Logster
|
|
219
249
|
end
|
220
250
|
message
|
221
251
|
end
|
252
|
+
messages.compact!
|
253
|
+
messages
|
222
254
|
end
|
223
255
|
|
224
256
|
def get_env(message_key)
|
@@ -301,6 +333,45 @@ module Logster
|
|
301
333
|
limited
|
302
334
|
end
|
303
335
|
|
336
|
+
def find_pattern_groups(load_messages: false)
|
337
|
+
patterns = @patterns_cache.fetch(Logster::GroupingPattern::CACHE_KEY) do
|
338
|
+
Logster::GroupingPattern.find_all(store: self)
|
339
|
+
end
|
340
|
+
patterns = patterns.select do |pattern|
|
341
|
+
if block_given?
|
342
|
+
yield(pattern)
|
343
|
+
else
|
344
|
+
true
|
345
|
+
end
|
346
|
+
end
|
347
|
+
return [] if patterns.size == 0
|
348
|
+
mapped = patterns.map(&:inspect)
|
349
|
+
jsons = @redis.hmget(pattern_groups_key, mapped)
|
350
|
+
jsons.map! do |json|
|
351
|
+
if json && json.size > 0
|
352
|
+
group = Logster::Group.from_json(json)
|
353
|
+
if load_messages
|
354
|
+
group.messages = bulk_get(group.messages_keys, with_env: false)
|
355
|
+
end
|
356
|
+
group
|
357
|
+
end
|
358
|
+
end
|
359
|
+
jsons.compact!
|
360
|
+
jsons
|
361
|
+
end
|
362
|
+
|
363
|
+
def save_pattern_group(group)
|
364
|
+
if group.count == 0
|
365
|
+
@redis.hdel(pattern_groups_key, group.key)
|
366
|
+
else
|
367
|
+
@redis.hset(pattern_groups_key, group.key, group.to_json)
|
368
|
+
end
|
369
|
+
end
|
370
|
+
|
371
|
+
def remove_pattern_group(pattern)
|
372
|
+
@redis.hdel(pattern_groups_key, pattern.inspect)
|
373
|
+
end
|
374
|
+
|
304
375
|
protected
|
305
376
|
|
306
377
|
def clear_solved(count = nil)
|
@@ -325,9 +396,7 @@ module Logster
|
|
325
396
|
while removed_key = @redis.lpop(list_key)
|
326
397
|
unless @redis.sismember(protected_key, removed_key)
|
327
398
|
rmsg = get removed_key
|
328
|
-
|
329
|
-
@redis.hdel(env_key, rmsg.key)
|
330
|
-
@redis.hdel(grouping_key, rmsg.grouping_key)
|
399
|
+
delete(rmsg)
|
331
400
|
break
|
332
401
|
else
|
333
402
|
removed_keys << removed_key
|
@@ -515,8 +584,28 @@ module Logster
|
|
515
584
|
"__LOGSTER__IP_RATE_LIMIT_#{ip_address}"
|
516
585
|
end
|
517
586
|
|
587
|
+
def pattern_groups_key
|
588
|
+
@pattern_groups_key ||= "__LOGSTER__PATTERN_GROUPS_KEY__MAP"
|
589
|
+
end
|
590
|
+
|
518
591
|
private
|
519
592
|
|
593
|
+
def serialize_group(group, row_id)
|
594
|
+
# row_id should be the key of the most recent *message* that is
|
595
|
+
# included in the group.
|
596
|
+
# It's used by the client in the before (not after) query param
|
597
|
+
# when you hit load more and the first row is a group.
|
598
|
+
# The server uses this info (row_id) to know where it needs to
|
599
|
+
# start scanning messages when looking up older messages.
|
600
|
+
Logster::Group::GroupWeb.new(
|
601
|
+
group.key,
|
602
|
+
group.count,
|
603
|
+
group.timestamp,
|
604
|
+
group.messages,
|
605
|
+
row_id
|
606
|
+
)
|
607
|
+
end
|
608
|
+
|
520
609
|
def apply_max_size_limit(message)
|
521
610
|
size = message.to_json(exclude_env: true).bytesize
|
522
611
|
env_size = message.env_json.bytesize
|
data/lib/logster/scheduler.rb
CHANGED
@@ -1,19 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Logster
|
2
4
|
class SuppressionPattern < Pattern
|
5
|
+
CACHE_KEY = :suppression
|
3
6
|
def self.set_name
|
4
7
|
"__LOGSTER__suppression_patterns_set".freeze
|
5
8
|
end
|
6
9
|
|
7
10
|
def save(args = {})
|
8
11
|
super
|
9
|
-
@store.
|
12
|
+
@store.clear_patterns_cache(CACHE_KEY)
|
10
13
|
retro_delete_messages if args[:retroactive]
|
11
14
|
end
|
12
15
|
|
13
16
|
def destroy(clear_cache: true) # arg used in tests
|
14
17
|
super()
|
15
18
|
@store.remove_ignore_count(self.to_s)
|
16
|
-
@store.
|
19
|
+
@store.clear_patterns_cache(CACHE_KEY) if clear_cache
|
17
20
|
end
|
18
21
|
|
19
22
|
private
|
data/lib/logster/version.rb
CHANGED
data/lib/logster/web.rb
CHANGED
data/logster.gemspec
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
2
4
|
lib = File.expand_path('../lib', __FILE__)
|
3
5
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
6
|
require 'logster/version'
|
@@ -31,5 +33,6 @@ Gem::Specification.new do |spec|
|
|
31
33
|
spec.add_development_dependency "guard-minitest"
|
32
34
|
spec.add_development_dependency "timecop"
|
33
35
|
spec.add_development_dependency "byebug"
|
34
|
-
spec.add_development_dependency "rubocop", "~> 0.
|
36
|
+
spec.add_development_dependency "rubocop", "~> 0.69.0"
|
37
|
+
spec.add_development_dependency "rubocop-discourse"
|
35
38
|
end
|
data/test/fake_data/Gemfile
CHANGED
data/test/fake_data/generate.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../test_helper'
|
2
4
|
require 'rack'
|
3
5
|
require 'logster/redis_store'
|
@@ -126,7 +128,7 @@ class TestViewer < Minitest::Test
|
|
126
128
|
params: { pattern: "disallowedpattern" }
|
127
129
|
)
|
128
130
|
assert_equal(404, response.status)
|
129
|
-
Logster::
|
131
|
+
Logster::Pattern.child_classes.each do |klass|
|
130
132
|
assert_equal(0, klass.find_all.size)
|
131
133
|
end
|
132
134
|
ensure
|
data/test/logster/test_cache.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../test_helper'
|
2
4
|
require 'logster/cache'
|
3
5
|
|
@@ -7,32 +9,37 @@ class TestCache < Minitest::Test
|
|
7
9
|
end
|
8
10
|
|
9
11
|
def test_cache_works
|
10
|
-
|
11
|
-
|
12
|
-
@cache.fetch do
|
12
|
+
prc = Proc.new do |key, value|
|
13
|
+
@cache.fetch(key) do
|
13
14
|
value
|
14
15
|
end
|
15
16
|
end
|
16
|
-
|
17
|
+
value = "I should be retured"
|
18
|
+
assert_equal(value, prc.call(:key1, value))
|
17
19
|
cached_value = value
|
18
20
|
value = "I shouldn't be returned"
|
19
|
-
assert_equal(cached_value, prc.call)
|
21
|
+
assert_equal(cached_value, prc.call(:key1, value))
|
22
|
+
value2 = "value for key2"
|
23
|
+
assert_equal(value2, prc.call(:key2, value2))
|
20
24
|
|
21
|
-
value = "Now I should be returned
|
25
|
+
value = value2 = "Now I should be returned"
|
22
26
|
Process.stub :clock_gettime, Process.clock_gettime(Process::CLOCK_MONOTONIC) + 6 do
|
23
|
-
assert_equal(value, prc.call)
|
27
|
+
assert_equal(value, prc.call(:key1, value))
|
28
|
+
assert_equal(value2, prc.call(:key2, value2))
|
24
29
|
end
|
25
30
|
end
|
26
31
|
|
27
32
|
def test_cache_can_be_cleared
|
28
33
|
value = "cached"
|
29
|
-
prc = Proc.new do
|
30
|
-
@cache.fetch {
|
34
|
+
prc = Proc.new do |key, val|
|
35
|
+
@cache.fetch(key) { val }
|
31
36
|
end
|
32
|
-
assert_equal(value, prc.call)
|
37
|
+
assert_equal(value, prc.call(:key1, value))
|
38
|
+
assert_equal("v2", prc.call(:key2, "v2"))
|
33
39
|
|
34
40
|
value = "new value"
|
35
|
-
@cache.clear
|
36
|
-
assert_equal(value, prc.call)
|
41
|
+
@cache.clear(:key1)
|
42
|
+
assert_equal(value, prc.call(:key1, value))
|
43
|
+
assert_equal("v2", prc.call(:key2, "v2.2"))
|
37
44
|
end
|
38
45
|
end
|