dead_bro 0.2.32 → 0.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/FEATURES.md +333 -0
- data/README.md +2 -2
- data/lib/dead_bro/ar_object_tracker.rb +1 -0
- data/lib/dead_bro/cache_subscriber.rb +4 -1
- data/lib/dead_bro/client.rb +40 -2
- data/lib/dead_bro/configuration.rb +23 -11
- data/lib/dead_bro/elasticsearch_subscriber.rb +1 -1
- data/lib/dead_bro/error_middleware.rb +14 -48
- data/lib/dead_bro/http_instrumentation.rb +3 -3
- data/lib/dead_bro/job_sql_tracking_middleware.rb +1 -0
- data/lib/dead_bro/job_subscriber.rb +16 -8
- data/lib/dead_bro/logger.rb +4 -1
- data/lib/dead_bro/memory_phase_tracker.rb +2 -2
- data/lib/dead_bro/memory_tracking_subscriber.rb +1 -0
- data/lib/dead_bro/redis_subscriber.rb +12 -45
- data/lib/dead_bro/sanitizer.rb +56 -0
- data/lib/dead_bro/sql_subscriber.rb +12 -4
- data/lib/dead_bro/sql_tracking_middleware.rb +29 -7
- data/lib/dead_bro/subscriber.rb +14 -7
- data/lib/dead_bro/version.rb +1 -1
- data/lib/dead_bro/view_rendering_subscriber.rb +3 -0
- data/lib/dead_bro.rb +19 -0
- metadata +4 -2
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "digest"
|
|
4
3
|
require "rack"
|
|
5
4
|
|
|
6
5
|
module DeadBro
|
|
@@ -21,8 +20,9 @@ module DeadBro
|
|
|
21
20
|
event_name = exception.class.name.to_s
|
|
22
21
|
event_name = EVENT_NAME if event_name.empty?
|
|
23
22
|
@client.post_metric(event_name: event_name, payload: payload, force: true)
|
|
24
|
-
rescue
|
|
25
|
-
# Never let APM reporting interfere with the host app
|
|
23
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
24
|
+
# Never let APM reporting interfere with the host app — in particular,
|
|
25
|
+
# never replace the host's exception with one of ours.
|
|
26
26
|
end
|
|
27
27
|
raise
|
|
28
28
|
end
|
|
@@ -32,12 +32,12 @@ module DeadBro
|
|
|
32
32
|
def build_payload(exception, env)
|
|
33
33
|
req = rack_request(env)
|
|
34
34
|
|
|
35
|
-
{
|
|
35
|
+
payload = {
|
|
36
36
|
exception_class: exception.class.name,
|
|
37
37
|
message: truncate(exception.message.to_s, 1000),
|
|
38
38
|
backtrace: safe_backtrace(exception),
|
|
39
|
-
fingerprint:
|
|
40
|
-
cause_chain: build_cause_chain(exception),
|
|
39
|
+
fingerprint: DeadBro::Subscriber.compute_error_fingerprint(exception),
|
|
40
|
+
cause_chain: DeadBro::Subscriber.build_cause_chain(exception),
|
|
41
41
|
occurred_at: Time.now.utc.to_i,
|
|
42
42
|
rack:
|
|
43
43
|
{
|
|
@@ -57,6 +57,10 @@ module DeadBro
|
|
|
57
57
|
process_kind: DeadBro.process_kind,
|
|
58
58
|
logs: DeadBro.logger.logs
|
|
59
59
|
}
|
|
60
|
+
# Every field above can carry raw client bytes (headers are binary strings),
|
|
61
|
+
# so scrub the whole payload rather than field by field. This path only
|
|
62
|
+
# runs for uncaught exceptions, so the extra walk is cheap.
|
|
63
|
+
DeadBro::Sanitizer.deep(payload)
|
|
60
64
|
end
|
|
61
65
|
|
|
62
66
|
def rack_request(env)
|
|
@@ -71,61 +75,23 @@ module DeadBro
|
|
|
71
75
|
[]
|
|
72
76
|
end
|
|
73
77
|
|
|
74
|
-
def normalize_message(msg)
|
|
75
|
-
msg.to_s
|
|
76
|
-
.gsub(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/i, "UUID")
|
|
77
|
-
.gsub(/\b\d+\b/, "N")
|
|
78
|
-
.gsub(/"[^"]*"/, '"?"')
|
|
79
|
-
.gsub(/'[^']*'/, "'?'")
|
|
80
|
-
.strip
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def compute_fingerprint(exception)
|
|
84
|
-
top_frame = Array(exception.backtrace).first.to_s.gsub(/:\d+:in /, ":N:in ")
|
|
85
|
-
input = "#{exception.class.name}|#{normalize_message(exception.message)}|#{top_frame}"
|
|
86
|
-
Digest::SHA256.hexdigest(input)[0, 16]
|
|
87
|
-
rescue
|
|
88
|
-
nil
|
|
89
|
-
end
|
|
90
|
-
|
|
91
|
-
def build_cause_chain(exception)
|
|
92
|
-
return [] unless exception
|
|
93
|
-
chain = []
|
|
94
|
-
cause = exception.cause
|
|
95
|
-
depth = 0
|
|
96
|
-
while cause && depth < 5
|
|
97
|
-
chain << {
|
|
98
|
-
exception_class: cause.class.name,
|
|
99
|
-
message: truncate(cause.message.to_s, 500),
|
|
100
|
-
backtrace_top: Array(cause.backtrace).first(3)
|
|
101
|
-
}
|
|
102
|
-
cause = cause.cause
|
|
103
|
-
depth += 1
|
|
104
|
-
end
|
|
105
|
-
chain
|
|
106
|
-
rescue
|
|
107
|
-
[]
|
|
108
|
-
end
|
|
109
|
-
|
|
110
78
|
def safe_params(req)
|
|
111
79
|
return {} unless req
|
|
112
80
|
|
|
113
81
|
params = req.params || {}
|
|
114
82
|
# Redact at every nesting level (e.g. user[password]) before serializing.
|
|
115
|
-
JSON.parse(JSON.dump(redact_sensitive(params)))
|
|
83
|
+
JSON.parse(JSON.dump(DeadBro::Sanitizer.deep(redact_sensitive(params))))
|
|
116
84
|
rescue
|
|
117
85
|
{}
|
|
118
86
|
end
|
|
119
87
|
|
|
120
|
-
# Matches a key segment so nested/prefixed/suffixed sensitive keys are caught
|
|
121
|
-
# without redacting innocent keys like passenger_count.
|
|
122
|
-
SENSITIVE_SEGMENT_RE = /(?:\A|[_\-\[])(password|passwd|secret|token|api_?key|access_?key|auth|authorization|credential|ssn|credit_?card|card_?number|cvv|cvc)(?:\z|[_\-\]])/i
|
|
123
|
-
|
|
124
88
|
def redact_sensitive(value)
|
|
125
89
|
case value
|
|
126
90
|
when Hash
|
|
127
91
|
value.each_with_object({}) do |(k, v), memo|
|
|
128
|
-
|
|
92
|
+
# Scrubbed first: the sensitive-key regex raises on a key that isn't valid UTF-8.
|
|
93
|
+
k = DeadBro::Sanitizer.string(k) if k.is_a?(String)
|
|
94
|
+
memo[k] = DeadBro::Subscriber.sensitive_key?(k) ? "[FILTERED]" : redact_sensitive(v)
|
|
129
95
|
end
|
|
130
96
|
when Array
|
|
131
97
|
value.map { |v| redact_sensitive(v) }
|
|
@@ -79,7 +79,7 @@ module DeadBro
|
|
|
79
79
|
Thread.current[THREAD_LOCAL_KEY] << payload
|
|
80
80
|
end
|
|
81
81
|
end
|
|
82
|
-
rescue
|
|
82
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
83
83
|
end
|
|
84
84
|
end
|
|
85
85
|
end
|
|
@@ -120,7 +120,7 @@ module DeadBro
|
|
|
120
120
|
Thread.current[THREAD_LOCAL_KEY] << payload
|
|
121
121
|
end
|
|
122
122
|
end
|
|
123
|
-
rescue
|
|
123
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
124
124
|
end
|
|
125
125
|
end
|
|
126
126
|
end
|
|
@@ -177,7 +177,7 @@ module DeadBro
|
|
|
177
177
|
Thread.current[key] << payload
|
|
178
178
|
end
|
|
179
179
|
end
|
|
180
|
-
rescue
|
|
180
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
181
181
|
end
|
|
182
182
|
end
|
|
183
183
|
end
|
|
@@ -15,7 +15,7 @@ module DeadBro
|
|
|
15
15
|
ActiveSupport::Notifications.subscribe("perform_start.active_job") do |_name, _started, _finished, _unique_id, _data|
|
|
16
16
|
DeadBro::GcTracker.start_request_tracking if defined?(DeadBro::GcTracker)
|
|
17
17
|
DeadBro::ArObjectTracker.start_request_tracking if defined?(DeadBro::ArObjectTracker)
|
|
18
|
-
rescue
|
|
18
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
# Track job execution — success AND failure both land here. ActiveJob wraps
|
|
@@ -156,8 +156,8 @@ module DeadBro
|
|
|
156
156
|
|
|
157
157
|
if has_error
|
|
158
158
|
payload[:exception_class] = exception.class.name
|
|
159
|
-
payload[:message] = exception.message.to_s[0, 1000]
|
|
160
|
-
payload[:backtrace] = Array(exception.backtrace).first(50)
|
|
159
|
+
payload[:message] = DeadBro::Sanitizer.string(exception.message.to_s[0, 1000])
|
|
160
|
+
payload[:backtrace] = Array(exception.backtrace).first(50).map { |line| DeadBro::Sanitizer.string(line) }
|
|
161
161
|
payload[:fingerprint] = DeadBro::Subscriber.compute_error_fingerprint(exception)
|
|
162
162
|
payload[:cause_chain] = DeadBro::Subscriber.build_cause_chain(exception)
|
|
163
163
|
payload[:error] = true
|
|
@@ -167,6 +167,11 @@ module DeadBro
|
|
|
167
167
|
# completions the sampling decision above already accounted for any
|
|
168
168
|
# per-job-type override, so client#post_metric must not re-roll it globally.
|
|
169
169
|
client.post_metric(event_name: name, payload: payload, force: true)
|
|
170
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
171
|
+
# Anything raised here would fail the host's job (see
|
|
172
|
+
# DeadBro::CONTAINED_ERRORS) — drop this job's metrics instead. Jobs have
|
|
173
|
+
# no Rack middleware to reset thread-locals afterwards, so drain here.
|
|
174
|
+
drain_job_tracking
|
|
170
175
|
end
|
|
171
176
|
rescue
|
|
172
177
|
# Never raise from instrumentation install
|
|
@@ -259,17 +264,19 @@ module DeadBro
|
|
|
259
264
|
def self.safe_arguments(arguments)
|
|
260
265
|
return [] unless arguments.is_a?(Array)
|
|
261
266
|
|
|
262
|
-
# Limit and sanitize job arguments
|
|
267
|
+
# Limit and sanitize job arguments. Strings are scrubbed to valid UTF-8 here
|
|
268
|
+
# so one binary argument doesn't send the payload down the client's slow retry path.
|
|
263
269
|
arguments.first(10).map do |arg|
|
|
264
270
|
case arg
|
|
265
271
|
when String
|
|
266
|
-
(arg
|
|
272
|
+
str = DeadBro::Sanitizer.string(arg)
|
|
273
|
+
(str.length > 200) ? str[0, 200] + "..." : str
|
|
267
274
|
when Hash
|
|
268
275
|
# Filter sensitive keys and limit size
|
|
269
276
|
filtered = arg.reject { |k, _| %w[password token secret key].include?(k.to_s) }
|
|
270
|
-
(filtered.keys.size > 20) ? filtered.first(20).to_h : filtered
|
|
277
|
+
DeadBro::Sanitizer.deep((filtered.keys.size > 20) ? filtered.first(20).to_h : filtered)
|
|
271
278
|
when Array
|
|
272
|
-
arg.first(5)
|
|
279
|
+
DeadBro::Sanitizer.deep(arg.first(5))
|
|
273
280
|
when ActiveRecord::Base
|
|
274
281
|
# Handle ActiveRecord objects safely
|
|
275
282
|
"#{arg.class.name}##{begin
|
|
@@ -279,7 +286,8 @@ module DeadBro
|
|
|
279
286
|
end}"
|
|
280
287
|
else
|
|
281
288
|
# Convert to string and truncate, but avoid object inspection
|
|
282
|
-
|
|
289
|
+
str = DeadBro::Sanitizer.string(arg)
|
|
290
|
+
(str.length > 200) ? str[0, 200] + "..." : str
|
|
283
291
|
end
|
|
284
292
|
end
|
|
285
293
|
rescue
|
data/lib/dead_bro/logger.rb
CHANGED
|
@@ -72,6 +72,9 @@ module DeadBro
|
|
|
72
72
|
|
|
73
73
|
def log(severity, message)
|
|
74
74
|
timestamp = Time.now.utc
|
|
75
|
+
# Messages often interpolate request data; scrubbed here so one bad byte
|
|
76
|
+
# doesn't send the payload down the client's slow retry path.
|
|
77
|
+
message = DeadBro::Sanitizer.string(message)
|
|
75
78
|
|
|
76
79
|
buffer = (Thread.current[@thread_logs_key] ||= [])
|
|
77
80
|
if buffer.length >= MAX_LOG_ENTRIES
|
|
@@ -80,7 +83,7 @@ module DeadBro
|
|
|
80
83
|
else
|
|
81
84
|
buffer << {
|
|
82
85
|
sev: severity.to_s,
|
|
83
|
-
msg: message
|
|
86
|
+
msg: message,
|
|
84
87
|
time: timestamp.iso8601(3) # Include milliseconds for better precision
|
|
85
88
|
}
|
|
86
89
|
end
|
|
@@ -91,7 +91,7 @@ module DeadBro
|
|
|
91
91
|
state[:buckets][parent[:phase]] += now - parent[:checkpoint]
|
|
92
92
|
end
|
|
93
93
|
stack << {phase: phase, checkpoint: now}
|
|
94
|
-
rescue
|
|
94
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
95
95
|
# Best-effort only.
|
|
96
96
|
end
|
|
97
97
|
|
|
@@ -110,7 +110,7 @@ module DeadBro
|
|
|
110
110
|
if (parent = stack.last)
|
|
111
111
|
parent[:checkpoint] = now
|
|
112
112
|
end
|
|
113
|
-
rescue
|
|
113
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
114
114
|
# Best-effort only.
|
|
115
115
|
end
|
|
116
116
|
|
|
@@ -77,7 +77,7 @@ module DeadBro
|
|
|
77
77
|
duration_ms = ((finish_time - start_time) * 1000.0).round(2)
|
|
78
78
|
|
|
79
79
|
begin
|
|
80
|
-
cmd_info =
|
|
80
|
+
cmd_info = RedisSubscriber.extract_command(command: command)
|
|
81
81
|
tracking_start = Thread.current[DeadBro::TRACKING_START_TIME_KEY]
|
|
82
82
|
start_offset_ms = tracking_start ? ((wall_start - tracking_start) * 1000.0).round(2) : nil
|
|
83
83
|
event = {
|
|
@@ -87,14 +87,14 @@ module DeadBro
|
|
|
87
87
|
args_count: cmd_info[:args_count],
|
|
88
88
|
duration_ms: duration_ms,
|
|
89
89
|
start_offset_ms: start_offset_ms,
|
|
90
|
-
db: safe_db(@db),
|
|
90
|
+
db: RedisSubscriber.safe_db(@db),
|
|
91
91
|
error: error ? error.class.name : nil
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
if Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] && RedisSubscriber.should_continue_tracking?
|
|
95
95
|
Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] << event
|
|
96
96
|
end
|
|
97
|
-
rescue
|
|
97
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
98
98
|
end
|
|
99
99
|
end
|
|
100
100
|
end
|
|
@@ -120,13 +120,13 @@ module DeadBro
|
|
|
120
120
|
commands_count: commands_count,
|
|
121
121
|
duration_ms: duration_ms,
|
|
122
122
|
start_offset_ms: start_offset_ms,
|
|
123
|
-
db: safe_db(@db)
|
|
123
|
+
db: RedisSubscriber.safe_db(@db)
|
|
124
124
|
}
|
|
125
125
|
|
|
126
126
|
if Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] && RedisSubscriber.should_continue_tracking?
|
|
127
127
|
Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] << event
|
|
128
128
|
end
|
|
129
|
-
rescue
|
|
129
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
130
130
|
end
|
|
131
131
|
end
|
|
132
132
|
end
|
|
@@ -152,52 +152,16 @@ module DeadBro
|
|
|
152
152
|
commands_count: commands_count,
|
|
153
153
|
duration_ms: duration_ms,
|
|
154
154
|
start_offset_ms: start_offset_ms,
|
|
155
|
-
db: safe_db(@db)
|
|
155
|
+
db: RedisSubscriber.safe_db(@db)
|
|
156
156
|
}
|
|
157
157
|
|
|
158
158
|
if Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] && RedisSubscriber.should_continue_tracking?
|
|
159
159
|
Thread.current[RedisSubscriber::THREAD_LOCAL_KEY] << event
|
|
160
160
|
end
|
|
161
|
-
rescue
|
|
161
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
162
162
|
end
|
|
163
163
|
end
|
|
164
164
|
end
|
|
165
|
-
|
|
166
|
-
def extract_command_info(command)
|
|
167
|
-
parts = Array(command).map(&:to_s)
|
|
168
|
-
command_name = parts.first&.upcase
|
|
169
|
-
key = parts[1]
|
|
170
|
-
args_count = (parts.length > 1) ? parts.length - 1 : 0
|
|
171
|
-
|
|
172
|
-
{
|
|
173
|
-
command: safe_command(command_name),
|
|
174
|
-
key: safe_key(key),
|
|
175
|
-
args_count: args_count
|
|
176
|
-
}
|
|
177
|
-
rescue
|
|
178
|
-
{command: nil, key: nil, args_count: nil}
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
def safe_command(cmd)
|
|
182
|
-
return nil if cmd.nil?
|
|
183
|
-
cmd.to_s[0, 20]
|
|
184
|
-
rescue
|
|
185
|
-
nil
|
|
186
|
-
end
|
|
187
|
-
|
|
188
|
-
def safe_key(key)
|
|
189
|
-
return nil if key.nil?
|
|
190
|
-
s = key.to_s
|
|
191
|
-
(s.length > 200) ? s[0, 200] + "…" : s
|
|
192
|
-
rescue
|
|
193
|
-
nil
|
|
194
|
-
end
|
|
195
|
-
|
|
196
|
-
def safe_db(db)
|
|
197
|
-
Integer(db)
|
|
198
|
-
rescue
|
|
199
|
-
nil
|
|
200
|
-
end
|
|
201
165
|
end
|
|
202
166
|
|
|
203
167
|
::Redis::Client.prepend(mod) unless ::Redis::Client.ancestors.include?(mod)
|
|
@@ -219,6 +183,7 @@ module DeadBro
|
|
|
219
183
|
if event && should_continue_tracking?
|
|
220
184
|
Thread.current[THREAD_LOCAL_KEY] << event
|
|
221
185
|
end
|
|
186
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
222
187
|
end
|
|
223
188
|
rescue
|
|
224
189
|
end
|
|
@@ -296,14 +261,16 @@ module DeadBro
|
|
|
296
261
|
|
|
297
262
|
def self.safe_command(cmd)
|
|
298
263
|
return nil if cmd.nil?
|
|
299
|
-
cmd
|
|
264
|
+
DeadBro::Sanitizer.string(cmd)[0, 20]
|
|
300
265
|
rescue
|
|
301
266
|
nil
|
|
302
267
|
end
|
|
303
268
|
|
|
269
|
+
# Redis keys can be binary (digests, packed ids) or embed request data;
|
|
270
|
+
# scrubbing here keeps one bad byte off the client's slow retry path.
|
|
304
271
|
def self.safe_key(key)
|
|
305
272
|
return nil if key.nil?
|
|
306
|
-
s = key
|
|
273
|
+
s = DeadBro::Sanitizer.string(key)
|
|
307
274
|
(s.length > 200) ? s[0, 200] + "…" : s
|
|
308
275
|
rescue
|
|
309
276
|
nil
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module DeadBro
|
|
4
|
+
# Makes values safe to JSON-encode and to store on the backend: every String
|
|
5
|
+
# comes out as valid UTF-8 without NUL bytes.
|
|
6
|
+
#
|
|
7
|
+
# Rack hands header values over as binary strings holding whatever bytes the
|
|
8
|
+
# client sent, and JSON.dump raises on any that aren't valid UTF-8 — a bot's
|
|
9
|
+
# user agent carrying a stray "\xA1" was enough to turn a host app's 200 into
|
|
10
|
+
# a 500. NUL bytes are valid JSON, but PostgreSQL rejects them in text columns.
|
|
11
|
+
module Sanitizer
|
|
12
|
+
REPLACEMENT = "�"
|
|
13
|
+
NUL = "\u0000"
|
|
14
|
+
|
|
15
|
+
module_function
|
|
16
|
+
|
|
17
|
+
# Returns value.to_s as valid UTF-8 without NUL bytes. A string that is
|
|
18
|
+
# already clean is returned as is, so the common case allocates nothing.
|
|
19
|
+
def string(value)
|
|
20
|
+
str = value.to_s
|
|
21
|
+
str = to_utf8(str) unless str.ascii_only? || (str.encoding == Encoding::UTF_8 && str.valid_encoding?)
|
|
22
|
+
str.include?(NUL) ? str.delete(NUL) : str
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Copies Hashes and Arrays, running every String (keys included) through
|
|
26
|
+
# .string. Anything else is returned untouched — Symbols too, since they come
|
|
27
|
+
# from code rather than from clients, and payload code looks keys up by Symbol.
|
|
28
|
+
def deep(value)
|
|
29
|
+
case value
|
|
30
|
+
when String
|
|
31
|
+
string(value)
|
|
32
|
+
when Hash
|
|
33
|
+
value.each_with_object({}) { |(k, v), out| out[deep(k)] = deep(v) }
|
|
34
|
+
when Array
|
|
35
|
+
value.map { |v| deep(v) }
|
|
36
|
+
else
|
|
37
|
+
value
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def to_utf8(str)
|
|
42
|
+
case str.encoding
|
|
43
|
+
when Encoding::BINARY, Encoding::US_ASCII, Encoding::UTF_8
|
|
44
|
+
# Raw bytes of unknown origin, or a broken UTF-8 string: read the bytes
|
|
45
|
+
# as UTF-8 and replace only the sequences that aren't.
|
|
46
|
+
utf8 = str.dup.force_encoding(Encoding::UTF_8)
|
|
47
|
+
utf8.valid_encoding? ? utf8 : utf8.scrub(REPLACEMENT)
|
|
48
|
+
else
|
|
49
|
+
str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: REPLACEMENT)
|
|
50
|
+
end
|
|
51
|
+
rescue EncodingError
|
|
52
|
+
# No converter from this encoding — keep whatever reads as UTF-8.
|
|
53
|
+
str.dup.force_encoding(Encoding::UTF_8).scrub(REPLACEMENT)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -271,6 +271,7 @@ module DeadBro
|
|
|
271
271
|
if should_continue_tracking?(current, MAX_TRACKED_QUERIES)
|
|
272
272
|
current << query_info
|
|
273
273
|
end
|
|
274
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
274
275
|
end
|
|
275
276
|
end
|
|
276
277
|
|
|
@@ -379,6 +380,9 @@ module DeadBro
|
|
|
379
380
|
# are big UPDATE/INSERT with long literal blobs; don't burn regex time on
|
|
380
381
|
# those when we're going to truncate anyway.
|
|
381
382
|
sql = sql[0..SANITIZE_MAX_LENGTH] + "..." if sql.length > SANITIZE_MAX_LENGTH
|
|
383
|
+
# The regexes below raise on invalid UTF-8 (e.g. a binary literal inlined
|
|
384
|
+
# by an adapter without prepared statements).
|
|
385
|
+
sql = DeadBro::Sanitizer.string(sql)
|
|
382
386
|
|
|
383
387
|
# Only scan for sensitive KV pairs if one of the keywords is actually
|
|
384
388
|
# present — saves two regex passes on the vast majority of queries.
|
|
@@ -406,7 +410,7 @@ module DeadBro
|
|
|
406
410
|
EXPLAIN_DML_KEYWORD_RE = /\b(INSERT|UPDATE|DELETE|MERGE)\b/i.freeze
|
|
407
411
|
|
|
408
412
|
def self.should_explain_query?(duration_ms, sql)
|
|
409
|
-
return false unless DeadBro.configuration.
|
|
413
|
+
return false unless DeadBro.configuration.explain_active?
|
|
410
414
|
return false if duration_ms < DeadBro.configuration.slow_query_threshold_ms
|
|
411
415
|
return false unless sql.is_a?(String)
|
|
412
416
|
|
|
@@ -419,6 +423,10 @@ module DeadBro
|
|
|
419
423
|
return false if stripped.match?(/\AWITH\b/i) && stripped.match?(EXPLAIN_DML_KEYWORD_RE)
|
|
420
424
|
|
|
421
425
|
true
|
|
426
|
+
rescue
|
|
427
|
+
# e.g. invalid UTF-8 in the SQL, which strip and match? raise on — when in
|
|
428
|
+
# doubt, don't run anything against the customer's database.
|
|
429
|
+
false
|
|
422
430
|
end
|
|
423
431
|
|
|
424
432
|
def self.start_explain_background(sql, connection_id, query_info, binds = nil)
|
|
@@ -494,7 +502,7 @@ module DeadBro
|
|
|
494
502
|
timestamp = Time.now.utc
|
|
495
503
|
log_entry = {
|
|
496
504
|
sev: severity.to_s,
|
|
497
|
-
msg: message
|
|
505
|
+
msg: DeadBro::Sanitizer.string(message),
|
|
498
506
|
time: timestamp.iso8601(3)
|
|
499
507
|
}
|
|
500
508
|
|
|
@@ -778,7 +786,7 @@ module DeadBro
|
|
|
778
786
|
# dominating CPU on N+1-heavy requests (100s of full Thread#backtrace
|
|
779
787
|
# allocations). The main subscriber now captures a trimmed backtrace
|
|
780
788
|
# lazily — and only when a query exceeds slow_query_threshold_ms.
|
|
781
|
-
rescue
|
|
789
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
782
790
|
end
|
|
783
791
|
|
|
784
792
|
def finish(name, id, payload)
|
|
@@ -796,7 +804,7 @@ module DeadBro
|
|
|
796
804
|
delta = end_count - start_count
|
|
797
805
|
results = (Thread.current[DeadBro::SqlSubscriber::THREAD_LOCAL_ALLOC_RESULTS_KEY] ||= {})
|
|
798
806
|
results[id] = delta
|
|
799
|
-
rescue
|
|
807
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
800
808
|
end
|
|
801
809
|
end
|
|
802
810
|
end
|
|
@@ -9,6 +9,18 @@ module DeadBro
|
|
|
9
9
|
def call(env)
|
|
10
10
|
return @app.call(env) if DeadBro.configuration.skip_tracking?
|
|
11
11
|
|
|
12
|
+
start_tracking(env)
|
|
13
|
+
@app.call(env)
|
|
14
|
+
ensure
|
|
15
|
+
stop_tracking
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
private
|
|
19
|
+
|
|
20
|
+
# Setup and teardown both run inside the host's request: if either raised,
|
|
21
|
+
# the host would get a 500 (or, from teardown, have its own response or
|
|
22
|
+
# exception replaced by ours). Tracking is best-effort, the request is not.
|
|
23
|
+
def start_tracking(env)
|
|
12
24
|
# Capture rack entry time before any setup so middleware overhead is accurately measured.
|
|
13
25
|
rack_entry = Time.now
|
|
14
26
|
Thread.current[DeadBro::TRACKING_START_TIME_KEY] = rack_entry
|
|
@@ -87,9 +99,10 @@ module DeadBro
|
|
|
87
99
|
|
|
88
100
|
# Start outgoing HTTP accumulation for this request
|
|
89
101
|
Thread.current[:dead_bro_http_events] = []
|
|
102
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
103
|
+
end
|
|
90
104
|
|
|
91
|
-
|
|
92
|
-
ensure
|
|
105
|
+
def stop_tracking
|
|
93
106
|
# Clean up thread-local storage
|
|
94
107
|
if defined?(DeadBro::SqlSubscriber)
|
|
95
108
|
Thread.current[:dead_bro_sql_queries]
|
|
@@ -117,11 +130,16 @@ module DeadBro
|
|
|
117
130
|
Thread.current[:dead_bro_lightweight_memory] = nil
|
|
118
131
|
end
|
|
119
132
|
|
|
120
|
-
# Clean up HTTP events, ES events, DB connection tracking, and tracking start time
|
|
133
|
+
# Clean up HTTP events, ES events, DB connection tracking, and tracking start time.
|
|
134
|
+
# The calls below rescue on their own: teardown has to finish, so one
|
|
135
|
+
# failing step must not skip the ones after it.
|
|
121
136
|
Thread.current[:dead_bro_elasticsearch_events] = nil
|
|
122
137
|
Thread.current[:dead_bro_http_events] = nil
|
|
123
138
|
Thread.current[:dead_bro_queue_duration_ms] = nil
|
|
124
|
-
|
|
139
|
+
begin
|
|
140
|
+
DeadBro::DbConnectionSubscriber.stop_request_tracking if defined?(DeadBro::DbConnectionSubscriber)
|
|
141
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
142
|
+
end
|
|
125
143
|
Thread.current[DeadBro::GcTracker::THREAD_KEY] = nil if defined?(DeadBro::GcTracker)
|
|
126
144
|
# Bypass stop_request_tracking intentionally — cleanup only, no return value needed here.
|
|
127
145
|
Thread.current[DeadBro::ArObjectTracker::THREAD_KEY] = nil if defined?(DeadBro::ArObjectTracker)
|
|
@@ -130,14 +148,18 @@ module DeadBro
|
|
|
130
148
|
# Safety net: ensure allocation tracing is never left running across
|
|
131
149
|
# requests (Subscriber normally stops it after analyzing).
|
|
132
150
|
if Thread.current[:dead_bro_alloc_active]
|
|
133
|
-
|
|
151
|
+
begin
|
|
152
|
+
DeadBro::AllocationSourceSampler.stop if defined?(DeadBro::AllocationSourceSampler)
|
|
153
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
154
|
+
end
|
|
134
155
|
end
|
|
156
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
157
|
+
# Backstop for anything unforeseen above.
|
|
158
|
+
ensure
|
|
135
159
|
Thread.current[:dead_bro_alloc_active] = nil
|
|
136
160
|
Thread.current[DeadBro::TRACKING_START_TIME_KEY] = nil
|
|
137
161
|
end
|
|
138
162
|
|
|
139
|
-
private
|
|
140
|
-
|
|
141
163
|
def parse_queue_start(env, rack_entry)
|
|
142
164
|
raw = env["HTTP_X_REQUEST_START"] || env["HTTP_X_QUEUE_START"]
|
|
143
165
|
return nil if raw.nil? || raw.empty?
|
data/lib/dead_bro/subscriber.rb
CHANGED
|
@@ -189,7 +189,7 @@ module DeadBro
|
|
|
189
189
|
begin
|
|
190
190
|
exception_class, exception_message = data[:exception] if data[:exception]
|
|
191
191
|
exception_obj = data[:exception_object]
|
|
192
|
-
backtrace = Array(exception_obj&.backtrace).first(50)
|
|
192
|
+
backtrace = Array(exception_obj&.backtrace).first(50).map { |line| sanitize_string(line) }
|
|
193
193
|
|
|
194
194
|
error_payload = detail_fields.merge(
|
|
195
195
|
controller: data[:controller],
|
|
@@ -207,7 +207,7 @@ module DeadBro
|
|
|
207
207
|
user_agent: safe_user_agent(data),
|
|
208
208
|
user_id: extract_user_id(data),
|
|
209
209
|
exception_class: exception_class || exception_obj&.class&.name,
|
|
210
|
-
message: (exception_message || exception_obj&.message).to_s[0, 1000],
|
|
210
|
+
message: sanitize_string((exception_message || exception_obj&.message).to_s[0, 1000]),
|
|
211
211
|
backtrace: backtrace,
|
|
212
212
|
fingerprint: compute_error_fingerprint(exception_obj),
|
|
213
213
|
cause_chain: build_cause_chain(exception_obj),
|
|
@@ -243,6 +243,10 @@ module DeadBro
|
|
|
243
243
|
# already made above; client#post_metric must not re-roll it with the
|
|
244
244
|
# global-only rate, which would silently override a per-type sample rate.
|
|
245
245
|
client.post_metric(event_name: name, payload: payload, force: true)
|
|
246
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
247
|
+
# Anything raised here would fail the host's request (see
|
|
248
|
+
# DeadBro::CONTAINED_ERRORS) — drop this request's metrics instead.
|
|
249
|
+
drain_request_tracking
|
|
246
250
|
end
|
|
247
251
|
end
|
|
248
252
|
|
|
@@ -309,7 +313,7 @@ module DeadBro
|
|
|
309
313
|
end
|
|
310
314
|
|
|
311
315
|
def self.sanitize_string(str)
|
|
312
|
-
|
|
316
|
+
DeadBro::Sanitizer.string(str)
|
|
313
317
|
end
|
|
314
318
|
|
|
315
319
|
# Matched against a key segment (delimited by start/end, underscore, dash, or
|
|
@@ -336,6 +340,9 @@ module DeadBro
|
|
|
336
340
|
when Hash
|
|
337
341
|
entries = value.to_a[0, max_hash_keys]
|
|
338
342
|
entries.each_with_object({}) do |(k, v), memo|
|
|
343
|
+
# Scrubbed first: the sensitive-key regex raises on a key that isn't
|
|
344
|
+
# valid UTF-8, which would drop the whole params hash.
|
|
345
|
+
k = sanitize_string(k) if k.is_a?(String)
|
|
339
346
|
memo[k] = sensitive_key?(k) ? "[FILTERED]" : truncate_value(v, max_str: max_str, max_array: max_array, max_hash_keys: max_hash_keys)
|
|
340
347
|
end
|
|
341
348
|
else
|
|
@@ -497,7 +504,7 @@ module DeadBro
|
|
|
497
504
|
end
|
|
498
505
|
|
|
499
506
|
def self.normalize_error_message(msg)
|
|
500
|
-
msg
|
|
507
|
+
sanitize_string(msg)
|
|
501
508
|
.gsub(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/i, "UUID")
|
|
502
509
|
.gsub(/\b\d+\b/, "N")
|
|
503
510
|
.gsub(/"[^"]*"/, '"?"')
|
|
@@ -507,7 +514,7 @@ module DeadBro
|
|
|
507
514
|
|
|
508
515
|
def self.compute_error_fingerprint(exception)
|
|
509
516
|
return nil unless exception
|
|
510
|
-
top_frame = Array(exception.backtrace).first.
|
|
517
|
+
top_frame = sanitize_string(Array(exception.backtrace).first).gsub(/:\d+:in /, ":N:in ")
|
|
511
518
|
input = "#{exception.class.name}|#{normalize_error_message(exception.message)}|#{top_frame}"
|
|
512
519
|
Digest::SHA256.hexdigest(input)[0, 16]
|
|
513
520
|
rescue
|
|
@@ -522,8 +529,8 @@ module DeadBro
|
|
|
522
529
|
while cause && depth < 5
|
|
523
530
|
chain << {
|
|
524
531
|
exception_class: cause.class.name,
|
|
525
|
-
message: cause.message.to_s[0, 500],
|
|
526
|
-
backtrace_top: Array(cause.backtrace).first(3)
|
|
532
|
+
message: sanitize_string(cause.message.to_s[0, 500]),
|
|
533
|
+
backtrace_top: Array(cause.backtrace).first(3).map { |line| sanitize_string(line) }
|
|
527
534
|
}
|
|
528
535
|
cause = cause.cause
|
|
529
536
|
depth += 1
|
data/lib/dead_bro/version.rb
CHANGED
|
@@ -19,6 +19,7 @@ module DeadBro
|
|
|
19
19
|
duration_ms: ((finished - started) * 1000.0).round(2),
|
|
20
20
|
rendered_at: Time.now.utc.to_i,
|
|
21
21
|
start_offset_ms: tracking_start ? ((started - tracking_start) * 1000.0).round(2) : nil)
|
|
22
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
22
23
|
end
|
|
23
24
|
|
|
24
25
|
ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_EVENT) do |_name, started, finished, _uid, data|
|
|
@@ -28,6 +29,7 @@ module DeadBro
|
|
|
28
29
|
cache_key: data[:cache_key],
|
|
29
30
|
rendered_at: Time.now.utc.to_i,
|
|
30
31
|
start_offset_ms: tracking_start ? ((started - tracking_start) * 1000.0).round(2) : nil)
|
|
32
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
31
33
|
end
|
|
32
34
|
|
|
33
35
|
ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_EVENT) do |_name, started, finished, _uid, data|
|
|
@@ -38,6 +40,7 @@ module DeadBro
|
|
|
38
40
|
collection_cached_count: (data[:cached_count] || 0).to_i,
|
|
39
41
|
rendered_at: Time.now.utc.to_i,
|
|
40
42
|
start_offset_ms: tracking_start ? ((started - tracking_start) * 1000.0).round(2) : nil)
|
|
43
|
+
rescue *DeadBro::CONTAINED_ERRORS
|
|
41
44
|
end
|
|
42
45
|
rescue
|
|
43
46
|
end
|