devbench 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +107 -20
- data/lib/devbench/config.rb +109 -27
- data/lib/devbench/direct.rb +172 -23
- data/lib/devbench/egress_policy.rb +101 -0
- data/lib/devbench/logs.rb +413 -0
- data/lib/devbench/railtie.rb +21 -0
- data/lib/devbench/scrub.rb +46 -8
- data/lib/devbench/sidekiq_hooks.rb +4 -0
- data/lib/devbench/transport.rb +65 -10
- data/lib/devbench/version.rb +1 -1
- data/lib/devbench/view_helper.rb +62 -0
- data/lib/devbench.rb +7 -2
- data/lib/generators/devbench/devbench_generator.rb +210 -0
- metadata +9 -3
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
require_relative 'current'
|
|
5
|
+
|
|
6
|
+
module Devbench
|
|
7
|
+
# In-process log capture (docs/SERVER_SDK_SPEC.md, "In-process log capture
|
|
8
|
+
# (direct mode)"; DECISIONS #161): recent log lines kept in memory, keyed
|
|
9
|
+
# by the trace that was current when each was written, so triage can be
|
|
10
|
+
# handed the server lines of one user action without a sidecar.
|
|
11
|
+
#
|
|
12
|
+
# Two ways in, both installed by the Railtie in direct mode only:
|
|
13
|
+
#
|
|
14
|
+
# * Rails 7.1+ (Rails.logger is an ActiveSupport::BroadcastLogger):
|
|
15
|
+
# broadcast_to a CaptureLogger — Rails' own public API for "also send
|
|
16
|
+
# these lines there". It writes nothing anywhere; it only remembers.
|
|
17
|
+
# * Any other ::Logger (Rails < 7.1, Sidekiq.logger, a plain Logger):
|
|
18
|
+
# a Tee prepended to that one logger object. It calls the logger's own
|
|
19
|
+
# #add first, unchanged, and remembers the line afterwards; a block
|
|
20
|
+
# message is evaluated once, by the logger, and its value reused.
|
|
21
|
+
#
|
|
22
|
+
# Never harms the host: what the app's logger writes is untouched; a line
|
|
23
|
+
# with no current trace costs one thread-local read; the buffer lock is
|
|
24
|
+
# held for an Array push, never across I/O or redaction; nothing raises
|
|
25
|
+
# out of a logging call. Fork-safe: a child starts with an empty buffer.
|
|
26
|
+
module Logs
|
|
27
|
+
MAX_LINES = 10_000
|
|
28
|
+
MAX_BYTES = 4 * 1024 * 1024
|
|
29
|
+
MAX_AGE = 15 * 60
|
|
30
|
+
MAX_LINE_BYTES = 4096
|
|
31
|
+
# Lines held from one multi-line message (an exception and its
|
|
32
|
+
# backtrace): the top of it, so one framework backtrace cannot push the
|
|
33
|
+
# rest of the request out of a 200-line slice.
|
|
34
|
+
MAX_MESSAGE_LINES = 50
|
|
35
|
+
|
|
36
|
+
SEVERITY = %w[DEBUG INFO WARN ERROR FATAL].freeze
|
|
37
|
+
# The last line captured on this thread, and by which hook: see #capture.
|
|
38
|
+
LAST = :__devbench_last_log
|
|
39
|
+
|
|
40
|
+
# Bounded, thread-safe, per-trace store of recent lines. Oldest evicted
|
|
41
|
+
# first, by count, bytes and age.
|
|
42
|
+
class Buffer
|
|
43
|
+
Entry = Struct.new(:key, :at, :text)
|
|
44
|
+
|
|
45
|
+
def initialize(max_lines: MAX_LINES, max_bytes: MAX_BYTES, max_age: MAX_AGE,
|
|
46
|
+
clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
|
|
47
|
+
@max_lines = max_lines
|
|
48
|
+
@max_bytes = max_bytes
|
|
49
|
+
@max_age = max_age
|
|
50
|
+
@clock = clock
|
|
51
|
+
@lock = Mutex.new
|
|
52
|
+
reset
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Keeps one line under a trace key.
|
|
56
|
+
def push(key, text)
|
|
57
|
+
now = @clock.call
|
|
58
|
+
entry = Entry.new(key, now, text)
|
|
59
|
+
@lock.synchronize do
|
|
60
|
+
reset if @pid != Process.pid
|
|
61
|
+
@entries << entry
|
|
62
|
+
(@by_key[key] ||= []) << entry
|
|
63
|
+
@bytes += text.bytesize
|
|
64
|
+
evict(now)
|
|
65
|
+
end
|
|
66
|
+
nil
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# The lines held for a key, oldest first, at most `limit` (the most
|
|
70
|
+
# recent ones when there are more). A copy: the caller redacts it
|
|
71
|
+
# outside the lock.
|
|
72
|
+
def lookup(key, limit)
|
|
73
|
+
@lock.synchronize do
|
|
74
|
+
reset if @pid != Process.pid
|
|
75
|
+
evict(@clock.call)
|
|
76
|
+
list = @by_key[key]
|
|
77
|
+
return [] if list.nil?
|
|
78
|
+
|
|
79
|
+
(list.length > limit ? list.last(limit) : list).map(&:text)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def size
|
|
84
|
+
@lock.synchronize { @entries.length }
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Whether any line younger than the age bound is held (by this
|
|
88
|
+
# process).
|
|
89
|
+
def any?
|
|
90
|
+
@lock.synchronize do
|
|
91
|
+
reset if @pid != Process.pid
|
|
92
|
+
evict(@clock.call)
|
|
93
|
+
!@entries.empty?
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def bytes
|
|
98
|
+
@lock.synchronize { @bytes }
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def clear
|
|
102
|
+
@lock.synchronize { reset }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
private
|
|
106
|
+
|
|
107
|
+
def reset
|
|
108
|
+
@pid = Process.pid
|
|
109
|
+
@entries = []
|
|
110
|
+
@by_key = {}
|
|
111
|
+
@bytes = 0
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Every entry is the oldest of its own key when it is the oldest
|
|
115
|
+
# overall, so removing it from its key's list is a shift.
|
|
116
|
+
def evict(now)
|
|
117
|
+
cutoff = now - @max_age
|
|
118
|
+
while (oldest = @entries.first) &&
|
|
119
|
+
(@entries.length > @max_lines || @bytes > @max_bytes || oldest.at < cutoff)
|
|
120
|
+
@entries.shift
|
|
121
|
+
list = @by_key[oldest.key]
|
|
122
|
+
list.shift
|
|
123
|
+
@by_key.delete(oldest.key) if list.empty?
|
|
124
|
+
@bytes -= oldest.text.bytesize
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Rails 7.1+: the logger handed to BroadcastLogger#broadcast_to. Its
|
|
130
|
+
# level mirrors the app's other loggers, so adding it never turns on a
|
|
131
|
+
# level the app has off (BroadcastLogger#level is the minimum of its
|
|
132
|
+
# loggers, and #debug? asks whether any is at debug). It has no #tagged:
|
|
133
|
+
# BroadcastLogger would run a tagged block once per logger that has one.
|
|
134
|
+
module CaptureLogger
|
|
135
|
+
attr_accessor :devbench_broadcast
|
|
136
|
+
|
|
137
|
+
def level
|
|
138
|
+
own = super
|
|
139
|
+
others = devbench_broadcast&.broadcasts&.reject { |l| l.equal?(self) }
|
|
140
|
+
return own if others.nil? || others.empty?
|
|
141
|
+
|
|
142
|
+
mirrored = others.map(&:level).min
|
|
143
|
+
local = respond_to?(:local_level) ? local_level : nil
|
|
144
|
+
local.nil? ? mirrored : [local, mirrored].max
|
|
145
|
+
rescue StandardError, SystemStackError
|
|
146
|
+
::Logger::FATAL
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def add(severity, message = nil, progname = nil)
|
|
150
|
+
severity ||= ::Logger::UNKNOWN
|
|
151
|
+
return true if severity < level || Current.trace.nil?
|
|
152
|
+
|
|
153
|
+
if message.nil?
|
|
154
|
+
message = block_given? ? yield : progname
|
|
155
|
+
end
|
|
156
|
+
Logs.capture(severity, message, :broadcast, Logs.broadcast_tags(devbench_broadcast, self))
|
|
157
|
+
true
|
|
158
|
+
rescue StandardError, SystemStackError
|
|
159
|
+
true
|
|
160
|
+
end
|
|
161
|
+
alias log add
|
|
162
|
+
|
|
163
|
+
def <<(message)
|
|
164
|
+
Logs.capture(nil, message, :broadcast, Logs.broadcast_tags(devbench_broadcast, self)) unless Current.trace.nil?
|
|
165
|
+
self
|
|
166
|
+
rescue StandardError, SystemStackError
|
|
167
|
+
self
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Every other ::Logger: prepended to the one logger object's singleton
|
|
172
|
+
# class. The logger's own #add runs first, exactly as before; only then
|
|
173
|
+
# is the line remembered.
|
|
174
|
+
module Tee
|
|
175
|
+
def add(severity, message = nil, progname = nil, &block)
|
|
176
|
+
return super if Current.trace.nil? || !Logs.active?
|
|
177
|
+
|
|
178
|
+
yielded = false
|
|
179
|
+
value = nil
|
|
180
|
+
if block
|
|
181
|
+
original = block
|
|
182
|
+
block = proc do
|
|
183
|
+
value = original.call
|
|
184
|
+
yielded = true
|
|
185
|
+
value
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
result = super(severity, message, progname, &block)
|
|
189
|
+
Logs.tee(self, severity, message, progname, !block.nil?, yielded, value)
|
|
190
|
+
result
|
|
191
|
+
end
|
|
192
|
+
alias log add
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
class << self
|
|
196
|
+
def buffer
|
|
197
|
+
@buffer ||= Buffer.new
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# For tests: a buffer with other bounds or a controllable clock.
|
|
201
|
+
attr_writer :buffer
|
|
202
|
+
|
|
203
|
+
# Whether capture is on: direct mode, after install. Hooks stay in
|
|
204
|
+
# place when it is turned off, and do nothing.
|
|
205
|
+
def active?
|
|
206
|
+
@active ? true : false
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# A new transport (Devbench.configure) must be started again by the
|
|
210
|
+
# next line held, so both reset the per-process start.
|
|
211
|
+
def activate!
|
|
212
|
+
@polling_pid = nil
|
|
213
|
+
@active = true
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# Whether any logger was hooked in this process.
|
|
217
|
+
def hooked?
|
|
218
|
+
@hooked ? true : false
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def deactivate!
|
|
222
|
+
@polling_pid = nil
|
|
223
|
+
@active = false
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# Hooks one logger. Returns :broadcast, :tee, or nil (not a logger we
|
|
227
|
+
# can hook — left alone). Idempotent.
|
|
228
|
+
def install(logger)
|
|
229
|
+
return nil if logger.nil?
|
|
230
|
+
|
|
231
|
+
if logger.respond_to?(:broadcast_to) && logger.respond_to?(:broadcasts)
|
|
232
|
+
return :broadcast if logger.broadcasts.any? { |l| l.is_a?(CaptureLogger) }
|
|
233
|
+
|
|
234
|
+
capture = capture_logger_class.new(nil)
|
|
235
|
+
capture.level = logger.level
|
|
236
|
+
capture.devbench_broadcast = logger
|
|
237
|
+
logger.broadcast_to(capture)
|
|
238
|
+
@hooked = true
|
|
239
|
+
:broadcast
|
|
240
|
+
elsif logger.is_a?(::Logger)
|
|
241
|
+
logger.singleton_class.prepend(Tee) unless logger.singleton_class.include?(Tee)
|
|
242
|
+
@hooked = true
|
|
243
|
+
:tee
|
|
244
|
+
end
|
|
245
|
+
rescue StandardError, SystemStackError
|
|
246
|
+
nil
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# Whether this process holds any traced line: the transport polls
|
|
250
|
+
# ingest for log requests only then.
|
|
251
|
+
def holding?
|
|
252
|
+
buffer.any?
|
|
253
|
+
rescue StandardError, SystemStackError
|
|
254
|
+
false
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# The lines held for one trace key (session/intent), oldest first.
|
|
258
|
+
def lookup(key, limit)
|
|
259
|
+
buffer.lookup(key, limit)
|
|
260
|
+
rescue StandardError, SystemStackError
|
|
261
|
+
[]
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# The tag prefixes of the app's loggers in a broadcast (not ours): the
|
|
265
|
+
# one Rails wrote after each newline of a multi-line message is the
|
|
266
|
+
# first broadcast's, but which logger is first is the app's choice.
|
|
267
|
+
# A Proc, so a single-line message never computes it.
|
|
268
|
+
def broadcast_tags(broadcast, own)
|
|
269
|
+
lambda do
|
|
270
|
+
loggers = broadcast&.broadcasts || []
|
|
271
|
+
loggers.reject { |l| l.equal?(own) }.filter_map { |l| tags_of(l) }
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Called by Tee after the logger's own #add returned.
|
|
276
|
+
def tee(logger, severity, message, progname, had_block, yielded, value)
|
|
277
|
+
severity ||= ::Logger::UNKNOWN
|
|
278
|
+
return if severity < logger.level
|
|
279
|
+
|
|
280
|
+
if message.nil?
|
|
281
|
+
return if had_block && !yielded
|
|
282
|
+
|
|
283
|
+
message = had_block ? value : progname
|
|
284
|
+
end
|
|
285
|
+
capture(severity, message, :tee, -> { tags_of(logger) })
|
|
286
|
+
rescue StandardError, SystemStackError
|
|
287
|
+
nil
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# Keeps one message under the current trace — one entry per line of
|
|
291
|
+
# it (see #lines_for). Never raises. `tags` is the tag prefix of the
|
|
292
|
+
# logger written to (an Array of candidates, or a Proc giving one),
|
|
293
|
+
# needed only for a multi-line message.
|
|
294
|
+
#
|
|
295
|
+
# One line can reach both hooks: in a Sidekiq process Rails.logger may
|
|
296
|
+
# broadcast to Sidekiq.logger. Both run on this thread, one right
|
|
297
|
+
# after the other, so a line equal to the one just captured *by the
|
|
298
|
+
# other hook* is that same write and is skipped. The same hook
|
|
299
|
+
# repeating a line is a real repeat and is kept.
|
|
300
|
+
def capture(severity, message, source, tags = nil)
|
|
301
|
+
return nil unless active?
|
|
302
|
+
|
|
303
|
+
trace = Current.trace
|
|
304
|
+
return nil if trace.nil?
|
|
305
|
+
|
|
306
|
+
lines = lines_for(trace, severity, message, tags)
|
|
307
|
+
return nil if lines.empty?
|
|
308
|
+
|
|
309
|
+
written = lines.join("\n")
|
|
310
|
+
last = Thread.current[LAST]
|
|
311
|
+
if last && last[0] != source && last[1] == written
|
|
312
|
+
Thread.current[LAST] = nil
|
|
313
|
+
return nil
|
|
314
|
+
end
|
|
315
|
+
Thread.current[LAST] = [source, written]
|
|
316
|
+
lines.each { |line| buffer.push(trace.key, line) }
|
|
317
|
+
start_polling if @polling_pid != Process.pid
|
|
318
|
+
nil
|
|
319
|
+
rescue StandardError, SystemStackError, ThreadError
|
|
320
|
+
nil
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
private
|
|
324
|
+
|
|
325
|
+
# The first line this process holds starts the transport's flush
|
|
326
|
+
# thread, so a process with nothing to report still polls for log
|
|
327
|
+
# requests (a forked worker included: the PID is per process).
|
|
328
|
+
def start_polling
|
|
329
|
+
@polling_pid = Process.pid
|
|
330
|
+
transport = Devbench.transport
|
|
331
|
+
transport.start if transport.respond_to?(:start)
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# "[a] [b] " — what ActiveSupport::TaggedLogging puts before a line,
|
|
335
|
+
# and what ActionDispatch::DebugExceptions puts after every newline of
|
|
336
|
+
# the exception it logs as one message. nil when the logger has none.
|
|
337
|
+
def tags_of(logger)
|
|
338
|
+
formatter = logger.respond_to?(:formatter) ? logger.formatter : nil
|
|
339
|
+
return nil unless formatter.respond_to?(:tags_text)
|
|
340
|
+
|
|
341
|
+
text = formatter.tags_text.to_s
|
|
342
|
+
text.empty? ? nil : text
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
# "[v1/s/i/h] INFO message", at most MAX_LINE_BYTES, valid UTF-8 —
|
|
346
|
+
# one per line of the message. A multi-line message (Rails logs an
|
|
347
|
+
# exception and its backtrace as one) is split so each line reaches
|
|
348
|
+
# triage on its own, with the logger's tags removed from the start of
|
|
349
|
+
# continuation lines (the trace frames each line once) and blank
|
|
350
|
+
# lines dropped. At most MAX_MESSAGE_LINES; when more, the last says
|
|
351
|
+
# how many were left out.
|
|
352
|
+
def lines_for(trace, severity, message, tags)
|
|
353
|
+
label = severity.is_a?(Integer) ? SEVERITY[severity] : nil
|
|
354
|
+
prefix = label ? "[#{trace}] #{label} " : "[#{trace}] "
|
|
355
|
+
text = message_text(message)
|
|
356
|
+
text = text.dup.force_encoding(Encoding::UTF_8) unless text.encoding == Encoding::UTF_8
|
|
357
|
+
text = text.scrub('') unless text.valid_encoding?
|
|
358
|
+
return [bound_line(prefix + text)] unless text.include?("\n")
|
|
359
|
+
|
|
360
|
+
prefixes = tag_prefixes(tags)
|
|
361
|
+
parts = text.split("\n").each_with_index.filter_map do |part, i|
|
|
362
|
+
part = untag(part, prefixes) if i.positive?
|
|
363
|
+
part = part.chomp("\r")
|
|
364
|
+
part unless part.strip.empty?
|
|
365
|
+
end
|
|
366
|
+
return [] if parts.empty?
|
|
367
|
+
|
|
368
|
+
if parts.length > MAX_MESSAGE_LINES
|
|
369
|
+
omitted = parts.length - (MAX_MESSAGE_LINES - 1)
|
|
370
|
+
parts = parts.first(MAX_MESSAGE_LINES - 1) << "... (#{omitted} more lines)"
|
|
371
|
+
end
|
|
372
|
+
parts.map { |part| bound_line(prefix + part) }
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# A logger whose formatter fails to give its tags costs the untagging,
|
|
376
|
+
# never the line.
|
|
377
|
+
def tag_prefixes(tags)
|
|
378
|
+
Array(tags.respond_to?(:call) ? tags.call : tags).select { |t| t.is_a?(::String) && !t.empty? }
|
|
379
|
+
rescue StandardError, SystemStackError
|
|
380
|
+
[]
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
def untag(part, prefixes)
|
|
384
|
+
found = prefixes.find { |p| part.start_with?(p) }
|
|
385
|
+
found ? part[found.length..] : part
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def bound_line(line)
|
|
389
|
+
line = line.byteslice(0, MAX_LINE_BYTES) if line.bytesize > MAX_LINE_BYTES
|
|
390
|
+
line = line.dup.force_encoding(Encoding::UTF_8) unless line.encoding == Encoding::UTF_8
|
|
391
|
+
line.valid_encoding? ? line : line.scrub('')
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
# As ::Logger::Formatter#msg2str.
|
|
395
|
+
def message_text(message)
|
|
396
|
+
text = case message
|
|
397
|
+
when ::String then message
|
|
398
|
+
when ::Exception
|
|
399
|
+
"#{message.message} (#{message.class})\n#{(message.backtrace || []).join("\n")}"
|
|
400
|
+
else message.inspect
|
|
401
|
+
end
|
|
402
|
+
text.end_with?("\n") ? text.chomp : text
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
def capture_logger_class
|
|
406
|
+
@capture_logger_class ||= begin
|
|
407
|
+
base = defined?(::ActiveSupport::Logger) ? ::ActiveSupport::Logger : ::Logger
|
|
408
|
+
Class.new(base) { include CaptureLogger }
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
end
|
data/lib/devbench/railtie.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative 'middleware'
|
|
4
4
|
require_relative 'rails_hooks'
|
|
5
5
|
require_relative 'sidekiq_hooks'
|
|
6
|
+
require_relative 'view_helper'
|
|
6
7
|
|
|
7
8
|
module Devbench
|
|
8
9
|
# Hooks Dev Bench into Rails at boot. Loaded by lib/devbench.rb only when
|
|
@@ -40,6 +41,11 @@ module Devbench
|
|
|
40
41
|
nil
|
|
41
42
|
end
|
|
42
43
|
|
|
44
|
+
# <%= devbench_script_tag %> in any view or layout.
|
|
45
|
+
initializer 'devbench.view_helper' do
|
|
46
|
+
ActiveSupport.on_load(:action_view) { include Devbench::ViewHelper }
|
|
47
|
+
end
|
|
48
|
+
|
|
43
49
|
# rake devbench:test — check the DSN without waiting for a flush.
|
|
44
50
|
rake_tasks do
|
|
45
51
|
namespace :devbench do
|
|
@@ -62,12 +68,27 @@ module Devbench
|
|
|
62
68
|
# Native Sidekiq jobs never reach the ActiveJob hook. after_initialize
|
|
63
69
|
# runs after every gem is required, so Gemfile order does not matter.
|
|
64
70
|
Devbench::SidekiqHooks.install if defined?(::Sidekiq)
|
|
71
|
+
|
|
72
|
+
# Server log lines for triage, kept in-process (direct mode only).
|
|
73
|
+
# After the app's initializers, so the logger they configured is the
|
|
74
|
+
# one hooked.
|
|
75
|
+
Devbench::Railtie.capture_logs
|
|
65
76
|
rescue StandardError, SystemStackError
|
|
66
77
|
nil
|
|
67
78
|
end
|
|
68
79
|
end
|
|
69
80
|
|
|
70
81
|
class << self
|
|
82
|
+
# Rails.logger, plus Sidekiq's logger when Sidekiq is loaded. A no-op
|
|
83
|
+
# unless reporting is direct.
|
|
84
|
+
def capture_logs
|
|
85
|
+
loggers = [::Rails.logger]
|
|
86
|
+
loggers << ::Sidekiq.logger if defined?(::Sidekiq) && ::Sidekiq.respond_to?(:logger)
|
|
87
|
+
Devbench.capture_logs(loggers.compact)
|
|
88
|
+
rescue StandardError, SystemStackError
|
|
89
|
+
false
|
|
90
|
+
end
|
|
91
|
+
|
|
71
92
|
# Records the conditional insert on a MiddlewareStackProxy. Returns
|
|
72
93
|
# true when recorded.
|
|
73
94
|
def auto_insert(proxy)
|
data/lib/devbench/scrub.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative 'fingerprint'
|
|
4
|
+
require_relative 'egress_policy'
|
|
4
5
|
|
|
5
6
|
module Devbench
|
|
6
7
|
# A port of the sidecar's strict egress (internal/scrub and
|
|
@@ -109,25 +110,62 @@ module Devbench
|
|
|
109
110
|
end
|
|
110
111
|
|
|
111
112
|
# The sidecar's forEgress in strict mode, for one line: any trace kept
|
|
112
|
-
# as is, the rest templated, scrubbed and prose-masked
|
|
113
|
-
|
|
113
|
+
# as is, the rest templated, scrubbed and prose-masked — then the
|
|
114
|
+
# learned rules (an EgressPolicy), exactly as Sidecar.redactBody
|
|
115
|
+
# applies them:
|
|
116
|
+
#
|
|
117
|
+
# shape 'none' the content never leaves: <withheld:shape:<fp12>>
|
|
118
|
+
# shape 'full' lifts only the prose heuristic, never base redaction
|
|
119
|
+
# shape 'masked' prose heuristic (already on in strict mode)
|
|
120
|
+
# field / term masked afterwards, whatever the shape rule says
|
|
121
|
+
#
|
|
122
|
+
# Shape rules are keyed by the line's shape fingerprint, as the sidecar
|
|
123
|
+
# computes it (log_template / sidecar / service / the trace-stripped
|
|
124
|
+
# line).
|
|
125
|
+
def egress(line, policy = nil, service = '')
|
|
114
126
|
line = utf8(line)
|
|
115
127
|
match = TRACE.match(line)
|
|
116
128
|
body = match ? line.gsub(TRACE, '<trace>') : line
|
|
117
|
-
|
|
129
|
+
fp, rule = shape_rule(policy, service, body)
|
|
130
|
+
redacted = if rule == EgressPolicy::NONE
|
|
131
|
+
withheld(fp)
|
|
132
|
+
else
|
|
133
|
+
redact_body(body, rule, policy)
|
|
134
|
+
end
|
|
118
135
|
match ? "[#{match[0]}] #{redacted}" : redacted
|
|
119
136
|
end
|
|
120
137
|
|
|
121
138
|
# A bundle's template text: as the sidecar's answerEvidenceRequests
|
|
122
|
-
# does,
|
|
123
|
-
|
|
124
|
-
|
|
139
|
+
# does, the shape's own rule (by the signal's fingerprint), strict
|
|
140
|
+
# redaction, and then a second shape pass. Empty when the shape's
|
|
141
|
+
# content never leaves.
|
|
142
|
+
def template_text(text, policy = nil, fp = nil)
|
|
143
|
+
rule = policy && fp ? policy.rule_for(fp) : nil
|
|
144
|
+
return '' if rule == EgressPolicy::NONE
|
|
145
|
+
|
|
146
|
+
self.text(redact_body(utf8(text), rule, policy))
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# What stands in for a line whose shape is marked 'none'.
|
|
150
|
+
def withheld(fp)
|
|
151
|
+
"<withheld:shape:#{fp.to_s[0, 12]}>"
|
|
125
152
|
end
|
|
126
153
|
|
|
127
154
|
private
|
|
128
155
|
|
|
129
|
-
|
|
130
|
-
|
|
156
|
+
# Base redaction (template + scrub), the prose heuristic unless the
|
|
157
|
+
# shape is marked 'full', then field and term rules.
|
|
158
|
+
def redact_body(body, rule, policy)
|
|
159
|
+
redacted = text(Fingerprint.template(body))
|
|
160
|
+
redacted = prose(redacted) unless rule == EgressPolicy::FULL
|
|
161
|
+
policy ? policy.mask(redacted) : redacted
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def shape_rule(policy, service, body)
|
|
165
|
+
return [nil, nil] if policy.nil? || !policy.shapes?
|
|
166
|
+
|
|
167
|
+
fp = Fingerprint.compute(kind: 'log_template', source: 'sidecar', service: service.to_s, message: body)
|
|
168
|
+
fp ? [fp, policy.rule_for(fp)] : [nil, nil]
|
|
131
169
|
end
|
|
132
170
|
|
|
133
171
|
def proper_noun?(token)
|
|
@@ -101,6 +101,10 @@ module Devbench
|
|
|
101
101
|
|
|
102
102
|
handlers = config.error_handlers
|
|
103
103
|
handlers << ERROR_HANDLER unless handlers.any? { |h| h.equal?(ERROR_HANDLER) }
|
|
104
|
+
|
|
105
|
+
# The job's own log lines (Sidekiq::Job#logger), kept under the
|
|
106
|
+
# job's trace — direct mode only, a no-op otherwise.
|
|
107
|
+
Devbench.capture_logs(config.logger) if config.respond_to?(:logger)
|
|
104
108
|
true
|
|
105
109
|
rescue StandardError, SystemStackError
|
|
106
110
|
false
|
data/lib/devbench/transport.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative 'config'
|
|
4
4
|
require_relative 'sidecar_transport'
|
|
5
5
|
require_relative 'direct'
|
|
6
|
+
require_relative 'logs'
|
|
6
7
|
|
|
7
8
|
# Configuration and transport selection (docs/SERVER_SDK_SPEC.md,
|
|
8
9
|
# "Transports: direct (default) or sidecar (optional)").
|
|
@@ -17,8 +18,6 @@ require_relative 'direct'
|
|
|
17
18
|
# Never both: a sidecar on the same host still reads logs, but writing to
|
|
18
19
|
# its socket as well would count every report twice.
|
|
19
20
|
module Devbench
|
|
20
|
-
# Raised (and rescued) by Devbench.test! to make its synthetic report.
|
|
21
|
-
class TestException < StandardError; end
|
|
22
21
|
|
|
23
22
|
TRANSPORT_LOCK = Mutex.new
|
|
24
23
|
CONFIG_LOCK = Mutex.new
|
|
@@ -42,6 +41,9 @@ module Devbench
|
|
|
42
41
|
def configure
|
|
43
42
|
yield config if block_given?
|
|
44
43
|
reset_transport!
|
|
44
|
+
# Hooks already in place follow the new configuration: on in direct
|
|
45
|
+
# mode, inert otherwise.
|
|
46
|
+
Logs.hooked? && direct? ? Logs.activate! : Logs.deactivate!
|
|
45
47
|
nil
|
|
46
48
|
rescue StandardError, SystemStackError => e
|
|
47
49
|
warn_once(:configure, "Devbench.configure raised #{e.class}; reporting is unchanged")
|
|
@@ -60,6 +62,64 @@ module Devbench
|
|
|
60
62
|
@transport || TRANSPORT_LOCK.synchronize { @transport ||= build_transport }
|
|
61
63
|
end
|
|
62
64
|
|
|
65
|
+
# The DSN the browser sensor may hold — https://<public>@host — derived
|
|
66
|
+
# from DEVBENCH_DSN, so a page needs no setting of its own. Nil when the
|
|
67
|
+
# DSN has no public part (a 0.5 single-key DSN: that key is a server
|
|
68
|
+
# secret), does not parse, or reporting is disabled. Never raises, and
|
|
69
|
+
# never returns anything carrying the secret.
|
|
70
|
+
def browser_dsn
|
|
71
|
+
cfg = config
|
|
72
|
+
return nil unless cfg.enabled?
|
|
73
|
+
|
|
74
|
+
raw = cfg.dsn.to_s
|
|
75
|
+
cached = @browser_dsn
|
|
76
|
+
return cached[1] if cached && cached[0] == raw
|
|
77
|
+
|
|
78
|
+
value = raw.strip.empty? ? nil : DSN.parse(raw).browser_dsn
|
|
79
|
+
@browser_dsn = [raw, value]
|
|
80
|
+
value
|
|
81
|
+
rescue StandardError, SystemStackError
|
|
82
|
+
nil
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Where the browser sensor connects, for the app's CSP connect_src:
|
|
86
|
+
# the ingest host from DEVBENCH_DSN and evidence storage. Read when the
|
|
87
|
+
# CSP initializer runs, so the policy follows the DSN instead of a host
|
|
88
|
+
# frozen at generate time; empty when there is no usable DSN, so an app
|
|
89
|
+
# without Dev Bench configured (e.g. production) keeps its policy as is.
|
|
90
|
+
# `bin/rails generate devbench` adds `*Devbench.csp_connect_sources`.
|
|
91
|
+
STORAGE_SOURCE = 'https://*.storage.supabase.co'
|
|
92
|
+
|
|
93
|
+
def csp_connect_sources
|
|
94
|
+
return [] if browser_dsn.nil?
|
|
95
|
+
|
|
96
|
+
[DSN.parse(config.dsn.to_s).base, STORAGE_SOURCE]
|
|
97
|
+
rescue StandardError, SystemStackError
|
|
98
|
+
[]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# True when this process reports straight to Dev Bench (a DSN is set
|
|
102
|
+
# and parses). Builds the transport if it was not built yet.
|
|
103
|
+
def direct?
|
|
104
|
+
transport.is_a?(DirectTransport)
|
|
105
|
+
rescue StandardError, SystemStackError
|
|
106
|
+
false
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Keeps recent lines from these loggers, per trace, for triage to ask
|
|
110
|
+
# for (direct mode only; elsewhere a no-op). The Railtie does this for
|
|
111
|
+
# Rails.logger and Sidekiq.logger; a Rack app without Rails calls it
|
|
112
|
+
# itself. Returns true when capture is on. Never raises.
|
|
113
|
+
def capture_logs(*loggers)
|
|
114
|
+
return false unless enabled? && direct?
|
|
115
|
+
|
|
116
|
+
hooked = loggers.flatten.map { |l| Logs.install(l) }.compact
|
|
117
|
+
Logs.activate! unless hooked.empty?
|
|
118
|
+
Logs.active?
|
|
119
|
+
rescue StandardError, SystemStackError
|
|
120
|
+
false
|
|
121
|
+
end
|
|
122
|
+
|
|
63
123
|
# Sends whatever direct mode has counted, now, waiting at most `timeout`
|
|
64
124
|
# seconds. A no-op in sidecar mode. Returns true when nothing is left
|
|
65
125
|
# unsent. Never raises.
|
|
@@ -81,7 +141,7 @@ module Devbench
|
|
|
81
141
|
end
|
|
82
142
|
if cfg.dsn.nil? || cfg.dsn.strip.empty?
|
|
83
143
|
io.puts 'DEVBENCH_DSN is not set. Set it to the DSN Dev Bench gave you for this ' \
|
|
84
|
-
'environment (https://<
|
|
144
|
+
'environment (https://<public>:<secret>@<host>, from `adt dsn create`), or call Devbench.configure { |c| c.dsn = ... }.'
|
|
85
145
|
return false
|
|
86
146
|
end
|
|
87
147
|
|
|
@@ -92,14 +152,8 @@ module Devbench
|
|
|
92
152
|
return false
|
|
93
153
|
end
|
|
94
154
|
|
|
95
|
-
error = begin
|
|
96
|
-
raise TestException, 'Dev Bench test exception: if you can read this, the DSN works'
|
|
97
|
-
rescue TestException => e
|
|
98
|
-
e
|
|
99
|
-
end
|
|
100
|
-
payload = Reporter.report_for(error, context: 'explicit', handled: true, symbol: 'devbench:test')
|
|
101
155
|
client = DirectTransport.new(dsn: dsn, service: cfg.resolved_service, release: cfg.resolved_release)
|
|
102
|
-
client.self_test(
|
|
156
|
+
client.self_test(io)
|
|
103
157
|
rescue StandardError, SystemStackError => e
|
|
104
158
|
io.puts "Dev Bench test failed: #{e.class}: #{e.message}"
|
|
105
159
|
false
|
|
@@ -110,6 +164,7 @@ module Devbench
|
|
|
110
164
|
# Devbench.configure.
|
|
111
165
|
def reset!
|
|
112
166
|
reset_transport!
|
|
167
|
+
Logs.deactivate!
|
|
113
168
|
CONFIG_LOCK.synchronize { @config = nil }
|
|
114
169
|
nil
|
|
115
170
|
end
|
data/lib/devbench/version.rb
CHANGED