devbench 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +21 -0
- data/README.md +83 -4
- data/lib/devbench/active_job_hooks.rb +121 -0
- data/lib/devbench/config.rb +49 -7
- data/lib/devbench/current.rb +49 -0
- data/lib/devbench/direct.rb +30 -4
- data/lib/devbench/identity.rb +17 -3
- data/lib/devbench/logs.rb +84 -13
- data/lib/devbench/middleware.rb +58 -2
- data/lib/devbench/rails_hooks.rb +1 -1
- data/lib/devbench/railtie.rb +7 -1
- data/lib/devbench/reporter.rb +17 -5
- data/lib/devbench/request_signals.rb +359 -0
- data/lib/devbench/scrub.rb +69 -3
- data/lib/devbench/session.rb +26 -5
- data/lib/devbench/sidecar_transport.rb +1 -1
- data/lib/devbench/sidekiq_hooks.rb +1 -1
- data/lib/devbench/trace.rb +1 -1
- data/lib/devbench/transport.rb +9 -1
- data/lib/devbench/version.rb +18 -6
- data/lib/devbench/view_helper.rb +24 -1
- data/lib/devbench/widget_identity.rb +125 -0
- data/lib/devbench.rb +5 -0
- metadata +7 -3
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'fingerprint'
|
|
4
|
+
|
|
5
|
+
module Devbench
|
|
6
|
+
# Request signals (docs/server-sdk/failed-and-slow-requests.md, "Capability
|
|
7
|
+
# 2b — Failed and slow requests"; DECISIONS #171): what a request does
|
|
8
|
+
# without raising.
|
|
9
|
+
#
|
|
10
|
+
# * failed: the application answered 500–599 and no exception was
|
|
11
|
+
# reported during the request;
|
|
12
|
+
# * slow: it answered 100–499, not streaming, after more than 3,000 ms;
|
|
13
|
+
# * rate_limit_spike: one route's 429s in one unix minute met the spike
|
|
14
|
+
# rule (at least ten, and at least 10% of that route's requests).
|
|
15
|
+
#
|
|
16
|
+
# A port of the backend's internal/reqsignal (and the Go SDK's copy,
|
|
17
|
+
# sdk/server/go/reqsignal.go). A port can drift, and a drifted port reports
|
|
18
|
+
# a different set of problems by language, so it is held to the shared
|
|
19
|
+
# vectors (testdata/request_signal_vectors.json) every implementation must
|
|
20
|
+
# reproduce. Change nothing here without changing those in lockstep.
|
|
21
|
+
#
|
|
22
|
+
# Devbench::Middleware observes; this module classifies, builds and keeps
|
|
23
|
+
# the per-process spike detector. Nothing here raises into the
|
|
24
|
+
# application.
|
|
25
|
+
module RequestSignals
|
|
26
|
+
# The thresholds: constants, not configuration. Tenant-configurable
|
|
27
|
+
# thresholds are a later issue (DECISIONS #171), and every SDK must apply
|
|
28
|
+
# the same numbers or the same app reports differently by language.
|
|
29
|
+
SLOW_THRESHOLD_MS = 3000
|
|
30
|
+
SPIKE_WINDOW_SECONDS = 60
|
|
31
|
+
SPIKE_MIN_COUNT = 10
|
|
32
|
+
SPIKE_MIN_PERCENT = 10
|
|
33
|
+
MAX_ROUTES_PER_WINDOW = 512
|
|
34
|
+
MAX_ROUTE_CHARS = 255
|
|
35
|
+
|
|
36
|
+
# A request matched to no route pattern: one literal, never the raw path
|
|
37
|
+
# (a path carries ids and slugs).
|
|
38
|
+
UNKNOWN_ROUTE = '*'
|
|
39
|
+
# Any method outside the standard set, so a client sending invented
|
|
40
|
+
# methods cannot mint fingerprints.
|
|
41
|
+
OTHER_METHOD = 'OTHER'
|
|
42
|
+
METHODS = %w[GET HEAD POST PUT PATCH DELETE OPTIONS TRACE CONNECT].freeze
|
|
43
|
+
|
|
44
|
+
# The `signal` field of the `request` control message.
|
|
45
|
+
FAILED = 'failed'
|
|
46
|
+
SLOW = 'slow'
|
|
47
|
+
RATE_LIMIT_SPIKE = 'rate_limit_spike'
|
|
48
|
+
|
|
49
|
+
# Everything the direct transport needs to count a request signal: the
|
|
50
|
+
# count's kind, the fingerprint's type and message, the template text
|
|
51
|
+
# kept for evidence, and the raw exemplar (scrubbed at egress).
|
|
52
|
+
Built = Struct.new(:kind, :type, :message, :text, :exemplar)
|
|
53
|
+
|
|
54
|
+
# One (method, route) whose window met the spike rule. `window` is the
|
|
55
|
+
# window's start, unix seconds.
|
|
56
|
+
Spike = Struct.new(:method, :route, :window, :count, :total)
|
|
57
|
+
|
|
58
|
+
# The clock the middleware reads: monotonic milliseconds for a request's
|
|
59
|
+
# duration, wall milliseconds for the spike window. Replaceable only so
|
|
60
|
+
# tests can pin a duration to the millisecond.
|
|
61
|
+
module SystemClock
|
|
62
|
+
def self.monotonic_ms
|
|
63
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def self.wall_ms
|
|
67
|
+
Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The 429 spike rule for one process's requests. Thread-safe: every
|
|
72
|
+
# request's thread observes, the flush thread ticks.
|
|
73
|
+
#
|
|
74
|
+
# One window is open at a time. An observation in a later window closes
|
|
75
|
+
# the open one first; one in an earlier window (a clock step back) counts
|
|
76
|
+
# in the open window. Memory is bounded by MAX_ROUTES_PER_WINDOW, and
|
|
77
|
+
# nothing is kept past the open window. A forked child starts empty: the
|
|
78
|
+
# parent reports what it saw.
|
|
79
|
+
class SpikeDetector
|
|
80
|
+
Tally = Struct.new(:count, :total)
|
|
81
|
+
|
|
82
|
+
def initialize
|
|
83
|
+
@lock = Mutex.new
|
|
84
|
+
reset
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Records one finished request at t_ms (unix milliseconds) and returns
|
|
88
|
+
# the spikes of any window it closed.
|
|
89
|
+
def observe(t_ms, method, route, status)
|
|
90
|
+
w = RequestSignals.window_of(t_ms)
|
|
91
|
+
key = [RequestSignals.normalize_method(method), RequestSignals.normalize_route(route)].freeze
|
|
92
|
+
|
|
93
|
+
@lock.synchronize do
|
|
94
|
+
check_fork
|
|
95
|
+
closed = []
|
|
96
|
+
if @window.nil?
|
|
97
|
+
start(w)
|
|
98
|
+
elsif w > @window
|
|
99
|
+
closed = evaluate
|
|
100
|
+
start(w)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
tally = @routes[key]
|
|
104
|
+
if tally.nil?
|
|
105
|
+
return closed if @routes.size >= MAX_ROUTES_PER_WINDOW
|
|
106
|
+
|
|
107
|
+
tally = @routes[key] = Tally.new(0, 0)
|
|
108
|
+
end
|
|
109
|
+
tally.total += 1
|
|
110
|
+
tally.count += 1 if status == 429
|
|
111
|
+
closed
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Closes the open window if now_ms is past it (at each flush) and
|
|
116
|
+
# returns its spikes.
|
|
117
|
+
def tick(now_ms)
|
|
118
|
+
@lock.synchronize do
|
|
119
|
+
check_fork
|
|
120
|
+
return [] if @window.nil? || RequestSignals.window_of(now_ms) <= @window
|
|
121
|
+
|
|
122
|
+
out = evaluate
|
|
123
|
+
reset
|
|
124
|
+
out
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Evaluates the open window as it stands (process exit, a new
|
|
129
|
+
# configuration) and closes it.
|
|
130
|
+
def finish
|
|
131
|
+
@lock.synchronize do
|
|
132
|
+
check_fork
|
|
133
|
+
return [] if @window.nil?
|
|
134
|
+
|
|
135
|
+
out = evaluate
|
|
136
|
+
reset
|
|
137
|
+
out
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
private
|
|
142
|
+
|
|
143
|
+
def reset
|
|
144
|
+
@pid = Process.pid
|
|
145
|
+
@window = nil
|
|
146
|
+
@routes = {}
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def check_fork
|
|
150
|
+
reset unless @pid == Process.pid
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def start(window)
|
|
154
|
+
@window = window
|
|
155
|
+
@routes = {}
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# The open window's spikes, sorted by method then route (byte order,
|
|
159
|
+
# as Go's string comparison). Caller holds @lock.
|
|
160
|
+
def evaluate
|
|
161
|
+
@routes.filter_map do |(method, route), tally|
|
|
162
|
+
next unless RequestSignals.spike?(tally.count, tally.total)
|
|
163
|
+
|
|
164
|
+
Spike.new(method, route, @window * SPIKE_WINDOW_SECONDS, tally.count, tally.total)
|
|
165
|
+
end.sort_by { |s| [s.method.b, s.route.b] }
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
class << self
|
|
170
|
+
attr_writer :clock
|
|
171
|
+
|
|
172
|
+
def clock
|
|
173
|
+
@clock || SystemClock
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# The process's spike detector, in both transports.
|
|
177
|
+
def detector
|
|
178
|
+
@detector ||= SpikeDetector.new
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# The signal one finished request produces: FAILED, SLOW or nil. At
|
|
182
|
+
# most one per request. 429s are not classified here; they feed the
|
|
183
|
+
# spike detector.
|
|
184
|
+
def classify(status:, duration_ms:, exception_reported:, streaming:)
|
|
185
|
+
return nil if status < 100 || status > 599
|
|
186
|
+
# A 5xx after a reported exception: the exception is the report.
|
|
187
|
+
return (exception_reported ? nil : FAILED) if status >= 500
|
|
188
|
+
return nil if streaming || status == 101
|
|
189
|
+
|
|
190
|
+
duration_ms > SLOW_THRESHOLD_MS ? SLOW : nil
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# Uppercased and trimmed; anything outside the standard set is OTHER.
|
|
194
|
+
def normalize_method(method)
|
|
195
|
+
m = Fingerprint.trim(utf8(method)).upcase
|
|
196
|
+
METHODS.include?(m) ? m : OTHER_METHOD
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Trimmed and bounded to MAX_ROUTE_CHARS code points; empty is
|
|
200
|
+
# UNKNOWN_ROUTE.
|
|
201
|
+
def normalize_route(route)
|
|
202
|
+
r = Fingerprint.trim(utf8(route))
|
|
203
|
+
return UNKNOWN_ROUTE if r.empty?
|
|
204
|
+
|
|
205
|
+
r.length > MAX_ROUTE_CHARS ? r[0, MAX_ROUTE_CHARS] : r
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# Fingerprint inputs, template text and exemplar for a request signal,
|
|
209
|
+
# as the sidecar builds them from the `request` control message. Nil
|
|
210
|
+
# for a signal/status pair a sensor must refuse.
|
|
211
|
+
def build(signal:, method:, route:, status:, duration_ms: 0, count: 0, total: 0)
|
|
212
|
+
status = status.to_i
|
|
213
|
+
message = "#{normalize_method(method)} #{normalize_route(route)}"
|
|
214
|
+
case signal
|
|
215
|
+
when FAILED
|
|
216
|
+
return nil unless status.between?(500, 599)
|
|
217
|
+
|
|
218
|
+
type = "http_#{status}"
|
|
219
|
+
new_built('error', type, message, "#{type}: #{message} in #{non_negative(duration_ms)} ms")
|
|
220
|
+
when SLOW
|
|
221
|
+
return nil unless status.between?(100, 499)
|
|
222
|
+
|
|
223
|
+
type = 'slow_request'
|
|
224
|
+
new_built('degradation', type, message,
|
|
225
|
+
"#{type}: #{message} in #{non_negative(duration_ms)} ms (status #{status})")
|
|
226
|
+
when RATE_LIMIT_SPIKE
|
|
227
|
+
type = 'http_429_spike'
|
|
228
|
+
new_built('error', type, message,
|
|
229
|
+
"#{type}: #{message} #{non_negative(count)} of #{non_negative(total)} " \
|
|
230
|
+
'requests answered 429 in one minute')
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# The rule, in integer arithmetic: at least SPIKE_MIN_COUNT 429s, and
|
|
235
|
+
# at least SPIKE_MIN_PERCENT of the route's requests in the window.
|
|
236
|
+
def spike?(count, total)
|
|
237
|
+
count >= SPIKE_MIN_COUNT && count * 100 >= total * SPIKE_MIN_PERCENT
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Floors toward negative infinity (Ruby's integer division does), so a
|
|
241
|
+
# pre-1970 clock still maps to one window per minute.
|
|
242
|
+
def window_of(t_ms)
|
|
243
|
+
t_ms.to_i / (SPIKE_WINDOW_SECONDS * 1000)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# ---- Reporting (called by Devbench::Middleware and the flushes)
|
|
247
|
+
|
|
248
|
+
# Classifies a finished request, reports its signal, and feeds the spike
|
|
249
|
+
# detector. Called inside the request's scope, so a signal carries its
|
|
250
|
+
# trace and user. Never raises.
|
|
251
|
+
def observe(status:, method:, route:, duration_ms:, streaming:, exception_reported:, finished_ms:)
|
|
252
|
+
return nil unless status.is_a?(Integer)
|
|
253
|
+
|
|
254
|
+
method = normalize_method(method)
|
|
255
|
+
route = normalize_route(route)
|
|
256
|
+
signal = classify(status: status, duration_ms: duration_ms,
|
|
257
|
+
exception_reported: exception_reported, streaming: streaming)
|
|
258
|
+
emit(request_payload(signal, method, route, status, duration_ms)) unless signal.nil?
|
|
259
|
+
return nil unless status.between?(100, 599)
|
|
260
|
+
|
|
261
|
+
install_exit_hook
|
|
262
|
+
report_spikes(detector.observe(finished_ms, method, route, status))
|
|
263
|
+
start_flusher if status == 429
|
|
264
|
+
nil
|
|
265
|
+
rescue StandardError, SystemStackError
|
|
266
|
+
nil
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# Closes the spike window if the clock has left it (at a flush) and
|
|
270
|
+
# reports its spikes to `transport`.
|
|
271
|
+
def tick!(transport = nil)
|
|
272
|
+
report_spikes(detector.tick(clock.wall_ms), transport)
|
|
273
|
+
rescue StandardError, SystemStackError
|
|
274
|
+
nil
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# Evaluates the open window as it stands and reports its spikes (process
|
|
278
|
+
# exit, a replaced configuration). Returns how many were reported.
|
|
279
|
+
def finish!(transport = nil)
|
|
280
|
+
report_spikes(detector.finish, transport)
|
|
281
|
+
rescue StandardError, SystemStackError
|
|
282
|
+
0
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# The `request` control message for a spike: an aggregate of many
|
|
286
|
+
# requests, so no trace and no user.
|
|
287
|
+
def spike_payload(spike)
|
|
288
|
+
{ v: PROTOCOL_VERSION, kind: 'request', signal: RATE_LIMIT_SPIKE, method: spike.method,
|
|
289
|
+
route: spike.route, status: 429, count: spike.count, total: spike.total }
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
private
|
|
293
|
+
|
|
294
|
+
def new_built(kind, type, message, exemplar)
|
|
295
|
+
Built.new(kind, type, message, "#{type} at #{message}", exemplar)
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
def non_negative(n)
|
|
299
|
+
n = n.to_i
|
|
300
|
+
n.negative? ? 0 : n
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
def utf8(text)
|
|
304
|
+
text.to_s.encode('UTF-8', invalid: :replace, undef: :replace).scrub
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
def request_payload(signal, method, route, status, duration_ms)
|
|
308
|
+
{
|
|
309
|
+
v: PROTOCOL_VERSION, kind: 'request', signal: signal, method: method, route: route,
|
|
310
|
+
status: status, duration_ms: non_negative(duration_ms),
|
|
311
|
+
trace: safely { Current.trace&.to_s }, user: safely { Current.user }
|
|
312
|
+
}.compact
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
def report_spikes(spikes, transport = nil)
|
|
316
|
+
spikes.each { |s| emit(spike_payload(s), transport) }
|
|
317
|
+
spikes.length
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
def emit(payload, transport = nil)
|
|
321
|
+
(transport || Devbench.transport).deliver(payload)
|
|
322
|
+
rescue StandardError, SystemStackError
|
|
323
|
+
nil
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
# Direct mode: a process whose only traffic is a burst of 429s has
|
|
327
|
+
# counted nothing, so its flush thread may not be running; without it
|
|
328
|
+
# the spike would wait for the next request or the process's exit.
|
|
329
|
+
def start_flusher
|
|
330
|
+
transport = Devbench.transport
|
|
331
|
+
transport.start if transport.respond_to?(:start)
|
|
332
|
+
rescue StandardError, SystemStackError
|
|
333
|
+
nil
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
# The open window is evaluated at process exit, in either transport:
|
|
337
|
+
# the spikes are delivered, then direct mode flushes them (bounded, as
|
|
338
|
+
# the transport's own exit flush is). Inherited by forked children,
|
|
339
|
+
# which report their own window.
|
|
340
|
+
def install_exit_hook
|
|
341
|
+
return if @exit_hook_installed
|
|
342
|
+
|
|
343
|
+
@exit_hook_installed = true
|
|
344
|
+
at_exit do
|
|
345
|
+
reported = finish!
|
|
346
|
+
Devbench.transport.flush!(timeout: DirectTransport::EXIT_TIMEOUT) if reported.positive?
|
|
347
|
+
rescue StandardError, SystemStackError
|
|
348
|
+
nil
|
|
349
|
+
end
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
def safely
|
|
353
|
+
yield
|
|
354
|
+
rescue StandardError, SystemStackError
|
|
355
|
+
nil
|
|
356
|
+
end
|
|
357
|
+
end
|
|
358
|
+
end
|
|
359
|
+
end
|
data/lib/devbench/scrub.rb
CHANGED
|
@@ -25,6 +25,25 @@ module Devbench
|
|
|
25
25
|
end
|
|
26
26
|
private_class_method :go_re
|
|
27
27
|
|
|
28
|
+
# The card and phone rules (#186; internal/scrub/numbers.go, the
|
|
29
|
+
# `numbers` vectors): a candidate regex plus checks on the whole
|
|
30
|
+
# candidate, run in the gsub block. A rejected candidate is left as
|
|
31
|
+
# written and matching carries on after it; it is never retried shorter.
|
|
32
|
+
# No lookaround in the candidates (the Go SDK is RE2), so every SDK finds
|
|
33
|
+
# the same ones.
|
|
34
|
+
CARD = '<redacted:card>'
|
|
35
|
+
PHONE = '<redacted:phone>'
|
|
36
|
+
CARD_CANDIDATE = go_re('\b(?:\d[ \-]*?){13,19}\b')
|
|
37
|
+
# Group 1 is the character before the number (or the start of the text):
|
|
38
|
+
# a phone number is not glued to a letter, digit, `_` or `-`, which keeps
|
|
39
|
+
# a UUID's tail and `ord_123456789` out. Group 2 is the candidate.
|
|
40
|
+
PHONE_CANDIDATE = /(\A|[^A-Za-z0-9_\-])(\+?\d[\d\t\n\f\r ().\-]{7,}\d)/
|
|
41
|
+
ISO_DATE_PREFIX = /\A\d{4}-\d{2}-\d{2}(?:\z|\D)/
|
|
42
|
+
DECIMAL = /\A\d+\.\d+\z/
|
|
43
|
+
WORD_BYTE = /\A[A-Za-z0-9_]/
|
|
44
|
+
# E.164 caps a phone number at 15 digits; anything longer is an id.
|
|
45
|
+
PHONE_DIGITS = (7..15)
|
|
46
|
+
|
|
28
47
|
# Ordered most-specific first, and the order is load-bearing: see the
|
|
29
48
|
# comments on internal/scrub's rules (a credential containing an email,
|
|
30
49
|
# a phone pattern eating an SSN).
|
|
@@ -41,9 +60,10 @@ module Devbench
|
|
|
41
60
|
"(#{WS}*[:=]#{WS}*)(\"[^\"]*\"|'[^']*'|#{NOT_WS}+)", Regexp::IGNORECASE), '\1\2<redacted:secret>'],
|
|
42
61
|
[go_re('\b[^\t\n\f\r <>@]+@[^\t\n\f\r <>@]+\.[A-Za-z]{2,}\b'), '<redacted:email>'],
|
|
43
62
|
[go_re('\b\d{3}-\d{2}-\d{4}\b'), '<redacted:ssn>'],
|
|
44
|
-
|
|
63
|
+
# The card and phone rules (#186) judge each candidate in the block.
|
|
64
|
+
[CARD_CANDIDATE, ->(m) { card?(m[0]) ? CARD : m[0] }],
|
|
45
65
|
[go_re('\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b'), '<redacted:ip>'],
|
|
46
|
-
[
|
|
66
|
+
[PHONE_CANDIDATE, ->(m) { phone?(m[2], m.string[m.end(0)].to_s) ? "#{m[1]}#{PHONE}" : m[0] }]
|
|
47
67
|
].freeze
|
|
48
68
|
|
|
49
69
|
# internal/sidecar/trace.go's traceRe.
|
|
@@ -75,7 +95,33 @@ module Devbench
|
|
|
75
95
|
class << self
|
|
76
96
|
# internal/scrub.Text: every shape rule, in order.
|
|
77
97
|
def text(line)
|
|
78
|
-
RULES.reduce(utf8(line))
|
|
98
|
+
RULES.reduce(utf8(line)) do |out, (re, with)|
|
|
99
|
+
with.is_a?(Proc) ? out.gsub(re) { with.call(Regexp.last_match) } : out.gsub(re, with)
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# A card: its digits pass the Luhn check, and it is not a bare 13-digit
|
|
104
|
+
# run starting with 1 (a millisecond timestamp).
|
|
105
|
+
def card?(candidate)
|
|
106
|
+
digits = candidate.delete('^0-9')
|
|
107
|
+
return false if digits == candidate && timestamp?(digits)
|
|
108
|
+
|
|
109
|
+
luhn?(digits)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# A phone, judged with the character after it ('' at the end of the
|
|
113
|
+
# text; one character is all the check reads). Not one when a letter, digit
|
|
114
|
+
# or `_` follows; it starts with an ISO date; it is a decimal; it has
|
|
115
|
+
# fewer than 7 or more than 15 digits; or it is a bare 10- or 13-digit
|
|
116
|
+
# run starting with 1 (a timestamp).
|
|
117
|
+
def phone?(candidate, after)
|
|
118
|
+
return false if WORD_BYTE.match?(after)
|
|
119
|
+
return false if ISO_DATE_PREFIX.match?(candidate) || DECIMAL.match?(candidate)
|
|
120
|
+
|
|
121
|
+
digits = candidate.delete('^0-9')
|
|
122
|
+
return false unless PHONE_DIGITS.cover?(digits.length)
|
|
123
|
+
|
|
124
|
+
!(digits == candidate && timestamp?(digits))
|
|
79
125
|
end
|
|
80
126
|
|
|
81
127
|
# internal/scrub.Prose: runs of two or more capitalised words that are
|
|
@@ -153,6 +199,26 @@ module Devbench
|
|
|
153
199
|
|
|
154
200
|
private
|
|
155
201
|
|
|
202
|
+
# A Unix timestamp in seconds or milliseconds (2001 to 2033).
|
|
203
|
+
def timestamp?(digits)
|
|
204
|
+
(digits.length == 10 || digits.length == 13) && digits.start_with?('1')
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def luhn?(digits)
|
|
208
|
+
return false if digits.empty?
|
|
209
|
+
|
|
210
|
+
sum = 0
|
|
211
|
+
digits.reverse.each_char.with_index do |c, i|
|
|
212
|
+
d = c.ord - 48
|
|
213
|
+
if i.odd?
|
|
214
|
+
d *= 2
|
|
215
|
+
d -= 9 if d > 9
|
|
216
|
+
end
|
|
217
|
+
sum += d
|
|
218
|
+
end
|
|
219
|
+
(sum % 10).zero?
|
|
220
|
+
end
|
|
221
|
+
|
|
156
222
|
# Base redaction (template + scrub), the prose heuristic unless the
|
|
157
223
|
# shape is marked 'full', then field and term rules.
|
|
158
224
|
def redact_body(body, rule, policy)
|
data/lib/devbench/session.rb
CHANGED
|
@@ -12,9 +12,11 @@ module Devbench
|
|
|
12
12
|
# trace propagation mints one and injects it into HTML responses.
|
|
13
13
|
#
|
|
14
14
|
# This must agree byte for byte with the Go SDK, because a token minted by a
|
|
15
|
-
# Rails app is verified by ingest
|
|
16
|
-
#
|
|
17
|
-
#
|
|
15
|
+
# Rails app is verified by ingest, which uses the Go SDK's VerifySession.
|
|
16
|
+
# Divergence produces a request that succeeds locally and is rejected in
|
|
17
|
+
# production, with nothing to say why. There are no shared testdata/ vectors
|
|
18
|
+
# for session tokens yet: test/session_test.rb pins known-answer tokens the Go
|
|
19
|
+
# SDK minted, and sdk/server/go/session_kat_test.go pins the same table.
|
|
18
20
|
module Session
|
|
19
21
|
HEADER = 'x-adt-session'
|
|
20
22
|
META_NAME = 'adt-session'
|
|
@@ -25,6 +27,7 @@ module Devbench
|
|
|
25
27
|
# rarely — a short TTL would expire mid-session and force a refresh endpoint,
|
|
26
28
|
# which is more surface in the customer's app for no security gain.
|
|
27
29
|
DEFAULT_TTL = 3600
|
|
30
|
+
# Bytes, not characters, as in the Go SDK.
|
|
28
31
|
MAX_SUBJECT = 128
|
|
29
32
|
|
|
30
33
|
class Expired < StandardError; end
|
|
@@ -38,7 +41,7 @@ module Devbench
|
|
|
38
41
|
raise ArgumentError, 'secret is required' if secret.nil? || secret.strip.empty?
|
|
39
42
|
raise ArgumentError, 'tenant is required' if tenant.nil? || tenant.strip.empty?
|
|
40
43
|
|
|
41
|
-
subject = subject.to_s
|
|
44
|
+
subject = truncate_subject(subject.to_s)
|
|
42
45
|
ttl = DEFAULT_TTL if ttl.nil? || ttl <= 0
|
|
43
46
|
|
|
44
47
|
# The payload is pipe-delimited, so a pipe in a field would let a crafted
|
|
@@ -47,7 +50,9 @@ module Devbench
|
|
|
47
50
|
raise ArgumentError, "tenant and subject must not contain '|'"
|
|
48
51
|
end
|
|
49
52
|
|
|
50
|
-
|
|
53
|
+
# Bytes throughout: the signature covers the base64 of these exact bytes,
|
|
54
|
+
# whatever encoding the caller's strings carry.
|
|
55
|
+
payload = [tenant.b, (Time.now.to_i + ttl).to_s, subject.b].join('|')
|
|
51
56
|
encoded = b64(payload)
|
|
52
57
|
|
|
53
58
|
"#{PREFIX}.#{encoded}.#{sign(secret, encoded)}"
|
|
@@ -81,6 +86,22 @@ module Devbench
|
|
|
81
86
|
Token.new(fields[0], fields[2], Time.at(expires))
|
|
82
87
|
end
|
|
83
88
|
|
|
89
|
+
# At most MAX_SUBJECT bytes without splitting a UTF-8 character: backs off
|
|
90
|
+
# over at most three continuation bytes, exactly as the Go SDK's
|
|
91
|
+
# truncateSubject does, so a long subject mints the same token in both.
|
|
92
|
+
def self.truncate_subject(subject)
|
|
93
|
+
bytes = subject.b
|
|
94
|
+
return subject if bytes.bytesize <= MAX_SUBJECT
|
|
95
|
+
|
|
96
|
+
cut = MAX_SUBJECT
|
|
97
|
+
3.times do
|
|
98
|
+
break if cut.zero? || (bytes.getbyte(cut) & 0xC0) != 0x80
|
|
99
|
+
|
|
100
|
+
cut -= 1
|
|
101
|
+
end
|
|
102
|
+
bytes.byteslice(0, cut).force_encoding(subject.encoding)
|
|
103
|
+
end
|
|
104
|
+
|
|
84
105
|
def self.sign(secret, encoded_payload)
|
|
85
106
|
b64(OpenSSL::HMAC.digest('SHA256', secret, encoded_payload))
|
|
86
107
|
end
|
|
@@ -5,7 +5,7 @@ require 'socket'
|
|
|
5
5
|
|
|
6
6
|
module Devbench
|
|
7
7
|
# Sidecar mode: each report is one line of JSON on the local sidecar's
|
|
8
|
-
# unix socket (docs/
|
|
8
|
+
# unix socket (docs/server-sdk/control-socket.md, "Control socket — v1"). Used when no
|
|
9
9
|
# DSN is set; unchanged from 0.4.
|
|
10
10
|
#
|
|
11
11
|
# Never blocks for long: this runs inside rescue blocks and exception
|
|
@@ -5,7 +5,7 @@ require_relative 'current'
|
|
|
5
5
|
require_relative 'reporter'
|
|
6
6
|
|
|
7
7
|
module Devbench
|
|
8
|
-
# Native Sidekiq jobs (docs/
|
|
8
|
+
# Native Sidekiq jobs (docs/server-sdk/unhandled-failures.md, Capability 2, `job`).
|
|
9
9
|
#
|
|
10
10
|
# A class that includes Sidekiq::Job never touches ActiveJob, so the
|
|
11
11
|
# perform.active_job hook sees none of its failures. This is the
|
data/lib/devbench/trace.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Devbench
|
|
4
4
|
# The correlation id that ties a user action together across every service.
|
|
5
5
|
#
|
|
6
|
-
# Wire form (docs/
|
|
6
|
+
# Wire form (docs/server-sdk/trace-propagation.md): v1/<session>/<intent>/<hop>
|
|
7
7
|
#
|
|
8
8
|
# This must agree exactly with the Go SDK and the sidecar's extraction regex.
|
|
9
9
|
# A trace the browser writes and Rails mangles is worse than no trace: the
|
data/lib/devbench/transport.rb
CHANGED
|
@@ -4,8 +4,9 @@ require_relative 'config'
|
|
|
4
4
|
require_relative 'sidecar_transport'
|
|
5
5
|
require_relative 'direct'
|
|
6
6
|
require_relative 'logs'
|
|
7
|
+
require_relative 'request_signals'
|
|
7
8
|
|
|
8
|
-
# Configuration and transport selection (docs/
|
|
9
|
+
# Configuration and transport selection (docs/server-sdk/transports.md,
|
|
9
10
|
# "Transports: direct (default) or sidecar (optional)").
|
|
10
11
|
#
|
|
11
12
|
# Exactly one transport per process, decided at the first report:
|
|
@@ -124,6 +125,9 @@ module Devbench
|
|
|
124
125
|
# seconds. A no-op in sidecar mode. Returns true when nothing is left
|
|
125
126
|
# unsent. Never raises.
|
|
126
127
|
def flush!(timeout: 5)
|
|
128
|
+
# Closes a 429 spike window the clock has left, in either transport
|
|
129
|
+
# (sidecar mode has no flush thread of its own to do it).
|
|
130
|
+
RequestSignals.tick!
|
|
127
131
|
transport.flush!(timeout: timeout) ? true : false
|
|
128
132
|
rescue StandardError, SystemStackError
|
|
129
133
|
false
|
|
@@ -190,6 +194,10 @@ module Devbench
|
|
|
190
194
|
private
|
|
191
195
|
|
|
192
196
|
def reset_transport!
|
|
197
|
+
# A replaced configuration ends the 429 spike window, as process exit
|
|
198
|
+
# does: evaluated as it stands and reported under the transport that
|
|
199
|
+
# saw its requests, before that transport's last flush.
|
|
200
|
+
RequestSignals.finish!(@transport)
|
|
193
201
|
old = TRANSPORT_LOCK.synchronize do
|
|
194
202
|
previous = @transport
|
|
195
203
|
@transport = nil
|
data/lib/devbench/version.rb
CHANGED
|
@@ -1,10 +1,22 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Devbench
|
|
4
|
-
#
|
|
5
|
-
#
|
|
6
|
-
# The wire format
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
4
|
+
# The gem's release version. Each SDK has its own version and release tag;
|
|
5
|
+
# the Ruby release (tag ruby-vX.Y.Z in the public repo) checks this equals
|
|
6
|
+
# the tag. The wire format is PROTOCOL_VERSION below.
|
|
7
|
+
VERSION = '0.8.0'
|
|
8
|
+
|
|
9
|
+
# This SDK's name in every flush's `sdk` field: the key
|
|
10
|
+
# testdata/protocol_vectors.json `sdk.names` gives sdk/server/ruby
|
|
11
|
+
# (docs/server-sdk/wire-protocol.md, "the SDK's name and version"). The
|
|
12
|
+
# field's version is VERSION, never a second literal.
|
|
13
|
+
SDK_NAME = 'ruby'
|
|
14
|
+
|
|
15
|
+
# The wire format this speaks: the `v` every payload carries (the flush
|
|
16
|
+
# body, the evidence bundle, every control-socket message). Not the gem's
|
|
17
|
+
# release VERSION above: it changes only for a breaking wire change, and
|
|
18
|
+
# ingest tells formats apart by it. It must equal `current` in
|
|
19
|
+
# testdata/protocol_vectors.json (docs/server-sdk/protocol-version.md, "Protocol
|
|
20
|
+
# version"); send it, never a literal.
|
|
21
|
+
PROTOCOL_VERSION = 1
|
|
10
22
|
end
|
data/lib/devbench/view_helper.rb
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
require 'erb'
|
|
4
4
|
require_relative 'version'
|
|
5
|
+
require_relative 'current'
|
|
6
|
+
require_relative 'widget_identity'
|
|
5
7
|
|
|
6
8
|
module Devbench
|
|
7
9
|
# The browser sensor's script tag, for the layout's <head>:
|
|
@@ -10,7 +12,7 @@ module Devbench
|
|
|
10
12
|
#
|
|
11
13
|
# renders
|
|
12
14
|
#
|
|
13
|
-
# <script src="https://unpkg.com/devbench@0.
|
|
15
|
+
# <script src="https://unpkg.com/devbench@0.8.0/dist/devbench.min.js"
|
|
14
16
|
# data-dsn="https://<public>@<host>" data-release="<release>" defer></script>
|
|
15
17
|
#
|
|
16
18
|
# The DSN is the public part of DEVBENCH_DSN only (Devbench.browser_dsn),
|
|
@@ -21,6 +23,14 @@ module Devbench
|
|
|
21
23
|
# CSP nonce (content_security_policy_nonce_generator), the tag carries it,
|
|
22
24
|
# as Rails' own `javascript_include_tag nonce: true` does.
|
|
23
25
|
#
|
|
26
|
+
# Chat widget (docs/server-sdk/chat-widget.md, Capability 6): when
|
|
27
|
+
# DEVBENCH_WIDGET_SECRET is set and Devbench.set_user was called for this
|
|
28
|
+
# request, the tag also carries data-widget-identity="<token>", the user
|
|
29
|
+
# (and their role and chat permission) signed at render time. No secret, no
|
|
30
|
+
# user (a signed-out page) or an identity the signer refuses: no
|
|
31
|
+
# attribute, so no widget. A page carrying one is per-user; don't serve it
|
|
32
|
+
# from a shared cache.
|
|
33
|
+
#
|
|
24
34
|
# Included into ActionView by the Railtie. Never raises.
|
|
25
35
|
module ViewHelper
|
|
26
36
|
CDN = 'https://unpkg.com'
|
|
@@ -34,6 +44,8 @@ module Devbench
|
|
|
34
44
|
return devbench_safe('') if dsn.nil?
|
|
35
45
|
|
|
36
46
|
attrs = [['src', ViewHelper.src], ['data-dsn', dsn], ['data-release', Devbench.config.resolved_release]]
|
|
47
|
+
identity = devbench_widget_identity
|
|
48
|
+
attrs << ['data-widget-identity', identity] if identity
|
|
37
49
|
nonce = devbench_nonce
|
|
38
50
|
attrs << ['nonce', nonce] if nonce
|
|
39
51
|
html = attrs.map { |name, value| %(#{name}="#{ERB::Util.html_escape(value)}") }.join(' ')
|
|
@@ -44,6 +56,17 @@ module Devbench
|
|
|
44
56
|
|
|
45
57
|
private
|
|
46
58
|
|
|
59
|
+
# The signed widget identity for this request's user, or nil.
|
|
60
|
+
def devbench_widget_identity
|
|
61
|
+
secret = Devbench.config.widget_secret
|
|
62
|
+
user = Current.widget_user
|
|
63
|
+
return nil if secret.nil? || secret.to_s.empty? || user.nil?
|
|
64
|
+
|
|
65
|
+
WidgetIdentity.sign(secret, email: user[:email], account: user[:account], role: user[:role], chat: user[:chat])
|
|
66
|
+
rescue StandardError, SystemStackError
|
|
67
|
+
nil
|
|
68
|
+
end
|
|
69
|
+
|
|
47
70
|
# Rails' content_security_policy_nonce: nil unless the app configured a
|
|
48
71
|
# nonce generator.
|
|
49
72
|
def devbench_nonce
|