closeyourit-ruby 0.9.4 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +6 -0
- data/lib/closeyourit/background_worker.rb +28 -9
- data/lib/closeyourit/breadcrumb.rb +2 -2
- data/lib/closeyourit/breadcrumb_buffer.rb +2 -2
- data/lib/closeyourit/client.rb +42 -29
- data/lib/closeyourit/configuration.rb +105 -101
- data/lib/closeyourit/event.rb +6 -6
- data/lib/closeyourit/events/error_event.rb +16 -16
- data/lib/closeyourit/events/job_metric_event.rb +8 -8
- data/lib/closeyourit/events/log_event.rb +17 -17
- data/lib/closeyourit/events/message_event.rb +4 -4
- data/lib/closeyourit/events/performance_issue_event.rb +4 -4
- data/lib/closeyourit/events/slow_method_event.rb +6 -6
- data/lib/closeyourit/events/slow_query_event.rb +4 -4
- data/lib/closeyourit/instrumenter.rb +12 -4
- data/lib/closeyourit/line_cache.rb +4 -4
- data/lib/closeyourit/log_buffer.rb +10 -10
- data/lib/closeyourit/log_device.rb +18 -18
- data/lib/closeyourit/monitor.rb +2 -2
- data/lib/closeyourit/performance/request_profile.rb +5 -5
- data/lib/closeyourit/performance/rollup.rb +3 -3
- data/lib/closeyourit/rails/active_job_extension.rb +33 -31
- data/lib/closeyourit/rails/capture_exceptions.rb +3 -3
- data/lib/closeyourit/rails/error_subscriber.rb +4 -4
- data/lib/closeyourit/rails/log_broadcast.rb +12 -12
- data/lib/closeyourit/rails/net_http_patch.rb +22 -22
- data/lib/closeyourit/rails/query_source.rb +3 -3
- data/lib/closeyourit/rails/railtie.rb +32 -52
- data/lib/closeyourit/rails/request_body.rb +7 -7
- data/lib/closeyourit/rails/request_context.rb +27 -27
- data/lib/closeyourit/scope.rb +29 -29
- data/lib/closeyourit/scrubber.rb +47 -48
- data/lib/closeyourit/sidekiq/error_handler.rb +2 -2
- data/lib/closeyourit/sidekiq/job_metrics_middleware.rb +10 -11
- data/lib/closeyourit/stats.rb +8 -8
- data/lib/closeyourit/subscribers/job_performance.rb +30 -19
- data/lib/closeyourit/subscribers/request_performance.rb +4 -4
- data/lib/closeyourit/subscribers/slow_query.rb +42 -20
- data/lib/closeyourit/trace_context.rb +22 -22
- data/lib/closeyourit/transport.rb +25 -20
- data/lib/closeyourit/usage_registry.rb +17 -16
- data/lib/closeyourit/version.rb +1 -1
- data/lib/closeyourit-ruby.rb +125 -125
- metadata +1 -1
data/lib/closeyourit/scrubber.rb
CHANGED
|
@@ -1,73 +1,72 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module CloseYourIt
|
|
4
|
-
#
|
|
5
|
-
# Privacy-by-default —
|
|
4
|
+
# PII removal from payloads: sensitive key filter, SQL normalization, message scrubbing.
|
|
5
|
+
# Privacy-by-default — see PDR §9.
|
|
6
6
|
class Scrubber
|
|
7
7
|
FILTERED = "[FILTERED]"
|
|
8
8
|
|
|
9
|
-
#
|
|
10
|
-
# SENSITIVE_KEY
|
|
11
|
-
# Errors/Logs::Ingest::Normalize::SENSITIVE_KEY.
|
|
9
|
+
# Key tokens that are always redacted (substring match, normalized). Aligned with the
|
|
10
|
+
# SENSITIVE_KEY regex of the backend and the Dart client (client-side parity) — see
|
|
11
|
+
# Errors/Logs::Ingest::Normalize::SENSITIVE_KEY. Covers credentials/secrets and PII:
|
|
12
12
|
# /pass|secret|token|api[_-]?key|apikey|authorization|cookie|csrf|credit|card|cvv|ssn|iban|
|
|
13
13
|
# email|phone|telephone|mobile|dob|birth|passport|bearer|session|pin|pan/i
|
|
14
|
-
# `pass`
|
|
15
|
-
# `credit`+`card`
|
|
16
|
-
# date_of_birth.
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
# CYRB-3:
|
|
14
|
+
# `pass` covers password/passwd/pass_code/passkey/passphrase; `cookie` covers set-cookie;
|
|
15
|
+
# `credit`+`card` cover credit_card; `phone` covers telephone/smartphone; `birth` covers
|
|
16
|
+
# date_of_birth. Substring match → favors over-redaction (privacy-by-default):
|
|
17
|
+
# e.g. `company_name` (contains `pan`) or `shipping` (contains `pin`) get redacted — that is
|
|
18
|
+
# acceptable, better to redact too much than to leak PII.
|
|
19
|
+
# CYRB-3: the list omitted email/phone/dob → slow query binds leaked them into the panel.
|
|
20
20
|
DENYLIST = %w[
|
|
21
21
|
pass secret token api_key apikey authorization
|
|
22
22
|
cookie csrf credit card cvv ssn iban
|
|
23
23
|
email phone telephone mobile dob birth passport bearer session pin pan
|
|
24
24
|
].freeze
|
|
25
25
|
|
|
26
|
-
# CYRB-23 —
|
|
27
|
-
# `filter_params`/`filter_value`
|
|
28
|
-
# DENYLIST);
|
|
29
|
-
#
|
|
30
|
-
#
|
|
26
|
+
# CYRB-23 — BUILT-IN redaction of authentication credentials in free text.
|
|
27
|
+
# `filter_params`/`filter_value` cover STRUCTURED headers (the `authorization` key is in
|
|
28
|
+
# DENYLIST); an `Authorization: Bearer <token>` pasted into `exception.message` or into a log
|
|
29
|
+
# line is text, not a key, and before this ticket it was sent in clear because `scrub_message`
|
|
30
|
+
# applied only the user-configured patterns (default `[]`).
|
|
31
31
|
#
|
|
32
|
-
#
|
|
33
|
-
# CYPY-15):
|
|
34
|
-
# `Authorization: Bearer [FILTERED]`.
|
|
35
|
-
#
|
|
32
|
+
# Shape of the redacted text (decided here once for all SDKs — twins CYDA-26/CYJS-38/
|
|
33
|
+
# CYPY-15): key and scheme stay readable, ONLY the credential disappears →
|
|
34
|
+
# `Authorization: Bearer [FILTERED]`. The message stays useful for diagnosis (you see THAT
|
|
35
|
+
# there was a header and with which scheme) without carrying the secret.
|
|
36
36
|
AUTH_SCHEME = /(?i:Bearer|Basic|Token)/
|
|
37
37
|
|
|
38
|
-
# `token68` (RFC 7235)
|
|
39
|
-
#
|
|
40
|
-
#
|
|
38
|
+
# `token68` (RFC 7235) with trailing `=` padding: the shape of a real credential. Minimum 4
|
|
39
|
+
# characters — a `Basic` with short credentials is short (`dTpw` = `u:p`); filtering out prose
|
|
40
|
+
# is NOT_PROSE's job, not the length's.
|
|
41
41
|
CREDENTIAL_TOKEN = %r{[A-Za-z0-9\-._~+/]{4,}={0,2}}
|
|
42
42
|
|
|
43
|
-
#
|
|
44
|
-
#
|
|
45
|
-
# `
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
# (`dTpw`, base64)
|
|
49
|
-
#
|
|
50
|
-
# Trade-off accettato: un CamelCase subito dopo lo schema (`Token MyAppName`) viene redatto.
|
|
43
|
+
# False-positive guard for the BARE scheme: what follows is NOT a credential if it has the shape
|
|
44
|
+
# of a human-written word, i.e. letters only in ONE of three forms — `lowercase`, `Capitalized`,
|
|
45
|
+
# `UPPERCASE` — and shorter than 20 characters. It spares "the bearer of bad news",
|
|
46
|
+
# "invalid bearer credentials", "Basic HTTP authentication", "Bearer Token expired".
|
|
47
|
+
# Everything else is a credential: entropy (digits, `_`, `-`, `.`), mixed inner capitals
|
|
48
|
+
# (`dTpw`, base64) or length ≥ 20 (no word is that long, while an opaque lowercase-only token is).
|
|
49
|
+
# Accepted trade-off: a CamelCase word right after the scheme (`Token MyAppName`) gets redacted.
|
|
51
50
|
PROSE_WORD = /(?:[a-z]{1,19}|[A-Z][a-z]{0,18}|[A-Z]{1,19})/
|
|
52
51
|
NOT_PROSE = %r{(?!\[FILTERED\])(?!#{PROSE_WORD}(?![A-Za-z0-9\-._~+/=]))}
|
|
53
52
|
|
|
54
|
-
#
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
#
|
|
53
|
+
# Value of a header pasted into text: a sequence of non-delimiter pieces, where a quoted string
|
|
54
|
+
# counts as ONE piece. NOT `\S+`, which would eat the `)` of "(Authorization: Bearer <token>)";
|
|
55
|
+
# but not a plain class either, which would stop at the first quote leaving in clear the secret
|
|
56
|
+
# of `Token token="<token>"` (RFC 7235 syntax and Rails' `authenticate_with_http_token`) and of a
|
|
57
|
+
# JSON fragment pasted into the message.
|
|
59
58
|
QUOTED = /"[^"\n]*"|'[^'\n]*'/
|
|
60
59
|
HEADER_VALUE = /(?:#{QUOTED}|[^\s,;)\]}"'<>])+/
|
|
61
60
|
|
|
62
|
-
# `Authorization:`/`authorization=` (
|
|
63
|
-
#
|
|
64
|
-
# over-redaction
|
|
65
|
-
#
|
|
66
|
-
#
|
|
61
|
+
# `Authorization:`/`authorization=` (also `Proxy-Authorization`, also `authorization_header`)
|
|
62
|
+
# followed by the value: here the key is explicit, so it is redacted without prose heuristics —
|
|
63
|
+
# intentional over-redaction, consistent with the DENYLIST. `[ \t]` not `\s`: never past newline.
|
|
64
|
+
# The lookahead covers the OPTIONAL scheme: without it, on already redacted text the engine would
|
|
65
|
+
# backtrack onto the "no scheme" branch and redact the word `Bearer` itself (not idempotent).
|
|
67
66
|
AUTH_HEADER = /((?i:(?:proxy[_-]?)?authorization)[\w-]*["']?[ \t]*[:=][ \t]*)(?!(?:#{AUTH_SCHEME}[ \t]+)?\[FILTERED\])(#{AUTH_SCHEME}[ \t]+)?#{HEADER_VALUE}/
|
|
68
67
|
|
|
69
|
-
#
|
|
70
|
-
#
|
|
68
|
+
# Bare scheme (`Bearer <token>` without the header name) followed by something shaped like a
|
|
69
|
+
# credential: here the prose guard is needed, the word "bearer" shows up in messages.
|
|
71
70
|
AUTH_CREDENTIAL = /(\b#{AUTH_SCHEME}[ \t]+)#{NOT_PROSE}#{CREDENTIAL_TOKEN}/
|
|
72
71
|
|
|
73
72
|
STRING_LITERAL = /'(?:[^']|'')*'/
|
|
@@ -77,7 +76,7 @@ module CloseYourIt
|
|
|
77
76
|
@configuration = configuration
|
|
78
77
|
end
|
|
79
78
|
|
|
80
|
-
#
|
|
79
|
+
# Recursively filters Hash/Array replacing the values of sensitive keys.
|
|
81
80
|
def filter_params(value)
|
|
82
81
|
case value
|
|
83
82
|
when Hash
|
|
@@ -91,15 +90,15 @@ module CloseYourIt
|
|
|
91
90
|
end
|
|
92
91
|
end
|
|
93
92
|
|
|
94
|
-
#
|
|
93
|
+
# Masks the (string/numeric) literals in SQL, preserving the structure.
|
|
95
94
|
def obfuscate_sql(sql)
|
|
96
95
|
return sql if sql.nil? || !@configuration.obfuscate_sql
|
|
97
96
|
|
|
98
97
|
sql.to_s.gsub(STRING_LITERAL, "?").gsub(NUMERIC_LITERAL, "?")
|
|
99
98
|
end
|
|
100
99
|
|
|
101
|
-
#
|
|
102
|
-
#
|
|
100
|
+
# Built-in rule (authentication schemes) BEFORE the user patterns: the two protections add up,
|
|
101
|
+
# the first cannot be disabled — privacy-by-default (CYRB-23).
|
|
103
102
|
def scrub_message(message)
|
|
104
103
|
return message if message.nil?
|
|
105
104
|
|
|
@@ -112,7 +111,7 @@ module CloseYourIt
|
|
|
112
111
|
end
|
|
113
112
|
end
|
|
114
113
|
|
|
115
|
-
#
|
|
114
|
+
# Value of a single bind/argument: redacted when the name (column/parameter) is sensitive.
|
|
116
115
|
def filter_value(key, value)
|
|
117
116
|
sensitive_key?(key) ? FILTERED : value
|
|
118
117
|
end
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module CloseYourIt
|
|
4
4
|
module Sidekiq
|
|
5
|
-
#
|
|
6
|
-
# `call(exception, context, config)`
|
|
5
|
+
# Sidekiq error handler (registered by the railtie only when Sidekiq is present). Sidekiq calls
|
|
6
|
+
# `call(exception, context, config)` and does NOT re-raise → here we just capture.
|
|
7
7
|
class ErrorHandler
|
|
8
8
|
def call(exception, context, _config = nil)
|
|
9
9
|
apply_job_scope(context)
|
|
@@ -4,13 +4,12 @@ require_relative "../subscribers/job_performance"
|
|
|
4
4
|
|
|
5
5
|
module CloseYourIt
|
|
6
6
|
module Sidekiq
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
# decisione vive in #record).
|
|
7
|
+
# Sidekiq server middleware (registered by the railtie only when Sidekiq is present) that measures
|
|
8
|
+
# the job's execution duration and queue wait, then delegates emitting over-threshold metrics to
|
|
9
|
+
# Subscribers::JobPerformance. It does not alter the job: it times around `yield` and re-raises
|
|
10
|
+
# any error unchanged (error capture belongs to the ErrorHandler). Measurement happens in `ensure`,
|
|
11
|
+
# so it is taken for jobs that raise too; telemetry is isolated (an error in our code never
|
|
12
|
+
# disturbs the host job). Effectively a no-op when `monitor_jobs` is OFF (the decision lives in #record).
|
|
14
13
|
class JobMetricsMiddleware
|
|
15
14
|
def initialize(subscriber = nil)
|
|
16
15
|
@subscriber = subscriber
|
|
@@ -18,8 +17,8 @@ module CloseYourIt
|
|
|
18
17
|
|
|
19
18
|
def call(_worker, job, queue)
|
|
20
19
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
21
|
-
#
|
|
22
|
-
#
|
|
20
|
+
# The queue wait is known at the START (now - enqueued_at); Sidekiq stores enqueued_at as epoch
|
|
21
|
+
# seconds. Computed before the yield so it does not include the job duration.
|
|
23
22
|
latency = Subscribers::JobPerformance.latency_ms(job["enqueued_at"], now: Time.now.utc)
|
|
24
23
|
yield
|
|
25
24
|
ensure
|
|
@@ -47,8 +46,8 @@ module CloseYourIt
|
|
|
47
46
|
@subscriber ||= Subscribers::JobPerformance.new
|
|
48
47
|
end
|
|
49
48
|
|
|
50
|
-
#
|
|
51
|
-
#
|
|
49
|
+
# 1-based execution number. Sidekiq does NOT set `retry_count` on the first run (nil), sets it to 0
|
|
50
|
+
# on the first retry, 1 on the second, ... → attempt = retry_count + 2 when present, 1 on the first run.
|
|
52
51
|
def attempt(job)
|
|
53
52
|
count = job["retry_count"]
|
|
54
53
|
count.nil? ? 1 : count + 2
|
data/lib/closeyourit/stats.rb
CHANGED
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
require "concurrent"
|
|
4
4
|
|
|
5
5
|
module CloseYourIt
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
6
|
+
# Thread-safe client diagnostic counters: how many events were enqueued, dropped (full queue /
|
|
7
|
+
# before_send / sampling), sent successfully, failed (network or non-2xx status) and, among the
|
|
8
|
+
# failed, how many by network timeout. They make the silent failures of the fire-and-forget
|
|
9
|
+
# transport visible. `timeout` is a sub-count of `failed` (a timeout is still a send failure):
|
|
10
|
+
# we keep them apart to isolate connectivity problems from non-2xx.
|
|
11
11
|
#
|
|
12
12
|
# CloseYourIt.stats.to_h # => { enqueued: 12, dropped: 0, sent: 11, failed: 1, timeout: 1 }
|
|
13
13
|
class Stats
|
|
@@ -31,9 +31,9 @@ module CloseYourIt
|
|
|
31
31
|
@counters.transform_values(&:value)
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
-
#
|
|
35
|
-
# `to_h`,
|
|
36
|
-
#
|
|
34
|
+
# Thread-safe snapshot of the counters (each value read atomically). It is the same Hash as
|
|
35
|
+
# `to_h`, with an explicit name for the "read the local diagnostics" use case (CYRB-12): a pure
|
|
36
|
+
# in-memory read, it never sends telemetry.
|
|
37
37
|
alias_method :snapshot, :to_h
|
|
38
38
|
|
|
39
39
|
def reset!
|
|
@@ -5,23 +5,23 @@ require_relative "../events/job_metric_event"
|
|
|
5
5
|
|
|
6
6
|
module CloseYourIt
|
|
7
7
|
module Subscribers
|
|
8
|
-
#
|
|
9
|
-
# Sidekiq —
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
# ActiveSupport::Notifications
|
|
13
|
-
#
|
|
8
|
+
# Measures execution duration and queue wait (queue latency) of background jobs — ActiveJob and
|
|
9
|
+
# Sidekiq — and, beyond the configured thresholds, emits performance_issue metrics (subtypes
|
|
10
|
+
# `slow_job` and `job_queue_latency`). PURE logic WITHOUT shared state: everything comes in as
|
|
11
|
+
# parameters, so concurrent jobs on the same thread/process do not contaminate each other. The
|
|
12
|
+
# wiring to ActiveSupport::Notifications and the Sidekiq middleware lives elsewhere (Railtie /
|
|
13
|
+
# JobMetricsMiddleware). Honors the `monitor_jobs` master switch, the thresholds and `jobs_sample_rate`.
|
|
14
14
|
class JobPerformance
|
|
15
15
|
def initialize(configuration = nil)
|
|
16
16
|
@configuration = configuration
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
# fire-and-forget. `duration_ms`
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
19
|
+
# Single decision point: from the measured values (duration and/or wait) it builds 0..2 metrics,
|
|
20
|
+
# applies the thresholds (strict `>`, so X makes no noise and X+1 does) and sampling, then sends
|
|
21
|
+
# them fire-and-forget. `duration_ms` and `queue_latency_ms` are optional: ActiveJob provides them
|
|
22
|
+
# from two separate hooks (perform_start → wait, perform → duration), Sidekiq both in one call.
|
|
23
|
+
# Sampling is applied ONLY to candidates already beyond the threshold (normal jobs neither
|
|
24
|
+
# consume nor generate anything).
|
|
25
25
|
def record(job_class:, queue: nil, adapter: nil, duration_ms: nil, queue_latency_ms: nil,
|
|
26
26
|
attempt: nil, trace_id: nil)
|
|
27
27
|
config = configuration
|
|
@@ -38,19 +38,20 @@ module CloseYourIt
|
|
|
38
38
|
nil
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
#
|
|
41
|
+
# `perform_start.active_job` hook: the queue wait is known as soon as the job starts (now - the
|
|
42
|
+
# moment it was due to run).
|
|
42
43
|
def active_job_started(job, now: Time.now.utc)
|
|
43
44
|
record(
|
|
44
45
|
job_class: job.class.name,
|
|
45
46
|
queue: (job.queue_name if job.respond_to?(:queue_name)),
|
|
46
47
|
adapter: "active_job",
|
|
47
|
-
queue_latency_ms: self.class.latency_ms(
|
|
48
|
+
queue_latency_ms: self.class.latency_ms(due_at(job), now: now),
|
|
48
49
|
attempt: (job.executions if job.respond_to?(:executions)),
|
|
49
50
|
trace_id: (job.job_id if job.respond_to?(:job_id))
|
|
50
51
|
)
|
|
51
52
|
end
|
|
52
53
|
|
|
53
|
-
#
|
|
54
|
+
# `perform.active_job` hook: at the end of execution the duration is `event.duration` (ms).
|
|
54
55
|
def active_job_performed(job, duration_ms)
|
|
55
56
|
record(
|
|
56
57
|
job_class: job.class.name,
|
|
@@ -62,10 +63,10 @@ module CloseYourIt
|
|
|
62
63
|
)
|
|
63
64
|
end
|
|
64
65
|
|
|
65
|
-
#
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
#
|
|
66
|
+
# Normalizes `enqueued_at` (Time, Numeric epoch in seconds like Sidekiq, or ISO8601 String) into
|
|
67
|
+
# a wait (ms) relative to `now`. nil or unparsable → nil: no wait metric (adapter that does not
|
|
68
|
+
# populate the enqueue instant). Clamped to 0 when negative (clock skew, enqueue "in the future"):
|
|
69
|
+
# the ingest schema requires `duration_ms >= 0`.
|
|
69
70
|
def self.latency_ms(enqueued_at, now:)
|
|
70
71
|
started = to_time(enqueued_at)
|
|
71
72
|
return nil if started.nil?
|
|
@@ -94,6 +95,16 @@ module CloseYourIt
|
|
|
94
95
|
job.enqueued_at if job.respond_to?(:enqueued_at)
|
|
95
96
|
end
|
|
96
97
|
|
|
98
|
+
# A job delayed on purpose (set(wait:), retry_on backoff) is due at scheduled_at, not when it was
|
|
99
|
+
# enqueued: the planned delay is not queue wait (CYRB-27). No enqueued_at still means no metric.
|
|
100
|
+
def due_at(job)
|
|
101
|
+
enqueued = self.class.to_time(enqueued_at(job))
|
|
102
|
+
scheduled = self.class.to_time(job.scheduled_at) if job.respond_to?(:scheduled_at)
|
|
103
|
+
return enqueued if enqueued.nil? || scheduled.nil?
|
|
104
|
+
|
|
105
|
+
[ enqueued, scheduled ].max
|
|
106
|
+
end
|
|
107
|
+
|
|
97
108
|
def configuration
|
|
98
109
|
@configuration || CloseYourIt.configuration
|
|
99
110
|
end
|
|
@@ -5,10 +5,10 @@ require_relative "../performance/rollup"
|
|
|
5
5
|
|
|
6
6
|
module CloseYourIt
|
|
7
7
|
module Subscribers
|
|
8
|
-
#
|
|
9
|
-
# Scope
|
|
10
|
-
#
|
|
11
|
-
# ActiveSupport::Notifications
|
|
8
|
+
# At the end of the request (process_action.action_controller) turns the RequestProfile collected
|
|
9
|
+
# in the Scope into performance_issue verdicts and sends them (fire-and-forget). The Scope — and so
|
|
10
|
+
# the profile — is reset right after by RequestContext#call (ensure). Pure logic: the wiring to
|
|
11
|
+
# ActiveSupport::Notifications lives in the Railtie.
|
|
12
12
|
class RequestPerformance
|
|
13
13
|
def initialize(configuration = nil)
|
|
14
14
|
@configuration = configuration
|
|
@@ -6,9 +6,9 @@ require_relative "../scope"
|
|
|
6
6
|
|
|
7
7
|
module CloseYourIt
|
|
8
8
|
module Subscribers
|
|
9
|
-
#
|
|
10
|
-
# (
|
|
11
|
-
#
|
|
9
|
+
# Receives the data of a `sql.active_record` event and, if the query exceeds the threshold
|
|
10
|
+
# (excluding SCHEMA/CACHE/TRANSACTION), sends a `slow_query` event.
|
|
11
|
+
# Pure logic: the wiring to ActiveSupport::Notifications lives in the Railtie.
|
|
12
12
|
class SlowQuery
|
|
13
13
|
IGNORED_NAMES = %w[SCHEMA CACHE TRANSACTION].freeze
|
|
14
14
|
|
|
@@ -16,6 +16,26 @@ module CloseYourIt
|
|
|
16
16
|
@configuration = configuration
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
+
# Entry point of the Railtie's sql.active_record block. `source` is a callable: the call site costs
|
|
20
|
+
# a full backtrace plus the Rails cleaner, so it is resolved at most once and only when needed.
|
|
21
|
+
def notify(payload, duration_ms, source)
|
|
22
|
+
resolved = false
|
|
23
|
+
site = nil
|
|
24
|
+
lazy_source = lambda do
|
|
25
|
+
site = source.call unless resolved
|
|
26
|
+
resolved = true
|
|
27
|
+
site
|
|
28
|
+
end
|
|
29
|
+
name = payload[:name]
|
|
30
|
+
sql = payload[:sql]
|
|
31
|
+
cached = payload.fetch(:cached, false)
|
|
32
|
+
record(name: name, duration_ms: duration_ms, sql: sql, cached: cached,
|
|
33
|
+
connection: payload[:connection], binds: payload[:binds],
|
|
34
|
+
type_casted_binds: payload[:type_casted_binds], source: lazy_source)
|
|
35
|
+
breadcrumb(name: name, sql: sql, duration_ms: duration_ms, cached: cached)
|
|
36
|
+
profile(name: name, sql: sql, duration_ms: duration_ms, cached: cached, source: lazy_source)
|
|
37
|
+
end
|
|
38
|
+
|
|
19
39
|
def record(name:, duration_ms:, sql:, cached: false, connection: nil,
|
|
20
40
|
binds: nil, type_casted_binds: nil, source: nil)
|
|
21
41
|
config = @configuration || CloseYourIt.configuration
|
|
@@ -25,18 +45,18 @@ module CloseYourIt
|
|
|
25
45
|
|
|
26
46
|
event = SlowQueryEvent.new(
|
|
27
47
|
{ name: name, sql: sql, cached: cached, connection: connection,
|
|
28
|
-
binds: binds, type_casted_binds: type_casted_binds, source: source },
|
|
48
|
+
binds: binds, type_casted_binds: type_casted_binds, source: resolve(source) },
|
|
29
49
|
duration_ms,
|
|
30
50
|
config
|
|
31
51
|
)
|
|
32
52
|
CloseYourIt.capture_event(event)
|
|
33
53
|
rescue StandardError
|
|
34
|
-
#
|
|
54
|
+
# Telemetry must never disturb the host query.
|
|
35
55
|
nil
|
|
36
56
|
end
|
|
37
57
|
|
|
38
|
-
# Breadcrumb
|
|
39
|
-
#
|
|
58
|
+
# Breadcrumb for EVERY non-system query (not only slow ones): obfuscated SQL, no binds.
|
|
59
|
+
# Gives the "which queries before the crash" history attached to the error event.
|
|
40
60
|
def breadcrumb(name:, sql:, duration_ms:, cached: false)
|
|
41
61
|
config = @configuration || CloseYourIt.configuration
|
|
42
62
|
return if ignored_name?(name)
|
|
@@ -49,12 +69,12 @@ module CloseYourIt
|
|
|
49
69
|
data: { "name" => name, "duration_ms" => duration_ms.to_f.round(2), "cached" => cached }
|
|
50
70
|
)
|
|
51
71
|
rescue StandardError
|
|
52
|
-
#
|
|
72
|
+
# Telemetry must never disturb the host query.
|
|
53
73
|
nil
|
|
54
74
|
end
|
|
55
75
|
|
|
56
|
-
#
|
|
57
|
-
#
|
|
76
|
+
# Pushes EVERY non-system query into the Scope's RequestProfile (for N+1 detection at the end of
|
|
77
|
+
# the request). Only with detect_performance_issues: fingerprint = obfuscated SQL + call site.
|
|
58
78
|
def profile(name:, sql:, duration_ms:, cached: false, source: nil)
|
|
59
79
|
config = @configuration || CloseYourIt.configuration
|
|
60
80
|
return unless config.detect_performance_issues
|
|
@@ -62,15 +82,17 @@ module CloseYourIt
|
|
|
62
82
|
|
|
63
83
|
CloseYourIt::Scope.current.performance_profile.add_query(
|
|
64
84
|
fingerprint: scrubber(config).obfuscate_sql(sql),
|
|
65
|
-
source: source, duration_ms: duration_ms, cached: cached
|
|
85
|
+
source: resolve(source), duration_ms: duration_ms, cached: cached
|
|
66
86
|
)
|
|
67
87
|
rescue StandardError
|
|
68
|
-
#
|
|
88
|
+
# Telemetry must never disturb the host query.
|
|
69
89
|
nil
|
|
70
90
|
end
|
|
71
91
|
|
|
72
92
|
private
|
|
73
93
|
|
|
94
|
+
def resolve(source) = source.respond_to?(:call) ? source.call : source
|
|
95
|
+
|
|
74
96
|
def scrubber(config)
|
|
75
97
|
@scrubber ||= Scrubber.new(config)
|
|
76
98
|
end
|
|
@@ -79,16 +101,16 @@ module CloseYourIt
|
|
|
79
101
|
name.nil? || IGNORED_NAMES.include?(name)
|
|
80
102
|
end
|
|
81
103
|
|
|
82
|
-
# Query
|
|
83
|
-
#
|
|
104
|
+
# Query excluded from slowdown MEASUREMENT (config.excluded_query_patterns). The filter lives
|
|
105
|
+
# only here, not in #breadcrumb nor in #profile:
|
|
84
106
|
#
|
|
85
|
-
# - breadcrumb:
|
|
86
|
-
#
|
|
87
|
-
# - profile:
|
|
88
|
-
#
|
|
107
|
+
# - breadcrumb: the "which queries before the crash" history has diagnostic value even when the
|
|
108
|
+
# query belongs to the framework — hiding it would make the sequence incomplete and misleading;
|
|
109
|
+
# - profile: it is per-request and serves N+1 detection, where a service table read many times
|
|
110
|
+
# is itself a symptom worth seeing.
|
|
89
111
|
#
|
|
90
|
-
#
|
|
91
|
-
#
|
|
112
|
+
# A slowdown is worth measuring if someone can act on it; a breadcrumb is worth keeping if it
|
|
113
|
+
# helps understanding. They are two different questions, and this list answers only the first.
|
|
92
114
|
def excluded_sql?(config, sql)
|
|
93
115
|
return false if sql.nil?
|
|
94
116
|
|
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
require "securerandom"
|
|
4
4
|
|
|
5
5
|
module CloseYourIt
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
# "
|
|
9
|
-
#
|
|
10
|
-
#
|
|
6
|
+
# W3C trace context (`traceparent`/`tracestate`, https://www.w3.org/TR/trace-context/).
|
|
7
|
+
# It is NOT a tracer: it opens no spans and measures no times. It is a minimal "propagation bridge"
|
|
8
|
+
# (the ticket: "do not build full custom tracing") — it adopts a valid incoming context passing
|
|
9
|
+
# through trace-id/parent-id/flags, or generates a root one, and serializes into outgoing headers.
|
|
10
|
+
# Deliberately separate from proprietary formats: standard-based, with no vendor dependency.
|
|
11
11
|
class TraceContext
|
|
12
|
-
# traceparent = version "-" trace-id "-" parent-id "-" trace-flags (55
|
|
13
|
-
# `rest`
|
|
12
|
+
# traceparent = version "-" trace-id "-" parent-id "-" trace-flags (55 chars for version 00).
|
|
13
|
+
# `rest` captures any future fields: allowed only for versions > 00 (forward-compat), forbidden on 00.
|
|
14
14
|
TRACEPARENT = /
|
|
15
15
|
\A
|
|
16
16
|
(?<version>[0-9a-f]{2})-
|
|
@@ -27,10 +27,10 @@ module CloseYourIt
|
|
|
27
27
|
ZERO_PARENT_ID = ("0" * 16).freeze
|
|
28
28
|
FLAG_SAMPLED = 0x01
|
|
29
29
|
|
|
30
|
-
# tracestate:
|
|
30
|
+
# tracestate: comma-separated list of `key=value` members, max 32 (W3C §3.3.1).
|
|
31
31
|
TRACESTATE_MAX_MEMBERS = 32
|
|
32
|
-
# key: lowercase alnum
|
|
33
|
-
# value:
|
|
32
|
+
# key: leading lowercase alnum + restricted set (including `@`/`/` for tenant@vendor keys).
|
|
33
|
+
# value: printable characters 0x20–0x7E except `,` (0x2C) and `=` (0x3D).
|
|
34
34
|
TRACESTATE_MEMBER = %r{\A[a-z0-9][a-z0-9_\-*/@]*=[\x20-\x2b\x2d-\x3c\x3e-\x7e]+\z}
|
|
35
35
|
|
|
36
36
|
attr_reader :trace_id, :parent_id, :flags, :tracestate
|
|
@@ -43,9 +43,9 @@ module CloseYourIt
|
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
class << self
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
# bridge
|
|
46
|
+
# Adopts an incoming traceparent. Returns nil when malformed (→ the caller generates a root or
|
|
47
|
+
# leaves the context absent). Pass-through: keeps trace-id/parent-id/flags verbatim so the
|
|
48
|
+
# bridge invents no spans. The tracestate is sanitized (invalid members dropped, capped at 32).
|
|
49
49
|
def parse(traceparent, tracestate = nil)
|
|
50
50
|
match = TRACEPARENT.match(traceparent.to_s.strip)
|
|
51
51
|
return nil unless match
|
|
@@ -62,9 +62,9 @@ module CloseYourIt
|
|
|
62
62
|
)
|
|
63
63
|
end
|
|
64
64
|
|
|
65
|
-
#
|
|
66
|
-
# (8
|
|
67
|
-
# `sampled`
|
|
65
|
+
# New root context (no valid incoming traceparent). Generates random trace-id (16 bytes) and
|
|
66
|
+
# parent-id (8 bytes) — SecureRandom is fork-safe, so a forked worker does not reuse the parent's
|
|
67
|
+
# ids. `sampled` sets the flag: a root we open traces by default.
|
|
68
68
|
def generate(sampled: true)
|
|
69
69
|
new(
|
|
70
70
|
trace_id: SecureRandom.hex(16),
|
|
@@ -76,8 +76,8 @@ module CloseYourIt
|
|
|
76
76
|
|
|
77
77
|
private
|
|
78
78
|
|
|
79
|
-
#
|
|
80
|
-
#
|
|
79
|
+
# Keeps only well-formed members, in the original order, up to 32. Returns nil if none are left
|
|
80
|
+
# → so we never propagate a garbage or oversized tracestate.
|
|
81
81
|
def sanitize_tracestate(tracestate)
|
|
82
82
|
return nil if tracestate.nil?
|
|
83
83
|
|
|
@@ -92,14 +92,14 @@ module CloseYourIt
|
|
|
92
92
|
(flags & FLAG_SAMPLED) != 0
|
|
93
93
|
end
|
|
94
94
|
|
|
95
|
-
# traceparent
|
|
96
|
-
#
|
|
95
|
+
# Outgoing traceparent, always version 00 (the only one we can emit). Flags are re-emitted in full
|
|
96
|
+
# (reserved bits must be propagated as-is), formatted as two hex digits.
|
|
97
97
|
def traceparent
|
|
98
98
|
format("%s-%s-%s-%02x", CURRENT_VERSION, trace_id, parent_id, flags & 0xff)
|
|
99
99
|
end
|
|
100
100
|
|
|
101
|
-
#
|
|
102
|
-
#
|
|
101
|
+
# W3C propagation headers: traceparent (+ tracestate if present). NEVER `baggage`: it may carry
|
|
102
|
+
# internal context/PII and must not cross the boundary (ticket: "sensitive baggage gets no header").
|
|
103
103
|
def headers
|
|
104
104
|
result = { "traceparent" => traceparent }
|
|
105
105
|
result["tracestate"] = tracestate if tracestate && !tracestate.empty?
|