rails_error_dashboard 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +54 -1
- data/app/controllers/rails_error_dashboard/application_controller.rb +5 -0
- data/app/controllers/rails_error_dashboard/errors_controller.rb +14 -3
- data/app/jobs/rails_error_dashboard/rack_attack_flush_job.rb +31 -0
- data/app/jobs/rails_error_dashboard/retention_cleanup_job.rb +30 -0
- data/app/jobs/rails_error_dashboard/storm_flush_job.rb +19 -0
- data/app/jobs/rails_error_dashboard/storm_notification_job.rb +74 -0
- data/app/models/rails_error_dashboard/rack_attack_event.rb +36 -0
- data/app/models/rails_error_dashboard/storm_event.rb +34 -0
- data/app/views/layouts/rails_error_dashboard.html.erb +22 -1
- data/app/views/rails_error_dashboard/errors/rack_attack_summary.html.erb +4 -6
- data/app/views/rails_error_dashboard/errors/storms.html.erb +91 -0
- data/config/routes.rb +1 -0
- data/db/migrate/20260306000002_add_instance_variables_to_error_logs.rb +7 -1
- data/db/migrate/20260306000003_create_rails_error_dashboard_swallowed_exceptions.rb +4 -0
- data/db/migrate/20260307000001_create_rails_error_dashboard_diagnostic_dumps.rb +4 -0
- data/db/migrate/20260613000001_create_storm_events.rb +28 -0
- data/db/migrate/20260730000001_create_rails_error_dashboard_rack_attack_events.rb +45 -0
- data/lib/generators/rails_error_dashboard/install/templates/initializer.rb +36 -0
- data/lib/rails_error_dashboard/commands/flush_rack_attack_events.rb +84 -0
- data/lib/rails_error_dashboard/commands/flush_storm_counts.rb +188 -0
- data/lib/rails_error_dashboard/commands/log_error.rb +70 -12
- data/lib/rails_error_dashboard/configuration.rb +78 -7
- data/lib/rails_error_dashboard/engine.rb +2 -2
- data/lib/rails_error_dashboard/queries/rack_attack_summary.rb +55 -44
- data/lib/rails_error_dashboard/queries/storm_history.rb +39 -0
- data/lib/rails_error_dashboard/services/rack_attack_tracker.rb +194 -0
- data/lib/rails_error_dashboard/services/storm_protection/circuit_breaker.rb +195 -0
- data/lib/rails_error_dashboard/services/storm_protection/count_buffer.rb +100 -0
- data/lib/rails_error_dashboard/services/storm_protection/fingerprint_buckets.rb +123 -0
- data/lib/rails_error_dashboard/services/storm_protection/gate.rb +258 -0
- data/lib/rails_error_dashboard/subscribers/issue_tracker_subscriber.rb +12 -0
- data/lib/rails_error_dashboard/subscribers/rack_attack_subscriber.rb +16 -2
- data/lib/rails_error_dashboard/version.rb +1 -1
- data/lib/rails_error_dashboard.rb +8 -0
- metadata +37 -18
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreateRailsErrorDashboardRackAttackEvents < ActiveRecord::Migration[7.0]
|
|
4
|
+
def change
|
|
5
|
+
# Guard against the squashed schema migration having already created this
|
|
6
|
+
# table — without it, every later migration is silently cancelled.
|
|
7
|
+
return if table_exists?(:rails_error_dashboard_rack_attack_events)
|
|
8
|
+
|
|
9
|
+
create_table :rails_error_dashboard_rack_attack_events do |t|
|
|
10
|
+
t.string :rule, null: false, limit: 250
|
|
11
|
+
t.string :match_type, null: false, limit: 50
|
|
12
|
+
# discriminator and path are capped at 191 (not 250) to keep the unique
|
|
13
|
+
# upsert index within MySQL's 3072-byte utf8mb4 limit. Budget:
|
|
14
|
+
# 250+50+191+191 chars * 4 bytes + 2 length-prefix each = 2736 bytes.
|
|
15
|
+
# See issue #96 — the swallowed_exceptions index blew this limit at 5042.
|
|
16
|
+
t.string :discriminator, limit: 191
|
|
17
|
+
t.string :path, limit: 191
|
|
18
|
+
t.string :http_method, limit: 10
|
|
19
|
+
t.datetime :period_hour, null: false
|
|
20
|
+
t.integer :event_count, null: false, default: 0
|
|
21
|
+
t.datetime :last_seen_at
|
|
22
|
+
t.bigint :application_id
|
|
23
|
+
t.timestamps
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
add_index :rails_error_dashboard_rack_attack_events,
|
|
27
|
+
:period_hour,
|
|
28
|
+
name: "index_rack_attack_events_on_period_hour"
|
|
29
|
+
|
|
30
|
+
add_index :rails_error_dashboard_rack_attack_events,
|
|
31
|
+
[ :application_id, :period_hour ],
|
|
32
|
+
name: "index_rack_attack_events_on_app_and_hour"
|
|
33
|
+
|
|
34
|
+
add_index :rails_error_dashboard_rack_attack_events,
|
|
35
|
+
[ :rule, :period_hour ],
|
|
36
|
+
name: "index_rack_attack_events_on_rule_and_hour"
|
|
37
|
+
|
|
38
|
+
# http_method is intentionally NOT part of the upsert key — it would push
|
|
39
|
+
# the index over the MySQL byte limit and adds no aggregation value.
|
|
40
|
+
add_index :rails_error_dashboard_rack_attack_events,
|
|
41
|
+
[ :rule, :match_type, :discriminator, :path, :period_hour, :application_id ],
|
|
42
|
+
unique: true,
|
|
43
|
+
name: "index_rack_attack_events_upsert_key"
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -518,6 +518,42 @@ RailsErrorDashboard.configure do |config|
|
|
|
518
518
|
# No PII or request bodies in span attributes — just metadata + timing.
|
|
519
519
|
# Safe to enable on production OTel pipelines.
|
|
520
520
|
|
|
521
|
+
# ============================================================================
|
|
522
|
+
# STORM PROTECTION (circuit breaker + adaptive sampling) — ON by default
|
|
523
|
+
# ============================================================================
|
|
524
|
+
#
|
|
525
|
+
# When the error rate spikes (bad deploy, dependency outage), storm
|
|
526
|
+
# protection limits the gem's own database writes so it never amplifies
|
|
527
|
+
# the incident. Occurrences are ALWAYS counted exactly — only per-event
|
|
528
|
+
# detail (context payloads, occurrence rows) is sampled under load.
|
|
529
|
+
#
|
|
530
|
+
# How it degrades, in order:
|
|
531
|
+
# 1. Per-fingerprint cap: past N/min, context is shed, then rows sampled
|
|
532
|
+
# 2. Global breaker: shedding (context off) → open (count-only mode)
|
|
533
|
+
# 3. Per-error notifications replaced by ONE "storm in progress" message
|
|
534
|
+
# 4. Counts reconciled onto error records every flush interval
|
|
535
|
+
#
|
|
536
|
+
# All thresholds are PER PROCESS (each Puma worker runs its own breaker).
|
|
537
|
+
#
|
|
538
|
+
# config.enable_storm_protection = true
|
|
539
|
+
# config.storm_fingerprint_full_per_minute = 30 # full-fidelity captures per fingerprint/min
|
|
540
|
+
# config.storm_occurrence_sample_keep_every = 10 # past the cap, keep every Nth occurrence
|
|
541
|
+
# config.storm_shedding_threshold_per_second = 10 # global rate entering shedding state
|
|
542
|
+
# config.storm_open_threshold_per_second = 50 # global rate opening the breaker (count-only)
|
|
543
|
+
# config.storm_cooldown_seconds = 60 # open → half-open probe delay
|
|
544
|
+
# config.storm_notification = true # one notification per storm episode
|
|
545
|
+
#
|
|
546
|
+
# Always-on issue cap (a storm of NEW critical errors must not open
|
|
547
|
+
# hundreds of GitHub/Linear issues):
|
|
548
|
+
# config.auto_issue_rate_limit_count = 5
|
|
549
|
+
# config.auto_issue_rate_limit_window_minutes = 10
|
|
550
|
+
#
|
|
551
|
+
# Calm-weather context economy: an error seen 1000x/day doesn't need 1000
|
|
552
|
+
# breadcrumb trails. After N full-context captures per fingerprint per day,
|
|
553
|
+
# context is kept every Mth time (occurrence rows are unaffected):
|
|
554
|
+
# config.context_sampling_threshold_per_day = 25
|
|
555
|
+
# config.context_sampling_keep_every = 10
|
|
556
|
+
|
|
521
557
|
# ============================================================================
|
|
522
558
|
# ISSUE TRACKING (GitHub / GitLab / Codeberg / Linear)
|
|
523
559
|
# ============================================================================
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
module Commands
|
|
5
|
+
# Command: Upsert buffered Rack::Attack event counts into the database.
|
|
6
|
+
#
|
|
7
|
+
# Receives a snapshot hash from RackAttackTracker and merges it into
|
|
8
|
+
# hourly-bucketed rows. Uses find_or_initialize_by + increment for
|
|
9
|
+
# cross-database compatibility (no raw SQL upsert).
|
|
10
|
+
#
|
|
11
|
+
# counts keys: "rule\x1Fmatch_type\x1Fdiscriminator\x1Fpath\x1Fhttp_method"
|
|
12
|
+
class FlushRackAttackEvents
|
|
13
|
+
def self.call(counts:)
|
|
14
|
+
new(counts: counts).call
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def initialize(counts:)
|
|
18
|
+
@counts = counts || {}
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def call
|
|
22
|
+
return if @counts.empty?
|
|
23
|
+
|
|
24
|
+
period = Time.current.beginning_of_hour
|
|
25
|
+
app_id = current_application_id
|
|
26
|
+
|
|
27
|
+
@counts.each do |key, count|
|
|
28
|
+
rule, match_type, discriminator, path, http_method =
|
|
29
|
+
Services::RackAttackTracker.parse_key(key)
|
|
30
|
+
|
|
31
|
+
next if rule.blank? || match_type.blank?
|
|
32
|
+
|
|
33
|
+
upsert_event(
|
|
34
|
+
rule: rule,
|
|
35
|
+
match_type: match_type,
|
|
36
|
+
discriminator: discriminator,
|
|
37
|
+
path: path,
|
|
38
|
+
http_method: http_method,
|
|
39
|
+
period: period,
|
|
40
|
+
app_id: app_id,
|
|
41
|
+
count: count
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
rescue => e
|
|
45
|
+
RailsErrorDashboard::Logger.debug(
|
|
46
|
+
"[RailsErrorDashboard] FlushRackAttackEvents failed: #{e.class} - #{e.message}"
|
|
47
|
+
)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def upsert_event(rule:, match_type:, discriminator:, path:, http_method:, period:, app_id:, count:)
|
|
53
|
+
# nil and "" must map to the same row — the unique index treats them as
|
|
54
|
+
# distinct in some adapters, so normalize blanks to nil consistently.
|
|
55
|
+
record = RackAttackEvent.find_or_initialize_by(
|
|
56
|
+
rule: rule,
|
|
57
|
+
match_type: match_type,
|
|
58
|
+
discriminator: discriminator.presence,
|
|
59
|
+
path: path.presence,
|
|
60
|
+
period_hour: period,
|
|
61
|
+
application_id: app_id
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
record.http_method = http_method.presence if record.http_method.blank?
|
|
65
|
+
record.event_count = (record.event_count || 0) + count
|
|
66
|
+
record.last_seen_at = Time.current
|
|
67
|
+
record.save!
|
|
68
|
+
rescue => e
|
|
69
|
+
RailsErrorDashboard::Logger.debug(
|
|
70
|
+
"[RailsErrorDashboard] FlushRackAttackEvents.upsert_event failed for #{rule}: #{e.message}"
|
|
71
|
+
)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def current_application_id
|
|
75
|
+
app_name = RailsErrorDashboard.configuration.application_name
|
|
76
|
+
return nil unless app_name.present?
|
|
77
|
+
|
|
78
|
+
Application.find_by(name: app_name)&.id
|
|
79
|
+
rescue => e
|
|
80
|
+
nil
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
module Commands
|
|
5
|
+
# Command: Reconcile counted-not-stored storm events onto ErrorLog rows
|
|
6
|
+
# and maintain the storm_events episode record.
|
|
7
|
+
#
|
|
8
|
+
# Runs in a background job (DB allowed). For each counted fingerprint:
|
|
9
|
+
# 1. Recompute the canonical error_hash from the stored identity parts
|
|
10
|
+
# (the gate's key deliberately omits application_id — resolved here)
|
|
11
|
+
# 2. Unresolved match → single UPDATE: occurrence_count += N
|
|
12
|
+
# 3. Resolved match → reopen (mirrors FindOrIncrementError semantics)
|
|
13
|
+
# 4. No match → create a minimal ErrorLog from the exemplar
|
|
14
|
+
#
|
|
15
|
+
# Counts are exact. Notifications are NOT dispatched from here — during a
|
|
16
|
+
# storm they're suppressed by design; the storm notification covers it.
|
|
17
|
+
class FlushStormCounts
|
|
18
|
+
def self.call(entries:, overflow: 0, episode: nil)
|
|
19
|
+
new(entries: entries, overflow: overflow, episode: episode).call
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def initialize(entries:, overflow: 0, episode: nil)
|
|
23
|
+
@entries = Array(entries)
|
|
24
|
+
@overflow = overflow.to_i
|
|
25
|
+
@episode = episode
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def call
|
|
29
|
+
application = resolve_application
|
|
30
|
+
counted = 0
|
|
31
|
+
|
|
32
|
+
@entries.each do |entry|
|
|
33
|
+
entry = entry.with_indifferent_access if entry.respond_to?(:with_indifferent_access)
|
|
34
|
+
counted += reconcile_entry(entry, application)
|
|
35
|
+
rescue => e
|
|
36
|
+
# A corrupt (non-Hash) entry must not abort the whole batch — and the
|
|
37
|
+
# log line itself must not assume `entry` is subscriptable (an Integer
|
|
38
|
+
# from a broken serializer would raise again here, escaping this rescue).
|
|
39
|
+
error_class = entry.is_a?(Hash) ? entry["error_class"] : entry.class
|
|
40
|
+
RailsErrorDashboard::Logger.error(
|
|
41
|
+
"[RailsErrorDashboard] Storm count reconcile failed for #{error_class}: #{e.class} - #{e.message}"
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
upsert_storm_event(counted)
|
|
46
|
+
{ success: true, reconciled: counted, overflow: @overflow }
|
|
47
|
+
rescue => e
|
|
48
|
+
RailsErrorDashboard::Logger.error(
|
|
49
|
+
"[RailsErrorDashboard] FlushStormCounts failed: #{e.class} - #{e.message}"
|
|
50
|
+
)
|
|
51
|
+
{ success: false, error: "#{e.class}: #{e.message}" }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def reconcile_entry(entry, application)
|
|
57
|
+
count = entry["count"].to_i
|
|
58
|
+
return 0 if count <= 0
|
|
59
|
+
|
|
60
|
+
error_hash = canonical_hash(entry, application)
|
|
61
|
+
last_seen = parse_time(entry["last_seen_at"]) || Time.current
|
|
62
|
+
|
|
63
|
+
# Priority 1: unresolved match — one UPDATE, no row instantiation
|
|
64
|
+
updated = ErrorLog.unresolved
|
|
65
|
+
.where(error_hash: error_hash, application_id: application.id)
|
|
66
|
+
.update_all([ "occurrence_count = occurrence_count + ?, last_seen_at = ?", count, last_seen ])
|
|
67
|
+
return count if updated.positive?
|
|
68
|
+
|
|
69
|
+
# Priority 2: resolved/wont_fix match — reopen, mirroring
|
|
70
|
+
# FindOrIncrementError so storm recurrences don't stay buried
|
|
71
|
+
resolved = ErrorLog
|
|
72
|
+
.where(error_hash: error_hash, application_id: application.id)
|
|
73
|
+
.where(status: %w[resolved wont_fix])
|
|
74
|
+
.order(last_seen_at: :desc)
|
|
75
|
+
.first
|
|
76
|
+
if resolved
|
|
77
|
+
attrs = {
|
|
78
|
+
resolved: false,
|
|
79
|
+
status: "new",
|
|
80
|
+
resolved_at: nil,
|
|
81
|
+
occurrence_count: resolved.occurrence_count + count,
|
|
82
|
+
last_seen_at: last_seen
|
|
83
|
+
}
|
|
84
|
+
attrs[:reopened_at] = Time.current if ErrorLog.column_names.include?("reopened_at")
|
|
85
|
+
resolved.update!(attrs)
|
|
86
|
+
return count
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Priority 3: first seen during count-only mode — minimal ErrorLog
|
|
90
|
+
# from the exemplar (no backtrace/context was captured; the next
|
|
91
|
+
# occurrence after the storm fills in detail via the normal path)
|
|
92
|
+
ErrorLog.create!(
|
|
93
|
+
application_id: application.id,
|
|
94
|
+
error_type: entry["error_class"],
|
|
95
|
+
message: entry["message"],
|
|
96
|
+
backtrace: entry["first_app_frame"],
|
|
97
|
+
controller_name: entry["controller_name"],
|
|
98
|
+
action_name: entry["action_name"],
|
|
99
|
+
occurred_at: parse_time(entry["first_seen_at"]) || Time.current,
|
|
100
|
+
last_seen_at: last_seen,
|
|
101
|
+
occurrence_count: count,
|
|
102
|
+
error_hash: error_hash,
|
|
103
|
+
resolved: false
|
|
104
|
+
)
|
|
105
|
+
count
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Mirrors ErrorHashGenerator.call exactly: same fields, same order,
|
|
109
|
+
# same normalization — so counts land on the same ErrorLog the full
|
|
110
|
+
# capture path would have used.
|
|
111
|
+
def canonical_hash(entry, application)
|
|
112
|
+
return entry["custom_hash"] if entry["custom_hash"].present?
|
|
113
|
+
|
|
114
|
+
digest_input = [
|
|
115
|
+
entry["error_class"],
|
|
116
|
+
Services::ErrorHashGenerator.normalize_message(entry["message"]),
|
|
117
|
+
entry["first_app_frame"],
|
|
118
|
+
entry["controller_name"],
|
|
119
|
+
entry["action_name"],
|
|
120
|
+
application.id.to_s
|
|
121
|
+
].compact.join("|")
|
|
122
|
+
|
|
123
|
+
Digest::SHA256.hexdigest(digest_input)[0..15]
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def resolve_application
|
|
127
|
+
# Same chain LogError uses — app name is process-global
|
|
128
|
+
app_name = RailsErrorDashboard.configuration.application_name ||
|
|
129
|
+
ENV["APPLICATION_NAME"] ||
|
|
130
|
+
(defined?(Rails) && Rails.application.class.module_parent_name) ||
|
|
131
|
+
"Rails Application"
|
|
132
|
+
Application.find_or_create_by_name(app_name)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def upsert_storm_event(counted)
|
|
136
|
+
return unless @episode.is_a?(Hash)
|
|
137
|
+
return unless StormEvent.table_exists?
|
|
138
|
+
|
|
139
|
+
started_at = parse_time(@episode["started_at"])
|
|
140
|
+
return unless started_at
|
|
141
|
+
|
|
142
|
+
event = StormEvent.active.recent_first.first || StormEvent.create!(started_at: started_at)
|
|
143
|
+
|
|
144
|
+
event.events_counted_only = event.events_counted_only.to_i + counted
|
|
145
|
+
event.events_overflow = event.events_overflow.to_i + @overflow
|
|
146
|
+
# events_total is the count-only total: in-map reconciled + overflow.
|
|
147
|
+
# It deliberately excludes :lite/:full admissions (those became real
|
|
148
|
+
# ErrorLog rows on the hot path and are never counted here), so it is
|
|
149
|
+
# always events_counted_only + events_overflow. Derive it rather than
|
|
150
|
+
# accumulate so it can't drift from its two components.
|
|
151
|
+
event.events_total = event.events_counted_only.to_i + event.events_overflow.to_i
|
|
152
|
+
event.fingerprints_affected = [ event.fingerprints_affected.to_i, @entries.size ].max
|
|
153
|
+
event.peak_rate_per_minute = [ event.peak_rate_per_minute.to_i, @episode["peak_rate_per_minute"].to_i ].max
|
|
154
|
+
event.reached_open ||= @episode["reached_open"] == true
|
|
155
|
+
event.top_fingerprints = top_fingerprints_json(event)
|
|
156
|
+
event.ended_at = parse_time(@episode["ended_at"]) if @episode["ended_at"]
|
|
157
|
+
event.save!
|
|
158
|
+
rescue => e
|
|
159
|
+
RailsErrorDashboard::Logger.error(
|
|
160
|
+
"[RailsErrorDashboard] Storm event upsert failed: #{e.class} - #{e.message}"
|
|
161
|
+
)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def top_fingerprints_json(event)
|
|
165
|
+
existing = event.top_fingerprints_list
|
|
166
|
+
fresh = @entries.map { |e|
|
|
167
|
+
e = e.with_indifferent_access if e.respond_to?(:with_indifferent_access)
|
|
168
|
+
{ "class" => e["error_class"], "message" => e["message"].to_s[0, 120], "count" => e["count"].to_i }
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
merged = (existing + fresh)
|
|
172
|
+
.group_by { |f| [ f["class"], f["message"] ] }
|
|
173
|
+
.map { |_k, group| group.first.merge("count" => group.sum { |f| f["count"].to_i }) }
|
|
174
|
+
|
|
175
|
+
merged.sort_by { |f| -f["count"].to_i }.first(5).to_json
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def parse_time(value)
|
|
179
|
+
return value if value.is_a?(Time) || value.is_a?(ActiveSupport::TimeWithZone)
|
|
180
|
+
return nil if value.blank?
|
|
181
|
+
|
|
182
|
+
Time.zone.parse(value.to_s)
|
|
183
|
+
rescue ArgumentError
|
|
184
|
+
nil
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
@@ -6,15 +6,59 @@ module RailsErrorDashboard
|
|
|
6
6
|
# This is a write operation that creates an ErrorLog record
|
|
7
7
|
class LogError
|
|
8
8
|
def self.call(exception, context = {})
|
|
9
|
-
#
|
|
9
|
+
# Filter FIRST (ignore list + static sampling) so ignored exceptions
|
|
10
|
+
# never count toward storm state. _pre_filtered prevents the sync path
|
|
11
|
+
# from re-rolling the sampling dice (rate would square otherwise).
|
|
12
|
+
# The filter + gate run inside this method's rescue: nothing in the
|
|
13
|
+
# capture path may ever raise into the host app.
|
|
14
|
+
begin
|
|
15
|
+
unless Services::ExceptionFilter.should_log?(exception)
|
|
16
|
+
# Preserve the OTel contract: filtered captures still emit a span
|
|
17
|
+
# tagged filtered=true (no-op when OTel export is disabled).
|
|
18
|
+
Integrations::Tracer.in_span(
|
|
19
|
+
"capture_error",
|
|
20
|
+
kind: :capture,
|
|
21
|
+
attributes: build_capture_span_attributes(exception, was_async: false)
|
|
22
|
+
) do |span|
|
|
23
|
+
span&.set_attribute("rails_error_dashboard.filtered", true)
|
|
24
|
+
end
|
|
25
|
+
return nil
|
|
26
|
+
end
|
|
27
|
+
context = context.merge(_pre_filtered: true)
|
|
28
|
+
|
|
29
|
+
# Storm protection gate — BEFORE the async branch, because with
|
|
30
|
+
# SolidQueue the enqueue itself is a DB write. :count_only events are
|
|
31
|
+
# tallied in memory and reconciled by StormFlushJob; nothing else
|
|
32
|
+
# happens for them (that's the point).
|
|
33
|
+
storm_decision = Services::StormProtection::Gate.admit!(exception, context)
|
|
34
|
+
return nil if storm_decision == :count_only
|
|
35
|
+
context = context.merge(_storm_decision: storm_decision) if storm_decision == :lite
|
|
36
|
+
rescue => e
|
|
37
|
+
RailsErrorDashboard::Logger.error(
|
|
38
|
+
"[RailsErrorDashboard] Capture pre-checks failed: #{e.class} - #{e.message}"
|
|
39
|
+
)
|
|
40
|
+
# Fall through and attempt full capture — fail open, never raise
|
|
41
|
+
end
|
|
42
|
+
|
|
10
43
|
if RailsErrorDashboard.configuration.async_logging
|
|
11
44
|
# For async logging, just enqueue the job
|
|
12
|
-
# All filtering happens when the job runs
|
|
13
45
|
call_async(exception, context)
|
|
14
46
|
else
|
|
15
47
|
# For sync logging, execute immediately
|
|
16
48
|
new(exception, context).call
|
|
17
49
|
end
|
|
50
|
+
rescue => e
|
|
51
|
+
RailsErrorDashboard::Logger.error(
|
|
52
|
+
"[RailsErrorDashboard] LogError.call failed: #{e.class} - #{e.message}"
|
|
53
|
+
)
|
|
54
|
+
nil
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# :lite captures shed context (breadcrumbs/health/locals/ivars) — the
|
|
58
|
+
# storm shedding ladder's first economy. Symbol or string key: the
|
|
59
|
+
# async job round-trips context through the queue serializer.
|
|
60
|
+
def self.storm_lite?(context)
|
|
61
|
+
context[:_storm_decision].to_s == "lite"
|
|
18
62
|
end
|
|
19
63
|
|
|
20
64
|
# Build the base OTel span attributes available before any work happens.
|
|
@@ -42,18 +86,22 @@ module RailsErrorDashboard
|
|
|
42
86
|
cause_chain: serialize_cause_chain(exception)
|
|
43
87
|
}
|
|
44
88
|
|
|
89
|
+
# Storm shedding: :lite captures skip ALL pre-enqueue context harvest —
|
|
90
|
+
# this is request-thread CPU, the most valuable thing to shed.
|
|
91
|
+
lite = storm_lite?(context)
|
|
92
|
+
|
|
45
93
|
# Harvest breadcrumbs NOW (before job dispatch — different thread won't have them)
|
|
46
|
-
if RailsErrorDashboard.configuration.enable_breadcrumbs
|
|
94
|
+
if !lite && RailsErrorDashboard.configuration.enable_breadcrumbs
|
|
47
95
|
context = context.merge(_serialized_breadcrumbs: Services::BreadcrumbCollector.harvest)
|
|
48
96
|
end
|
|
49
97
|
|
|
50
98
|
# Capture system health NOW (metrics are time-sensitive, different thread = different state)
|
|
51
|
-
if RailsErrorDashboard.configuration.enable_system_health
|
|
99
|
+
if !lite && RailsErrorDashboard.configuration.enable_system_health
|
|
52
100
|
context = context.merge(_serialized_system_health: Services::SystemHealthSnapshot.capture)
|
|
53
101
|
end
|
|
54
102
|
|
|
55
103
|
# Capture local variables NOW (TracePoint attaches to exception, must extract before job dispatch)
|
|
56
|
-
if RailsErrorDashboard.configuration.enable_local_variables
|
|
104
|
+
if !lite && RailsErrorDashboard.configuration.enable_local_variables
|
|
57
105
|
begin
|
|
58
106
|
raw_locals = Services::LocalVariableCapturer.extract(exception)
|
|
59
107
|
if raw_locals.is_a?(Hash) && raw_locals.any?
|
|
@@ -65,7 +113,7 @@ module RailsErrorDashboard
|
|
|
65
113
|
end
|
|
66
114
|
|
|
67
115
|
# Capture instance variables NOW (same reason — attached to exception object)
|
|
68
|
-
if RailsErrorDashboard.configuration.enable_instance_variables
|
|
116
|
+
if !lite && RailsErrorDashboard.configuration.enable_instance_variables
|
|
69
117
|
begin
|
|
70
118
|
raw_ivars = Services::LocalVariableCapturer.extract_instance_vars(exception)
|
|
71
119
|
if raw_ivars.is_a?(Hash) && raw_ivars.any?
|
|
@@ -157,12 +205,19 @@ module RailsErrorDashboard
|
|
|
157
205
|
kind: :capture,
|
|
158
206
|
attributes: self.class.build_capture_span_attributes(@exception, was_async: false)
|
|
159
207
|
) do |span|
|
|
160
|
-
# Check if this exception should be logged (ignore list + sampling)
|
|
161
|
-
|
|
208
|
+
# Check if this exception should be logged (ignore list + sampling).
|
|
209
|
+
# Skipped when self.call already filtered (re-rolling the sampling
|
|
210
|
+
# dice here would square the effective rate).
|
|
211
|
+
if !@context[:_pre_filtered] && !Services::ExceptionFilter.should_log?(@exception)
|
|
162
212
|
span&.set_attribute("rails_error_dashboard.filtered", true)
|
|
163
213
|
next nil
|
|
164
214
|
end
|
|
165
215
|
|
|
216
|
+
# Storm shedding: :lite captures keep the error + occurrence row but
|
|
217
|
+
# shed context payloads (breadcrumbs/health/locals/ivars).
|
|
218
|
+
storm_lite = self.class.storm_lite?(@context)
|
|
219
|
+
span&.set_attribute("rails_error_dashboard.storm_degraded", true) if storm_lite
|
|
220
|
+
|
|
166
221
|
error_context = ValueObjects::ErrorContext.new(@context, @context[:source])
|
|
167
222
|
|
|
168
223
|
# Find or create application (cached lookup)
|
|
@@ -239,7 +294,7 @@ module RailsErrorDashboard
|
|
|
239
294
|
attributes = Services::SensitiveDataFilter.filter_attributes(attributes)
|
|
240
295
|
|
|
241
296
|
# Harvest breadcrumbs (if enabled and column exists)
|
|
242
|
-
if ErrorLog.column_names.include?("breadcrumbs") && RailsErrorDashboard.configuration.enable_breadcrumbs
|
|
297
|
+
if !storm_lite && ErrorLog.column_names.include?("breadcrumbs") && RailsErrorDashboard.configuration.enable_breadcrumbs
|
|
243
298
|
# Sync path: harvest from current thread
|
|
244
299
|
raw_breadcrumbs = Services::BreadcrumbCollector.harvest
|
|
245
300
|
|
|
@@ -256,13 +311,13 @@ module RailsErrorDashboard
|
|
|
256
311
|
end
|
|
257
312
|
|
|
258
313
|
# Capture system health snapshot (if enabled and column exists)
|
|
259
|
-
if ErrorLog.column_names.include?("system_health") && RailsErrorDashboard.configuration.enable_system_health
|
|
314
|
+
if !storm_lite && ErrorLog.column_names.include?("system_health") && RailsErrorDashboard.configuration.enable_system_health
|
|
260
315
|
health_data = @context[:_serialized_system_health] || Services::SystemHealthSnapshot.capture
|
|
261
316
|
attributes[:system_health] = health_data.to_json
|
|
262
317
|
end
|
|
263
318
|
|
|
264
319
|
# Capture local variables (if enabled and column exists)
|
|
265
|
-
if ErrorLog.column_names.include?("local_variables") && RailsErrorDashboard.configuration.enable_local_variables
|
|
320
|
+
if !storm_lite && ErrorLog.column_names.include?("local_variables") && RailsErrorDashboard.configuration.enable_local_variables
|
|
266
321
|
begin
|
|
267
322
|
# Sync path: extract from exception ivar
|
|
268
323
|
raw_locals = Services::LocalVariableCapturer.extract(@exception)
|
|
@@ -278,7 +333,7 @@ module RailsErrorDashboard
|
|
|
278
333
|
end
|
|
279
334
|
|
|
280
335
|
# Capture instance variables (if enabled and column exists)
|
|
281
|
-
if ErrorLog.column_names.include?("instance_variables") && RailsErrorDashboard.configuration.enable_instance_variables
|
|
336
|
+
if !storm_lite && ErrorLog.column_names.include?("instance_variables") && RailsErrorDashboard.configuration.enable_instance_variables
|
|
282
337
|
begin
|
|
283
338
|
# Sync path: extract from exception ivar
|
|
284
339
|
raw_ivars = Services::LocalVariableCapturer.extract_instance_vars(@exception)
|
|
@@ -364,8 +419,11 @@ module RailsErrorDashboard
|
|
|
364
419
|
|
|
365
420
|
# Dispatch notification if error is not muted and the throttle check passes.
|
|
366
421
|
# Muted errors skip notifications but still fire plugin events/callbacks.
|
|
422
|
+
# During a storm (breaker not closed) per-error notifications are
|
|
423
|
+
# suppressed — a single storm notification replaces them.
|
|
367
424
|
def maybe_notify(error_log)
|
|
368
425
|
return if error_log.muted?
|
|
426
|
+
return if Services::StormProtection::Gate.notifications_suppressed?
|
|
369
427
|
return unless yield
|
|
370
428
|
|
|
371
429
|
Services::ErrorNotificationDispatcher.call(error_log)
|
|
@@ -67,6 +67,25 @@ module RailsErrorDashboard
|
|
|
67
67
|
# Sampling rate for non-critical errors (0.0 to 1.0, default 1.0 = 100%)
|
|
68
68
|
attr_accessor :sampling_rate
|
|
69
69
|
|
|
70
|
+
# Storm protection — circuit breaker + adaptive sampling for error floods.
|
|
71
|
+
# Protects the HOST APP from the gem's own writes during an error storm
|
|
72
|
+
# (bad deploy throwing thousands of errors/minute). Default ON: this is
|
|
73
|
+
# the feature that makes the gem quieter, so ON is the conservative choice.
|
|
74
|
+
# All thresholds are PER PROCESS (no cross-process coordination by design).
|
|
75
|
+
attr_accessor :enable_storm_protection # Master switch (default: true)
|
|
76
|
+
attr_accessor :storm_fingerprint_full_per_minute # Full-fidelity captures per fingerprint per minute (default: 30)
|
|
77
|
+
attr_accessor :storm_occurrence_sample_keep_every # Past the cap, keep every Nth occurrence (default: 10)
|
|
78
|
+
attr_accessor :storm_shedding_threshold_per_second # Global rate that enters shedding state (default: 10)
|
|
79
|
+
attr_accessor :storm_open_threshold_per_second # Global rate that opens the breaker = count-only (default: 50)
|
|
80
|
+
attr_accessor :storm_cooldown_seconds # Open → half-open probe delay (default: 60)
|
|
81
|
+
attr_accessor :storm_max_tracked_fingerprints # Bounded in-memory map size; beyond = overflow bucket (default: 1000)
|
|
82
|
+
attr_accessor :storm_flush_interval_seconds # Count-buffer flush cadence (default: 30)
|
|
83
|
+
attr_accessor :storm_notification # Single "storm in progress" notification per episode (default: true)
|
|
84
|
+
attr_accessor :auto_issue_rate_limit_count # Max auto-created issues per window — applies always (default: 5)
|
|
85
|
+
attr_accessor :auto_issue_rate_limit_window_minutes # Window for the above (default: 10)
|
|
86
|
+
attr_accessor :context_sampling_threshold_per_day # Full-context captures per fingerprint per day before sampling (default: 25)
|
|
87
|
+
attr_accessor :context_sampling_keep_every # After threshold, keep full context every Nth (default: 10)
|
|
88
|
+
|
|
70
89
|
# Async logging configuration
|
|
71
90
|
attr_accessor :async_logging
|
|
72
91
|
attr_accessor :async_adapter # :sidekiq, :solid_queue, or :async
|
|
@@ -176,8 +195,11 @@ module RailsErrorDashboard
|
|
|
176
195
|
# Code path coverage (diagnostic mode — Ruby 3.2+)
|
|
177
196
|
attr_accessor :enable_coverage_tracking # Master switch (default: false)
|
|
178
197
|
|
|
179
|
-
# Rack Attack event tracking
|
|
198
|
+
# Rack Attack event tracking — persists throttle/blocklist/track events to
|
|
199
|
+
# their own table, independent of error capture (breadcrumbs optional).
|
|
180
200
|
attr_accessor :enable_rack_attack_tracking # Master switch (default: false)
|
|
201
|
+
attr_accessor :rack_attack_max_cache_size # Max buffered keys per thread (default: 1000)
|
|
202
|
+
attr_accessor :rack_attack_flush_interval # Seconds between DB flushes (default: 60)
|
|
181
203
|
|
|
182
204
|
# ActionCable event tracking (requires enable_breadcrumbs = true)
|
|
183
205
|
attr_accessor :enable_actioncable_tracking # Master switch (default: false)
|
|
@@ -268,6 +290,21 @@ module RailsErrorDashboard
|
|
|
268
290
|
@ignored_exceptions = []
|
|
269
291
|
@custom_fingerprint = nil # Lambda: ->(exception, context) { "custom_key" }
|
|
270
292
|
@sampling_rate = 1.0 # 100% by default
|
|
293
|
+
|
|
294
|
+
# Storm protection defaults (thresholds tuned via chaos Phase G — see ROADMAP)
|
|
295
|
+
@enable_storm_protection = true
|
|
296
|
+
@storm_fingerprint_full_per_minute = 30
|
|
297
|
+
@storm_occurrence_sample_keep_every = 10
|
|
298
|
+
@storm_shedding_threshold_per_second = 10
|
|
299
|
+
@storm_open_threshold_per_second = 50
|
|
300
|
+
@storm_cooldown_seconds = 60
|
|
301
|
+
@storm_max_tracked_fingerprints = 1000
|
|
302
|
+
@storm_flush_interval_seconds = 30
|
|
303
|
+
@storm_notification = true
|
|
304
|
+
@auto_issue_rate_limit_count = 5
|
|
305
|
+
@auto_issue_rate_limit_window_minutes = 10
|
|
306
|
+
@context_sampling_threshold_per_day = 25
|
|
307
|
+
@context_sampling_keep_every = 10
|
|
271
308
|
@async_logging = false
|
|
272
309
|
@async_adapter = :sidekiq # Battle-tested default
|
|
273
310
|
@max_backtrace_lines = 100 # Matches industry standard (Rollbar, Airbrake)
|
|
@@ -366,8 +403,11 @@ module RailsErrorDashboard
|
|
|
366
403
|
# Code path coverage defaults - OFF by default (opt-in, Ruby 3.2+)
|
|
367
404
|
@enable_coverage_tracking = false
|
|
368
405
|
|
|
369
|
-
# Rack Attack event tracking defaults - OFF by default (opt-in
|
|
406
|
+
# Rack Attack event tracking defaults - OFF by default (opt-in).
|
|
407
|
+
# Persists to its own table; does NOT require breadcrumbs.
|
|
370
408
|
@enable_rack_attack_tracking = false
|
|
409
|
+
@rack_attack_max_cache_size = 1000 # Max buffered keys per thread (LRU eviction)
|
|
410
|
+
@rack_attack_flush_interval = 60 # Seconds between DB flushes
|
|
371
411
|
|
|
372
412
|
# ActionCable event tracking defaults - OFF by default (opt-in, requires breadcrumbs)
|
|
373
413
|
@enable_actioncable_tracking = false
|
|
@@ -540,11 +580,16 @@ module RailsErrorDashboard
|
|
|
540
580
|
end
|
|
541
581
|
end
|
|
542
582
|
|
|
543
|
-
#
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
583
|
+
# Rack Attack tracking no longer requires breadcrumbs — events are persisted
|
|
584
|
+
# to their own table (issue #143). Breadcrumbs only add the event to the
|
|
585
|
+
# activity trail on error detail pages.
|
|
586
|
+
if enable_rack_attack_tracking
|
|
587
|
+
if rack_attack_max_cache_size && rack_attack_max_cache_size < 1
|
|
588
|
+
errors << "rack_attack_max_cache_size must be at least 1 (got: #{rack_attack_max_cache_size})"
|
|
589
|
+
end
|
|
590
|
+
if rack_attack_flush_interval && rack_attack_flush_interval < 1
|
|
591
|
+
errors << "rack_attack_flush_interval must be at least 1 (got: #{rack_attack_flush_interval})"
|
|
592
|
+
end
|
|
548
593
|
end
|
|
549
594
|
|
|
550
595
|
# Validate actioncable tracking requires breadcrumbs
|
|
@@ -683,6 +728,32 @@ module RailsErrorDashboard
|
|
|
683
728
|
end
|
|
684
729
|
end
|
|
685
730
|
|
|
731
|
+
# Validate storm protection thresholds (all must be positive when protection is on)
|
|
732
|
+
if enable_storm_protection
|
|
733
|
+
{
|
|
734
|
+
storm_fingerprint_full_per_minute: storm_fingerprint_full_per_minute,
|
|
735
|
+
storm_occurrence_sample_keep_every: storm_occurrence_sample_keep_every,
|
|
736
|
+
storm_shedding_threshold_per_second: storm_shedding_threshold_per_second,
|
|
737
|
+
storm_open_threshold_per_second: storm_open_threshold_per_second,
|
|
738
|
+
storm_cooldown_seconds: storm_cooldown_seconds,
|
|
739
|
+
storm_max_tracked_fingerprints: storm_max_tracked_fingerprints,
|
|
740
|
+
storm_flush_interval_seconds: storm_flush_interval_seconds,
|
|
741
|
+
auto_issue_rate_limit_count: auto_issue_rate_limit_count,
|
|
742
|
+
auto_issue_rate_limit_window_minutes: auto_issue_rate_limit_window_minutes,
|
|
743
|
+
context_sampling_threshold_per_day: context_sampling_threshold_per_day,
|
|
744
|
+
context_sampling_keep_every: context_sampling_keep_every
|
|
745
|
+
}.each do |name, value|
|
|
746
|
+
if value.nil? || value.to_i < 1
|
|
747
|
+
errors << "#{name} must be a positive integer (got: #{value.inspect})"
|
|
748
|
+
end
|
|
749
|
+
end
|
|
750
|
+
|
|
751
|
+
if storm_open_threshold_per_second.to_i < storm_shedding_threshold_per_second.to_i
|
|
752
|
+
errors << "storm_open_threshold_per_second (#{storm_open_threshold_per_second}) must be >= " \
|
|
753
|
+
"storm_shedding_threshold_per_second (#{storm_shedding_threshold_per_second})"
|
|
754
|
+
end
|
|
755
|
+
end
|
|
756
|
+
|
|
686
757
|
# Validate total_users_for_impact (must be positive if set)
|
|
687
758
|
if total_users_for_impact && total_users_for_impact < 1
|
|
688
759
|
errors << "total_users_for_impact must be at least 1 (got: #{total_users_for_impact})"
|
|
@@ -80,9 +80,9 @@ module RailsErrorDashboard
|
|
|
80
80
|
RailsErrorDashboard::Subscribers::BreadcrumbSubscriber.subscribe!
|
|
81
81
|
end
|
|
82
82
|
|
|
83
|
-
# Subscribe to Rack Attack AS::Notifications events (requires
|
|
83
|
+
# Subscribe to Rack Attack AS::Notifications events (requires Rack::Attack).
|
|
84
|
+
# Breadcrumbs are NOT required — events persist to their own table (issue #143).
|
|
84
85
|
if RailsErrorDashboard.configuration.enable_rack_attack_tracking &&
|
|
85
|
-
RailsErrorDashboard.configuration.enable_breadcrumbs &&
|
|
86
86
|
defined?(Rack::Attack)
|
|
87
87
|
RailsErrorDashboard::Subscribers::RackAttackSubscriber.subscribe!
|
|
88
88
|
end
|