rails_error_dashboard 0.8.1 → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +54 -1
  3. data/app/controllers/rails_error_dashboard/application_controller.rb +5 -0
  4. data/app/controllers/rails_error_dashboard/errors_controller.rb +14 -3
  5. data/app/jobs/rails_error_dashboard/rack_attack_flush_job.rb +31 -0
  6. data/app/jobs/rails_error_dashboard/retention_cleanup_job.rb +30 -0
  7. data/app/jobs/rails_error_dashboard/storm_flush_job.rb +19 -0
  8. data/app/jobs/rails_error_dashboard/storm_notification_job.rb +74 -0
  9. data/app/models/rails_error_dashboard/rack_attack_event.rb +36 -0
  10. data/app/models/rails_error_dashboard/storm_event.rb +34 -0
  11. data/app/views/layouts/rails_error_dashboard.html.erb +22 -1
  12. data/app/views/rails_error_dashboard/errors/rack_attack_summary.html.erb +4 -6
  13. data/app/views/rails_error_dashboard/errors/storms.html.erb +91 -0
  14. data/config/routes.rb +1 -0
  15. data/db/migrate/20260306000002_add_instance_variables_to_error_logs.rb +7 -1
  16. data/db/migrate/20260306000003_create_rails_error_dashboard_swallowed_exceptions.rb +4 -0
  17. data/db/migrate/20260307000001_create_rails_error_dashboard_diagnostic_dumps.rb +4 -0
  18. data/db/migrate/20260613000001_create_storm_events.rb +28 -0
  19. data/db/migrate/20260730000001_create_rails_error_dashboard_rack_attack_events.rb +45 -0
  20. data/lib/generators/rails_error_dashboard/install/templates/initializer.rb +36 -0
  21. data/lib/rails_error_dashboard/commands/flush_rack_attack_events.rb +84 -0
  22. data/lib/rails_error_dashboard/commands/flush_storm_counts.rb +188 -0
  23. data/lib/rails_error_dashboard/commands/log_error.rb +70 -12
  24. data/lib/rails_error_dashboard/configuration.rb +78 -7
  25. data/lib/rails_error_dashboard/engine.rb +2 -2
  26. data/lib/rails_error_dashboard/queries/rack_attack_summary.rb +55 -44
  27. data/lib/rails_error_dashboard/queries/storm_history.rb +39 -0
  28. data/lib/rails_error_dashboard/services/rack_attack_tracker.rb +194 -0
  29. data/lib/rails_error_dashboard/services/storm_protection/circuit_breaker.rb +195 -0
  30. data/lib/rails_error_dashboard/services/storm_protection/count_buffer.rb +100 -0
  31. data/lib/rails_error_dashboard/services/storm_protection/fingerprint_buckets.rb +123 -0
  32. data/lib/rails_error_dashboard/services/storm_protection/gate.rb +258 -0
  33. data/lib/rails_error_dashboard/subscribers/issue_tracker_subscriber.rb +12 -0
  34. data/lib/rails_error_dashboard/subscribers/rack_attack_subscriber.rb +16 -2
  35. data/lib/rails_error_dashboard/version.rb +1 -1
  36. data/lib/rails_error_dashboard.rb +8 -0
  37. metadata +37 -18
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ class CreateRailsErrorDashboardRackAttackEvents < ActiveRecord::Migration[7.0]
4
+ def change
5
+ # Guard against the squashed schema migration having already created this
6
+ # table — without it, every later migration is silently cancelled.
7
+ return if table_exists?(:rails_error_dashboard_rack_attack_events)
8
+
9
+ create_table :rails_error_dashboard_rack_attack_events do |t|
10
+ t.string :rule, null: false, limit: 250
11
+ t.string :match_type, null: false, limit: 50
12
+ # discriminator and path are capped at 191 (not 250) to keep the unique
13
+ # upsert index within MySQL's 3072-byte utf8mb4 limit. Budget:
14
+ # 250+50+191+191 chars * 4 bytes + 2 length-prefix each = 2736 bytes.
15
+ # See issue #96 — the swallowed_exceptions index blew this limit at 5042.
16
+ t.string :discriminator, limit: 191
17
+ t.string :path, limit: 191
18
+ t.string :http_method, limit: 10
19
+ t.datetime :period_hour, null: false
20
+ t.integer :event_count, null: false, default: 0
21
+ t.datetime :last_seen_at
22
+ t.bigint :application_id
23
+ t.timestamps
24
+ end
25
+
26
+ add_index :rails_error_dashboard_rack_attack_events,
27
+ :period_hour,
28
+ name: "index_rack_attack_events_on_period_hour"
29
+
30
+ add_index :rails_error_dashboard_rack_attack_events,
31
+ [ :application_id, :period_hour ],
32
+ name: "index_rack_attack_events_on_app_and_hour"
33
+
34
+ add_index :rails_error_dashboard_rack_attack_events,
35
+ [ :rule, :period_hour ],
36
+ name: "index_rack_attack_events_on_rule_and_hour"
37
+
38
+ # http_method is intentionally NOT part of the upsert key — it would push
39
+ # the index over the MySQL byte limit and adds no aggregation value.
40
+ add_index :rails_error_dashboard_rack_attack_events,
41
+ [ :rule, :match_type, :discriminator, :path, :period_hour, :application_id ],
42
+ unique: true,
43
+ name: "index_rack_attack_events_upsert_key"
44
+ end
45
+ end
@@ -518,6 +518,42 @@ RailsErrorDashboard.configure do |config|
518
518
  # No PII or request bodies in span attributes — just metadata + timing.
519
519
  # Safe to enable on production OTel pipelines.
520
520
 
521
+ # ============================================================================
522
+ # STORM PROTECTION (circuit breaker + adaptive sampling) — ON by default
523
+ # ============================================================================
524
+ #
525
+ # When the error rate spikes (bad deploy, dependency outage), storm
526
+ # protection limits the gem's own database writes so it never amplifies
527
+ # the incident. Occurrences are ALWAYS counted exactly — only per-event
528
+ # detail (context payloads, occurrence rows) is sampled under load.
529
+ #
530
+ # How it degrades, in order:
531
+ # 1. Per-fingerprint cap: past N/min, context is shed, then rows sampled
532
+ # 2. Global breaker: shedding (context off) → open (count-only mode)
533
+ # 3. Per-error notifications replaced by ONE "storm in progress" message
534
+ # 4. Counts reconciled onto error records every flush interval
535
+ #
536
+ # All thresholds are PER PROCESS (each Puma worker runs its own breaker).
537
+ #
538
+ # config.enable_storm_protection = true
539
+ # config.storm_fingerprint_full_per_minute = 30 # full-fidelity captures per fingerprint/min
540
+ # config.storm_occurrence_sample_keep_every = 10 # past the cap, keep every Nth occurrence
541
+ # config.storm_shedding_threshold_per_second = 10 # global rate entering shedding state
542
+ # config.storm_open_threshold_per_second = 50 # global rate opening the breaker (count-only)
543
+ # config.storm_cooldown_seconds = 60 # open → half-open probe delay
544
+ # config.storm_notification = true # one notification per storm episode
545
+ #
546
+ # Always-on issue cap (a storm of NEW critical errors must not open
547
+ # hundreds of GitHub/Linear issues):
548
+ # config.auto_issue_rate_limit_count = 5
549
+ # config.auto_issue_rate_limit_window_minutes = 10
550
+ #
551
+ # Calm-weather context economy: an error seen 1000x/day doesn't need 1000
552
+ # breadcrumb trails. After N full-context captures per fingerprint per day,
553
+ # context is kept every Mth time (occurrence rows are unaffected):
554
+ # config.context_sampling_threshold_per_day = 25
555
+ # config.context_sampling_keep_every = 10
556
+
521
557
  # ============================================================================
522
558
  # ISSUE TRACKING (GitHub / GitLab / Codeberg / Linear)
523
559
  # ============================================================================
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RailsErrorDashboard
4
+ module Commands
5
+ # Command: Upsert buffered Rack::Attack event counts into the database.
6
+ #
7
+ # Receives a snapshot hash from RackAttackTracker and merges it into
8
+ # hourly-bucketed rows. Uses find_or_initialize_by + increment for
9
+ # cross-database compatibility (no raw SQL upsert).
10
+ #
11
+ # counts keys: "rule\x1Fmatch_type\x1Fdiscriminator\x1Fpath\x1Fhttp_method"
12
+ class FlushRackAttackEvents
13
+ def self.call(counts:)
14
+ new(counts: counts).call
15
+ end
16
+
17
+ def initialize(counts:)
18
+ @counts = counts || {}
19
+ end
20
+
21
+ def call
22
+ return if @counts.empty?
23
+
24
+ period = Time.current.beginning_of_hour
25
+ app_id = current_application_id
26
+
27
+ @counts.each do |key, count|
28
+ rule, match_type, discriminator, path, http_method =
29
+ Services::RackAttackTracker.parse_key(key)
30
+
31
+ next if rule.blank? || match_type.blank?
32
+
33
+ upsert_event(
34
+ rule: rule,
35
+ match_type: match_type,
36
+ discriminator: discriminator,
37
+ path: path,
38
+ http_method: http_method,
39
+ period: period,
40
+ app_id: app_id,
41
+ count: count
42
+ )
43
+ end
44
+ rescue => e
45
+ RailsErrorDashboard::Logger.debug(
46
+ "[RailsErrorDashboard] FlushRackAttackEvents failed: #{e.class} - #{e.message}"
47
+ )
48
+ end
49
+
50
+ private
51
+
52
+ def upsert_event(rule:, match_type:, discriminator:, path:, http_method:, period:, app_id:, count:)
53
+ # nil and "" must map to the same row — the unique index treats them as
54
+ # distinct in some adapters, so normalize blanks to nil consistently.
55
+ record = RackAttackEvent.find_or_initialize_by(
56
+ rule: rule,
57
+ match_type: match_type,
58
+ discriminator: discriminator.presence,
59
+ path: path.presence,
60
+ period_hour: period,
61
+ application_id: app_id
62
+ )
63
+
64
+ record.http_method = http_method.presence if record.http_method.blank?
65
+ record.event_count = (record.event_count || 0) + count
66
+ record.last_seen_at = Time.current
67
+ record.save!
68
+ rescue => e
69
+ RailsErrorDashboard::Logger.debug(
70
+ "[RailsErrorDashboard] FlushRackAttackEvents.upsert_event failed for #{rule}: #{e.message}"
71
+ )
72
+ end
73
+
74
+ def current_application_id
75
+ app_name = RailsErrorDashboard.configuration.application_name
76
+ return nil unless app_name.present?
77
+
78
+ Application.find_by(name: app_name)&.id
79
+ rescue => e
80
+ nil
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,188 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RailsErrorDashboard
4
+ module Commands
5
+ # Command: Reconcile counted-not-stored storm events onto ErrorLog rows
6
+ # and maintain the storm_events episode record.
7
+ #
8
+ # Runs in a background job (DB allowed). For each counted fingerprint:
9
+ # 1. Recompute the canonical error_hash from the stored identity parts
10
+ # (the gate's key deliberately omits application_id — resolved here)
11
+ # 2. Unresolved match → single UPDATE: occurrence_count += N
12
+ # 3. Resolved match → reopen (mirrors FindOrIncrementError semantics)
13
+ # 4. No match → create a minimal ErrorLog from the exemplar
14
+ #
15
+ # Counts are exact. Notifications are NOT dispatched from here — during a
16
+ # storm they're suppressed by design; the storm notification covers it.
17
+ class FlushStormCounts
18
+ def self.call(entries:, overflow: 0, episode: nil)
19
+ new(entries: entries, overflow: overflow, episode: episode).call
20
+ end
21
+
22
+ def initialize(entries:, overflow: 0, episode: nil)
23
+ @entries = Array(entries)
24
+ @overflow = overflow.to_i
25
+ @episode = episode
26
+ end
27
+
28
+ def call
29
+ application = resolve_application
30
+ counted = 0
31
+
32
+ @entries.each do |entry|
33
+ entry = entry.with_indifferent_access if entry.respond_to?(:with_indifferent_access)
34
+ counted += reconcile_entry(entry, application)
35
+ rescue => e
36
+ # A corrupt (non-Hash) entry must not abort the whole batch — and the
37
+ # log line itself must not assume `entry` is subscriptable (an Integer
38
+ # from a broken serializer would raise again here, escaping this rescue).
39
+ error_class = entry.is_a?(Hash) ? entry["error_class"] : entry.class
40
+ RailsErrorDashboard::Logger.error(
41
+ "[RailsErrorDashboard] Storm count reconcile failed for #{error_class}: #{e.class} - #{e.message}"
42
+ )
43
+ end
44
+
45
+ upsert_storm_event(counted)
46
+ { success: true, reconciled: counted, overflow: @overflow }
47
+ rescue => e
48
+ RailsErrorDashboard::Logger.error(
49
+ "[RailsErrorDashboard] FlushStormCounts failed: #{e.class} - #{e.message}"
50
+ )
51
+ { success: false, error: "#{e.class}: #{e.message}" }
52
+ end
53
+
54
+ private
55
+
56
+ def reconcile_entry(entry, application)
57
+ count = entry["count"].to_i
58
+ return 0 if count <= 0
59
+
60
+ error_hash = canonical_hash(entry, application)
61
+ last_seen = parse_time(entry["last_seen_at"]) || Time.current
62
+
63
+ # Priority 1: unresolved match — one UPDATE, no row instantiation
64
+ updated = ErrorLog.unresolved
65
+ .where(error_hash: error_hash, application_id: application.id)
66
+ .update_all([ "occurrence_count = occurrence_count + ?, last_seen_at = ?", count, last_seen ])
67
+ return count if updated.positive?
68
+
69
+ # Priority 2: resolved/wont_fix match — reopen, mirroring
70
+ # FindOrIncrementError so storm recurrences don't stay buried
71
+ resolved = ErrorLog
72
+ .where(error_hash: error_hash, application_id: application.id)
73
+ .where(status: %w[resolved wont_fix])
74
+ .order(last_seen_at: :desc)
75
+ .first
76
+ if resolved
77
+ attrs = {
78
+ resolved: false,
79
+ status: "new",
80
+ resolved_at: nil,
81
+ occurrence_count: resolved.occurrence_count + count,
82
+ last_seen_at: last_seen
83
+ }
84
+ attrs[:reopened_at] = Time.current if ErrorLog.column_names.include?("reopened_at")
85
+ resolved.update!(attrs)
86
+ return count
87
+ end
88
+
89
+ # Priority 3: first seen during count-only mode — minimal ErrorLog
90
+ # from the exemplar (no backtrace/context was captured; the next
91
+ # occurrence after the storm fills in detail via the normal path)
92
+ ErrorLog.create!(
93
+ application_id: application.id,
94
+ error_type: entry["error_class"],
95
+ message: entry["message"],
96
+ backtrace: entry["first_app_frame"],
97
+ controller_name: entry["controller_name"],
98
+ action_name: entry["action_name"],
99
+ occurred_at: parse_time(entry["first_seen_at"]) || Time.current,
100
+ last_seen_at: last_seen,
101
+ occurrence_count: count,
102
+ error_hash: error_hash,
103
+ resolved: false
104
+ )
105
+ count
106
+ end
107
+
108
+ # Mirrors ErrorHashGenerator.call exactly: same fields, same order,
109
+ # same normalization — so counts land on the same ErrorLog the full
110
+ # capture path would have used.
111
+ def canonical_hash(entry, application)
112
+ return entry["custom_hash"] if entry["custom_hash"].present?
113
+
114
+ digest_input = [
115
+ entry["error_class"],
116
+ Services::ErrorHashGenerator.normalize_message(entry["message"]),
117
+ entry["first_app_frame"],
118
+ entry["controller_name"],
119
+ entry["action_name"],
120
+ application.id.to_s
121
+ ].compact.join("|")
122
+
123
+ Digest::SHA256.hexdigest(digest_input)[0..15]
124
+ end
125
+
126
+ def resolve_application
127
+ # Same chain LogError uses — app name is process-global
128
+ app_name = RailsErrorDashboard.configuration.application_name ||
129
+ ENV["APPLICATION_NAME"] ||
130
+ (defined?(Rails) && Rails.application.class.module_parent_name) ||
131
+ "Rails Application"
132
+ Application.find_or_create_by_name(app_name)
133
+ end
134
+
135
+ def upsert_storm_event(counted)
136
+ return unless @episode.is_a?(Hash)
137
+ return unless StormEvent.table_exists?
138
+
139
+ started_at = parse_time(@episode["started_at"])
140
+ return unless started_at
141
+
142
+ event = StormEvent.active.recent_first.first || StormEvent.create!(started_at: started_at)
143
+
144
+ event.events_counted_only = event.events_counted_only.to_i + counted
145
+ event.events_overflow = event.events_overflow.to_i + @overflow
146
+ # events_total is the count-only total: in-map reconciled + overflow.
147
+ # It deliberately excludes :lite/:full admissions (those became real
148
+ # ErrorLog rows on the hot path and are never counted here), so it is
149
+ # always events_counted_only + events_overflow. Derive it rather than
150
+ # accumulate so it can't drift from its two components.
151
+ event.events_total = event.events_counted_only.to_i + event.events_overflow.to_i
152
+ event.fingerprints_affected = [ event.fingerprints_affected.to_i, @entries.size ].max
153
+ event.peak_rate_per_minute = [ event.peak_rate_per_minute.to_i, @episode["peak_rate_per_minute"].to_i ].max
154
+ event.reached_open ||= @episode["reached_open"] == true
155
+ event.top_fingerprints = top_fingerprints_json(event)
156
+ event.ended_at = parse_time(@episode["ended_at"]) if @episode["ended_at"]
157
+ event.save!
158
+ rescue => e
159
+ RailsErrorDashboard::Logger.error(
160
+ "[RailsErrorDashboard] Storm event upsert failed: #{e.class} - #{e.message}"
161
+ )
162
+ end
163
+
164
+ def top_fingerprints_json(event)
165
+ existing = event.top_fingerprints_list
166
+ fresh = @entries.map { |e|
167
+ e = e.with_indifferent_access if e.respond_to?(:with_indifferent_access)
168
+ { "class" => e["error_class"], "message" => e["message"].to_s[0, 120], "count" => e["count"].to_i }
169
+ }
170
+
171
+ merged = (existing + fresh)
172
+ .group_by { |f| [ f["class"], f["message"] ] }
173
+ .map { |_k, group| group.first.merge("count" => group.sum { |f| f["count"].to_i }) }
174
+
175
+ merged.sort_by { |f| -f["count"].to_i }.first(5).to_json
176
+ end
177
+
178
+ def parse_time(value)
179
+ return value if value.is_a?(Time) || value.is_a?(ActiveSupport::TimeWithZone)
180
+ return nil if value.blank?
181
+
182
+ Time.zone.parse(value.to_s)
183
+ rescue ArgumentError
184
+ nil
185
+ end
186
+ end
187
+ end
188
+ end
@@ -6,15 +6,59 @@ module RailsErrorDashboard
6
6
  # This is a write operation that creates an ErrorLog record
7
7
  class LogError
8
8
  def self.call(exception, context = {})
9
- # Check if async logging is enabled
9
+ # Filter FIRST (ignore list + static sampling) so ignored exceptions
10
+ # never count toward storm state. _pre_filtered prevents the sync path
11
+ # from re-rolling the sampling dice (rate would square otherwise).
12
+ # The filter + gate run inside this method's rescue: nothing in the
13
+ # capture path may ever raise into the host app.
14
+ begin
15
+ unless Services::ExceptionFilter.should_log?(exception)
16
+ # Preserve the OTel contract: filtered captures still emit a span
17
+ # tagged filtered=true (no-op when OTel export is disabled).
18
+ Integrations::Tracer.in_span(
19
+ "capture_error",
20
+ kind: :capture,
21
+ attributes: build_capture_span_attributes(exception, was_async: false)
22
+ ) do |span|
23
+ span&.set_attribute("rails_error_dashboard.filtered", true)
24
+ end
25
+ return nil
26
+ end
27
+ context = context.merge(_pre_filtered: true)
28
+
29
+ # Storm protection gate — BEFORE the async branch, because with
30
+ # SolidQueue the enqueue itself is a DB write. :count_only events are
31
+ # tallied in memory and reconciled by StormFlushJob; nothing else
32
+ # happens for them (that's the point).
33
+ storm_decision = Services::StormProtection::Gate.admit!(exception, context)
34
+ return nil if storm_decision == :count_only
35
+ context = context.merge(_storm_decision: storm_decision) if storm_decision == :lite
36
+ rescue => e
37
+ RailsErrorDashboard::Logger.error(
38
+ "[RailsErrorDashboard] Capture pre-checks failed: #{e.class} - #{e.message}"
39
+ )
40
+ # Fall through and attempt full capture — fail open, never raise
41
+ end
42
+
10
43
  if RailsErrorDashboard.configuration.async_logging
11
44
  # For async logging, just enqueue the job
12
- # All filtering happens when the job runs
13
45
  call_async(exception, context)
14
46
  else
15
47
  # For sync logging, execute immediately
16
48
  new(exception, context).call
17
49
  end
50
+ rescue => e
51
+ RailsErrorDashboard::Logger.error(
52
+ "[RailsErrorDashboard] LogError.call failed: #{e.class} - #{e.message}"
53
+ )
54
+ nil
55
+ end
56
+
57
+ # :lite captures shed context (breadcrumbs/health/locals/ivars) — the
58
+ # storm shedding ladder's first economy. Symbol or string key: the
59
+ # async job round-trips context through the queue serializer.
60
+ def self.storm_lite?(context)
61
+ context[:_storm_decision].to_s == "lite"
18
62
  end
19
63
 
20
64
  # Build the base OTel span attributes available before any work happens.
@@ -42,18 +86,22 @@ module RailsErrorDashboard
42
86
  cause_chain: serialize_cause_chain(exception)
43
87
  }
44
88
 
89
+ # Storm shedding: :lite captures skip ALL pre-enqueue context harvest —
90
+ # this is request-thread CPU, the most valuable thing to shed.
91
+ lite = storm_lite?(context)
92
+
45
93
  # Harvest breadcrumbs NOW (before job dispatch — different thread won't have them)
46
- if RailsErrorDashboard.configuration.enable_breadcrumbs
94
+ if !lite && RailsErrorDashboard.configuration.enable_breadcrumbs
47
95
  context = context.merge(_serialized_breadcrumbs: Services::BreadcrumbCollector.harvest)
48
96
  end
49
97
 
50
98
  # Capture system health NOW (metrics are time-sensitive, different thread = different state)
51
- if RailsErrorDashboard.configuration.enable_system_health
99
+ if !lite && RailsErrorDashboard.configuration.enable_system_health
52
100
  context = context.merge(_serialized_system_health: Services::SystemHealthSnapshot.capture)
53
101
  end
54
102
 
55
103
  # Capture local variables NOW (TracePoint attaches to exception, must extract before job dispatch)
56
- if RailsErrorDashboard.configuration.enable_local_variables
104
+ if !lite && RailsErrorDashboard.configuration.enable_local_variables
57
105
  begin
58
106
  raw_locals = Services::LocalVariableCapturer.extract(exception)
59
107
  if raw_locals.is_a?(Hash) && raw_locals.any?
@@ -65,7 +113,7 @@ module RailsErrorDashboard
65
113
  end
66
114
 
67
115
  # Capture instance variables NOW (same reason — attached to exception object)
68
- if RailsErrorDashboard.configuration.enable_instance_variables
116
+ if !lite && RailsErrorDashboard.configuration.enable_instance_variables
69
117
  begin
70
118
  raw_ivars = Services::LocalVariableCapturer.extract_instance_vars(exception)
71
119
  if raw_ivars.is_a?(Hash) && raw_ivars.any?
@@ -157,12 +205,19 @@ module RailsErrorDashboard
157
205
  kind: :capture,
158
206
  attributes: self.class.build_capture_span_attributes(@exception, was_async: false)
159
207
  ) do |span|
160
- # Check if this exception should be logged (ignore list + sampling)
161
- if !Services::ExceptionFilter.should_log?(@exception)
208
+ # Check if this exception should be logged (ignore list + sampling).
209
+ # Skipped when self.call already filtered (re-rolling the sampling
210
+ # dice here would square the effective rate).
211
+ if !@context[:_pre_filtered] && !Services::ExceptionFilter.should_log?(@exception)
162
212
  span&.set_attribute("rails_error_dashboard.filtered", true)
163
213
  next nil
164
214
  end
165
215
 
216
+ # Storm shedding: :lite captures keep the error + occurrence row but
217
+ # shed context payloads (breadcrumbs/health/locals/ivars).
218
+ storm_lite = self.class.storm_lite?(@context)
219
+ span&.set_attribute("rails_error_dashboard.storm_degraded", true) if storm_lite
220
+
166
221
  error_context = ValueObjects::ErrorContext.new(@context, @context[:source])
167
222
 
168
223
  # Find or create application (cached lookup)
@@ -239,7 +294,7 @@ module RailsErrorDashboard
239
294
  attributes = Services::SensitiveDataFilter.filter_attributes(attributes)
240
295
 
241
296
  # Harvest breadcrumbs (if enabled and column exists)
242
- if ErrorLog.column_names.include?("breadcrumbs") && RailsErrorDashboard.configuration.enable_breadcrumbs
297
+ if !storm_lite && ErrorLog.column_names.include?("breadcrumbs") && RailsErrorDashboard.configuration.enable_breadcrumbs
243
298
  # Sync path: harvest from current thread
244
299
  raw_breadcrumbs = Services::BreadcrumbCollector.harvest
245
300
 
@@ -256,13 +311,13 @@ module RailsErrorDashboard
256
311
  end
257
312
 
258
313
  # Capture system health snapshot (if enabled and column exists)
259
- if ErrorLog.column_names.include?("system_health") && RailsErrorDashboard.configuration.enable_system_health
314
+ if !storm_lite && ErrorLog.column_names.include?("system_health") && RailsErrorDashboard.configuration.enable_system_health
260
315
  health_data = @context[:_serialized_system_health] || Services::SystemHealthSnapshot.capture
261
316
  attributes[:system_health] = health_data.to_json
262
317
  end
263
318
 
264
319
  # Capture local variables (if enabled and column exists)
265
- if ErrorLog.column_names.include?("local_variables") && RailsErrorDashboard.configuration.enable_local_variables
320
+ if !storm_lite && ErrorLog.column_names.include?("local_variables") && RailsErrorDashboard.configuration.enable_local_variables
266
321
  begin
267
322
  # Sync path: extract from exception ivar
268
323
  raw_locals = Services::LocalVariableCapturer.extract(@exception)
@@ -278,7 +333,7 @@ module RailsErrorDashboard
278
333
  end
279
334
 
280
335
  # Capture instance variables (if enabled and column exists)
281
- if ErrorLog.column_names.include?("instance_variables") && RailsErrorDashboard.configuration.enable_instance_variables
336
+ if !storm_lite && ErrorLog.column_names.include?("instance_variables") && RailsErrorDashboard.configuration.enable_instance_variables
282
337
  begin
283
338
  # Sync path: extract from exception ivar
284
339
  raw_ivars = Services::LocalVariableCapturer.extract_instance_vars(@exception)
@@ -364,8 +419,11 @@ module RailsErrorDashboard
364
419
 
365
420
  # Dispatch notification if error is not muted and the throttle check passes.
366
421
  # Muted errors skip notifications but still fire plugin events/callbacks.
422
+ # During a storm (breaker not closed) per-error notifications are
423
+ # suppressed — a single storm notification replaces them.
367
424
  def maybe_notify(error_log)
368
425
  return if error_log.muted?
426
+ return if Services::StormProtection::Gate.notifications_suppressed?
369
427
  return unless yield
370
428
 
371
429
  Services::ErrorNotificationDispatcher.call(error_log)
@@ -67,6 +67,25 @@ module RailsErrorDashboard
67
67
  # Sampling rate for non-critical errors (0.0 to 1.0, default 1.0 = 100%)
68
68
  attr_accessor :sampling_rate
69
69
 
70
+ # Storm protection — circuit breaker + adaptive sampling for error floods.
71
+ # Protects the HOST APP from the gem's own writes during an error storm
72
+ # (bad deploy throwing thousands of errors/minute). Default ON: this is
73
+ # the feature that makes the gem quieter, so ON is the conservative choice.
74
+ # All thresholds are PER PROCESS (no cross-process coordination by design).
75
+ attr_accessor :enable_storm_protection # Master switch (default: true)
76
+ attr_accessor :storm_fingerprint_full_per_minute # Full-fidelity captures per fingerprint per minute (default: 30)
77
+ attr_accessor :storm_occurrence_sample_keep_every # Past the cap, keep every Nth occurrence (default: 10)
78
+ attr_accessor :storm_shedding_threshold_per_second # Global rate that enters shedding state (default: 10)
79
+ attr_accessor :storm_open_threshold_per_second # Global rate that opens the breaker = count-only (default: 50)
80
+ attr_accessor :storm_cooldown_seconds # Open → half-open probe delay (default: 60)
81
+ attr_accessor :storm_max_tracked_fingerprints # Bounded in-memory map size; beyond = overflow bucket (default: 1000)
82
+ attr_accessor :storm_flush_interval_seconds # Count-buffer flush cadence (default: 30)
83
+ attr_accessor :storm_notification # Single "storm in progress" notification per episode (default: true)
84
+ attr_accessor :auto_issue_rate_limit_count # Max auto-created issues per window — applies always (default: 5)
85
+ attr_accessor :auto_issue_rate_limit_window_minutes # Window for the above (default: 10)
86
+ attr_accessor :context_sampling_threshold_per_day # Full-context captures per fingerprint per day before sampling (default: 25)
87
+ attr_accessor :context_sampling_keep_every # After threshold, keep full context every Nth (default: 10)
88
+
70
89
  # Async logging configuration
71
90
  attr_accessor :async_logging
72
91
  attr_accessor :async_adapter # :sidekiq, :solid_queue, or :async
@@ -176,8 +195,11 @@ module RailsErrorDashboard
176
195
  # Code path coverage (diagnostic mode — Ruby 3.2+)
177
196
  attr_accessor :enable_coverage_tracking # Master switch (default: false)
178
197
 
179
- # Rack Attack event tracking (requires enable_breadcrumbs = true)
198
+ # Rack Attack event tracking persists throttle/blocklist/track events to
199
+ # their own table, independent of error capture (breadcrumbs optional).
180
200
  attr_accessor :enable_rack_attack_tracking # Master switch (default: false)
201
+ attr_accessor :rack_attack_max_cache_size # Max buffered keys per thread (default: 1000)
202
+ attr_accessor :rack_attack_flush_interval # Seconds between DB flushes (default: 60)
181
203
 
182
204
  # ActionCable event tracking (requires enable_breadcrumbs = true)
183
205
  attr_accessor :enable_actioncable_tracking # Master switch (default: false)
@@ -268,6 +290,21 @@ module RailsErrorDashboard
268
290
  @ignored_exceptions = []
269
291
  @custom_fingerprint = nil # Lambda: ->(exception, context) { "custom_key" }
270
292
  @sampling_rate = 1.0 # 100% by default
293
+
294
+ # Storm protection defaults (thresholds tuned via chaos Phase G — see ROADMAP)
295
+ @enable_storm_protection = true
296
+ @storm_fingerprint_full_per_minute = 30
297
+ @storm_occurrence_sample_keep_every = 10
298
+ @storm_shedding_threshold_per_second = 10
299
+ @storm_open_threshold_per_second = 50
300
+ @storm_cooldown_seconds = 60
301
+ @storm_max_tracked_fingerprints = 1000
302
+ @storm_flush_interval_seconds = 30
303
+ @storm_notification = true
304
+ @auto_issue_rate_limit_count = 5
305
+ @auto_issue_rate_limit_window_minutes = 10
306
+ @context_sampling_threshold_per_day = 25
307
+ @context_sampling_keep_every = 10
271
308
  @async_logging = false
272
309
  @async_adapter = :sidekiq # Battle-tested default
273
310
  @max_backtrace_lines = 100 # Matches industry standard (Rollbar, Airbrake)
@@ -366,8 +403,11 @@ module RailsErrorDashboard
366
403
  # Code path coverage defaults - OFF by default (opt-in, Ruby 3.2+)
367
404
  @enable_coverage_tracking = false
368
405
 
369
- # Rack Attack event tracking defaults - OFF by default (opt-in, requires breadcrumbs)
406
+ # Rack Attack event tracking defaults - OFF by default (opt-in).
407
+ # Persists to its own table; does NOT require breadcrumbs.
370
408
  @enable_rack_attack_tracking = false
409
+ @rack_attack_max_cache_size = 1000 # Max buffered keys per thread (LRU eviction)
410
+ @rack_attack_flush_interval = 60 # Seconds between DB flushes
371
411
 
372
412
  # ActionCable event tracking defaults - OFF by default (opt-in, requires breadcrumbs)
373
413
  @enable_actioncable_tracking = false
@@ -540,11 +580,16 @@ module RailsErrorDashboard
540
580
  end
541
581
  end
542
582
 
543
- # Validate rack_attack tracking requires breadcrumbs
544
- if enable_rack_attack_tracking && !enable_breadcrumbs
545
- warnings << "enable_rack_attack_tracking requires enable_breadcrumbs = true. " \
546
- "Rack Attack tracking has been auto-disabled."
547
- @enable_rack_attack_tracking = false
583
+ # Rack Attack tracking no longer requires breadcrumbs — events are persisted
584
+ # to their own table (issue #143). Breadcrumbs only add the event to the
585
+ # activity trail on error detail pages.
586
+ if enable_rack_attack_tracking
587
+ if rack_attack_max_cache_size && rack_attack_max_cache_size < 1
588
+ errors << "rack_attack_max_cache_size must be at least 1 (got: #{rack_attack_max_cache_size})"
589
+ end
590
+ if rack_attack_flush_interval && rack_attack_flush_interval < 1
591
+ errors << "rack_attack_flush_interval must be at least 1 (got: #{rack_attack_flush_interval})"
592
+ end
548
593
  end
549
594
 
550
595
  # Validate actioncable tracking requires breadcrumbs
@@ -683,6 +728,32 @@ module RailsErrorDashboard
683
728
  end
684
729
  end
685
730
 
731
+ # Validate storm protection thresholds (all must be positive when protection is on)
732
+ if enable_storm_protection
733
+ {
734
+ storm_fingerprint_full_per_minute: storm_fingerprint_full_per_minute,
735
+ storm_occurrence_sample_keep_every: storm_occurrence_sample_keep_every,
736
+ storm_shedding_threshold_per_second: storm_shedding_threshold_per_second,
737
+ storm_open_threshold_per_second: storm_open_threshold_per_second,
738
+ storm_cooldown_seconds: storm_cooldown_seconds,
739
+ storm_max_tracked_fingerprints: storm_max_tracked_fingerprints,
740
+ storm_flush_interval_seconds: storm_flush_interval_seconds,
741
+ auto_issue_rate_limit_count: auto_issue_rate_limit_count,
742
+ auto_issue_rate_limit_window_minutes: auto_issue_rate_limit_window_minutes,
743
+ context_sampling_threshold_per_day: context_sampling_threshold_per_day,
744
+ context_sampling_keep_every: context_sampling_keep_every
745
+ }.each do |name, value|
746
+ if value.nil? || value.to_i < 1
747
+ errors << "#{name} must be a positive integer (got: #{value.inspect})"
748
+ end
749
+ end
750
+
751
+ if storm_open_threshold_per_second.to_i < storm_shedding_threshold_per_second.to_i
752
+ errors << "storm_open_threshold_per_second (#{storm_open_threshold_per_second}) must be >= " \
753
+ "storm_shedding_threshold_per_second (#{storm_shedding_threshold_per_second})"
754
+ end
755
+ end
756
+
686
757
  # Validate total_users_for_impact (must be positive if set)
687
758
  if total_users_for_impact && total_users_for_impact < 1
688
759
  errors << "total_users_for_impact must be at least 1 (got: #{total_users_for_impact})"
@@ -80,9 +80,9 @@ module RailsErrorDashboard
80
80
  RailsErrorDashboard::Subscribers::BreadcrumbSubscriber.subscribe!
81
81
  end
82
82
 
83
- # Subscribe to Rack Attack AS::Notifications events (requires breadcrumbs + Rack::Attack)
83
+ # Subscribe to Rack Attack AS::Notifications events (requires Rack::Attack).
84
+ # Breadcrumbs are NOT required — events persist to their own table (issue #143).
84
85
  if RailsErrorDashboard.configuration.enable_rack_attack_tracking &&
85
- RailsErrorDashboard.configuration.enable_breadcrumbs &&
86
86
  defined?(Rack::Attack)
87
87
  RailsErrorDashboard::Subscribers::RackAttackSubscriber.subscribe!
88
88
  end