rails_error_dashboard 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +34 -1
- data/app/controllers/rails_error_dashboard/application_controller.rb +5 -0
- data/app/controllers/rails_error_dashboard/errors_controller.rb +12 -0
- data/app/controllers/rails_error_dashboard/webhooks_controller.rb +35 -1
- data/app/jobs/rails_error_dashboard/storm_flush_job.rb +19 -0
- data/app/jobs/rails_error_dashboard/storm_notification_job.rb +74 -0
- data/app/models/rails_error_dashboard/storm_event.rb +34 -0
- data/app/views/layouts/rails_error_dashboard.html.erb +21 -0
- data/app/views/rails_error_dashboard/errors/_issue_section.html.erb +1 -0
- data/app/views/rails_error_dashboard/errors/storms.html.erb +91 -0
- data/config/routes.rb +1 -0
- data/db/migrate/20260306000002_add_instance_variables_to_error_logs.rb +7 -1
- data/db/migrate/20260306000003_create_rails_error_dashboard_swallowed_exceptions.rb +4 -0
- data/db/migrate/20260307000001_create_rails_error_dashboard_diagnostic_dumps.rb +4 -0
- data/db/migrate/20260613000001_create_storm_events.rb +28 -0
- data/lib/generators/rails_error_dashboard/install/templates/initializer.rb +49 -2
- data/lib/rails_error_dashboard/commands/create_issue.rb +1 -1
- data/lib/rails_error_dashboard/commands/flush_storm_counts.rb +188 -0
- data/lib/rails_error_dashboard/commands/link_existing_issue.rb +3 -1
- data/lib/rails_error_dashboard/commands/log_error.rb +70 -12
- data/lib/rails_error_dashboard/configuration.rb +73 -7
- data/lib/rails_error_dashboard/queries/storm_history.rb +39 -0
- data/lib/rails_error_dashboard/services/issue_tracker_client.rb +8 -5
- data/lib/rails_error_dashboard/services/linear_issue_client.rb +248 -0
- data/lib/rails_error_dashboard/services/storm_protection/circuit_breaker.rb +195 -0
- data/lib/rails_error_dashboard/services/storm_protection/count_buffer.rb +100 -0
- data/lib/rails_error_dashboard/services/storm_protection/fingerprint_buckets.rb +123 -0
- data/lib/rails_error_dashboard/services/storm_protection/gate.rb +258 -0
- data/lib/rails_error_dashboard/subscribers/issue_tracker_subscriber.rb +12 -0
- data/lib/rails_error_dashboard/version.rb +1 -1
- data/lib/rails_error_dashboard.rb +7 -0
- metadata +15 -3
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
module Commands
|
|
5
|
+
# Command: Reconcile counted-not-stored storm events onto ErrorLog rows
|
|
6
|
+
# and maintain the storm_events episode record.
|
|
7
|
+
#
|
|
8
|
+
# Runs in a background job (DB allowed). For each counted fingerprint:
|
|
9
|
+
# 1. Recompute the canonical error_hash from the stored identity parts
|
|
10
|
+
# (the gate's key deliberately omits application_id — resolved here)
|
|
11
|
+
# 2. Unresolved match → single UPDATE: occurrence_count += N
|
|
12
|
+
# 3. Resolved match → reopen (mirrors FindOrIncrementError semantics)
|
|
13
|
+
# 4. No match → create a minimal ErrorLog from the exemplar
|
|
14
|
+
#
|
|
15
|
+
# Counts are exact. Notifications are NOT dispatched from here — during a
|
|
16
|
+
# storm they're suppressed by design; the storm notification covers it.
|
|
17
|
+
class FlushStormCounts
|
|
18
|
+
def self.call(entries:, overflow: 0, episode: nil)
|
|
19
|
+
new(entries: entries, overflow: overflow, episode: episode).call
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def initialize(entries:, overflow: 0, episode: nil)
|
|
23
|
+
@entries = Array(entries)
|
|
24
|
+
@overflow = overflow.to_i
|
|
25
|
+
@episode = episode
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def call
|
|
29
|
+
application = resolve_application
|
|
30
|
+
counted = 0
|
|
31
|
+
|
|
32
|
+
@entries.each do |entry|
|
|
33
|
+
entry = entry.with_indifferent_access if entry.respond_to?(:with_indifferent_access)
|
|
34
|
+
counted += reconcile_entry(entry, application)
|
|
35
|
+
rescue => e
|
|
36
|
+
# A corrupt (non-Hash) entry must not abort the whole batch — and the
|
|
37
|
+
# log line itself must not assume `entry` is subscriptable (an Integer
|
|
38
|
+
# from a broken serializer would raise again here, escaping this rescue).
|
|
39
|
+
error_class = entry.is_a?(Hash) ? entry["error_class"] : entry.class
|
|
40
|
+
RailsErrorDashboard::Logger.error(
|
|
41
|
+
"[RailsErrorDashboard] Storm count reconcile failed for #{error_class}: #{e.class} - #{e.message}"
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
upsert_storm_event(counted)
|
|
46
|
+
{ success: true, reconciled: counted, overflow: @overflow }
|
|
47
|
+
rescue => e
|
|
48
|
+
RailsErrorDashboard::Logger.error(
|
|
49
|
+
"[RailsErrorDashboard] FlushStormCounts failed: #{e.class} - #{e.message}"
|
|
50
|
+
)
|
|
51
|
+
{ success: false, error: "#{e.class}: #{e.message}" }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def reconcile_entry(entry, application)
|
|
57
|
+
count = entry["count"].to_i
|
|
58
|
+
return 0 if count <= 0
|
|
59
|
+
|
|
60
|
+
error_hash = canonical_hash(entry, application)
|
|
61
|
+
last_seen = parse_time(entry["last_seen_at"]) || Time.current
|
|
62
|
+
|
|
63
|
+
# Priority 1: unresolved match — one UPDATE, no row instantiation
|
|
64
|
+
updated = ErrorLog.unresolved
|
|
65
|
+
.where(error_hash: error_hash, application_id: application.id)
|
|
66
|
+
.update_all([ "occurrence_count = occurrence_count + ?, last_seen_at = ?", count, last_seen ])
|
|
67
|
+
return count if updated.positive?
|
|
68
|
+
|
|
69
|
+
# Priority 2: resolved/wont_fix match — reopen, mirroring
|
|
70
|
+
# FindOrIncrementError so storm recurrences don't stay buried
|
|
71
|
+
resolved = ErrorLog
|
|
72
|
+
.where(error_hash: error_hash, application_id: application.id)
|
|
73
|
+
.where(status: %w[resolved wont_fix])
|
|
74
|
+
.order(last_seen_at: :desc)
|
|
75
|
+
.first
|
|
76
|
+
if resolved
|
|
77
|
+
attrs = {
|
|
78
|
+
resolved: false,
|
|
79
|
+
status: "new",
|
|
80
|
+
resolved_at: nil,
|
|
81
|
+
occurrence_count: resolved.occurrence_count + count,
|
|
82
|
+
last_seen_at: last_seen
|
|
83
|
+
}
|
|
84
|
+
attrs[:reopened_at] = Time.current if ErrorLog.column_names.include?("reopened_at")
|
|
85
|
+
resolved.update!(attrs)
|
|
86
|
+
return count
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Priority 3: first seen during count-only mode — minimal ErrorLog
|
|
90
|
+
# from the exemplar (no backtrace/context was captured; the next
|
|
91
|
+
# occurrence after the storm fills in detail via the normal path)
|
|
92
|
+
ErrorLog.create!(
|
|
93
|
+
application_id: application.id,
|
|
94
|
+
error_type: entry["error_class"],
|
|
95
|
+
message: entry["message"],
|
|
96
|
+
backtrace: entry["first_app_frame"],
|
|
97
|
+
controller_name: entry["controller_name"],
|
|
98
|
+
action_name: entry["action_name"],
|
|
99
|
+
occurred_at: parse_time(entry["first_seen_at"]) || Time.current,
|
|
100
|
+
last_seen_at: last_seen,
|
|
101
|
+
occurrence_count: count,
|
|
102
|
+
error_hash: error_hash,
|
|
103
|
+
resolved: false
|
|
104
|
+
)
|
|
105
|
+
count
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Mirrors ErrorHashGenerator.call exactly: same fields, same order,
|
|
109
|
+
# same normalization — so counts land on the same ErrorLog the full
|
|
110
|
+
# capture path would have used.
|
|
111
|
+
def canonical_hash(entry, application)
|
|
112
|
+
return entry["custom_hash"] if entry["custom_hash"].present?
|
|
113
|
+
|
|
114
|
+
digest_input = [
|
|
115
|
+
entry["error_class"],
|
|
116
|
+
Services::ErrorHashGenerator.normalize_message(entry["message"]),
|
|
117
|
+
entry["first_app_frame"],
|
|
118
|
+
entry["controller_name"],
|
|
119
|
+
entry["action_name"],
|
|
120
|
+
application.id.to_s
|
|
121
|
+
].compact.join("|")
|
|
122
|
+
|
|
123
|
+
Digest::SHA256.hexdigest(digest_input)[0..15]
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def resolve_application
|
|
127
|
+
# Same chain LogError uses — app name is process-global
|
|
128
|
+
app_name = RailsErrorDashboard.configuration.application_name ||
|
|
129
|
+
ENV["APPLICATION_NAME"] ||
|
|
130
|
+
(defined?(Rails) && Rails.application.class.module_parent_name) ||
|
|
131
|
+
"Rails Application"
|
|
132
|
+
Application.find_or_create_by_name(app_name)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def upsert_storm_event(counted)
|
|
136
|
+
return unless @episode.is_a?(Hash)
|
|
137
|
+
return unless StormEvent.table_exists?
|
|
138
|
+
|
|
139
|
+
started_at = parse_time(@episode["started_at"])
|
|
140
|
+
return unless started_at
|
|
141
|
+
|
|
142
|
+
event = StormEvent.active.recent_first.first || StormEvent.create!(started_at: started_at)
|
|
143
|
+
|
|
144
|
+
event.events_counted_only = event.events_counted_only.to_i + counted
|
|
145
|
+
event.events_overflow = event.events_overflow.to_i + @overflow
|
|
146
|
+
# events_total is the count-only total: in-map reconciled + overflow.
|
|
147
|
+
# It deliberately excludes :lite/:full admissions (those became real
|
|
148
|
+
# ErrorLog rows on the hot path and are never counted here), so it is
|
|
149
|
+
# always events_counted_only + events_overflow. Derive it rather than
|
|
150
|
+
# accumulate so it can't drift from its two components.
|
|
151
|
+
event.events_total = event.events_counted_only.to_i + event.events_overflow.to_i
|
|
152
|
+
event.fingerprints_affected = [ event.fingerprints_affected.to_i, @entries.size ].max
|
|
153
|
+
event.peak_rate_per_minute = [ event.peak_rate_per_minute.to_i, @episode["peak_rate_per_minute"].to_i ].max
|
|
154
|
+
event.reached_open ||= @episode["reached_open"] == true
|
|
155
|
+
event.top_fingerprints = top_fingerprints_json(event)
|
|
156
|
+
event.ended_at = parse_time(@episode["ended_at"]) if @episode["ended_at"]
|
|
157
|
+
event.save!
|
|
158
|
+
rescue => e
|
|
159
|
+
RailsErrorDashboard::Logger.error(
|
|
160
|
+
"[RailsErrorDashboard] Storm event upsert failed: #{e.class} - #{e.message}"
|
|
161
|
+
)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def top_fingerprints_json(event)
|
|
165
|
+
existing = event.top_fingerprints_list
|
|
166
|
+
fresh = @entries.map { |e|
|
|
167
|
+
e = e.with_indifferent_access if e.respond_to?(:with_indifferent_access)
|
|
168
|
+
{ "class" => e["error_class"], "message" => e["message"].to_s[0, 120], "count" => e["count"].to_i }
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
merged = (existing + fresh)
|
|
172
|
+
.group_by { |f| [ f["class"], f["message"] ] }
|
|
173
|
+
.map { |_k, group| group.first.merge("count" => group.sum { |f| f["count"].to_i }) }
|
|
174
|
+
|
|
175
|
+
merged.sort_by { |f| -f["count"].to_i }.first(5).to_json
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def parse_time(value)
|
|
179
|
+
return value if value.is_a?(Time) || value.is_a?(ActiveSupport::TimeWithZone)
|
|
180
|
+
return nil if value.blank?
|
|
181
|
+
|
|
182
|
+
Time.zone.parse(value.to_s)
|
|
183
|
+
rescue ArgumentError
|
|
184
|
+
nil
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
@@ -14,7 +14,9 @@ module RailsErrorDashboard
|
|
|
14
14
|
PROVIDER_PATTERNS = {
|
|
15
15
|
github: %r{github\.com/([^/]+/[^/]+)/issues/(\d+)}i,
|
|
16
16
|
gitlab: %r{gitlab\.com/([^/]+/[^/]+)/-/issues/(\d+)}i,
|
|
17
|
-
codeberg: %r{codeberg\.org/([^/]+/[^/]+)/issues/(\d+)}i
|
|
17
|
+
codeberg: %r{codeberg\.org/([^/]+/[^/]+)/issues/(\d+)}i,
|
|
18
|
+
# https://linear.app/<workspace>/issue/ENG-123/<slug> — capture team key + number
|
|
19
|
+
linear: %r{linear\.app/[^/]+/issue/([A-Za-z][A-Za-z0-9]*)-(\d+)}i
|
|
18
20
|
}.freeze
|
|
19
21
|
|
|
20
22
|
def self.call(error_id, issue_url:)
|
|
@@ -6,15 +6,59 @@ module RailsErrorDashboard
|
|
|
6
6
|
# This is a write operation that creates an ErrorLog record
|
|
7
7
|
class LogError
|
|
8
8
|
def self.call(exception, context = {})
|
|
9
|
-
#
|
|
9
|
+
# Filter FIRST (ignore list + static sampling) so ignored exceptions
|
|
10
|
+
# never count toward storm state. _pre_filtered prevents the sync path
|
|
11
|
+
# from re-rolling the sampling dice (rate would square otherwise).
|
|
12
|
+
# The filter + gate run inside this method's rescue: nothing in the
|
|
13
|
+
# capture path may ever raise into the host app.
|
|
14
|
+
begin
|
|
15
|
+
unless Services::ExceptionFilter.should_log?(exception)
|
|
16
|
+
# Preserve the OTel contract: filtered captures still emit a span
|
|
17
|
+
# tagged filtered=true (no-op when OTel export is disabled).
|
|
18
|
+
Integrations::Tracer.in_span(
|
|
19
|
+
"capture_error",
|
|
20
|
+
kind: :capture,
|
|
21
|
+
attributes: build_capture_span_attributes(exception, was_async: false)
|
|
22
|
+
) do |span|
|
|
23
|
+
span&.set_attribute("rails_error_dashboard.filtered", true)
|
|
24
|
+
end
|
|
25
|
+
return nil
|
|
26
|
+
end
|
|
27
|
+
context = context.merge(_pre_filtered: true)
|
|
28
|
+
|
|
29
|
+
# Storm protection gate — BEFORE the async branch, because with
|
|
30
|
+
# SolidQueue the enqueue itself is a DB write. :count_only events are
|
|
31
|
+
# tallied in memory and reconciled by StormFlushJob; nothing else
|
|
32
|
+
# happens for them (that's the point).
|
|
33
|
+
storm_decision = Services::StormProtection::Gate.admit!(exception, context)
|
|
34
|
+
return nil if storm_decision == :count_only
|
|
35
|
+
context = context.merge(_storm_decision: storm_decision) if storm_decision == :lite
|
|
36
|
+
rescue => e
|
|
37
|
+
RailsErrorDashboard::Logger.error(
|
|
38
|
+
"[RailsErrorDashboard] Capture pre-checks failed: #{e.class} - #{e.message}"
|
|
39
|
+
)
|
|
40
|
+
# Fall through and attempt full capture — fail open, never raise
|
|
41
|
+
end
|
|
42
|
+
|
|
10
43
|
if RailsErrorDashboard.configuration.async_logging
|
|
11
44
|
# For async logging, just enqueue the job
|
|
12
|
-
# All filtering happens when the job runs
|
|
13
45
|
call_async(exception, context)
|
|
14
46
|
else
|
|
15
47
|
# For sync logging, execute immediately
|
|
16
48
|
new(exception, context).call
|
|
17
49
|
end
|
|
50
|
+
rescue => e
|
|
51
|
+
RailsErrorDashboard::Logger.error(
|
|
52
|
+
"[RailsErrorDashboard] LogError.call failed: #{e.class} - #{e.message}"
|
|
53
|
+
)
|
|
54
|
+
nil
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# :lite captures shed context (breadcrumbs/health/locals/ivars) — the
|
|
58
|
+
# storm shedding ladder's first economy. Symbol or string key: the
|
|
59
|
+
# async job round-trips context through the queue serializer.
|
|
60
|
+
def self.storm_lite?(context)
|
|
61
|
+
context[:_storm_decision].to_s == "lite"
|
|
18
62
|
end
|
|
19
63
|
|
|
20
64
|
# Build the base OTel span attributes available before any work happens.
|
|
@@ -42,18 +86,22 @@ module RailsErrorDashboard
|
|
|
42
86
|
cause_chain: serialize_cause_chain(exception)
|
|
43
87
|
}
|
|
44
88
|
|
|
89
|
+
# Storm shedding: :lite captures skip ALL pre-enqueue context harvest —
|
|
90
|
+
# this is request-thread CPU, the most valuable thing to shed.
|
|
91
|
+
lite = storm_lite?(context)
|
|
92
|
+
|
|
45
93
|
# Harvest breadcrumbs NOW (before job dispatch — different thread won't have them)
|
|
46
|
-
if RailsErrorDashboard.configuration.enable_breadcrumbs
|
|
94
|
+
if !lite && RailsErrorDashboard.configuration.enable_breadcrumbs
|
|
47
95
|
context = context.merge(_serialized_breadcrumbs: Services::BreadcrumbCollector.harvest)
|
|
48
96
|
end
|
|
49
97
|
|
|
50
98
|
# Capture system health NOW (metrics are time-sensitive, different thread = different state)
|
|
51
|
-
if RailsErrorDashboard.configuration.enable_system_health
|
|
99
|
+
if !lite && RailsErrorDashboard.configuration.enable_system_health
|
|
52
100
|
context = context.merge(_serialized_system_health: Services::SystemHealthSnapshot.capture)
|
|
53
101
|
end
|
|
54
102
|
|
|
55
103
|
# Capture local variables NOW (TracePoint attaches to exception, must extract before job dispatch)
|
|
56
|
-
if RailsErrorDashboard.configuration.enable_local_variables
|
|
104
|
+
if !lite && RailsErrorDashboard.configuration.enable_local_variables
|
|
57
105
|
begin
|
|
58
106
|
raw_locals = Services::LocalVariableCapturer.extract(exception)
|
|
59
107
|
if raw_locals.is_a?(Hash) && raw_locals.any?
|
|
@@ -65,7 +113,7 @@ module RailsErrorDashboard
|
|
|
65
113
|
end
|
|
66
114
|
|
|
67
115
|
# Capture instance variables NOW (same reason — attached to exception object)
|
|
68
|
-
if RailsErrorDashboard.configuration.enable_instance_variables
|
|
116
|
+
if !lite && RailsErrorDashboard.configuration.enable_instance_variables
|
|
69
117
|
begin
|
|
70
118
|
raw_ivars = Services::LocalVariableCapturer.extract_instance_vars(exception)
|
|
71
119
|
if raw_ivars.is_a?(Hash) && raw_ivars.any?
|
|
@@ -157,12 +205,19 @@ module RailsErrorDashboard
|
|
|
157
205
|
kind: :capture,
|
|
158
206
|
attributes: self.class.build_capture_span_attributes(@exception, was_async: false)
|
|
159
207
|
) do |span|
|
|
160
|
-
# Check if this exception should be logged (ignore list + sampling)
|
|
161
|
-
|
|
208
|
+
# Check if this exception should be logged (ignore list + sampling).
|
|
209
|
+
# Skipped when self.call already filtered (re-rolling the sampling
|
|
210
|
+
# dice here would square the effective rate).
|
|
211
|
+
if !@context[:_pre_filtered] && !Services::ExceptionFilter.should_log?(@exception)
|
|
162
212
|
span&.set_attribute("rails_error_dashboard.filtered", true)
|
|
163
213
|
next nil
|
|
164
214
|
end
|
|
165
215
|
|
|
216
|
+
# Storm shedding: :lite captures keep the error + occurrence row but
|
|
217
|
+
# shed context payloads (breadcrumbs/health/locals/ivars).
|
|
218
|
+
storm_lite = self.class.storm_lite?(@context)
|
|
219
|
+
span&.set_attribute("rails_error_dashboard.storm_degraded", true) if storm_lite
|
|
220
|
+
|
|
166
221
|
error_context = ValueObjects::ErrorContext.new(@context, @context[:source])
|
|
167
222
|
|
|
168
223
|
# Find or create application (cached lookup)
|
|
@@ -239,7 +294,7 @@ module RailsErrorDashboard
|
|
|
239
294
|
attributes = Services::SensitiveDataFilter.filter_attributes(attributes)
|
|
240
295
|
|
|
241
296
|
# Harvest breadcrumbs (if enabled and column exists)
|
|
242
|
-
if ErrorLog.column_names.include?("breadcrumbs") && RailsErrorDashboard.configuration.enable_breadcrumbs
|
|
297
|
+
if !storm_lite && ErrorLog.column_names.include?("breadcrumbs") && RailsErrorDashboard.configuration.enable_breadcrumbs
|
|
243
298
|
# Sync path: harvest from current thread
|
|
244
299
|
raw_breadcrumbs = Services::BreadcrumbCollector.harvest
|
|
245
300
|
|
|
@@ -256,13 +311,13 @@ module RailsErrorDashboard
|
|
|
256
311
|
end
|
|
257
312
|
|
|
258
313
|
# Capture system health snapshot (if enabled and column exists)
|
|
259
|
-
if ErrorLog.column_names.include?("system_health") && RailsErrorDashboard.configuration.enable_system_health
|
|
314
|
+
if !storm_lite && ErrorLog.column_names.include?("system_health") && RailsErrorDashboard.configuration.enable_system_health
|
|
260
315
|
health_data = @context[:_serialized_system_health] || Services::SystemHealthSnapshot.capture
|
|
261
316
|
attributes[:system_health] = health_data.to_json
|
|
262
317
|
end
|
|
263
318
|
|
|
264
319
|
# Capture local variables (if enabled and column exists)
|
|
265
|
-
if ErrorLog.column_names.include?("local_variables") && RailsErrorDashboard.configuration.enable_local_variables
|
|
320
|
+
if !storm_lite && ErrorLog.column_names.include?("local_variables") && RailsErrorDashboard.configuration.enable_local_variables
|
|
266
321
|
begin
|
|
267
322
|
# Sync path: extract from exception ivar
|
|
268
323
|
raw_locals = Services::LocalVariableCapturer.extract(@exception)
|
|
@@ -278,7 +333,7 @@ module RailsErrorDashboard
|
|
|
278
333
|
end
|
|
279
334
|
|
|
280
335
|
# Capture instance variables (if enabled and column exists)
|
|
281
|
-
if ErrorLog.column_names.include?("instance_variables") && RailsErrorDashboard.configuration.enable_instance_variables
|
|
336
|
+
if !storm_lite && ErrorLog.column_names.include?("instance_variables") && RailsErrorDashboard.configuration.enable_instance_variables
|
|
282
337
|
begin
|
|
283
338
|
# Sync path: extract from exception ivar
|
|
284
339
|
raw_ivars = Services::LocalVariableCapturer.extract_instance_vars(@exception)
|
|
@@ -364,8 +419,11 @@ module RailsErrorDashboard
|
|
|
364
419
|
|
|
365
420
|
# Dispatch notification if error is not muted and the throttle check passes.
|
|
366
421
|
# Muted errors skip notifications but still fire plugin events/callbacks.
|
|
422
|
+
# During a storm (breaker not closed) per-error notifications are
|
|
423
|
+
# suppressed — a single storm notification replaces them.
|
|
367
424
|
def maybe_notify(error_log)
|
|
368
425
|
return if error_log.muted?
|
|
426
|
+
return if Services::StormProtection::Gate.notifications_suppressed?
|
|
369
427
|
return unless yield
|
|
370
428
|
|
|
371
429
|
Services::ErrorNotificationDispatcher.call(error_log)
|
|
@@ -67,6 +67,25 @@ module RailsErrorDashboard
|
|
|
67
67
|
# Sampling rate for non-critical errors (0.0 to 1.0, default 1.0 = 100%)
|
|
68
68
|
attr_accessor :sampling_rate
|
|
69
69
|
|
|
70
|
+
# Storm protection — circuit breaker + adaptive sampling for error floods.
|
|
71
|
+
# Protects the HOST APP from the gem's own writes during an error storm
|
|
72
|
+
# (bad deploy throwing thousands of errors/minute). Default ON: this is
|
|
73
|
+
# the feature that makes the gem quieter, so ON is the conservative choice.
|
|
74
|
+
# All thresholds are PER PROCESS (no cross-process coordination by design).
|
|
75
|
+
attr_accessor :enable_storm_protection # Master switch (default: true)
|
|
76
|
+
attr_accessor :storm_fingerprint_full_per_minute # Full-fidelity captures per fingerprint per minute (default: 30)
|
|
77
|
+
attr_accessor :storm_occurrence_sample_keep_every # Past the cap, keep every Nth occurrence (default: 10)
|
|
78
|
+
attr_accessor :storm_shedding_threshold_per_second # Global rate that enters shedding state (default: 10)
|
|
79
|
+
attr_accessor :storm_open_threshold_per_second # Global rate that opens the breaker = count-only (default: 50)
|
|
80
|
+
attr_accessor :storm_cooldown_seconds # Open → half-open probe delay (default: 60)
|
|
81
|
+
attr_accessor :storm_max_tracked_fingerprints # Bounded in-memory map size; beyond = overflow bucket (default: 1000)
|
|
82
|
+
attr_accessor :storm_flush_interval_seconds # Count-buffer flush cadence (default: 30)
|
|
83
|
+
attr_accessor :storm_notification # Single "storm in progress" notification per episode (default: true)
|
|
84
|
+
attr_accessor :auto_issue_rate_limit_count # Max auto-created issues per window — applies always (default: 5)
|
|
85
|
+
attr_accessor :auto_issue_rate_limit_window_minutes # Window for the above (default: 10)
|
|
86
|
+
attr_accessor :context_sampling_threshold_per_day # Full-context captures per fingerprint per day before sampling (default: 25)
|
|
87
|
+
attr_accessor :context_sampling_keep_every # After threshold, keep full context every Nth (default: 10)
|
|
88
|
+
|
|
70
89
|
# Async logging configuration
|
|
71
90
|
attr_accessor :async_logging
|
|
72
91
|
attr_accessor :async_adapter # :sidekiq, :solid_queue, or :async
|
|
@@ -92,8 +111,8 @@ module RailsErrorDashboard
|
|
|
92
111
|
# issue_webhook_secret is set.
|
|
93
112
|
attr_accessor :enable_issue_tracking # Master switch (default: false) — enables all platform integration
|
|
94
113
|
attr_accessor :issue_tracker_token # String or lambda/proc for Rails credentials
|
|
95
|
-
attr_accessor :issue_tracker_provider # :github, :gitlab, :codeberg (auto-detected from git_repository_url)
|
|
96
|
-
attr_accessor :issue_tracker_repo # "owner/repo" (auto-extracted from git_repository_url)
|
|
114
|
+
attr_accessor :issue_tracker_provider # :github, :gitlab, :codeberg (auto-detected from git_repository_url), or :linear (explicit only)
|
|
115
|
+
attr_accessor :issue_tracker_repo # "owner/repo" (auto-extracted from git_repository_url), or Linear team key like "ENG"
|
|
97
116
|
attr_accessor :issue_tracker_labels # Array of label strings (default: ["bug"])
|
|
98
117
|
attr_accessor :issue_tracker_api_url # Custom API base URL for self-hosted instances
|
|
99
118
|
attr_accessor :issue_tracker_auto_create_severities # Auto-create for these severities (default: [:critical, :high])
|
|
@@ -268,6 +287,21 @@ module RailsErrorDashboard
|
|
|
268
287
|
@ignored_exceptions = []
|
|
269
288
|
@custom_fingerprint = nil # Lambda: ->(exception, context) { "custom_key" }
|
|
270
289
|
@sampling_rate = 1.0 # 100% by default
|
|
290
|
+
|
|
291
|
+
# Storm protection defaults (thresholds tuned via chaos Phase G — see ROADMAP)
|
|
292
|
+
@enable_storm_protection = true
|
|
293
|
+
@storm_fingerprint_full_per_minute = 30
|
|
294
|
+
@storm_occurrence_sample_keep_every = 10
|
|
295
|
+
@storm_shedding_threshold_per_second = 10
|
|
296
|
+
@storm_open_threshold_per_second = 50
|
|
297
|
+
@storm_cooldown_seconds = 60
|
|
298
|
+
@storm_max_tracked_fingerprints = 1000
|
|
299
|
+
@storm_flush_interval_seconds = 30
|
|
300
|
+
@storm_notification = true
|
|
301
|
+
@auto_issue_rate_limit_count = 5
|
|
302
|
+
@auto_issue_rate_limit_window_minutes = 10
|
|
303
|
+
@context_sampling_threshold_per_day = 25
|
|
304
|
+
@context_sampling_keep_every = 10
|
|
271
305
|
@async_logging = false
|
|
272
306
|
@async_adapter = :sidekiq # Battle-tested default
|
|
273
307
|
@max_backtrace_lines = 100 # Matches industry standard (Rollbar, Airbrake)
|
|
@@ -604,7 +638,8 @@ module RailsErrorDashboard
|
|
|
604
638
|
|
|
605
639
|
if enable_issue_tracking && effective_issue_tracker_provider.nil?
|
|
606
640
|
warnings << "enable_issue_tracking is true but provider could not be detected. " \
|
|
607
|
-
"Set issue_tracker_provider or git_repository_url."
|
|
641
|
+
"Set issue_tracker_provider (:github, :gitlab, :codeberg, :linear) or git_repository_url. " \
|
|
642
|
+
"Note: :linear is never auto-detected — set it explicitly with issue_tracker_repo as the team key."
|
|
608
643
|
end
|
|
609
644
|
end
|
|
610
645
|
|
|
@@ -682,6 +717,32 @@ module RailsErrorDashboard
|
|
|
682
717
|
end
|
|
683
718
|
end
|
|
684
719
|
|
|
720
|
+
# Validate storm protection thresholds (all must be positive when protection is on)
|
|
721
|
+
if enable_storm_protection
|
|
722
|
+
{
|
|
723
|
+
storm_fingerprint_full_per_minute: storm_fingerprint_full_per_minute,
|
|
724
|
+
storm_occurrence_sample_keep_every: storm_occurrence_sample_keep_every,
|
|
725
|
+
storm_shedding_threshold_per_second: storm_shedding_threshold_per_second,
|
|
726
|
+
storm_open_threshold_per_second: storm_open_threshold_per_second,
|
|
727
|
+
storm_cooldown_seconds: storm_cooldown_seconds,
|
|
728
|
+
storm_max_tracked_fingerprints: storm_max_tracked_fingerprints,
|
|
729
|
+
storm_flush_interval_seconds: storm_flush_interval_seconds,
|
|
730
|
+
auto_issue_rate_limit_count: auto_issue_rate_limit_count,
|
|
731
|
+
auto_issue_rate_limit_window_minutes: auto_issue_rate_limit_window_minutes,
|
|
732
|
+
context_sampling_threshold_per_day: context_sampling_threshold_per_day,
|
|
733
|
+
context_sampling_keep_every: context_sampling_keep_every
|
|
734
|
+
}.each do |name, value|
|
|
735
|
+
if value.nil? || value.to_i < 1
|
|
736
|
+
errors << "#{name} must be a positive integer (got: #{value.inspect})"
|
|
737
|
+
end
|
|
738
|
+
end
|
|
739
|
+
|
|
740
|
+
if storm_open_threshold_per_second.to_i < storm_shedding_threshold_per_second.to_i
|
|
741
|
+
errors << "storm_open_threshold_per_second (#{storm_open_threshold_per_second}) must be >= " \
|
|
742
|
+
"storm_shedding_threshold_per_second (#{storm_shedding_threshold_per_second})"
|
|
743
|
+
end
|
|
744
|
+
end
|
|
745
|
+
|
|
685
746
|
# Validate total_users_for_impact (must be positive if set)
|
|
686
747
|
if total_users_for_impact && total_users_for_impact < 1
|
|
687
748
|
errors << "total_users_for_impact must be at least 1 (got: #{total_users_for_impact})"
|
|
@@ -736,9 +797,11 @@ module RailsErrorDashboard
|
|
|
736
797
|
default || blank
|
|
737
798
|
end
|
|
738
799
|
|
|
739
|
-
# Resolve the effective issue tracker provider (auto-detect from git_repository_url)
|
|
800
|
+
# Resolve the effective issue tracker provider (auto-detect from git_repository_url).
|
|
801
|
+
# Linear is never auto-detected (it is not a git forge) — set issue_tracker_provider
|
|
802
|
+
# explicitly.
|
|
740
803
|
#
|
|
741
|
-
# @return [Symbol, nil] :github, :gitlab, :codeberg, or nil
|
|
804
|
+
# @return [Symbol, nil] :github, :gitlab, :codeberg, :linear, or nil
|
|
742
805
|
def effective_issue_tracker_provider
|
|
743
806
|
return issue_tracker_provider&.to_sym if issue_tracker_provider.present?
|
|
744
807
|
return nil if git_repository_url.blank?
|
|
@@ -751,11 +814,13 @@ module RailsErrorDashboard
|
|
|
751
814
|
end
|
|
752
815
|
end
|
|
753
816
|
|
|
754
|
-
# Resolve the effective issue tracker repository ("owner/repo")
|
|
817
|
+
# Resolve the effective issue tracker repository ("owner/repo", or Linear team key)
|
|
755
818
|
#
|
|
756
|
-
# @return [String, nil] "owner/repo" or nil
|
|
819
|
+
# @return [String, nil] "owner/repo", Linear team key, or nil
|
|
757
820
|
def effective_issue_tracker_repo
|
|
758
821
|
return issue_tracker_repo if issue_tracker_repo.present?
|
|
822
|
+
# A git URL can never yield a Linear team key — require explicit config
|
|
823
|
+
return nil if effective_issue_tracker_provider == :linear
|
|
759
824
|
return nil if git_repository_url.blank?
|
|
760
825
|
|
|
761
826
|
# Extract owner/repo from URL: https://github.com/owner/repo(.git)
|
|
@@ -783,6 +848,7 @@ module RailsErrorDashboard
|
|
|
783
848
|
when :github then "https://api.github.com"
|
|
784
849
|
when :gitlab then "https://gitlab.com/api/v4"
|
|
785
850
|
when :codeberg then "https://codeberg.org/api/v1"
|
|
851
|
+
when :linear then "https://api.linear.app/graphql"
|
|
786
852
|
end
|
|
787
853
|
end
|
|
788
854
|
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
module Queries
|
|
5
|
+
# Query: Storm protection episode history + active-storm lookup.
|
|
6
|
+
#
|
|
7
|
+
# Read-only. Powers the /errors/storms page and the layout banner.
|
|
8
|
+
class StormHistory
|
|
9
|
+
RECENT_BANNER_WINDOW = 24.hours
|
|
10
|
+
|
|
11
|
+
def self.call(limit: 50)
|
|
12
|
+
return { active: nil, recent: nil, events: [] } unless StormEvent.table_exists?
|
|
13
|
+
|
|
14
|
+
{
|
|
15
|
+
active: StormEvent.active.recent_first.first,
|
|
16
|
+
recent: StormEvent.ended_within(RECENT_BANNER_WINDOW).recent_first.first,
|
|
17
|
+
events: StormEvent.recent_first.limit(limit).to_a
|
|
18
|
+
}
|
|
19
|
+
rescue => e
|
|
20
|
+
RailsErrorDashboard::Logger.error(
|
|
21
|
+
"[RailsErrorDashboard] StormHistory query failed: #{e.message}"
|
|
22
|
+
)
|
|
23
|
+
{ active: nil, recent: nil, events: [] }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Cheap banner lookup for the layout — one indexed query on the happy
|
|
27
|
+
# path (no active storm), two when a banner is showing.
|
|
28
|
+
def self.banner_event
|
|
29
|
+
return nil unless RailsErrorDashboard.configuration.enable_storm_protection
|
|
30
|
+
return nil unless StormEvent.table_exists?
|
|
31
|
+
|
|
32
|
+
StormEvent.active.recent_first.first ||
|
|
33
|
+
StormEvent.ended_within(RECENT_BANNER_WINDOW).recent_first.first
|
|
34
|
+
rescue
|
|
35
|
+
nil
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -8,8 +8,9 @@ module RailsErrorDashboard
|
|
|
8
8
|
module Services
|
|
9
9
|
# Base class and factory for issue tracker API clients.
|
|
10
10
|
#
|
|
11
|
-
# Supports GitHub, GitLab,
|
|
12
|
-
# Each provider implements the same methods with provider-specific
|
|
11
|
+
# Supports GitHub, GitLab, Codeberg/Gitea/Forgejo, and Linear via a unified
|
|
12
|
+
# interface. Each provider implements the same methods with provider-specific
|
|
13
|
+
# API calls.
|
|
13
14
|
#
|
|
14
15
|
# @example
|
|
15
16
|
# client = IssueTrackerClient.for(:github, token: "ghp_xxx", repo: "user/repo")
|
|
@@ -23,9 +24,9 @@ module RailsErrorDashboard
|
|
|
23
24
|
|
|
24
25
|
# Factory method — returns the correct client for the provider
|
|
25
26
|
#
|
|
26
|
-
# @param provider [Symbol] :github, :gitlab, or :
|
|
27
|
+
# @param provider [Symbol] :github, :gitlab, :codeberg, or :linear
|
|
27
28
|
# @param token [String] API authentication token
|
|
28
|
-
# @param repo [String] Repository identifier ("owner/repo")
|
|
29
|
+
# @param repo [String] Repository identifier ("owner/repo"), or Linear team key ("ENG")
|
|
29
30
|
# @param api_url [String, nil] Custom API base URL (for self-hosted)
|
|
30
31
|
# @return [IssueTrackerClient] Provider-specific client instance
|
|
31
32
|
def self.for(provider, token:, repo:, api_url: nil)
|
|
@@ -36,8 +37,10 @@ module RailsErrorDashboard
|
|
|
36
37
|
GitLabIssueClient.new(token: token, repo: repo, api_url: api_url)
|
|
37
38
|
when :codeberg
|
|
38
39
|
CodebergIssueClient.new(token: token, repo: repo, api_url: api_url)
|
|
40
|
+
when :linear
|
|
41
|
+
LinearIssueClient.new(token: token, repo: repo, api_url: api_url)
|
|
39
42
|
else
|
|
40
|
-
raise ArgumentError, "Unknown issue tracker provider: #{provider}. Supported: :github, :gitlab, :codeberg"
|
|
43
|
+
raise ArgumentError, "Unknown issue tracker provider: #{provider}. Supported: :github, :gitlab, :codeberg, :linear"
|
|
41
44
|
end
|
|
42
45
|
end
|
|
43
46
|
|