rails_error_dashboard 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +54 -1
- data/app/controllers/rails_error_dashboard/application_controller.rb +5 -0
- data/app/controllers/rails_error_dashboard/errors_controller.rb +14 -3
- data/app/jobs/rails_error_dashboard/rack_attack_flush_job.rb +31 -0
- data/app/jobs/rails_error_dashboard/retention_cleanup_job.rb +30 -0
- data/app/jobs/rails_error_dashboard/storm_flush_job.rb +19 -0
- data/app/jobs/rails_error_dashboard/storm_notification_job.rb +74 -0
- data/app/models/rails_error_dashboard/rack_attack_event.rb +36 -0
- data/app/models/rails_error_dashboard/storm_event.rb +34 -0
- data/app/views/layouts/rails_error_dashboard.html.erb +22 -1
- data/app/views/rails_error_dashboard/errors/rack_attack_summary.html.erb +4 -6
- data/app/views/rails_error_dashboard/errors/storms.html.erb +91 -0
- data/config/routes.rb +1 -0
- data/db/migrate/20260306000002_add_instance_variables_to_error_logs.rb +7 -1
- data/db/migrate/20260306000003_create_rails_error_dashboard_swallowed_exceptions.rb +4 -0
- data/db/migrate/20260307000001_create_rails_error_dashboard_diagnostic_dumps.rb +4 -0
- data/db/migrate/20260613000001_create_storm_events.rb +28 -0
- data/db/migrate/20260730000001_create_rails_error_dashboard_rack_attack_events.rb +45 -0
- data/lib/generators/rails_error_dashboard/install/templates/initializer.rb +36 -0
- data/lib/rails_error_dashboard/commands/flush_rack_attack_events.rb +84 -0
- data/lib/rails_error_dashboard/commands/flush_storm_counts.rb +188 -0
- data/lib/rails_error_dashboard/commands/log_error.rb +70 -12
- data/lib/rails_error_dashboard/configuration.rb +78 -7
- data/lib/rails_error_dashboard/engine.rb +2 -2
- data/lib/rails_error_dashboard/queries/rack_attack_summary.rb +55 -44
- data/lib/rails_error_dashboard/queries/storm_history.rb +39 -0
- data/lib/rails_error_dashboard/services/rack_attack_tracker.rb +194 -0
- data/lib/rails_error_dashboard/services/storm_protection/circuit_breaker.rb +195 -0
- data/lib/rails_error_dashboard/services/storm_protection/count_buffer.rb +100 -0
- data/lib/rails_error_dashboard/services/storm_protection/fingerprint_buckets.rb +123 -0
- data/lib/rails_error_dashboard/services/storm_protection/gate.rb +258 -0
- data/lib/rails_error_dashboard/subscribers/issue_tracker_subscriber.rb +12 -0
- data/lib/rails_error_dashboard/subscribers/rack_attack_subscriber.rb +16 -2
- data/lib/rails_error_dashboard/version.rb +1 -1
- data/lib/rails_error_dashboard.rb +8 -0
- metadata +37 -18
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 270ac48ebee51c2f68cab29abdbf1169b7db7d7b860bf1fc5a43883960cdc0c6
|
|
4
|
+
data.tar.gz: 6def5b5a2bbcabffd1116f13607771f6923823dc65eec803e028ba911af85ec9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fd6984fc8ccb2a68f9543786ffd6157d1a975ed5d446a13e38b3a38b42cc93047117918186b7f3936c2ddc3a53c3609db63073abad5f25d02edba4961e7b1f30
|
|
7
|
+
data.tar.gz: 55ba8d78b0d06176ac80e63919cd29a6ca94423d8880712c8fb82979a17741b5881878ef2351efd89c468fbb3d03c6411c82c5c4668a687f4f441b512c169814
|
data/README.md
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
gem 'rails_error_dashboard'
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
-
**5-minute setup** · **Works out-of-the-box** · **
|
|
16
|
+
**5-minute setup** · **Works out-of-the-box** · **PostgreSQL, MySQL/Trilogy, SQLite — shared or separate database** · **No vendor lock-in**
|
|
17
17
|
|
|
18
18
|
[Full Documentation](https://anjanj.github.io/rails_error_dashboard/) · [Live Demo](https://rails-error-dashboard.anjan.dev) · [RubyGems](https://rubygems.org/gems/rails_error_dashboard)
|
|
19
19
|
|
|
@@ -79,6 +79,28 @@ Error capture from controllers, jobs, and middleware. Custom-designed dashboard
|
|
|
79
79
|
|
|
80
80
|
### Optional Features
|
|
81
81
|
|
|
82
|
+
<details>
|
|
83
|
+
<summary><strong>Storm Protection — Circuit Breaker + Adaptive Sampling</strong></summary>
|
|
84
|
+
|
|
85
|
+
When the error rate spikes (a bad deploy throwing thousands of errors a minute), the nightmare scenario for any in-process tracker is amplifying the outage with its own database writes. Storm protection makes the gem **provably degrade itself first** — ON by default.
|
|
86
|
+
|
|
87
|
+
- **Per-fingerprint caps:** past N occurrences/minute per error, context is shed, then rows are sampled deterministically (a fresh exemplar is always kept each minute)
|
|
88
|
+
- **Global circuit breaker:** sustained floods flip the gem to count-only mode — zero per-event I/O, exact in-memory counts reconciled onto error records every 30s. Async mode is gated too (a SolidQueue enqueue is itself a DB write)
|
|
89
|
+
- **One storm notification** replaces hundreds of per-error pings; auto-issue creation is capped (default 5 per 10 min) so a storm of new errors can't open 500 GitHub/Linear issues
|
|
90
|
+
- **Honest accounting:** a dashboard banner during/after the storm, a Storm History page with exact counts of everything shed, and `reached_open`/peak-rate per episode. Counts are never extrapolated
|
|
91
|
+
- **Calm-weather economy:** after 25 full-context captures of the same error per day, context is sampled (occurrence counting unaffected)
|
|
92
|
+
- **Fails open:** any internal storm-protection error means full capture. Protection can never be the thing that loses an error
|
|
93
|
+
|
|
94
|
+
```ruby
|
|
95
|
+
config.enable_storm_protection = true # default
|
|
96
|
+
config.storm_open_threshold_per_second = 50 # per process
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
All thresholds are per process and individually configurable. Disable with one flag.
|
|
100
|
+
|
|
101
|
+
**Measured overhead** (Apple Silicon, Ruby 4.0): 2.4µs/error with protection active and calm, 2.95µs in count-only mode, 0.2µs when disabled — against a 5µs budget. The check is a digest plus an atomic increment; there is no I/O on the hot path.
|
|
102
|
+
</details>
|
|
103
|
+
|
|
82
104
|
<details>
|
|
83
105
|
<summary><strong>Breadcrumbs — Request Activity Trail</strong></summary>
|
|
84
106
|
|
|
@@ -580,6 +602,37 @@ end
|
|
|
580
602
|
|
|
581
603
|
---
|
|
582
604
|
|
|
605
|
+
## FAQ
|
|
606
|
+
|
|
607
|
+
**Does Rails Error Dashboard support a separate database for errors?**
|
|
608
|
+
Yes. You can store errors in your app's existing database (shared) **or** in a dedicated, isolated database (separate). Set `config.use_separate_database = true` (or `USE_SEPARATE_ERROR_DB=true`) and point it at a separate connection — the engine routes all of its tables through `connects_to`, keeping error data fully isolated from your app data. Both modes are first-class and covered by the [Database Options guide](docs/guides/DATABASE_OPTIONS.md).
|
|
609
|
+
|
|
610
|
+
**Which databases does it work with?**
|
|
611
|
+
SQLite, PostgreSQL, and MySQL/Trilogy — in either shared or separate-database mode.
|
|
612
|
+
|
|
613
|
+
**Is this a self-hosted alternative to Sentry?**
|
|
614
|
+
Yes. It runs entirely inside your own Rails process — no external services, no SDK calling out, no per-event pricing. Error data never leaves your infrastructure.
|
|
615
|
+
|
|
616
|
+
**Does it capture local variables like Sentry?**
|
|
617
|
+
Yes — local **and** instance variables at the moment the exception is raised, via `TracePoint(:raise)`, with sensitive-data filtering and configurable limits. This is opt-in and a capability Sentry charges extra for.
|
|
618
|
+
|
|
619
|
+
**Will a flood of errors take down my app?**
|
|
620
|
+
No. Storm protection (a circuit breaker with adaptive sampling, **ON by default**) makes the gem degrade itself first during error floods — occurrence counts stay exact while it sheds the expensive work. Measured hot-path overhead is ~2.4µs/error.
|
|
621
|
+
|
|
622
|
+
**Does it work with my background jobs?**
|
|
623
|
+
Yes — it auto-detects and supports Sidekiq, SolidQueue, and GoodJob, and can log errors asynchronously through any of them.
|
|
624
|
+
|
|
625
|
+
**Does it work with my authentication?**
|
|
626
|
+
Yes — HTTP Basic Auth out of the box, or a custom `authenticate_with` lambda that integrates with Devise, Warden, or session-based auth.
|
|
627
|
+
|
|
628
|
+
**Can it track more than one app?**
|
|
629
|
+
Yes — multi-app support tracks errors from multiple Rails apps in one dashboard with per-app filtering.
|
|
630
|
+
|
|
631
|
+
**What Rails and Ruby versions are supported?**
|
|
632
|
+
Rails 7.0–8.1 and Ruby 3.2–4.0.
|
|
633
|
+
|
|
634
|
+
---
|
|
635
|
+
|
|
583
636
|
## Documentation
|
|
584
637
|
|
|
585
638
|
### Getting Started
|
|
@@ -77,6 +77,11 @@ module RailsErrorDashboard
|
|
|
77
77
|
def set_common_view_variables
|
|
78
78
|
@applications = Application.ordered_by_name.pluck(:name, :id) rescue []
|
|
79
79
|
@default_credentials_warning = RailsErrorDashboard.configuration.default_credentials? rescue false
|
|
80
|
+
# Only query when a before_action (e.g. ErrorsController#load_storm_banner)
|
|
81
|
+
# hasn't already loaded it this request — the error renderer runs after
|
|
82
|
+
# those callbacks, so reuse their result instead of querying a second time.
|
|
83
|
+
# Tracked by a flag rather than the value, since the common case is nil.
|
|
84
|
+
@storm_banner_event = Queries::StormHistory.banner_event unless @storm_banner_loaded
|
|
80
85
|
end
|
|
81
86
|
end
|
|
82
87
|
end
|
|
@@ -5,6 +5,7 @@ module RailsErrorDashboard
|
|
|
5
5
|
before_action :authenticate_dashboard_user!
|
|
6
6
|
before_action :set_application_context
|
|
7
7
|
before_action :check_default_credentials
|
|
8
|
+
before_action :load_storm_banner
|
|
8
9
|
|
|
9
10
|
FILTERABLE_PARAMS = %i[
|
|
10
11
|
error_type
|
|
@@ -329,6 +330,12 @@ module RailsErrorDashboard
|
|
|
329
330
|
@pagy, @releases = pagy(:offset, all_releases, limit: params[:per_page] || 25)
|
|
330
331
|
end
|
|
331
332
|
|
|
333
|
+
def storms
|
|
334
|
+
result = Queries::StormHistory.call
|
|
335
|
+
@active_storm = result[:active]
|
|
336
|
+
@storm_events = result[:events]
|
|
337
|
+
end
|
|
338
|
+
|
|
332
339
|
def user_impact
|
|
333
340
|
days = days_param(default: 30)
|
|
334
341
|
@days = days
|
|
@@ -477,9 +484,8 @@ module RailsErrorDashboard
|
|
|
477
484
|
end
|
|
478
485
|
|
|
479
486
|
def rack_attack_summary
|
|
480
|
-
unless RailsErrorDashboard.configuration.enable_rack_attack_tracking
|
|
481
|
-
|
|
482
|
-
flash[:alert] = "Rack Attack tracking is not enabled. Enable enable_rack_attack_tracking and enable_breadcrumbs in config/initializers/rails_error_dashboard.rb"
|
|
487
|
+
unless RailsErrorDashboard.configuration.enable_rack_attack_tracking
|
|
488
|
+
flash[:alert] = "Rack Attack tracking is not enabled. Set enable_rack_attack_tracking = true in config/initializers/rails_error_dashboard.rb"
|
|
483
489
|
redirect_to errors_path(**app_context_params)
|
|
484
490
|
return
|
|
485
491
|
end
|
|
@@ -729,6 +735,11 @@ module RailsErrorDashboard
|
|
|
729
735
|
@default_credentials_warning = RailsErrorDashboard.configuration.default_credentials?
|
|
730
736
|
end
|
|
731
737
|
|
|
738
|
+
def load_storm_banner
|
|
739
|
+
@storm_banner_loaded = true
|
|
740
|
+
@storm_banner_event = Queries::StormHistory.banner_event
|
|
741
|
+
end
|
|
742
|
+
|
|
732
743
|
def authenticate_dashboard_user!
|
|
733
744
|
auth_lambda = RailsErrorDashboard.configuration.authenticate_with
|
|
734
745
|
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
# Job: Persist buffered Rack::Attack event counts to the database.
|
|
5
|
+
#
|
|
6
|
+
# Two usage modes:
|
|
7
|
+
# 1. With a counts hash — dispatched by RackAttackTracker's periodic flush.
|
|
8
|
+
# Zero I/O in the request path; all DB writes happen here.
|
|
9
|
+
# 2. Without arguments — scheduled periodic sweep that flushes the current
|
|
10
|
+
# thread's buffer (useful as a cron safety net for low-traffic apps where
|
|
11
|
+
# the flush interval may not be reached during a request).
|
|
12
|
+
#
|
|
13
|
+
# Example cron (via solid_queue or whenever):
|
|
14
|
+
# every 5.minutes { RailsErrorDashboard::RackAttackFlushJob.perform_later }
|
|
15
|
+
class RackAttackFlushJob < ApplicationJob
|
|
16
|
+
queue_as :default
|
|
17
|
+
|
|
18
|
+
def perform(counts = nil)
|
|
19
|
+
return unless RailsErrorDashboard.configuration.enable_rack_attack_tracking
|
|
20
|
+
|
|
21
|
+
if counts
|
|
22
|
+
# Mode 1: Persist provided snapshot (dispatched from tracker flush)
|
|
23
|
+
Commands::FlushRackAttackEvents.call(counts: counts)
|
|
24
|
+
else
|
|
25
|
+
# Mode 2: Flush current thread's buffer (scheduled cron safety net).
|
|
26
|
+
# sync: true because we are already off the request path.
|
|
27
|
+
Services::RackAttackTracker.flush!(sync: true)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -20,6 +20,12 @@ module RailsErrorDashboard
|
|
|
20
20
|
return 0 if retention_days.blank?
|
|
21
21
|
|
|
22
22
|
cutoff = retention_days.days.ago
|
|
23
|
+
|
|
24
|
+
# Rack Attack events live in their own table and expire independently of
|
|
25
|
+
# errors — clean them up BEFORE the early return below, which fires
|
|
26
|
+
# whenever no error logs happen to be expired.
|
|
27
|
+
cleanup_rack_attack_events(cutoff)
|
|
28
|
+
|
|
23
29
|
expired_scope = ErrorLog.where("occurred_at < ?", cutoff)
|
|
24
30
|
return 0 if expired_scope.none?
|
|
25
31
|
|
|
@@ -52,5 +58,29 @@ module RailsErrorDashboard
|
|
|
52
58
|
RailsErrorDashboard::Logger.error("[RailsErrorDashboard] Retention cleanup failed: #{e.class} - #{e.message}")
|
|
53
59
|
0
|
|
54
60
|
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
# Expire aggregated Rack Attack event rows. Isolated in its own rescue so a
|
|
65
|
+
# failure here (e.g. table not yet migrated) never blocks error cleanup.
|
|
66
|
+
def cleanup_rack_attack_events(cutoff)
|
|
67
|
+
return unless RailsErrorDashboard.configuration.enable_rack_attack_tracking
|
|
68
|
+
return unless RackAttackEvent.table_exists?
|
|
69
|
+
|
|
70
|
+
deleted = 0
|
|
71
|
+
RackAttackEvent.where("period_hour < ?", cutoff).in_batches(of: 1000) do |batch|
|
|
72
|
+
deleted += batch.delete_all
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
if deleted > 0
|
|
76
|
+
RailsErrorDashboard::Logger.info(
|
|
77
|
+
"[RailsErrorDashboard] Retention cleanup: deleted #{deleted} rack attack events"
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
rescue => e
|
|
81
|
+
RailsErrorDashboard::Logger.debug(
|
|
82
|
+
"[RailsErrorDashboard] Rack attack retention cleanup failed: #{e.class} - #{e.message}"
|
|
83
|
+
)
|
|
84
|
+
end
|
|
55
85
|
end
|
|
56
86
|
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
# Persists storm-protection count snapshots (mirrors SwallowedExceptionFlushJob):
|
|
5
|
+
# the gate accumulates counts in memory with zero I/O, snapshots are handed
|
|
6
|
+
# to this job at most once per flush interval, and ALL DB writes happen here.
|
|
7
|
+
class StormFlushJob < ApplicationJob
|
|
8
|
+
queue_as :default
|
|
9
|
+
|
|
10
|
+
def perform(entries: [], overflow: 0, episode: nil)
|
|
11
|
+
entries = entries.map { |e| e.respond_to?(:stringify_keys) ? e.stringify_keys : e }
|
|
12
|
+
episode = episode.stringify_keys if episode.respond_to?(:stringify_keys)
|
|
13
|
+
|
|
14
|
+
Commands::FlushStormCounts.call(entries: entries, overflow: overflow, episode: episode)
|
|
15
|
+
rescue => e
|
|
16
|
+
Rails.logger.error("[RailsErrorDashboard] StormFlushJob failed: #{e.class} - #{e.message}")
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
# Sends the SINGLE "error storm in progress" notification per storm episode.
|
|
5
|
+
#
|
|
6
|
+
# During a storm, per-error notifications are suppressed (500 Slack pings
|
|
7
|
+
# help nobody) — this one message replaces them. The gate guarantees at
|
|
8
|
+
# most one enqueue per episode; this job just delivers.
|
|
9
|
+
class StormNotificationJob < ApplicationJob
|
|
10
|
+
queue_as :default
|
|
11
|
+
|
|
12
|
+
# @param started_at [String] ISO8601 episode start
|
|
13
|
+
# @param state [String] breaker state at notification time ("shedding"/"open")
|
|
14
|
+
def perform(started_at:, state: "shedding")
|
|
15
|
+
config = RailsErrorDashboard.configuration
|
|
16
|
+
message = build_message(started_at, state, config)
|
|
17
|
+
|
|
18
|
+
if config.enable_slack_notifications && config.slack_webhook_url.present?
|
|
19
|
+
post_json(config.slack_webhook_url, { text: message })
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
if config.enable_discord_notifications && config.discord_webhook_url.present?
|
|
23
|
+
post_json(config.discord_webhook_url, { content: message })
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
if config.enable_webhook_notifications && config.webhook_urls.any?
|
|
27
|
+
payload = {
|
|
28
|
+
event: "error_storm_detected",
|
|
29
|
+
started_at: started_at,
|
|
30
|
+
state: state,
|
|
31
|
+
application: app_name(config)
|
|
32
|
+
}
|
|
33
|
+
config.webhook_urls.each { |url| post_json(url, payload) }
|
|
34
|
+
end
|
|
35
|
+
rescue => e
|
|
36
|
+
Rails.logger.error("[RailsErrorDashboard] StormNotificationJob failed: #{e.class} - #{e.message}")
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def build_message(started_at, state, config)
|
|
42
|
+
mode = state == "open" ? "count-only mode (occurrences tallied, detail paused)" : "shedding mode (context sampling active)"
|
|
43
|
+
dashboard = (config.dashboard_base_url || "").chomp("/")
|
|
44
|
+
link = dashboard.present? ? " Dashboard: #{dashboard}/errors/storms" : ""
|
|
45
|
+
|
|
46
|
+
":warning: Error storm detected in #{app_name(config)} at #{started_at}. " \
|
|
47
|
+
"Storm protection engaged — #{mode}. Per-error notifications are " \
|
|
48
|
+
"suppressed until the storm subsides; exact counts are preserved.#{link}"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def app_name(config)
|
|
52
|
+
config.application_name || ENV["APPLICATION_NAME"] ||
|
|
53
|
+
(defined?(Rails) && Rails.application.class.module_parent_name) || "Rails Application"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def post_json(url, payload)
|
|
57
|
+
if defined?(HTTParty)
|
|
58
|
+
HTTParty.post(url, body: payload.to_json,
|
|
59
|
+
headers: { "Content-Type" => "application/json" }, timeout: 10)
|
|
60
|
+
else
|
|
61
|
+
uri = URI(url)
|
|
62
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
63
|
+
http.use_ssl = uri.scheme == "https"
|
|
64
|
+
http.open_timeout = 5
|
|
65
|
+
http.read_timeout = 10
|
|
66
|
+
request = Net::HTTP::Post.new(uri.path, { "Content-Type" => "application/json" })
|
|
67
|
+
request.body = payload.to_json
|
|
68
|
+
http.request(request)
|
|
69
|
+
end
|
|
70
|
+
rescue => e
|
|
71
|
+
Rails.logger.error("[RailsErrorDashboard] Storm notification post failed: #{e.message}")
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
# Stores aggregated Rack::Attack throttle/blocklist/track events per hourly bucket.
|
|
5
|
+
#
|
|
6
|
+
# Rack::Attack events are NOT errors — a throttled request returns HTTP 429 without
|
|
7
|
+
# raising. They are therefore persisted independently of error_logs rather than as
|
|
8
|
+
# a side-effect of error capture (see issue #143).
|
|
9
|
+
#
|
|
10
|
+
# Rows are aggregated hourly by (rule, match_type, discriminator, path) so that a
|
|
11
|
+
# rate-limit flood collapses to a handful of rows instead of one INSERT per request.
|
|
12
|
+
class RackAttackEvent < ErrorLogsRecord
|
|
13
|
+
self.table_name = "rails_error_dashboard_rack_attack_events"
|
|
14
|
+
|
|
15
|
+
# Event types emitted by Rack::Attack (v5.0+)
|
|
16
|
+
MATCH_TYPES = %w[throttle blocklist track safelist].freeze
|
|
17
|
+
|
|
18
|
+
belongs_to :application, optional: true
|
|
19
|
+
|
|
20
|
+
validates :rule, presence: true
|
|
21
|
+
validates :match_type, presence: true
|
|
22
|
+
validates :period_hour, presence: true
|
|
23
|
+
validates :event_count, presence: true, numericality: { greater_than_or_equal_to: 0 }
|
|
24
|
+
|
|
25
|
+
scope :for_application, ->(app_id) { where(application_id: app_id) }
|
|
26
|
+
scope :since, ->(time) { where("period_hour >= ?", time) }
|
|
27
|
+
scope :recent, -> { order(period_hour: :desc) }
|
|
28
|
+
scope :throttles, -> { where(match_type: "throttle") }
|
|
29
|
+
scope :blocklists, -> { where(match_type: "blocklist") }
|
|
30
|
+
|
|
31
|
+
# Whether this event represents a hard block rather than a rate limit
|
|
32
|
+
def blocked?
|
|
33
|
+
match_type == "blocklist"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RailsErrorDashboard
|
|
4
|
+
# One row per storm-protection episode (per process). Powers the dashboard
|
|
5
|
+
# banner and the storm history page. Counts are exact, not extrapolated.
|
|
6
|
+
# Inherits ErrorLogsRecord so separate-database routing applies.
|
|
7
|
+
class StormEvent < ErrorLogsRecord
|
|
8
|
+
self.table_name = "rails_error_dashboard_storm_events"
|
|
9
|
+
|
|
10
|
+
scope :active, -> { where(ended_at: nil) }
|
|
11
|
+
scope :recent_first, -> { order(started_at: :desc) }
|
|
12
|
+
scope :ended_within, ->(duration) { where.not(ended_at: nil).where(ended_at: duration.ago..) }
|
|
13
|
+
|
|
14
|
+
def active?
|
|
15
|
+
ended_at.nil?
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def duration_seconds
|
|
19
|
+
return nil unless ended_at
|
|
20
|
+
|
|
21
|
+
(ended_at - started_at).round
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# @return [Array<Hash>] top fingerprints by count, [] when absent/corrupt
|
|
25
|
+
def top_fingerprints_list
|
|
26
|
+
return [] if top_fingerprints.blank?
|
|
27
|
+
|
|
28
|
+
parsed = JSON.parse(top_fingerprints)
|
|
29
|
+
parsed.is_a?(Array) ? parsed : []
|
|
30
|
+
rescue JSON::ParserError
|
|
31
|
+
[]
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -993,7 +993,7 @@ tr[data-red-row-href]:hover .sev-bar { opacity: 1 !important; }
|
|
|
993
993
|
<% health_items << { path: job_health_summary_errors_path(nav_params), icon: 'bi-gear-wide-connected', label: 'Jobs' } %>
|
|
994
994
|
<% health_items << { path: database_health_summary_errors_path(nav_params), icon: 'bi-database', label: 'Database' } %>
|
|
995
995
|
<% end %>
|
|
996
|
-
<% if RailsErrorDashboard.configuration.enable_rack_attack_tracking
|
|
996
|
+
<% if RailsErrorDashboard.configuration.enable_rack_attack_tracking %>
|
|
997
997
|
<% health_items << { path: rack_attack_summary_errors_path(nav_params), icon: 'bi-shield-exclamation', label: 'Rate Limits' } %>
|
|
998
998
|
<% end %>
|
|
999
999
|
<% if RailsErrorDashboard.configuration.enable_actioncable_tracking && RailsErrorDashboard.configuration.enable_breadcrumbs %>
|
|
@@ -1036,6 +1036,9 @@ tr[data-red-row-href]:hover .sev-bar { opacity: 1 !important; }
|
|
|
1036
1036
|
<% if RailsErrorDashboard.configuration.enable_diagnostic_dump %>
|
|
1037
1037
|
<% diag_items << { path: diagnostic_dumps_errors_path(nav_params), icon: 'bi-clipboard-pulse', label: 'Diagnostics' } %>
|
|
1038
1038
|
<% end %>
|
|
1039
|
+
<% if RailsErrorDashboard.configuration.enable_storm_protection %>
|
|
1040
|
+
<% diag_items << { path: storms_errors_path(nav_params), icon: 'bi-cloud-lightning-rain', label: 'Storms' } %>
|
|
1041
|
+
<% end %>
|
|
1039
1042
|
|
|
1040
1043
|
<% if diag_items.any? %>
|
|
1041
1044
|
<div style="margin-bottom: var(--space-2);" id="navDiagSection">
|
|
@@ -1175,6 +1178,24 @@ tr[data-red-row-href]:hover .sev-bar { opacity: 1 !important; }
|
|
|
1175
1178
|
</div>
|
|
1176
1179
|
<script<%= " nonce=\"#{red_csp_nonce}\"".html_safe if red_csp_nonce %>>try { if (sessionStorage.getItem('red_dismiss_creds_warning') === '1') { document.getElementById('security-warning').style.display = 'none'; } } catch(e) {}</script>
|
|
1177
1180
|
<% end %>
|
|
1181
|
+
<% if defined?(@storm_banner_event) && @storm_banner_event %>
|
|
1182
|
+
<% storm_active = @storm_banner_event.active? %>
|
|
1183
|
+
<div id="storm-banner" class="alert <%= storm_active ? 'alert-danger' : 'alert-info' %>" style="display: flex; align-items: center; gap: 10px; margin-top: var(--space-2); margin-bottom: var(--space-4); border-left: 4px solid <%= storm_active ? 'var(--status-critical)' : 'var(--status-info)' %>;">
|
|
1184
|
+
<i class="bi <%= storm_active ? 'bi-cloud-lightning-rain-fill' : 'bi-cloud-check' %>" style="font-size: 18px; flex-shrink: 0;"></i>
|
|
1185
|
+
<div style="flex: 1;">
|
|
1186
|
+
<% if storm_active %>
|
|
1187
|
+
<strong>Error storm in progress</strong> (since <%= @storm_banner_event.started_at.strftime("%H:%M %Z") %>) —
|
|
1188
|
+
storm protection engaged. Occurrences are being counted exactly;
|
|
1189
|
+
<%= @storm_banner_event.reached_open ? "detail capture is paused (count-only mode)" : "context capture is sampled" %>.
|
|
1190
|
+
<% else %>
|
|
1191
|
+
<strong>Storm protection engaged recently</strong> (ended <%= @storm_banner_event.ended_at.strftime("%H:%M %Z") %>) —
|
|
1192
|
+
<%= number_with_delimiter(@storm_banner_event.events_counted_only.to_i + @storm_banner_event.events_overflow.to_i) %> occurrences
|
|
1193
|
+
were recorded as exact counts with sampled detail.
|
|
1194
|
+
<% end %>
|
|
1195
|
+
<a href="<%= storms_errors_path %>" style="margin-left: 4px;">View storm history</a>
|
|
1196
|
+
</div>
|
|
1197
|
+
</div>
|
|
1198
|
+
<% end %>
|
|
1178
1199
|
<%= yield %>
|
|
1179
1200
|
</main>
|
|
1180
1201
|
|
|
@@ -25,16 +25,16 @@
|
|
|
25
25
|
<i class="bi bi-shield-check display-1 text-success mb-3"></i>
|
|
26
26
|
<div class="red-empty-state-title">No Rate Limit Events Found</div>
|
|
27
27
|
<p class="text-muted">
|
|
28
|
-
No Rack Attack throttle, blocklist, or track events were
|
|
28
|
+
No Rack Attack throttle, blocklist, or track events were recorded over the last <%= @days %> days.
|
|
29
29
|
</p>
|
|
30
30
|
<div class="card mx-auto" style="max-width: 500px;">
|
|
31
31
|
<div class="card-body text-start">
|
|
32
32
|
<h6>How Rack Attack tracking works:</h6>
|
|
33
33
|
<ul class="mb-0">
|
|
34
|
-
<li>Breadcrumbs must be enabled (<code>enable_breadcrumbs = true</code>)</li>
|
|
35
34
|
<li>Rack Attack tracking must be enabled (<code>enable_rack_attack_tracking = true</code>)</li>
|
|
36
35
|
<li>Rack Attack must be installed and configured in your app</li>
|
|
37
|
-
<li>Throttle, blocklist, and track events are
|
|
36
|
+
<li>Throttle, blocklist, and track events are recorded whenever a rule matches — no error required</li>
|
|
37
|
+
<li>Events are buffered and written every <%= RailsErrorDashboard.configuration.rack_attack_flush_interval %>s, so allow a short delay before they appear</li>
|
|
38
38
|
</ul>
|
|
39
39
|
</div>
|
|
40
40
|
</div>
|
|
@@ -86,7 +86,6 @@
|
|
|
86
86
|
<th width="80">Count</th>
|
|
87
87
|
<th width="80">IPs</th>
|
|
88
88
|
<th>Top Path</th>
|
|
89
|
-
<th width="80">Errors</th>
|
|
90
89
|
<th width="140">Last Seen</th>
|
|
91
90
|
</tr>
|
|
92
91
|
</thead>
|
|
@@ -105,7 +104,6 @@
|
|
|
105
104
|
<td><strong><%= event[:count] %></strong></td>
|
|
106
105
|
<td><%= event[:unique_ips] %></td>
|
|
107
106
|
<td><code><%= event[:top_path] %></code></td>
|
|
108
|
-
<td><%= event[:error_count] %></td>
|
|
109
107
|
<td><%= local_time_ago(event[:last_seen]) %></td>
|
|
110
108
|
</tr>
|
|
111
109
|
<% end %>
|
|
@@ -116,7 +114,7 @@
|
|
|
116
114
|
<div class="card-footer border-top d-flex justify-content-between align-items-center">
|
|
117
115
|
<div>
|
|
118
116
|
<small class="text-muted">
|
|
119
|
-
<i class="bi bi-lightbulb text-warning"></i>
|
|
117
|
+
<i class="bi bi-lightbulb text-warning"></i> High counts may indicate abuse or misconfigured rules.
|
|
120
118
|
</small>
|
|
121
119
|
<small class="ms-3">
|
|
122
120
|
<a href="https://github.com/rack/rack-attack" target="_blank" rel="noopener" class="text-decoration-none">
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
<% content_for :page_title, "Storm History" %>
|
|
2
|
+
|
|
3
|
+
<div>
|
|
4
|
+
<div class="d-flex justify-content-between align-items-center mb-4">
|
|
5
|
+
<h1 style="font-size: 20px; font-weight: 700; margin: 0;">
|
|
6
|
+
<i class="bi bi-cloud-lightning-rain me-2"></i>
|
|
7
|
+
Storm History
|
|
8
|
+
</h1>
|
|
9
|
+
</div>
|
|
10
|
+
|
|
11
|
+
<p class="text-muted" style="font-size: 13px; margin-bottom: var(--space-4);">
|
|
12
|
+
When the error rate spikes (a bad deploy, a dependency outage), storm protection
|
|
13
|
+
limits the gem's own database writes so it never amplifies the incident.
|
|
14
|
+
Occurrences are always counted exactly — only per-event detail is sampled.
|
|
15
|
+
Thresholds are configurable per process via <code>storm_*</code> options.
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+
<% if @active_storm %>
|
|
19
|
+
<div class="alert alert-danger" style="border-left: 4px solid var(--status-critical);">
|
|
20
|
+
<i class="bi bi-cloud-lightning-rain-fill me-1"></i>
|
|
21
|
+
<strong>Storm in progress</strong> since <%= @active_storm.started_at.strftime("%Y-%m-%d %H:%M %Z") %> —
|
|
22
|
+
peak rate <%= number_with_delimiter(@active_storm.peak_rate_per_minute) %>/min.
|
|
23
|
+
</div>
|
|
24
|
+
<% end %>
|
|
25
|
+
|
|
26
|
+
<% if @storm_events.empty? %>
|
|
27
|
+
<div class="red-empty-state">
|
|
28
|
+
<i class="bi bi-cloud-sun display-1 text-muted mb-3"></i>
|
|
29
|
+
<div class="red-empty-state-title">No Storms Recorded</div>
|
|
30
|
+
<p class="text-muted">
|
|
31
|
+
Storm protection is standing by. If an error flood ever hits this app,
|
|
32
|
+
episodes will appear here with exact counts of what was shed.
|
|
33
|
+
</p>
|
|
34
|
+
</div>
|
|
35
|
+
<% else %>
|
|
36
|
+
<div class="card">
|
|
37
|
+
<div class="card-body" style="padding: 0;">
|
|
38
|
+
<table class="table" style="margin: 0;">
|
|
39
|
+
<thead>
|
|
40
|
+
<tr>
|
|
41
|
+
<th>Started</th>
|
|
42
|
+
<th>Duration</th>
|
|
43
|
+
<th>Peak Rate</th>
|
|
44
|
+
<th>Mode</th>
|
|
45
|
+
<th class="text-end">Counted (exact)</th>
|
|
46
|
+
<th class="text-end">Overflow</th>
|
|
47
|
+
<th>Top Errors</th>
|
|
48
|
+
</tr>
|
|
49
|
+
</thead>
|
|
50
|
+
<tbody>
|
|
51
|
+
<% @storm_events.each do |event| %>
|
|
52
|
+
<tr>
|
|
53
|
+
<td style="white-space: nowrap;">
|
|
54
|
+
<%= event.started_at.strftime("%Y-%m-%d %H:%M") %>
|
|
55
|
+
<% if event.active? %>
|
|
56
|
+
<span class="badge bg-danger ms-1">active</span>
|
|
57
|
+
<% end %>
|
|
58
|
+
</td>
|
|
59
|
+
<td>
|
|
60
|
+
<% if event.duration_seconds %>
|
|
61
|
+
<%= ActiveSupport::Duration.build(event.duration_seconds).inspect %>
|
|
62
|
+
<% else %>
|
|
63
|
+
—
|
|
64
|
+
<% end %>
|
|
65
|
+
</td>
|
|
66
|
+
<td><%= number_with_delimiter(event.peak_rate_per_minute) %>/min</td>
|
|
67
|
+
<td>
|
|
68
|
+
<% if event.reached_open %>
|
|
69
|
+
<span class="badge bg-danger">count-only</span>
|
|
70
|
+
<% else %>
|
|
71
|
+
<span class="badge bg-warning text-dark">shedding</span>
|
|
72
|
+
<% end %>
|
|
73
|
+
</td>
|
|
74
|
+
<td class="text-end"><%= number_with_delimiter(event.events_counted_only) %></td>
|
|
75
|
+
<td class="text-end"><%= number_with_delimiter(event.events_overflow) %></td>
|
|
76
|
+
<td style="max-width: 320px;">
|
|
77
|
+
<% event.top_fingerprints_list.first(3).each do |fp| %>
|
|
78
|
+
<div style="font-size: 12px; font-family: var(--font-mono); overflow: hidden; text-overflow: ellipsis; white-space: nowrap;" title="<%= fp["message"] %>">
|
|
79
|
+
<strong><%= fp["class"] %></strong>
|
|
80
|
+
<span class="text-muted">×<%= number_with_delimiter(fp["count"]) %></span>
|
|
81
|
+
</div>
|
|
82
|
+
<% end %>
|
|
83
|
+
</td>
|
|
84
|
+
</tr>
|
|
85
|
+
<% end %>
|
|
86
|
+
</tbody>
|
|
87
|
+
</table>
|
|
88
|
+
</div>
|
|
89
|
+
</div>
|
|
90
|
+
<% end %>
|
|
91
|
+
</div>
|
data/config/routes.rb
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
class AddInstanceVariablesToErrorLogs < ActiveRecord::Migration[7.0]
|
|
4
|
-
def
|
|
4
|
+
def up
|
|
5
|
+
return if column_exists?(:rails_error_dashboard_error_logs, :instance_variables)
|
|
6
|
+
|
|
5
7
|
add_column :rails_error_dashboard_error_logs, :instance_variables, :text
|
|
6
8
|
end
|
|
9
|
+
|
|
10
|
+
def down
|
|
11
|
+
remove_column :rails_error_dashboard_error_logs, :instance_variables if column_exists?(:rails_error_dashboard_error_logs, :instance_variables)
|
|
12
|
+
end
|
|
7
13
|
end
|
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
class CreateRailsErrorDashboardSwallowedExceptions < ActiveRecord::Migration[7.0]
|
|
4
4
|
def change
|
|
5
|
+
# Guard against the squashed schema migration having already created this
|
|
6
|
+
# table — without it, every later migration is silently cancelled.
|
|
7
|
+
return if table_exists?(:rails_error_dashboard_swallowed_exceptions)
|
|
8
|
+
|
|
5
9
|
create_table :rails_error_dashboard_swallowed_exceptions do |t|
|
|
6
10
|
t.string :exception_class, null: false, limit: 250
|
|
7
11
|
t.string :raise_location, null: false, limit: 250
|
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
class CreateRailsErrorDashboardDiagnosticDumps < ActiveRecord::Migration[7.0]
|
|
4
4
|
def change
|
|
5
|
+
# Guard against the squashed schema migration having already created this
|
|
6
|
+
# table — without it, every later migration is silently cancelled.
|
|
7
|
+
return if table_exists?(:rails_error_dashboard_diagnostic_dumps)
|
|
8
|
+
|
|
5
9
|
create_table :rails_error_dashboard_diagnostic_dumps do |t|
|
|
6
10
|
t.references :application, null: false,
|
|
7
11
|
foreign_key: { to_table: :rails_error_dashboard_applications }
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Storm protection honesty layer: one row per storm episode, powering the
|
|
4
|
+
# dashboard banner ("storm detected, counts recorded, detail sampled") and
|
|
5
|
+
# the storm history page. Small table — a few rows per incident.
|
|
6
|
+
class CreateStormEvents < ActiveRecord::Migration[7.0]
|
|
7
|
+
def change
|
|
8
|
+
return if table_exists?(:rails_error_dashboard_storm_events)
|
|
9
|
+
|
|
10
|
+
create_table :rails_error_dashboard_storm_events do |t|
|
|
11
|
+
t.datetime :started_at, null: false
|
|
12
|
+
t.datetime :ended_at # NULL while the storm is active
|
|
13
|
+
t.integer :peak_rate_per_minute, default: 0
|
|
14
|
+
t.boolean :reached_open, default: false # true if count-only mode engaged
|
|
15
|
+
t.bigint :events_total, default: 0 # count-only total = counted_only + overflow (excludes :lite/:full rows)
|
|
16
|
+
t.bigint :events_counted_only, default: 0 # counted in memory, no rows
|
|
17
|
+
t.bigint :events_overflow, default: 0 # beyond the bounded map — exact total, anonymous identity
|
|
18
|
+
t.integer :fingerprints_affected, default: 0
|
|
19
|
+
t.text :top_fingerprints # JSON: top 5 by count [{class, message, count}]
|
|
20
|
+
t.timestamps
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
add_index :rails_error_dashboard_storm_events, :ended_at,
|
|
24
|
+
name: "index_red_storm_events_on_ended_at"
|
|
25
|
+
add_index :rails_error_dashboard_storm_events, :started_at,
|
|
26
|
+
name: "index_red_storm_events_on_started_at"
|
|
27
|
+
end
|
|
28
|
+
end
|