rails_error_dashboard 0.8.1 → 0.8.2

data/lib/rails_error_dashboard/services/storm_protection/fingerprint_buckets.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    module StormProtection
+      # Layer 1: per-fingerprint rate limiting with graceful degradation.
+      #
+      # Each fingerprint gets a 60-second window. Within the window:
+      #   - first N events            → :full  (everything captured)
+      #   - past N, every Mth event   → :lite  (row captured, context shed)
+      #   - everything else           → :count_only (in-memory count, flushed later)
+      #
+      # The first event of every window is ALWAYS at least :lite, so a
+      # melting-down fingerprint still has a fresh exemplar each minute —
+      # deterministic, unlike rand-based sampling.
+      #
+      # Calm-mode adaptive context sampling rides the same entries: after K
+      # full-context captures per fingerprint per day, context is captured
+      # only every Mth time (an error firing 1000×/day doesn't need 1000
+      # breadcrumb trails). Occurrence rows are unaffected in calm mode.
+      #
+      # Concurrency: entries are mutable structs in a Concurrent::Map with
+      # plain (unlocked) field increments — races can miscount by a handful
+      # of events, which is acceptable for rate limiting. No mutex anywhere.
+      #
+      # Memory: the map is bounded. Once full, unseen fingerprints are NOT
+      # tracked and decide as :full — in calm weather that's harmless; in a
+      # storm of unique fingerprints the global breaker (Layer 2) takes over.
+      class FingerprintBuckets
+        WINDOW_SECONDS = 60
+        DAY_SECONDS = 86_400
+
+        Entry = Struct.new(:window_start, :window_count, :day_start, :day_full_context_count)
+
+        def initialize(clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+          @clock = clock
+          reset!
+        end
+
+        def reset!
+          @entries = Concurrent::Map.new
+        end
+
+        # Decide capture fidelity for one event of this fingerprint.
+        # @return [Symbol] :full, :lite, or :count_only
+        def decide(gate_key)
+          now = @clock.call
+          entry = fetch_entry(gate_key, now)
+          return :full unless entry # map full — Layer 2 owns the storm case
+
+          roll_windows(entry, now)
+          entry.window_count += 1
+          n = entry.window_count
+
+          if n <= full_per_minute
+            decide_calm_context(entry)
+          elsif n == full_per_minute + 1 || ((n - full_per_minute) % keep_every).zero?
+            :lite
+          else
+            :count_only
+          end
+        end
+
+        private
+
+        def fetch_entry(gate_key, now)
+          existing = @entries[gate_key]
+          return existing if existing
+
+          # Bounded: never insert past the cap (size check is approximate
+          # under concurrency — a few entries over the cap is fine)
+          return nil if @entries.size >= max_tracked
+
+          @entries.compute_if_absent(gate_key) { Entry.new(now, 0, now, 0) }
+        end
+
+        def roll_windows(entry, now)
+          if now - entry.window_start >= WINDOW_SECONDS
+            entry.window_start = now
+            entry.window_count = 0
+          end
+          if now - entry.day_start >= DAY_SECONDS
+            entry.day_start = now
+            entry.day_full_context_count = 0
+          end
+        end
+
+        # Under the per-minute cap: full context unless this fingerprint has
+        # already produced plenty of full-context captures today.
+        def decide_calm_context(entry)
+          entry.day_full_context_count += 1
+          k = entry.day_full_context_count
+
+          if k <= context_threshold_per_day || (k % context_keep_every).zero?
+            :full
+          else
+            :lite
+          end
+        end
+
+        def full_per_minute
+          RailsErrorDashboard.configuration.storm_fingerprint_full_per_minute.to_i
+        end
+
+        def keep_every
+          [ RailsErrorDashboard.configuration.storm_occurrence_sample_keep_every.to_i, 1 ].max
+        end
+
+        def context_threshold_per_day
+          RailsErrorDashboard.configuration.context_sampling_threshold_per_day.to_i
+        end
+
+        def context_keep_every
+          [ RailsErrorDashboard.configuration.context_sampling_keep_every.to_i, 1 ].max
+        end
+
+        def max_tracked
+          RailsErrorDashboard.configuration.storm_max_tracked_fingerprints.to_i
+        end
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/storm_protection/gate.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    module StormProtection
+      # Facade for the storm-protection hot path. One call per capture attempt:
+      #
+      #   Gate.admit!(exception, context) # => :full | :lite | :count_only
+      #
+      #   :full       — capture everything (the normal path)
+      #   :lite       — capture the error + occurrence row, shed context
+      #                 (breadcrumbs / system health / locals / ivars)
+      #   :count_only — nothing stored now; counted in memory, reconciled
+      #                 onto ErrorLog.occurrence_count by the flush job
+      #
+      # Safety contract (mirrors SwallowedExceptionTracker):
+      # - FAILS OPEN: any internal error → :full. Protection must never be
+      #   the thing that loses an error.
+      # - Zero I/O on the hot path. The only DB-adjacent work is enqueueing
+      #   the flush job at most once per flush interval.
+      # - Budget: digest + atomic increment + comparisons (~µs). Benchmarked.
+      # - Per-process state; Puma workers each run their own breaker. No
+      #   thread-locals — shared atomics, so no Thread.current cleanup needed.
+      #
+      # IMPORTANT ordering: callers must run ExceptionFilter (ignore list +
+      # static sampling) BEFORE this gate — ignored exceptions must never
+      # count toward storm state or be reconciled into ErrorLogs.
+      class Gate
+        class << self
+          def admit!(exception, context = {})
+            return :full unless enabled?
+
+            state = breaker.record!
+            maybe_storm_notification(state)
+
+            decision = decide(state, exception, context)
+            maybe_flush!
+            decision
+          rescue => e
+            RailsErrorDashboard::Logger.error(
+              "[RailsErrorDashboard] StormProtection failed open: #{e.class} - #{e.message}"
+            )
+            :full
+          end
+
+          # While the breaker is not closed, per-error notifications are
+          # suppressed (a single storm notification replaces them).
+          def notifications_suppressed?
+            enabled? && breaker.state != :closed
+          rescue
+            false
+          end
+
+          # Always-on cap for auto-created issues (a storm of NEW critical
+          # fingerprints must not open 500 GitHub/Linear issues). Token
+          # bucket: N per rolling window, per process. Each call consumes a
+          # token — call only when actually about to create an issue.
+          def issue_creation_allowed?
+            return true unless enabled?
+
+            now = monotonic_now
+            window = RailsErrorDashboard.configuration.auto_issue_rate_limit_window_minutes.to_i * 60
+            limit = RailsErrorDashboard.configuration.auto_issue_rate_limit_count.to_i
+
+            @issue_window_start ||= now
+            @issue_window_count ||= Concurrent::AtomicFixnum.new(0)
+
+            if now - @issue_window_start >= window
+              @issue_window_start = now
+              @issue_window_count = Concurrent::AtomicFixnum.new(0)
+            end
+
+            @issue_window_count.increment <= limit
+          rescue
+            true
+          end
+
+          def state
+            enabled? ? breaker.state : :closed
+          rescue
+            :closed
+          end
+
+          # Exposed for the flush job (episode metadata for storm_events).
+          def breaker
+            @breaker ||= CircuitBreaker.new
+          end
+
+          def count_buffer
+            @count_buffer ||= CountBuffer.new
+          end
+
+          def fingerprint_buckets
+            @fingerprint_buckets ||= FingerprintBuckets.new
+          end
+
+          # Test hook + fork hygiene: fresh state, no leftover episodes.
+          def reset!
+            @breaker = nil
+            @count_buffer = nil
+            @fingerprint_buckets = nil
+            @probe_counter = nil
+            @issue_window_start = nil
+            @issue_window_count = nil
+            @last_flush = nil
+            @storm_notified_episode = nil
+          end
+
+          private
+
+          def enabled?
+            RailsErrorDashboard.configuration.enable_storm_protection
+          end
+
+          # All identity computation happens here, once, in locals — no
+          # shared mutable caches (thread safety by construction).
+          def decide(state, exception, context)
+            case state
+            when :open
+              count!(exception, context)
+              :count_only
+            when :half_open
+              # Probe: a trickle of :lite captures tells us whether the storm
+              # has actually subsided; everything else stays counted.
+              if (probe_counter.increment % 10).zero?
+                :lite
+              else
+                count!(exception, context)
+                :count_only
+              end
+            when :shedding
+              # Never :full while shedding — context capture is request-thread
+              # CPU we can't afford. Buckets still apply their per-fingerprint
+              # row sampling underneath.
+              parts = gate_parts(exception, context)
+              if fingerprint_buckets.decide(gate_key(parts)) == :count_only
+                count_buffer.record(gate_key(parts), parts)
+                :count_only
+              else
+                :lite
+              end
+            else # :closed
+              parts = gate_parts(exception, context)
+              decision = fingerprint_buckets.decide(gate_key(parts))
+              count_buffer.record(gate_key(parts), parts) if decision == :count_only
+              decision
+            end
+          end
+
+          def count!(exception, context)
+            parts = gate_parts(exception, context)
+            count_buffer.record(gate_key(parts), parts)
+          end
+
+          # Cheap in-process bucketing key. Deliberately NOT the canonical
+          # error_hash (that needs application_id = DB); the flush job
+          # recomputes the canonical hash from the stored parts.
+          def gate_key(parts)
+            parts[:gate_key] ||= parts[:custom_hash] || Digest::SHA256.hexdigest(
+              "#{parts[:error_class]}|#{ErrorHashGenerator.normalize_message(parts[:message])}|" \
+              "#{parts[:first_app_frame]}|#{parts[:controller_name]}|#{parts[:action_name]}"
+            )[0..15]
+          end
+
+          def gate_parts(exception, context)
+            {
+              error_class: exception.class.name,
+              message: exception.message.to_s[0, 500],
+              first_app_frame: ErrorHashGenerator.extract_app_frame_from_locations(exception) ||
+                               ErrorHashGenerator.extract_app_frame(exception.backtrace),
+              controller_name: context[:controller_name]&.to_s,
+              action_name: context[:action_name]&.to_s,
+              custom_hash: custom_hash_for(exception, context)
+            }
+          end
+
+          # When a custom fingerprint lambda is configured the canonical hash
+          # doesn't include application_id, so the gate can compute it exactly
+          # — the flush job then reconciles by hash directly.
+          def custom_hash_for(exception, context)
+            return nil unless RailsErrorDashboard.configuration.custom_fingerprint
+
+            result = RailsErrorDashboard.configuration.custom_fingerprint.call(exception, context)
+            return nil unless result.is_a?(String) && !result.empty?
+
+            Digest::SHA256.hexdigest(result)[0..15]
+          rescue
+            nil
+          end
+
+          def probe_counter
+            @probe_counter ||= Concurrent::AtomicFixnum.new(0)
+          end
+
+          # Piggyback flush (SwallowedExceptionTracker pattern): cheap
+          # timestamp check per admit; enqueue at most once per interval.
+          def maybe_flush!
+            now = monotonic_now
+            interval = RailsErrorDashboard.configuration.storm_flush_interval_seconds.to_i
+            return if now - (@last_flush ||= now) < interval
+            return unless count_buffer.any? || breaker.episode_snapshot
+
+            @last_flush = now
+            snapshot = count_buffer.snapshot!
+            episode = breaker.episode_snapshot
+            breaker.clear_closed_episode!
+
+            StormFlushJob.perform_later(
+              entries: snapshot[:entries],
+              overflow: snapshot[:overflow],
+              episode: serialize_episode(episode)
+            )
+          rescue => e
+            RailsErrorDashboard::Logger.error(
+              "[RailsErrorDashboard] Storm flush enqueue failed: #{e.class} - #{e.message}"
+            )
+          end
+
+          def serialize_episode(episode)
+            return nil unless episode
+
+            {
+              "started_at" => episode[:started_at]&.iso8601,
+              "ended_at" => episode[:ended_at]&.iso8601,
+              "peak_rate_per_minute" => episode[:peak_rate_per_minute],
+              "reached_open" => episode[:reached_open]
+            }
+          end
+
+          # One notification per storm episode, on the first transition out
+          # of :closed.
+          def maybe_storm_notification(state)
+            return if state == :closed
+            return unless RailsErrorDashboard.configuration.storm_notification
+
+            episode = breaker.episode_snapshot
+            return unless episode
+            return if @storm_notified_episode == episode[:started_at]
+
+            @storm_notified_episode = episode[:started_at]
+            StormNotificationJob.perform_later(
+              started_at: episode[:started_at].iso8601,
+              state: state.to_s
+            )
+          rescue => e
+            RailsErrorDashboard::Logger.error(
+              "[RailsErrorDashboard] Storm notification enqueue failed: #{e.class} - #{e.message}"
+            )
+          end
+
+          def monotonic_now
+            Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/subscribers/issue_tracker_subscriber.rb

@@ -13,6 +13,18 @@ module RailsErrorDashboard
         # Called when a new error is first logged
         def on_error_logged(error_log)
           return unless should_auto_create?(error_log)
+
+          # Always-on rate cap (default 5 per 10 min): a storm of NEW distinct
+          # fingerprints must not open hundreds of issues. Checked LAST so a
+          # token is only consumed when we were actually about to create.
+          unless Services::StormProtection::Gate.issue_creation_allowed?
+            RailsErrorDashboard::Logger.warn(
+              "[RailsErrorDashboard] Auto-issue creation rate limit reached — " \
+              "skipping issue for ErrorLog ##{error_log.id} (#{error_log.error_type})"
+            )
+            return
+          end
+
           dashboard_url = Services::NotificationHelpers.dashboard_url(error_log)
           CreateIssueJob.perform_later(error_log.id, dashboard_url: dashboard_url)
         rescue => e

data/lib/rails_error_dashboard/version.rb

@@ -1,3 +1,3 @@
 module RailsErrorDashboard
-  VERSION = "0.8.1"
+  VERSION = "0.8.2"
 end

data/lib/rails_error_dashboard.rb

@@ -69,6 +69,10 @@ require "rails_error_dashboard/services/github_issue_client"
 require "rails_error_dashboard/services/gitlab_issue_client"
 require "rails_error_dashboard/services/codeberg_issue_client"
 require "rails_error_dashboard/services/linear_issue_client"
+require "rails_error_dashboard/services/storm_protection/circuit_breaker"
+require "rails_error_dashboard/services/storm_protection/fingerprint_buckets"
+require "rails_error_dashboard/services/storm_protection/count_buffer"
+require "rails_error_dashboard/services/storm_protection/gate"
 require "rails_error_dashboard/services/database_health_inspector"
 require "rails_error_dashboard/services/cache_analyzer"
 require "rails_error_dashboard/services/llm_summary"
@@ -80,6 +84,7 @@ require "rails_error_dashboard/services/diagnostic_dump_generator"
 require "rails_error_dashboard/services/coverage_tracker"
 require "rails_error_dashboard/services/digest_builder"
 require "rails_error_dashboard/queries/user_impact_summary"
+require "rails_error_dashboard/queries/storm_history"
 require "rails_error_dashboard/subscribers/breadcrumb_subscriber"
 require "rails_error_dashboard/subscribers/rack_attack_subscriber"
 require "rails_error_dashboard/subscribers/action_cable_subscriber"
@@ -97,6 +102,7 @@ require "rails_error_dashboard/commands/log_error"
 require "rails_error_dashboard/commands/resolve_error"
 require "rails_error_dashboard/commands/create_issue"
 require "rails_error_dashboard/commands/link_existing_issue"
+require "rails_error_dashboard/commands/flush_storm_counts"
 require "rails_error_dashboard/services/issue_body_formatter"
 require "rails_error_dashboard/commands/batch_resolve_errors"
 require "rails_error_dashboard/commands/batch_delete_errors"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails_error_dashboard
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Anjan Jagirdar
@@ -262,6 +262,8 @@ files:
 - app/jobs/rails_error_dashboard/retention_cleanup_job.rb
 - app/jobs/rails_error_dashboard/scheduled_digest_job.rb
 - app/jobs/rails_error_dashboard/slack_error_notification_job.rb
+- app/jobs/rails_error_dashboard/storm_flush_job.rb
+- app/jobs/rails_error_dashboard/storm_notification_job.rb
 - app/jobs/rails_error_dashboard/swallowed_exception_flush_job.rb
 - app/jobs/rails_error_dashboard/webhook_error_notification_job.rb
 - app/mailers/rails_error_dashboard/application_mailer.rb
@@ -275,6 +277,7 @@ files:
 - app/models/rails_error_dashboard/error_log.rb
 - app/models/rails_error_dashboard/error_logs_record.rb
 - app/models/rails_error_dashboard/error_occurrence.rb
+- app/models/rails_error_dashboard/storm_event.rb
 - app/models/rails_error_dashboard/swallowed_exception.rb
 - app/views/layouts/rails_error_dashboard.html.erb
 - app/views/rails_error_dashboard/digest_mailer/digest_summary.html.erb
@@ -321,6 +324,7 @@ files:
 - app/views/rails_error_dashboard/errors/settings.html.erb
 - app/views/rails_error_dashboard/errors/settings/_value_badge.html.erb
 - app/views/rails_error_dashboard/errors/show.html.erb
+- app/views/rails_error_dashboard/errors/storms.html.erb
 - app/views/rails_error_dashboard/errors/swallowed_exceptions.html.erb
 - app/views/rails_error_dashboard/errors/user_impact.html.erb
 - config/routes.rb
@@ -359,6 +363,7 @@ files:
 - db/migrate/20260325000001_fix_swallowed_exceptions_index_for_mysql.rb
 - db/migrate/20260326000001_add_issue_tracking_to_error_logs.rb
 - db/migrate/20260503000001_backfill_resolved_status.rb
+- db/migrate/20260613000001_create_storm_events.rb
 - lib/generators/rails_error_dashboard/install/install_generator.rb
 - lib/generators/rails_error_dashboard/install/templates/README
 - lib/generators/rails_error_dashboard/install/templates/initializer.rb
@@ -376,6 +381,7 @@ files:
 - lib/rails_error_dashboard/commands/create_issue.rb
 - lib/rails_error_dashboard/commands/find_or_create_application.rb
 - lib/rails_error_dashboard/commands/find_or_increment_error.rb
+- lib/rails_error_dashboard/commands/flush_storm_counts.rb
 - lib/rails_error_dashboard/commands/flush_swallowed_exceptions.rb
 - lib/rails_error_dashboard/commands/increment_cascade_detection.rb
 - lib/rails_error_dashboard/commands/link_existing_issue.rb
@@ -431,6 +437,7 @@ files:
 - lib/rails_error_dashboard/queries/recurring_issues.rb
 - lib/rails_error_dashboard/queries/release_timeline.rb
 - lib/rails_error_dashboard/queries/similar_errors.rb
+- lib/rails_error_dashboard/queries/storm_history.rb
 - lib/rails_error_dashboard/queries/swallowed_exception_summary.rb
 - lib/rails_error_dashboard/queries/user_impact_summary.rb
 - lib/rails_error_dashboard/services/analytics_cache_manager.rb
@@ -484,6 +491,10 @@ files:
 - lib/rails_error_dashboard/services/slack_payload_builder.rb
 - lib/rails_error_dashboard/services/source_code_reader.rb
 - lib/rails_error_dashboard/services/statistical_classifier.rb
+- lib/rails_error_dashboard/services/storm_protection/circuit_breaker.rb
+- lib/rails_error_dashboard/services/storm_protection/count_buffer.rb
+- lib/rails_error_dashboard/services/storm_protection/fingerprint_buckets.rb
+- lib/rails_error_dashboard/services/storm_protection/gate.rb
 - lib/rails_error_dashboard/services/swallowed_exception_tracker.rb
 - lib/rails_error_dashboard/services/system_health_snapshot.rb
 - lib/rails_error_dashboard/services/variable_serializer.rb
@@ -512,7 +523,7 @@ metadata:
   funding_uri: https://github.com/sponsors/AnjanJ
 post_install_message: |
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-    RED (Rails Error Dashboard) v0.8.1
+    RED (Rails Error Dashboard) v0.8.2
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
   First install: