rails_error_dashboard 0.8.1 → 0.8.2

data/lib/rails_error_dashboard/commands/log_error.rb

@@ -6,15 +6,59 @@ module RailsErrorDashboard
     # This is a write operation that creates an ErrorLog record
     class LogError
       def self.call(exception, context = {})
-        # Check if async logging is enabled
+        # Filter FIRST (ignore list + static sampling) so ignored exceptions
+        # never count toward storm state. _pre_filtered prevents the sync path
+        # from re-rolling the sampling dice (rate would square otherwise).
+        # The filter + gate run inside this method's rescue: nothing in the
+        # capture path may ever raise into the host app.
+        begin
+          unless Services::ExceptionFilter.should_log?(exception)
+            # Preserve the OTel contract: filtered captures still emit a span
+            # tagged filtered=true (no-op when OTel export is disabled).
+            Integrations::Tracer.in_span(
+              "capture_error",
+              kind: :capture,
+              attributes: build_capture_span_attributes(exception, was_async: false)
+            ) do |span|
+              span&.set_attribute("rails_error_dashboard.filtered", true)
+            end
+            return nil
+          end
+          context = context.merge(_pre_filtered: true)
+
+          # Storm protection gate — BEFORE the async branch, because with
+          # SolidQueue the enqueue itself is a DB write. :count_only events are
+          # tallied in memory and reconciled by StormFlushJob; nothing else
+          # happens for them (that's the point).
+          storm_decision = Services::StormProtection::Gate.admit!(exception, context)
+          return nil if storm_decision == :count_only
+          context = context.merge(_storm_decision: storm_decision) if storm_decision == :lite
+        rescue => e
+          RailsErrorDashboard::Logger.error(
+            "[RailsErrorDashboard] Capture pre-checks failed: #{e.class} - #{e.message}"
+          )
+          # Fall through and attempt full capture — fail open, never raise
+        end
+
         if RailsErrorDashboard.configuration.async_logging
           # For async logging, just enqueue the job
-          # All filtering happens when the job runs
           call_async(exception, context)
         else
           # For sync logging, execute immediately
           new(exception, context).call
         end
+      rescue => e
+        RailsErrorDashboard::Logger.error(
+          "[RailsErrorDashboard] LogError.call failed: #{e.class} - #{e.message}"
+        )
+        nil
+      end
+
+      # :lite captures shed context (breadcrumbs/health/locals/ivars) — the
+      # storm shedding ladder's first economy. Symbol or string key: the
+      # async job round-trips context through the queue serializer.
+      def self.storm_lite?(context)
+        context[:_storm_decision].to_s == "lite"
       end
 
       # Build the base OTel span attributes available before any work happens.
@@ -42,18 +86,22 @@ module RailsErrorDashboard
           cause_chain: serialize_cause_chain(exception)
         }
 
+        # Storm shedding: :lite captures skip ALL pre-enqueue context harvest —
+        # this is request-thread CPU, the most valuable thing to shed.
+        lite = storm_lite?(context)
+
         # Harvest breadcrumbs NOW (before job dispatch — different thread won't have them)
-        if RailsErrorDashboard.configuration.enable_breadcrumbs
+        if !lite && RailsErrorDashboard.configuration.enable_breadcrumbs
           context = context.merge(_serialized_breadcrumbs: Services::BreadcrumbCollector.harvest)
         end
 
         # Capture system health NOW (metrics are time-sensitive, different thread = different state)
-        if RailsErrorDashboard.configuration.enable_system_health
+        if !lite && RailsErrorDashboard.configuration.enable_system_health
           context = context.merge(_serialized_system_health: Services::SystemHealthSnapshot.capture)
         end
 
         # Capture local variables NOW (TracePoint attaches to exception, must extract before job dispatch)
-        if RailsErrorDashboard.configuration.enable_local_variables
+        if !lite && RailsErrorDashboard.configuration.enable_local_variables
           begin
             raw_locals = Services::LocalVariableCapturer.extract(exception)
             if raw_locals.is_a?(Hash) && raw_locals.any?
@@ -65,7 +113,7 @@ module RailsErrorDashboard
         end
 
         # Capture instance variables NOW (same reason — attached to exception object)
-        if RailsErrorDashboard.configuration.enable_instance_variables
+        if !lite && RailsErrorDashboard.configuration.enable_instance_variables
           begin
             raw_ivars = Services::LocalVariableCapturer.extract_instance_vars(exception)
             if raw_ivars.is_a?(Hash) && raw_ivars.any?
@@ -157,12 +205,19 @@ module RailsErrorDashboard
           kind: :capture,
           attributes: self.class.build_capture_span_attributes(@exception, was_async: false)
         ) do |span|
-        # Check if this exception should be logged (ignore list + sampling)
-        if !Services::ExceptionFilter.should_log?(@exception)
+        # Check if this exception should be logged (ignore list + sampling).
+        # Skipped when self.call already filtered (re-rolling the sampling
+        # dice here would square the effective rate).
+        if !@context[:_pre_filtered] && !Services::ExceptionFilter.should_log?(@exception)
           span&.set_attribute("rails_error_dashboard.filtered", true)
           next nil
         end
 
+        # Storm shedding: :lite captures keep the error + occurrence row but
+        # shed context payloads (breadcrumbs/health/locals/ivars).
+        storm_lite = self.class.storm_lite?(@context)
+        span&.set_attribute("rails_error_dashboard.storm_degraded", true) if storm_lite
+
         error_context = ValueObjects::ErrorContext.new(@context, @context[:source])
 
         # Find or create application (cached lookup)
@@ -239,7 +294,7 @@ module RailsErrorDashboard
         attributes = Services::SensitiveDataFilter.filter_attributes(attributes)
 
         # Harvest breadcrumbs (if enabled and column exists)
-        if ErrorLog.column_names.include?("breadcrumbs") && RailsErrorDashboard.configuration.enable_breadcrumbs
+        if !storm_lite && ErrorLog.column_names.include?("breadcrumbs") && RailsErrorDashboard.configuration.enable_breadcrumbs
           # Sync path: harvest from current thread
           raw_breadcrumbs = Services::BreadcrumbCollector.harvest
 
@@ -256,13 +311,13 @@ module RailsErrorDashboard
         end
 
         # Capture system health snapshot (if enabled and column exists)
-        if ErrorLog.column_names.include?("system_health") && RailsErrorDashboard.configuration.enable_system_health
+        if !storm_lite && ErrorLog.column_names.include?("system_health") && RailsErrorDashboard.configuration.enable_system_health
           health_data = @context[:_serialized_system_health] || Services::SystemHealthSnapshot.capture
           attributes[:system_health] = health_data.to_json
         end
 
         # Capture local variables (if enabled and column exists)
-        if ErrorLog.column_names.include?("local_variables") && RailsErrorDashboard.configuration.enable_local_variables
+        if !storm_lite && ErrorLog.column_names.include?("local_variables") && RailsErrorDashboard.configuration.enable_local_variables
           begin
             # Sync path: extract from exception ivar
             raw_locals = Services::LocalVariableCapturer.extract(@exception)
@@ -278,7 +333,7 @@ module RailsErrorDashboard
         end
 
         # Capture instance variables (if enabled and column exists)
-        if ErrorLog.column_names.include?("instance_variables") && RailsErrorDashboard.configuration.enable_instance_variables
+        if !storm_lite && ErrorLog.column_names.include?("instance_variables") && RailsErrorDashboard.configuration.enable_instance_variables
           begin
             # Sync path: extract from exception ivar
             raw_ivars = Services::LocalVariableCapturer.extract_instance_vars(@exception)
@@ -364,8 +419,11 @@ module RailsErrorDashboard
 
       # Dispatch notification if error is not muted and the throttle check passes.
       # Muted errors skip notifications but still fire plugin events/callbacks.
+      # During a storm (breaker not closed) per-error notifications are
+      # suppressed — a single storm notification replaces them.
       def maybe_notify(error_log)
         return if error_log.muted?
+        return if Services::StormProtection::Gate.notifications_suppressed?
         return unless yield
 
         Services::ErrorNotificationDispatcher.call(error_log)

data/lib/rails_error_dashboard/configuration.rb

@@ -67,6 +67,25 @@ module RailsErrorDashboard
     # Sampling rate for non-critical errors (0.0 to 1.0, default 1.0 = 100%)
     attr_accessor :sampling_rate
 
+    # Storm protection — circuit breaker + adaptive sampling for error floods.
+    # Protects the HOST APP from the gem's own writes during an error storm
+    # (bad deploy throwing thousands of errors/minute). Default ON: this is
+    # the feature that makes the gem quieter, so ON is the conservative choice.
+    # All thresholds are PER PROCESS (no cross-process coordination by design).
+    attr_accessor :enable_storm_protection            # Master switch (default: true)
+    attr_accessor :storm_fingerprint_full_per_minute  # Full-fidelity captures per fingerprint per minute (default: 30)
+    attr_accessor :storm_occurrence_sample_keep_every # Past the cap, keep every Nth occurrence (default: 10)
+    attr_accessor :storm_shedding_threshold_per_second # Global rate that enters shedding state (default: 10)
+    attr_accessor :storm_open_threshold_per_second    # Global rate that opens the breaker = count-only (default: 50)
+    attr_accessor :storm_cooldown_seconds             # Open → half-open probe delay (default: 60)
+    attr_accessor :storm_max_tracked_fingerprints     # Bounded in-memory map size; beyond = overflow bucket (default: 1000)
+    attr_accessor :storm_flush_interval_seconds       # Count-buffer flush cadence (default: 30)
+    attr_accessor :storm_notification                 # Single "storm in progress" notification per episode (default: true)
+    attr_accessor :auto_issue_rate_limit_count        # Max auto-created issues per window — applies always (default: 5)
+    attr_accessor :auto_issue_rate_limit_window_minutes # Window for the above (default: 10)
+    attr_accessor :context_sampling_threshold_per_day # Full-context captures per fingerprint per day before sampling (default: 25)
+    attr_accessor :context_sampling_keep_every        # After threshold, keep full context every Nth (default: 10)
+
     # Async logging configuration
     attr_accessor :async_logging
     attr_accessor :async_adapter # :sidekiq, :solid_queue, or :async
@@ -268,6 +287,21 @@ module RailsErrorDashboard
       @ignored_exceptions = []
       @custom_fingerprint = nil # Lambda: ->(exception, context) { "custom_key" }
       @sampling_rate = 1.0 # 100% by default
+
+      # Storm protection defaults (thresholds tuned via chaos Phase G — see ROADMAP)
+      @enable_storm_protection = true
+      @storm_fingerprint_full_per_minute = 30
+      @storm_occurrence_sample_keep_every = 10
+      @storm_shedding_threshold_per_second = 10
+      @storm_open_threshold_per_second = 50
+      @storm_cooldown_seconds = 60
+      @storm_max_tracked_fingerprints = 1000
+      @storm_flush_interval_seconds = 30
+      @storm_notification = true
+      @auto_issue_rate_limit_count = 5
+      @auto_issue_rate_limit_window_minutes = 10
+      @context_sampling_threshold_per_day = 25
+      @context_sampling_keep_every = 10
       @async_logging = false
       @async_adapter = :sidekiq # Battle-tested default
       @max_backtrace_lines = 100 # Matches industry standard (Rollbar, Airbrake)
@@ -683,6 +717,32 @@ module RailsErrorDashboard
         end
       end
 
+      # Validate storm protection thresholds (all must be positive when protection is on)
+      if enable_storm_protection
+        {
+          storm_fingerprint_full_per_minute: storm_fingerprint_full_per_minute,
+          storm_occurrence_sample_keep_every: storm_occurrence_sample_keep_every,
+          storm_shedding_threshold_per_second: storm_shedding_threshold_per_second,
+          storm_open_threshold_per_second: storm_open_threshold_per_second,
+          storm_cooldown_seconds: storm_cooldown_seconds,
+          storm_max_tracked_fingerprints: storm_max_tracked_fingerprints,
+          storm_flush_interval_seconds: storm_flush_interval_seconds,
+          auto_issue_rate_limit_count: auto_issue_rate_limit_count,
+          auto_issue_rate_limit_window_minutes: auto_issue_rate_limit_window_minutes,
+          context_sampling_threshold_per_day: context_sampling_threshold_per_day,
+          context_sampling_keep_every: context_sampling_keep_every
+        }.each do |name, value|
+          if value.nil? || value.to_i < 1
+            errors << "#{name} must be a positive integer (got: #{value.inspect})"
+          end
+        end
+
+        if storm_open_threshold_per_second.to_i < storm_shedding_threshold_per_second.to_i
+          errors << "storm_open_threshold_per_second (#{storm_open_threshold_per_second}) must be >= " \
+                    "storm_shedding_threshold_per_second (#{storm_shedding_threshold_per_second})"
+        end
+      end
+
       # Validate total_users_for_impact (must be positive if set)
       if total_users_for_impact && total_users_for_impact < 1
         errors << "total_users_for_impact must be at least 1 (got: #{total_users_for_impact})"

data/lib/rails_error_dashboard/queries/storm_history.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Queries
+    # Query: Storm protection episode history + active-storm lookup.
+    #
+    # Read-only. Powers the /errors/storms page and the layout banner.
+    class StormHistory
+      RECENT_BANNER_WINDOW = 24.hours
+
+      def self.call(limit: 50)
+        return { active: nil, recent: nil, events: [] } unless StormEvent.table_exists?
+
+        {
+          active: StormEvent.active.recent_first.first,
+          recent: StormEvent.ended_within(RECENT_BANNER_WINDOW).recent_first.first,
+          events: StormEvent.recent_first.limit(limit).to_a
+        }
+      rescue => e
+        RailsErrorDashboard::Logger.error(
+          "[RailsErrorDashboard] StormHistory query failed: #{e.message}"
+        )
+        { active: nil, recent: nil, events: [] }
+      end
+
+      # Cheap banner lookup for the layout — one indexed query on the happy
+      # path (no active storm), two when a banner is showing.
+      def self.banner_event
+        return nil unless RailsErrorDashboard.configuration.enable_storm_protection
+        return nil unless StormEvent.table_exists?
+
+        StormEvent.active.recent_first.first ||
+          StormEvent.ended_within(RECENT_BANNER_WINDOW).recent_first.first
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/storm_protection/circuit_breaker.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    module StormProtection
+      # Per-process circuit breaker for the error capture path.
+      #
+      # Counts capture attempts in fixed 10-second buckets and transitions
+      # between states based on the completed bucket's rate:
+      #
+      #   :closed    — normal operation, per-fingerprint buckets decide fidelity
+      #   :shedding  — elevated rate: context shed, notifications suppressed
+      #   :open      — storm: count-only mode, zero per-event I/O
+      #   :half_open — post-cooldown probe: small sample admitted, watching rate
+      #
+      # Hysteresis: opens FAST (a single hot bucket, or mid-bucket fast-trip),
+      # closes SLOW (two consecutive calm buckets) to prevent flapping.
+      #
+      # Concurrency: the hot path is one AtomicFixnum increment plus a float
+      # comparison. The mutex is taken only on bucket roll (once per 10s) and
+      # for state transitions — never per event.
+      class CircuitBreaker
+        BUCKET_SECONDS = 10
+        CALM_BUCKETS_TO_CLOSE = 2
+
+        attr_reader :state
+
+        # @param clock [#call] returns monotonic seconds; injectable for tests
+        def initialize(clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+          @clock = clock
+          @mutex = Mutex.new
+          reset!
+        end
+
+        def reset!
+          @mutex.synchronize do
+            @state = :closed
+            @bucket_start = @clock.call
+            @bucket_count = Concurrent::AtomicFixnum.new(0)
+            @calm_buckets = 0
+            @opened_at = nil
+            @episode = nil
+          end
+        end
+
+        # Count one capture attempt and return the state that should govern it.
+        # Called on EVERY capture — must stay allocation-free on the fast path.
+        def record!
+          now = @clock.call
+          roll!(now) if now - @bucket_start >= BUCKET_SECONDS
+
+          count = @bucket_count.increment
+
+          # Fast-trip: don't wait for the bucket to complete if it's already
+          # over the open threshold — at 50k errors/min a full 10s bucket
+          # would let ~8k events through before reacting.
+          if count >= open_threshold * BUCKET_SECONDS && @state != :open
+            trip_open!(now, count)
+          end
+
+          @state
+        end
+
+        # Episode metadata for the honesty layer (storm_events row).
+        # @return [Hash, nil] nil when no episode is active or recently closed
+        def episode_snapshot
+          @mutex.synchronize { @episode&.dup }
+        end
+
+        # Forget a closed episode once it has been persisted by the flush job.
+        def clear_closed_episode!
+          @mutex.synchronize do
+            @episode = nil if @episode && @episode[:ended_at]
+          end
+        end
+
+        private
+
+        def roll!(now)
+          @mutex.synchronize do
+            elapsed = now - @bucket_start
+            return if elapsed < BUCKET_SECONDS # another thread already rolled
+
+            rate = @bucket_count.value / elapsed.to_f
+            @bucket_start = now
+            @bucket_count = Concurrent::AtomicFixnum.new(0)
+            transition!(rate, now)
+          end
+        end
+
+        # Transition table — runs inside @mutex, once per bucket roll.
+        # track_peak runs AFTER the case: a transition out of :closed creates
+        # the episode, and the triggering bucket's rate must be its first peak.
+        def transition!(rate, now)
+          case @state
+          when :closed
+            if rate >= open_threshold
+              open!(now)
+            elsif rate >= shedding_threshold
+              enter!(:shedding, now)
+            end
+          when :shedding
+            if rate >= open_threshold
+              open!(now)
+            elsif rate < shedding_threshold / 2.0
+              calm_step!(now)
+            else
+              @calm_buckets = 0
+            end
+          when :open
+            if now - @opened_at >= cooldown_seconds && rate < shedding_threshold
+              @state = :half_open
+              @calm_buckets = 0
+            end
+          when :half_open
+            if rate >= shedding_threshold
+              open!(now)
+            else
+              calm_step!(now)
+            end
+          end
+
+          track_peak(rate)
+        end
+
+        def calm_step!(now)
+          @calm_buckets += 1
+          close!(now) if @calm_buckets >= CALM_BUCKETS_TO_CLOSE
+        end
+
+        def enter!(new_state, _now)
+          begin_episode! if @state == :closed
+          @state = new_state
+          @calm_buckets = 0
+        end
+
+        def open!(now)
+          begin_episode! if @state == :closed
+          @state = :open
+          @opened_at = now
+          @calm_buckets = 0
+          @episode[:reached_open] = true if @episode
+        end
+
+        # Mid-bucket fast trip — takes the mutex (rare: at most once per storm onset).
+        def trip_open!(now, count)
+          @mutex.synchronize do
+            return if @state == :open
+
+            begin_episode! if @state == :closed
+            track_peak(count / [ now - @bucket_start, 1.0 ].max)
+            @state = :open
+            @opened_at = now
+            @calm_buckets = 0
+            @episode[:reached_open] = true if @episode
+          end
+        end
+
+        def close!(_now)
+          @state = :closed
+          @calm_buckets = 0
+          @episode[:ended_at] = Time.current if @episode
+        end
+
+        def begin_episode!
+          @episode = {
+            started_at: Time.current,
+            ended_at: nil,
+            peak_rate_per_minute: 0,
+            reached_open: false
+          }
+        end
+
+        def track_peak(rate_per_second)
+          return unless @episode
+
+          per_minute = (rate_per_second * 60).round
+          @episode[:peak_rate_per_minute] = per_minute if per_minute > @episode[:peak_rate_per_minute]
+        end
+
+        def shedding_threshold
+          RailsErrorDashboard.configuration.storm_shedding_threshold_per_second.to_f
+        end
+
+        def open_threshold
+          RailsErrorDashboard.configuration.storm_open_threshold_per_second.to_f
+        end
+
+        def cooldown_seconds
+          RailsErrorDashboard.configuration.storm_cooldown_seconds.to_i
+        end
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/storm_protection/count_buffer.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    module StormProtection
+      # In-memory accumulator for events that are counted but not stored
+      # per-event (Layer 1 overflow and the breaker's count-only mode).
+      #
+      # Stores exact counts plus just enough identity to reconcile onto the
+      # right ErrorLog at flush time: the flush command recomputes the
+      # canonical error_hash from these parts (with application resolved in
+      # the background job, where DB access is allowed) and issues a single
+      # `occurrence_count = occurrence_count + N` UPDATE per fingerprint.
+      # Fingerprints first seen during count-only mode get a minimal ErrorLog
+      # created from the stored exemplar. Counting is exact — no extrapolation.
+      #
+      # Memory: bounded map; beyond the cap events land in a single overflow
+      # counter (still exact in total, anonymous in identity).
+      #
+      # Concurrency: snapshot! atomically swaps the whole map out via
+      # AtomicReference, so flushing never races with recording.
+      class CountBuffer
+        Entry = Struct.new(
+          :error_class, :message, :first_app_frame,
+          :controller_name, :action_name, :custom_hash,
+          :count, :first_seen_at, :last_seen_at
+        )
+
+        def initialize
+          reset!
+        end
+
+        def reset!
+          @map_ref = Concurrent::AtomicReference.new(Concurrent::Map.new)
+          @overflow = Concurrent::AtomicFixnum.new(0)
+        end
+
+        # Record one counted-not-stored event.
+        # @param gate_key [String] cheap in-process bucketing key
+        # @param parts [Hash] identity parts captured at the gate
+        def record(gate_key, parts)
+          map = @map_ref.get
+          entry = map[gate_key]
+
+          unless entry
+            if map.size >= max_tracked
+              @overflow.increment
+              return
+            end
+            entry = map.compute_if_absent(gate_key) do
+              Entry.new(
+                parts[:error_class], parts[:message], parts[:first_app_frame],
+                parts[:controller_name], parts[:action_name], parts[:custom_hash],
+                Concurrent::AtomicFixnum.new(0), Time.current, Time.current
+              )
+            end
+          end
+
+          entry.count.increment
+          entry.last_seen_at = Time.current
+        end
+
+        def any?
+          @overflow.value.positive? || !@map_ref.get.empty?
+        end
+
+        # Atomically swap the buffer out and return serializable entry hashes.
+        # @return [Hash] { entries: Array<Hash>, overflow: Integer }
+        def snapshot!
+          old_map = @map_ref.get_and_set(Concurrent::Map.new)
+          overflow = @overflow.value
+          @overflow.update { |v| v - overflow }
+
+          entries = []
+          old_map.each_pair do |_key, entry|
+            entries << {
+              "error_class" => entry.error_class,
+              "message" => entry.message,
+              "first_app_frame" => entry.first_app_frame,
+              "controller_name" => entry.controller_name,
+              "action_name" => entry.action_name,
+              "custom_hash" => entry.custom_hash,
+              "count" => entry.count.value,
+              "first_seen_at" => entry.first_seen_at.iso8601,
+              "last_seen_at" => entry.last_seen_at.iso8601
+            }
+          end
+
+          { entries: entries, overflow: overflow }
+        end
+
+        private
+
+        def max_tracked
+          RailsErrorDashboard.configuration.storm_max_tracked_fingerprints.to_i
+        end
+      end
+    end
+  end
+end