rails_error_dashboard 0.12.1 → 0.13.0

data/lib/rails_error_dashboard/services/notification_throttler.rb

@@ -2,21 +2,73 @@
 
 module RailsErrorDashboard
   module Services
-    # Pure algorithm: Throttle error notifications to prevent alert fatigue
+    # Throttle error notifications to prevent alert fatigue
     #
     # Checks severity minimum, per-error cooldown, and threshold milestones.
-    # Uses in-memory cache (same pattern as BaselineAlertThrottler).
+    #
+    # The cooldown is CLAIMED IN THE DATABASE (error_logs.last_notified_at, see
+    # claim!). A per-process Hash cannot do this job: every Puma worker and
+    # every job process has its own, so one error reopened by a bad deploy
+    # notified once per process, and a restart forgot the cooldown altogether.
+    # The Hash survives only as a bounded fallback for the window between
+    # upgrading the gem and running its migration, and for callers that pass
+    # something other than a persisted row.
+    #
     # Thread-safe via Mutex. Fail-open: returns true on any error.
     class NotificationThrottler
       # Severity levels ranked from lowest to highest
       SEVERITY_RANK = { low: 0, medium: 1, high: 2, critical: 3 }.freeze
 
+      # Hard cap on the in-process fallback. Expired entries are swept on every
+      # insert; the cap only bites when more than this many distinct errors are
+      # inside their cooldown at once, and then the oldest goes first.
+      MAX_TRACKED = 1_000
+
+      COOLDOWN_COLUMN = "last_notified_at"
+
       @last_notification_times = {}
       @mutex = Mutex.new
+      @burst_mutex = Mutex.new
+      @burst_window_start = nil
+      @burst_count = 0
 
       class << self
-        # Should we send a notification for this error?
-        # Checks: severity minimum + cooldown period
+        # Take the right to notify about this error, atomically.
+        #
+        # Call it immediately BEFORE dispatching, and dispatch only on true. It
+        # is claim-then-send: a claim followed by a failed send is not handed
+        # back, so that error stays quiet until the cooldown ends. The
+        # alternative (send, then record) is what let N processes all send.
+        #
+        # With the column present this is ONE statement and no read:
+        #
+        #   UPDATE error_logs SET last_notified_at = :now
+        #   WHERE id = :id AND (last_notified_at IS NULL OR last_notified_at < :cutoff)
+        #
+        # Exactly one connection can match the row, on SQLite, PostgreSQL and
+        # MySQL alike, so exactly one process notifies per cooldown window.
+        #
+        # @param error_log [ErrorLog] the row being notified about
+        # @param respect_cooldown [Boolean] false for notifications the cooldown
+        #   has never applied to (first occurrence, threshold milestones): always
+        #   granted, but still stamped, so a reopen soon afterwards is throttled
+        # @return [Boolean] true if the caller may notify
+        def claim!(error_log, respect_cooldown: true)
+          minutes = respect_cooldown ? cooldown_minutes : 0
+
+          if database_claim?(error_log)
+            claim_in_database(error_log, minutes)
+          else
+            claim_in_process(error_log, minutes)
+          end
+        rescue => e
+          # Fail-open: a throttler that cannot decide must not lose a page.
+          RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] NotificationThrottler.claim! failed: #{e.class}: #{e.message}")
+          true
+        end
+
+        # Should we send a notification for this error? Read-only (severity
+        # minimum + cooldown): it takes nothing. claim! is what notifies.
         # @param error_log [ErrorLog] The error to check
         # @return [Boolean] true if notification should be sent
         def should_notify?(error_log)
@@ -79,53 +131,145 @@ module RailsErrorDashboard
           false
         end
 
-        # Record that a notification was sent for this error
-        # @param error_log [ErrorLog] The error that was notified about
-        def record_notification(error_log)
-          key = error_log.error_hash
+        # May a FIRST-OCCURRENCE notification go out right now?
+        #
+        # The per-error cooldown cannot bound a bad deploy that produces hundreds
+        # of DISTINCT new errors: each is a first occurrence, and first
+        # occurrences always notify. This is a fixed window counter:
+        #
+        #   :notify    within config.notification_burst_limit for this window
+        #   :summarize the first one over the limit: send ONE summary instead
+        #   :suppress  everything after that, until the window ends
+        #
+        # Per process, like Gate.issue_creation_allowed?, because there is no
+        # store every deployment shares except the database and this is asked on
+        # the capture path. Worst case is limit x processes per window.
+        #
+        # Each call consumes a slot: ask only when about to notify. A limit or
+        # window of 0 / nil turns the cap off. Fails open to :notify.
+        #
+        # @return [Symbol] :notify, :summarize or :suppress
+        def burst_decision
+          limit = RailsErrorDashboard.configuration.notification_burst_limit.to_i
+          window = RailsErrorDashboard.configuration.notification_burst_window_seconds.to_i
+          return :notify unless limit.positive? && window.positive?
 
-          @mutex.synchronize do
-            @last_notification_times[key] = Time.current
+          now = monotonic_now
+          count = @burst_mutex.synchronize do
+            if @burst_window_start.nil? || now - @burst_window_start >= window
+              @burst_window_start = now
+              @burst_count = 0
+            end
+            @burst_count += 1
+          end
+
+          if count <= limit
+            :notify
+          elsif count == limit + 1
+            :summarize
+          else
+            :suppress
           end
         rescue => e
-          RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] NotificationThrottler.record_notification failed: #{e.message}")
+          RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] NotificationThrottler.burst_decision failed: #{e.class}: #{e.message}")
+          :notify
+        end
+
+        # Record that a notification was sent for this error, unconditionally.
+        # Kept for callers that notify outside LogError; LogError itself uses
+        # claim!, which decides and records in one step.
+        # @param error_log [ErrorLog] The error that was notified about
+        def record_notification(error_log)
+          claim!(error_log, respect_cooldown: false)
+          nil
         end
 
-        # Clear all throttle state (for testing)
+        # Clear all in-process throttle state (for testing)
         def clear!
           @mutex.synchronize do
             @last_notification_times.clear
           end
+          @burst_mutex.synchronize do
+            @burst_window_start = nil
+            @burst_count = 0
+          end
         end
 
-        # Remove old entries to prevent memory growth
-        # @param max_age_hours [Integer] Remove entries older than this (default: 24)
-        def cleanup!(max_age_hours: 24)
-          cutoff_time = max_age_hours.hours.ago
+        private
 
-          @mutex.synchronize do
-            @last_notification_times.delete_if { |_, time| time < cutoff_time }
-          end
+        def monotonic_now
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
         end
 
-        private
+        def cooldown_minutes
+          RailsErrorDashboard.configuration.notification_cooldown_minutes.to_i
+        end
 
-        # Is the error outside the cooldown window?
-        # @param error_log [ErrorLog] The error to check
-        # @return [Boolean] true if not in cooldown (ok to notify)
-        def cooldown_ok?(error_log)
-          cooldown_minutes = RailsErrorDashboard.configuration.notification_cooldown_minutes
-          return true if cooldown_minutes.nil? || cooldown_minutes <= 0
+        # A persisted row AND the column: before the migration has run the
+        # UPDATE would raise on every notification.
+        def database_claim?(error_log)
+          error_log.is_a?(ErrorLog) && error_log.persisted? &&
+            ErrorLog.column_names.include?(COOLDOWN_COLUMN)
+        end
 
+        def claim_in_database(error_log, minutes)
+          now = Time.current
+          scope = ErrorLog.where(id: error_log.id)
+          if minutes.positive?
+            scope = scope.where("#{COOLDOWN_COLUMN} IS NULL OR #{COOLDOWN_COLUMN} < ?", now - minutes.minutes)
+          end
+
+          granted = scope.update_all(COOLDOWN_COLUMN => now) == 1
+          # No cooldown to lose: a row deleted under us must not silence the page.
+          granted || !minutes.positive?
+        end
+
+        def claim_in_process(error_log, minutes)
           key = error_log.error_hash
+          now = Time.current
 
           @mutex.synchronize do
             last_time = @last_notification_times[key]
-            return true if last_time.nil?
+            next false if minutes.positive? && last_time && now <= last_time + minutes.minutes
 
-            Time.current > (last_time + cooldown_minutes.minutes)
+            remember(key, now)
+            true
           end
         end
+
+        # Caller holds @mutex. Delete-and-reinsert keeps the Hash in recency
+        # order, so "oldest" is simply the first key.
+        def remember(key, now)
+          window = cooldown_minutes
+          @last_notification_times.delete(key)
+          # With no cooldown nothing will ever read the entry back.
+          return unless window.positive?
+
+          cutoff = now - window.minutes
+          @last_notification_times.delete_if { |_, time| time < cutoff }
+          @last_notification_times.shift while @last_notification_times.size >= MAX_TRACKED
+          @last_notification_times[key] = now
+        end
+
+        # Is the error outside the cooldown window? Read-only.
+        # @param error_log [ErrorLog] The error to check
+        # @return [Boolean] true if not in cooldown (ok to notify)
+        def cooldown_ok?(error_log)
+          minutes = cooldown_minutes
+          return true unless minutes.positive?
+
+          last_time =
+            if database_claim?(error_log)
+              # Read the ROW, not the object: claim! stamps it with update_all,
+              # so the caller's copy -- and every other process's -- is stale.
+              ErrorLog.where(id: error_log.id).pick(COOLDOWN_COLUMN)
+            else
+              @mutex.synchronize { @last_notification_times[error_log.error_hash] }
+            end
+          return true if last_time.nil?
+
+          Time.current > (last_time + minutes.minutes)
+        end
       end
     end
   end

data/lib/rails_error_dashboard/services/sensitive_data_filter.rb

@@ -51,7 +51,53 @@ module RailsErrorDashboard
         attributes
       end
 
-      # Build and cache the ParameterFilter instance
+      SESSION_DIGEST_PREFIX = "h1:"
+      SESSION_DIGEST_FORMAT = /\Ah1:\h{32}\z/
+
+      # Keyed digest of a session ID, for storage.
+      #
+      # A session ID is a bearer credential, and the only thing the dashboard does
+      # with it is equality ("these occurrences were one session"). An HMAC keeps
+      # that and is useless for replay; keying it with secret_key_base means a
+      # guessed ID cannot be confirmed offline either. The prefix tells a digest
+      # from a raw Rails session ID (itself 32 hex characters), which makes
+      # digesting idempotent and leaves room for an "h2:".
+      #
+      # Rotating secret_key_base breaks correlation across the rotation.
+      #
+      # @param raw [#to_s, nil] raw session ID, or an existing digest
+      # @return [String, nil] "h1:" + 32 hex characters; nil for blank input or on
+      #   any failure (never the raw value, never an exception)
+      def self.digest_session_id(raw)
+        value = raw.to_s
+        return nil if value.empty?
+        # Exact shape only: a raw value that merely starts with "h1:" is digested.
+        return value if value.match?(SESSION_DIGEST_FORMAT)
+
+        SESSION_DIGEST_PREFIX + OpenSSL::HMAC.hexdigest("SHA256", session_digest_key, value)[0, 32]
+      rescue => e
+        RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] Session ID digest failed: #{e.class}")
+        nil
+      end
+
+      # What to write to error_occurrences.session_id: the digest while filtering
+      # is on, the raw value when the operator has turned filtering off.
+      def self.storable_session_id(raw)
+        return digest_session_id(raw) if RailsErrorDashboard.configuration.filter_sensitive_data
+
+        raw.nil? ? nil : raw.to_s
+      rescue => e
+        RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] Session ID handling failed: #{e.class}")
+        nil
+      end
+
+      def self.session_digest_key
+        key = Rails.application.secret_key_base if defined?(Rails) && Rails.application.respond_to?(:secret_key_base)
+        key.to_s.empty? ? "rails_error_dashboard" : key.to_s
+      rescue StandardError
+        "rails_error_dashboard"
+      end
+
       # @return [ActiveSupport::ParameterFilter, nil]
       def self.parameter_filter
         @parameter_filter ||= build_parameter_filter

data/lib/rails_error_dashboard/services/storm_protection/circuit_breaker.rb

@@ -22,8 +22,15 @@ module RailsErrorDashboard
       class CircuitBreaker
         BUCKET_SECONDS = 10
         CALM_BUCKETS_TO_CLOSE = 2
+        # Empty buckets replayed after a silence. Seven is enough to walk
+        # :open -> :half_open -> :closed; the cap keeps a roll O(1) however long
+        # the process sat idle.
+        MAX_CATCH_UP_BUCKETS = 7
 
-        attr_reader :state
+        # Incremented every time the breaker ENTERS :half_open. The gate compares
+        # it with the last value it saw to restart its probe counter, so each
+        # recovery attempt probes with its first event.
+        attr_reader :half_open_epoch
 
         # @param clock [#call] returns monotonic seconds; injectable for tests
         def initialize(clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
@@ -40,6 +47,7 @@ module RailsErrorDashboard
             @calm_buckets = 0
             @opened_at = nil
             @episode = nil
+            @half_open_epoch = 0
           end
         end
 
@@ -61,6 +69,25 @@ module RailsErrorDashboard
           @state
         end
 
+        # Current state, advanced by elapsed TIME as well as by events. Without the
+        # tick a storm that simply stopped left the breaker open for ever: only
+        # record! rolled buckets, and nothing calls record! when errors stop.
+        def state
+          tick!
+          @state
+        end
+
+        # Roll the bucket if one is due. Cheap when it is not: one clock read and
+        # a comparison, no lock. Safe to call from anywhere, any number of times.
+        def tick!
+          now = @clock.call
+          roll!(now) if now - @bucket_start >= BUCKET_SECONDS
+          nil
+        rescue => e
+          RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] CircuitBreaker#tick! failed: #{e.class}: #{e.message}")
+          nil
+        end
+
         # Episode metadata for the honesty layer (storm_events row).
         # @return [Hash, nil] nil when no episode is active or recently closed
         def episode_snapshot
@@ -81,10 +108,38 @@ module RailsErrorDashboard
             elapsed = now - @bucket_start
             return if elapsed < BUCKET_SECONDS # another thread already rolled
 
-            rate = @bucket_count.value / elapsed.to_f
+            count = @bucket_count.value
+            bucket_start = @bucket_start
             @bucket_start = now
             @bucket_count = Concurrent::AtomicFixnum.new(0)
-            transition!(rate, now)
+
+            buckets = (elapsed / BUCKET_SECONDS).floor
+            if buckets <= 1
+              transition!(count / elapsed.to_f, now)
+            else
+              catch_up!(count, elapsed, bucket_start, now, buckets)
+            end
+          end
+        end
+
+        # More than one bucket of wall time has passed since the last roll, so
+        # nobody called in between: replay the gap instead of treating it as one
+        # long bucket. Each step carries the time its bucket ENDED, which is what
+        # keeps the cooldown and the two-calm-bucket rule honest (10s past the
+        # cooldown is :half_open, not :closed).
+        #
+        # The measured bucket keeps the diluted rate (count / elapsed) it has
+        # always had, so a burst followed by silence never escalates a closed
+        # breaker after the fact. Only the most recent MAX_CATCH_UP_BUCKETS empty
+        # buckets are replayed; older ones could not change the outcome.
+        def catch_up!(count, elapsed, bucket_start, now, buckets)
+          transition!(count / elapsed.to_f, bucket_start + BUCKET_SECONDS)
+
+          empty = [ buckets - 1, MAX_CATCH_UP_BUCKETS ].min
+          (empty - 1).downto(0) do |back|
+            break if @state == :closed
+
+            transition!(0.0, now - (back * BUCKET_SECONDS))
           end
         end
 
@@ -111,6 +166,7 @@ module RailsErrorDashboard
             if now - @opened_at >= cooldown_seconds && rate < shedding_threshold
               @state = :half_open
               @calm_buckets = 0
+              @half_open_epoch += 1
             end
           when :half_open
             if rate >= shedding_threshold

data/lib/rails_error_dashboard/services/storm_protection/fingerprint_buckets.rb

@@ -39,6 +39,7 @@ module RailsErrorDashboard
 
         def reset!
           @entries = Concurrent::Map.new
+          @last_sweep = nil
         end
 
         # Decide capture fidelity for one event of this fingerprint.
@@ -68,12 +69,34 @@ module RailsErrorDashboard
           return existing if existing
 
           # Bounded: never insert past the cap (size check is approximate
-          # under concurrency — a few entries over the cap is fine)
-          return nil if @entries.size >= max_tracked
+          # under concurrency — a few entries over the cap is fine). Before
+          # declining, make room by dropping fingerprints that have gone quiet:
+          # without that the map filled once and stayed full for the life of
+          # the process, so the first N fingerprints a worker ever saw were the
+          # only ones it would ever rate-limit. If every tracked entry is still
+          # live, a new key is still declined — a storm of unique fingerprints
+          # is the global breaker's job (Layer 2), not this map's.
+          if @entries.size >= max_tracked
+            sweep_expired!(now)
+            return nil if @entries.size >= max_tracked
+          end
 
           @entries.compute_if_absent(gate_key) { Entry.new(now, 0, now, 0) }
         end
 
+        # Drop entries whose minute window has ended. At most once per window:
+        # while the map is full every unseen key lands here, and a full scan per
+        # miss would put an O(n) walk on the capture path. Racy by design (two
+        # threads may both sweep); deleting an expired entry twice is harmless.
+        def sweep_expired!(now)
+          return if @last_sweep && now - @last_sweep < WINDOW_SECONDS
+
+          @last_sweep = now
+          @entries.each_pair do |key, entry|
+            @entries.delete(key) if now - entry.window_start >= WINDOW_SECONDS
+          end
+        end
+
         def roll_windows(entry, now)
           if now - entry.window_start >= WINDOW_SECONDS
             entry.window_start = now

data/lib/rails_error_dashboard/services/storm_protection/gate.rb

@@ -108,6 +108,11 @@ module RailsErrorDashboard
           def flush_if_due!
             return unless enabled?
 
+            # Advance the breaker by the clock before flushing. When errors
+            # stop, this (end of every request and job) is the only thing left
+            # that can move it out of :open, and doing it first means an
+            # episode that has just ended is persisted by this very flush.
+            breaker.tick!
             maybe_flush!
             nil
           rescue => e
@@ -159,6 +164,7 @@ module RailsErrorDashboard
             @count_buffer = nil
             @fingerprint_buckets = nil
             @probe_counter = nil
+            @probe_epoch = nil
             @issue_window_start = nil
             @issue_window_count = nil
             @last_flush = nil
@@ -180,8 +186,11 @@ module RailsErrorDashboard
               :count_only
             when :half_open
               # Probe: a trickle of :lite captures tells us whether the storm
-              # has actually subsided; everything else stays counted.
-              if (probe_counter.increment % 10).zero?
+              # has actually subsided; everything else stays counted. The FIRST
+              # event of each half-open period is the probe (then every tenth):
+              # a recovering app with a slow trickle must not wait nine errors
+              # before we look at one.
+              if next_probe_index % 10 == 1
                 :lite
               else
                 count!(exception, context)
@@ -238,7 +247,11 @@ module RailsErrorDashboard
           def gate_parts(exception, context)
             raw_message = exception.message.to_s[0, ErrorHashGenerator::HASH_MESSAGE_LIMIT]
 
-            {
+            # Everything below is buffered, JSON-encoded for StormFlushJob and
+            # later INSERTed, so no String may keep an invalid byte. The identity
+            # digest is computed from the raw message first (it is hex, and must
+            # match what the sync and async paths hash).
+            EncodingSanitizer.scrub_deep(
               error_class: exception.class.name,
               # The identity is hashed from the RAW message here, on the hot
               # path, and only the digest is buffered. The message itself is
@@ -256,7 +269,8 @@ module RailsErrorDashboard
                 controller_name: context[:controller_name]&.to_s,
                 action_name: context[:action_name]&.to_s
               ),
-              message: redact(raw_message),
+              # Scrubbed before redaction: the filter's regexes raise on invalid bytes.
+              message: redact(EncodingSanitizer.scrub(raw_message)),
               first_app_frame: ErrorHashGenerator.extract_app_frame_from_locations(exception) ||
                                ErrorHashGenerator.extract_app_frame(exception.backtrace),
               controller_name: context[:controller_name]&.to_s,
@@ -266,7 +280,7 @@ module RailsErrorDashboard
               # worker's own environment", resolved at flush time exactly as
               # LogError resolves it for a full capture.
               environment: context[:environment].to_s.strip.presence&.[](0, 64)
-            }
+            )
           end
 
           # When a custom fingerprint lambda is configured the canonical hash
@@ -299,6 +313,19 @@ module RailsErrorDashboard
             @probe_counter ||= Concurrent::AtomicFixnum.new(0)
           end
 
+          # 1-based index of this event within the current half-open period.
+          # The reset is deliberately lock-free: two threads racing on a fresh
+          # epoch can at worst both start from a new counter, which admits one
+          # extra :lite probe. Nothing is lost and nothing can raise.
+          def next_probe_index
+            epoch = breaker.half_open_epoch
+            if @probe_epoch != epoch
+              @probe_epoch = epoch
+              @probe_counter = Concurrent::AtomicFixnum.new(0)
+            end
+            probe_counter.increment
+          end
+
           # Piggyback flush (SwallowedExceptionTracker pattern): cheap
           # timestamp check per admit; enqueue at most once per interval.
           def maybe_flush!