rails_error_dashboard 0.11.9 → 0.12.0

data/lib/rails_error_dashboard/commands/log_error.rb

@@ -5,6 +5,20 @@ module RailsErrorDashboard
     # Command: Log an error to the database
     # This is a write operation that creates an ErrorLog record
     class LogError
+      # Raised internally when perform_later did not reach the queue, so the
+      # one rescue below covers both failure shapes. Never escapes this class.
+      class EnqueueFailed < StandardError; end
+
+      # The error store is down right now and may be up in a moment. Worth
+      # another attempt from a background worker; indistinguishable from any
+      # other failure on a user's request, where nothing is ever re-raised.
+      RETRYABLE_STORE_ERRORS = [
+        ActiveRecord::ConnectionNotEstablished,
+        ActiveRecord::StatementInvalid,
+        ActiveRecord::LockWaitTimeout,
+        ActiveRecord::Deadlocked
+      ].freeze
+
       def self.call(exception, context = {})
         # Filter FIRST (ignore list + static sampling) so ignored exceptions
         # never count toward storm state. _pre_filtered prevents the sync path
@@ -79,6 +93,19 @@ module RailsErrorDashboard
       # Queue error logging as a background job
       def self.call_async(exception, context = {})
         # Serialize exception data for the job
+        # Grouping identity is computed from the RAW message, BEFORE redaction
+        # below. ErrorHashGenerator hashes a 500-char prefix of the unredacted
+        # message, so redacting first would silently re-group every error whose
+        # message contains a filtered key: "password=hunter2" and
+        # "password=[FILTERED]" are different fingerprints.
+        #
+        # application_id is deliberately absent. Resolving it means
+        # Application.find_or_create_by_name -- a DB write -- and this runs on
+        # the request thread, where the gem promises no I/O. The worker
+        # completes the hash with the application resolved, exactly as
+        # FlushStormCounts#canonical_hash does for storm counts.
+        identity_parts = capture_identity_parts(exception, context)
+
         exception_data = {
           class_name: exception.class.name,
           message: exception.message,
@@ -86,6 +113,20 @@ module RailsErrorDashboard
           cause_chain: serialize_cause_chain(exception)
         }
 
+        # Redact BEFORE the payload crosses the queue boundary. Until now the
+        # filter ran only just before the INSERT, so a durable adapter
+        # (Sidekiq/Redis, Solid Queue) persisted the raw secret in its own
+        # store, its backups, and any job-argument logging -- even though the
+        # error row itself was correctly redacted.
+        #
+        # Breadcrumbs, locals and instance variables are already filtered by
+        # their own collectors (BreadcrumbCollector.filter_sensitive,
+        # VariableSerializer.filter_serialized) before they are put in the
+        # context above, so this covers the rest: message, cause chain, request
+        # params and request URL -- the four keys filter_attributes touches.
+        exception_data, context = redact_async_payload(exception_data, context)
+        context = context.merge(_identity: identity_parts) if identity_parts
+
         # Storm shedding: :lite captures skip ALL pre-enqueue context harvest —
         # this is request-thread CPU, the most valuable thing to shed.
         lite = storm_lite?(context)
@@ -140,13 +181,23 @@ module RailsErrorDashboard
             kind: :capture,
             attributes: build_capture_span_attributes(exception, was_async: true)
           ) do |_span|
-            AsyncErrorLoggingJob.perform_later(exception_data, context)
+            job = AsyncErrorLoggingJob.perform_later(exception_data, context)
+
+            # A raise is not the only way a handoff fails. From Rails 7.2
+            # perform_later swallows ActiveJob::EnqueueError and returns false
+            # (an aborting enqueue callback does the same), so without this
+            # check the capture is dropped silently: no queued job, no row.
+            # The storm gate has always checked this; ordinary capture did not.
+            unless ApplicationJob.enqueued?(job)
+              raise EnqueueFailed, ApplicationJob.enqueue_failure_reason(job)
+            end
           end
         rescue => e
-          # Queue adapter failed (e.g., Redis down for Sidekiq). Fall back to
-          # sync logging so the error is still captured. Without this rescue,
-          # the exception propagates back to ErrorReporter, which re-reports it
-          # via Rails.error.report → infinite recursion (issue #114).
+          # Queue adapter failed (e.g., Redis down for Sidekiq), or the job
+          # never reached the queue. Fall back to sync logging so the error is
+          # still captured. Without this rescue, the exception propagates back
+          # to ErrorReporter, which re-reports it via Rails.error.report →
+          # infinite recursion (issue #114).
           RailsErrorDashboard::Logger.error(
             "[RailsErrorDashboard] Async enqueue failed (#{e.class}: #{e.message}), falling back to sync logging"
           )
@@ -154,6 +205,76 @@ module RailsErrorDashboard
         end
       end
 
+      # The opaque half of the canonical fingerprint, computed from the RAW
+      # exception before the payload is redacted for the queue.
+      #
+      # This is a digest, not the identity parts themselves: an earlier version
+      # shipped `normalized_message` and put the very secret the redaction had
+      # just removed straight back on the queue. normalize_message replaces
+      # hex, digits and quoted strings -- it has no notion of secrets.
+      #
+      # A custom fingerprint lambda already yields a complete, message-free
+      # value, so it is passed through unchanged.
+      def self.capture_identity_parts(exception, context)
+        custom = Services::ErrorHashGenerator.send(:try_custom_fingerprint, exception, context)
+        return custom if custom
+
+        Services::ErrorHashGenerator.opaque_identity(
+          error_class: exception.class.name,
+          normalized_message: Services::ErrorHashGenerator.normalize_message(exception.message),
+          frames: Services::ErrorHashGenerator.extract_app_frame_from_locations(exception) ||
+                  Services::ErrorHashGenerator.extract_app_frame(exception.backtrace),
+          controller_name: context[:controller_name]&.to_s,
+          action_name: context[:action_name]&.to_s
+        )
+      rescue => e
+        # No identity parts simply means the worker recomputes the hash from
+        # the (redacted) payload, which is the pre-existing behaviour.
+        RailsErrorDashboard::Logger.debug(
+          "[RailsErrorDashboard] capture_identity_parts failed: #{e.class} - #{e.message}"
+        )
+        nil
+      end
+
+      # Apply the storage filter to everything secret-bearing that crosses the
+      # queue, reusing SensitiveDataFilter so the queue and the database are
+      # redacted by ONE policy rather than two that can drift.
+      def self.redact_async_payload(exception_data, context)
+        return [ exception_data, context ] unless RailsErrorDashboard.configuration.filter_sensitive_data
+
+        filtered = Services::SensitiveDataFilter.filter_attributes(
+          message: exception_data[:message],
+          request_params: context[:request_params],
+          request_url: context[:request_url],
+          exception_cause: exception_data[:cause_chain]&.to_json
+        )
+
+        exception_data = exception_data.merge(message: filtered[:message])
+        if exception_data[:cause_chain] && filtered[:exception_cause]
+          begin
+            exception_data = exception_data.merge(
+              cause_chain: JSON.parse(filtered[:exception_cause], symbolize_names: true)
+            )
+          rescue JSON::ParserError
+            # Keep the filtered-but-unparsed chain out of the payload entirely
+            # rather than shipping the raw one.
+            exception_data = exception_data.merge(cause_chain: nil)
+          end
+        end
+
+        context = context.merge(request_params: filtered[:request_params]) if context.key?(:request_params)
+        context = context.merge(request_url: filtered[:request_url]) if context.key?(:request_url)
+
+        [ exception_data, context ]
+      rescue => e
+        # Never fail a capture over redaction. Fall back to the previous
+        # behaviour: the row itself is still filtered before the INSERT.
+        RailsErrorDashboard::Logger.error(
+          "[RailsErrorDashboard] Async payload redaction failed: #{e.class} - #{e.message}"
+        )
+        [ exception_data, context ]
+      end
+
       # Serialize cause chain for async job serialization
       # Returns an array of hashes (not JSON string) for ActiveJob compatibility
       def self.serialize_cause_chain(exception)
@@ -185,9 +306,19 @@ module RailsErrorDashboard
       end
       private_class_method :serialize_cause_chain
 
-      def initialize(exception, context = {})
+      # @param exception [Exception] the exception to capture
+      # @param context [Hash] request/job context
+      # @param worker [Boolean] true when a background job is the caller.
+      #   The capture path's blanket rescue exists so a failing capture can
+      #   never break a user's request (safety rule 1). A worker has the
+      #   opposite obligation: if the error store was unreachable, the job did
+      #   NOT deliver the capture, and saying otherwise discards the payload.
+      #   In worker mode an unreachable-store failure is re-raised so Active
+      #   Job can retry it; every other failure is still swallowed.
+      def initialize(exception, context = {}, worker: false)
         @exception = exception
         @context = context
+        @worker = worker
       end
 
       def call
@@ -196,10 +327,11 @@ module RailsErrorDashboard
         # tracing pipeline. Child spans (breadcrumbs, health, notifications)
         # nest under this one automatically via OTel context propagation.
         #
-        # The span lives INSIDE the rescue clause — if the span itself raises
-        # somehow, the outer rescue still catches it and returns nil. Defense
-        # in depth. When the block raises, the Tracer façade records the
-        # exception on the span and re-raises so the rescue can swallow it.
+        # The span lives INSIDE the rescue clause — if span setup itself fails,
+        # the Tracer runs this block once with a no-op span, and anything that
+        # still escapes is caught by the outer rescue below. Defense in depth.
+        # When this block raises, the façade records the exception on the span
+        # and re-raises it exactly once; it never re-runs the block.
         Integrations::Tracer.in_span(
           "capture_error",
           kind: :capture,
@@ -254,13 +386,19 @@ module RailsErrorDashboard
         end
 
         # Generate error hash for deduplication (including controller/action context and application)
-        error_hash = Services::ErrorHashGenerator.call(
-          @exception,
-          controller_name: error_context.controller_name,
-          action_name: error_context.action_name,
-          application_id: application.id,
-          context: @context
-        )
+        #
+        # On the async path the identity was captured from the RAW exception
+        # before the payload was redacted for the queue; completing it here
+        # with application_id keeps grouping identical to the sync path, where
+        # the hash is taken before filter_attributes runs.
+        error_hash = canonical_hash_from_identity(application) ||
+                     Services::ErrorHashGenerator.call(
+                       @exception,
+                       controller_name: error_context.controller_name,
+                       action_name: error_context.action_name,
+                       application_id: application.id,
+                       context: @context
+                     )
 
         #  Calculate backtrace signature for fuzzy matching (if column exists)
         if ErrorLog.column_names.include?("backtrace_signature")
@@ -366,8 +504,17 @@ module RailsErrorDashboard
         end
 
         # Find existing error or create new one
-        # This ensures accurate occurrence tracking
-        error_log = ErrorLog.find_or_increment_by_hash(error_hash, attributes.merge(error_hash: error_hash))
+        # This ensures accurate occurrence tracking.
+        #
+        # _context_fidelity travels with the attributes so the grouping command
+        # can tell a full capture from a shed one. A :lite capture carries no
+        # context payloads by design, and must not be recorded as though it
+        # refreshed the snapshot -- nor allowed to overwrite a good backtrace.
+        # It is stripped before the INSERT (it is a signal, not a column).
+        error_log = ErrorLog.find_or_increment_by_hash(
+          error_hash,
+          attributes.merge(error_hash: error_hash, _context_fidelity: storm_lite ? "lite" : "full")
+        )
 
         # OTel: now that the error_log exists, attach its id + dedup flag + severity
         # to the parent capture span so operators can correlate to dashboard URLs.
@@ -427,6 +574,13 @@ module RailsErrorDashboard
         RailsErrorDashboard::Logger.error("Original exception: #{@exception.class} - #{@exception.message}") if @exception
         RailsErrorDashboard::Logger.error("Context: #{@context.inspect.truncate(500)}") if @context
         RailsErrorDashboard::Logger.error(e.backtrace&.first(5)&.join("\n")) if e.backtrace
+
+        # A worker must not report a delivery it did not make. Only the
+        # store-unavailable failures are re-raised (they are worth another
+        # attempt); a payload problem would fail identically on every retry,
+        # so it stays swallowed here as it always has.
+        raise if @worker && RETRYABLE_STORE_ERRORS.any? { |klass| e.is_a?(klass) }
+
         nil # Explicitly return nil, never raise
       end
 
@@ -444,6 +598,14 @@ module RailsErrorDashboard
 
         Services::ErrorNotificationDispatcher.call(error_log)
         Services::NotificationThrottler.record_notification(error_log)
+      rescue => e
+        # The error row is already written by the time we get here. A channel
+        # that cannot be reached (Redis down for the Slack job's enqueue, a
+        # broken webhook config) must not take the capture down with it: the
+        # caller asked us to record an error, and we did. Log, don't raise.
+        RailsErrorDashboard::Logger.error(
+          "[RailsErrorDashboard] Failed to dispatch notification for error #{error_log&.id}: #{e.class} - #{e.message}"
+        )
       end
 
       # The environment this error is attributed to: an explicit context value
@@ -456,6 +618,23 @@ module RailsErrorDashboard
       end
 
       # Find or create application for multi-app support
+      # Complete the fingerprint the request thread started, if it sent one.
+      # The request thread hashed the identity parts into an opaque value (no
+      # message text crosses the queue); this adds the application, which only
+      # a worker can resolve. Same two stages as the sync path, so a capture
+      # groups onto the same row whether it travelled through the queue or not.
+      def canonical_hash_from_identity(application)
+        opaque = @context[:_identity]
+        return nil unless opaque.is_a?(String) && opaque.present?
+
+        Services::ErrorHashGenerator.complete(opaque, application.id)
+      rescue => e
+        RailsErrorDashboard::Logger.debug(
+          "[RailsErrorDashboard] canonical_hash_from_identity failed: #{e.class} - #{e.message}"
+        )
+        nil
+      end
+
       def find_or_create_application
         app_name = RailsErrorDashboard.configuration.application_name ||
                    ENV["APPLICATION_NAME"] ||
@@ -508,6 +687,14 @@ module RailsErrorDashboard
         if error_log.critical?
           ActiveSupport::Notifications.instrument("critical_error.rails_error_dashboard", payload)
         end
+      rescue => e
+        # AS::Notifications re-raises subscriber exceptions to the instrumenting
+        # caller (fanout.rb#iterate_guarding_exceptions), so a host subscriber
+        # on error_logged.rails_error_dashboard would otherwise abort a capture
+        # whose row is already persisted.
+        RailsErrorDashboard::Logger.error(
+          "[RailsErrorDashboard] Failed to emit instrumentation events for error #{error_log&.id}: #{e.class} - #{e.message}"
+        )
       end
 
       #  Check if error exceeds baseline and send alert if needed

data/lib/rails_error_dashboard/configuration.rb

@@ -233,8 +233,9 @@ module RailsErrorDashboard
     # Locale the dashboard renders in, independent of the host app's locale.
     #
     # Drives both Pagy's pagination labels and RED's own translation lookups.
-    # Ships "en", "de", "fr", "es", "pt-BR" and "ja". Everything but English is
-    # machine-translated and has NOT been reviewed by a native speaker — see
+    # Ships "en", "de", "es", "fr", "pt-BR", "ja", "ru", "uk", "pl", "it" and
+    # "zh-CN". "fr" has been reviewed by a native speaker; everything but
+    # English and French is machine-translated and has NOT been — see
     # docs/guides/TRANSLATIONS.md. A missing or wrong translation falls back to
     # English rather than breaking the page.
     #
@@ -467,7 +468,7 @@ module RailsErrorDashboard
 
       # Dashboard UI
       @accent_color = :crimson  # :crimson, :ruby, :ember, :violet
-      @dashboard_locale = "en"  # en, de, es, fr, pt-BR, ja, ru, uk, pl, it, zh-CN (non-English machine-translated)
+      @dashboard_locale = "en"  # en, de, es, fr, pt-BR, ja, ru, uk, pl, it, zh-CN (fr native-reviewed; other non-English machine-translated)
 
       # LLM-powered AI help defaults - OFF until provider and API key are configured
       @llm_provider = ENV["RED_LLM_PROVIDER"]&.to_sym

data/lib/rails_error_dashboard/engine.rb

@@ -109,6 +109,34 @@ module RailsErrorDashboard
         at_exit { RailsErrorDashboard::Services::RackAttackTracker.flush_all_threads! }
       end
 
+      # Drain the storm count buffer at the end of every request and job, and
+      # again at process exit.
+      #
+      # Counted-only events accumulate in this process's memory and were only
+      # ever written out by a LATER admit! reaching the flush interval (see
+      # Gate#maybe_flush!). That makes the drain conditional on the flood
+      # continuing: when the errors stop -- which is exactly when an operator
+      # starts looking -- the tail of the burst stays in memory indefinitely,
+      # and a deploy drops it.
+      #
+      # to_complete fires after the response body is closed, so the client
+      # already has its bytes and this never delays a request (safety rule 2).
+      # It also fires when the app raised, and is re-entrant, so nested
+      # executor blocks do not double-flush. The call is interval-gated, so a
+      # flood costs a clock read per request rather than an enqueue.
+      #
+      # at_exit drains unconditionally and writes synchronously: at shutdown a
+      # job handed to the queue may never be picked up. at_exit, not
+      # Signal.trap -- trapping would clobber Puma's USR1/USR2 handlers
+      # (safety rule 9).
+      if RailsErrorDashboard.configuration.enable_storm_protection
+        Rails.application.executor.to_complete do
+          RailsErrorDashboard::Services::StormProtection::Gate.flush_if_due!
+        end
+
+        at_exit { RailsErrorDashboard::Services::StormProtection::Gate.drain! }
+      end
+
       # Subscribe to ActionCable AS::Notifications events (requires breadcrumbs + ActionCable)
       if RailsErrorDashboard.configuration.enable_actioncable_tracking &&
          RailsErrorDashboard.configuration.enable_breadcrumbs &&

data/lib/rails_error_dashboard/integrations/tracer.rb

@@ -65,15 +65,33 @@ module RailsErrorDashboard
         # @yieldparam span [NoopSpan, ::OpenTelemetry::Trace::Span] real or no-op
         # @return [Object] whatever the block returns
         def in_span(name, kind: :capture, attributes: {})
-          return yield(NOOP_SPAN) unless emit?(kind)
-
-          tr = tracer
-          return yield(NOOP_SPAN) unless tr
+          # The work block must run AT MOST ONCE. Everything that can fail
+          # before it is entered (emit? checks, tracer construction, attribute
+          # merging, the SDK's own span setup) falls back to a no-op span and
+          # runs the block once, here. Once the block has been entered, its
+          # own exceptions belong to the caller and propagate untouched —
+          # they must never re-enter the block.
+          #
+          # This method previously wrapped the yield in a rescue that yielded
+          # again on failure. A downstream failure AFTER the work completed
+          # (a notification dispatcher raising, say) therefore re-ran the whole
+          # capture: one exception became two occurrences and two count
+          # increments, with OTel disabled, for every user. The `entered` flag
+          # is what makes "at most once" structural rather than incidental.
+          entered = false
+
+          tr = (tracer if emit?(kind))
+
+          unless tr
+            entered = true
+            return yield(NOOP_SPAN)
+          end
 
           full_name = "#{INSTRUMENTATION_NAME}.#{name}"
           merged = base_attributes.merge(safe_stringify(attributes))
 
           tr.in_span(full_name, attributes: merged) do |span|
+            entered = true
             begin
               yield span
             rescue StandardError => e
@@ -82,9 +100,10 @@ module RailsErrorDashboard
             end
           end
         rescue StandardError => e
-          # Tracer internals failed (e.g. OTel SDK threw on add_span). Fall back
-          # to running the block with a no-op so the host app never sees a crash
-          # caused by the tracer.
+          # Only reachable while the block has NOT run: tracer internals failed
+          # during setup. Re-raise anything that escaped the block itself.
+          raise if entered
+
           Logger.debug("[RailsErrorDashboard] Tracer.in_span(#{name.inspect}) failed: #{e.class}: #{e.message}")
           yield NOOP_SPAN
         end