rails_error_dashboard 0.11.9 → 0.12.0

data/db/migrate/20260915000003_add_context_provenance_to_error_logs.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# Say WHICH occurrence supplied the diagnostic snapshot a group displays.
+#
+# An ErrorLog row is a group, but its breadcrumbs, system health, locals,
+# instance variables and request context describe ONE moment of failure. They
+# are refreshed by each occurrence that carries them, and deliberately left
+# alone by one that does not (a storm :lite capture, or a feature switched
+# off) -- keeping a useful snapshot rather than blanking it is right.
+#
+# What was missing is provenance. The page labelled that collection as the
+# error's context without saying which event it came from, so a row could show
+# a new request URL beside a previous occurrence's user id and locals with
+# nothing to indicate they came from different requests.
+#
+#   context_captured_at  when the displayed snapshot was captured
+#   context_fidelity     how complete that capture was:
+#                          "full"    -- the normal path, everything captured
+#                          "lite"    -- storm shedding: error + occurrence row
+#                                       only, context payloads shed
+#                          "minimal" -- reconstructed by the storm flush from a
+#                                       counted-only exemplar (no backtrace,
+#                                       no context at all)
+#
+# Both are nullable: rows captured before this migration have no recorded
+# provenance, and the view says so rather than inventing one.
+class AddContextProvenanceToErrorLogs < ActiveRecord::Migration[7.0]
+  TABLE = :rails_error_dashboard_error_logs
+
+  def change
+    return unless table_exists?(TABLE)
+
+    unless column_exists?(TABLE, :context_captured_at)
+      add_column TABLE, :context_captured_at, :datetime
+    end
+
+    unless column_exists?(TABLE, :context_fidelity)
+      # 10: "full" / "lite" / "minimal".
+      add_column TABLE, :context_fidelity, :string, limit: 10
+    end
+  end
+end

data/db/migrate/20260915000004_create_storm_flush_batches.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Batch identity for storm count reconciliation, so a replayed batch is
+# applied exactly once.
+#
+# FlushStormCounts applies counts additively:
+#
+#   UPDATE error_logs SET occurrence_count = occurrence_count + N
+#
+# which is not idempotent. Delivering the identical five-event snapshot twice
+# persisted ten occurrences. The realistic replay path is the job's own
+# retry: StormFlushJob raises when a batch reconciles nothing, and
+# ApplicationJob retries it three times with the identical payload -- so a
+# batch that partially applied and then failed was re-applied in full on the
+# next attempt. A queue that redelivers does the same thing.
+#
+# This table is the ledger. A digest of the batch is inserted in the SAME
+# transaction as the increments, so either both land or neither does. A
+# replay hits the unique index, and the command reports the batch as already
+# applied instead of counting it again.
+#
+# Small and self-expiring: at most one row per flush interval per process
+# (~120/hour while a storm is actually running, none otherwise), pruned by
+# RetentionCleanupJob alongside the rack-attack events.
+class CreateStormFlushBatches < ActiveRecord::Migration[7.0]
+  def change
+    # Guard against a squashed schema migration having already created this
+    # table -- without it, every later migration is silently cancelled.
+    return if table_exists?(:rails_error_dashboard_storm_flush_batches)
+
+    create_table :rails_error_dashboard_storm_flush_batches do |t|
+      # SHA256 hex of the batch's identity parts. 64 chars, far inside
+      # MySQL's 3072-byte utf8mb4 index limit (64 * 4 + 2 = 258 bytes).
+      t.string :digest, null: false, limit: 64
+      # What the batch carried, for operators reading the ledger directly.
+      t.integer :entry_count, null: false, default: 0
+      t.bigint :occurrences_applied, null: false, default: 0
+      t.datetime :applied_at, null: false
+      t.timestamps
+    end
+
+    add_index :rails_error_dashboard_storm_flush_batches, :digest,
+              unique: true,
+              name: "index_storm_flush_batches_on_digest"
+
+    # Retention prunes on this column.
+    add_index :rails_error_dashboard_storm_flush_batches, :applied_at,
+              name: "index_storm_flush_batches_on_applied_at"
+  end
+end

data/lib/rails_error_dashboard/commands/create_issue.rb

@@ -42,11 +42,19 @@ module RailsErrorDashboard
         result = client.create_issue(title: title, body: body, labels: labels)
 
         if result[:success]
-          error.update!(
+          attrs = {
             external_issue_url: result[:url],
             external_issue_number: result[:number],
             external_issue_provider: config.effective_issue_tracker_provider.to_s
-          )
+          }
+          # Record WHICH repository (or Linear team) this issue was opened in,
+          # so a later webhook, comment or close targets that one rather than
+          # whatever the global configuration happens to say at the time.
+          if ErrorLog.column_names.include?("external_issue_repo")
+            attrs[:external_issue_repo] = config.effective_issue_tracker_repo
+          end
+
+          error.update!(attrs)
           { success: true, issue_url: result[:url], issue_number: result[:number] }
         else
           { success: false, error: result[:error] }

data/lib/rails_error_dashboard/commands/find_or_increment_error.rb

@@ -93,6 +93,71 @@ module RailsErrorDashboard
         end
       end
 
+      # The request-identity fields the detail page shows beside the context
+      # payloads. They are refreshed by the `||` chain in increment_existing,
+      # so a capture can move the displayed URL without touching any
+      # REFRESHED_CONTEXT key -- and the provenance has to follow the whole
+      # displayed snapshot, not just part of it.
+      REFRESHED_REQUEST_IDENTITY = %i[
+        user_id request_url request_params user_agent ip_address
+      ].freeze
+
+      # Provenance for the snapshot the row will DISPLAY.
+      #
+      # Stamped whenever this occurrence refreshed ANY displayed field. An
+      # occurrence that carried nothing (storm :lite, feature switched off)
+      # leaves both the snapshot and its provenance alone -- otherwise the row
+      # would claim a fresh capture time for evidence from an older event,
+      # which is precisely the confusion this exists to remove.
+      def context_provenance(refreshed)
+        return {} unless refreshed.any? || refreshed_request_identity?
+        return {} unless ErrorLog.column_names.include?("context_captured_at")
+
+        provenance = { context_captured_at: @attributes[:occurred_at] || Time.current }
+        if ErrorLog.column_names.include?("context_fidelity")
+          provenance[:context_fidelity] = @attributes[:_context_fidelity].presence || "full"
+        end
+        provenance
+      end
+
+      def refreshed_request_identity?
+        REFRESHED_REQUEST_IDENTITY.any? { |key| !@attributes[key].nil? }
+      end
+
+      # A group first seen during a storm has a MINIMAL exemplar: the flush job
+      # could only record the first app frame, because a counted-only event
+      # captures no backtrace. The comment there promises the next occurrence
+      # fills in detail -- it never did, because nothing replaced backtrace on
+      # an existing row, so the group kept a single bare path with no line
+      # number or caller frame for its whole life.
+      #
+      # A capture that HAS a real backtrace now upgrades it. Only a full one:
+      # a :lite capture sheds context by design and must not overwrite good
+      # evidence with less.
+      def backtrace_upgrade(error)
+        return {} unless @attributes[:backtrace].present?
+        return {} if storm_lite?
+
+        stored = error.backtrace.to_s
+        return {} if stored.include?("\n") # already a real stack
+
+        incoming = @attributes[:backtrace].to_s
+        return {} unless incoming.include?("\n") || incoming.length > stored.length
+
+        upgrade = { backtrace: @attributes[:backtrace] }
+        # The row is no longer a reconstructed exemplar: it now carries a real
+        # stack from a real capture, so it must stop describing itself as
+        # "minimal".
+        if ErrorLog.column_names.include?("context_fidelity") && error.context_fidelity.to_s == "minimal"
+          upgrade[:context_fidelity] = @attributes[:_context_fidelity].presence || "full"
+        end
+        upgrade
+      end
+
+      def storm_lite?
+        @attributes[:_context_fidelity].to_s == "lite"
+      end
+
       # {} unless this is a legacy NULL-environment row being claimed.
       def environment_adoption(error)
         return {} unless ErrorLog.column_names.include?("environment")
@@ -102,6 +167,7 @@ module RailsErrorDashboard
       end
 
       def increment_existing(error)
+        refreshed = latest_context
         error.update!(
           occurrence_count: error.occurrence_count + 1,
           last_seen_at: Time.current,
@@ -110,7 +176,9 @@ module RailsErrorDashboard
           request_params: @attributes[:request_params] || error.request_params,
           user_agent: @attributes[:user_agent] || error.user_agent,
           ip_address: @attributes[:ip_address] || error.ip_address,
-          **latest_context,
+          **refreshed,
+          **context_provenance(refreshed),
+          **backtrace_upgrade(error),
           **environment_adoption(error)
         )
         error
@@ -128,7 +196,9 @@ module RailsErrorDashboard
           request_params: @attributes[:request_params] || error.request_params,
           user_agent: @attributes[:user_agent] || error.user_agent,
           ip_address: @attributes[:ip_address] || error.ip_address,
-          **latest_context,
+          **(refreshed = latest_context),
+          **context_provenance(refreshed),
+          **backtrace_upgrade(error),
           **environment_adoption(error)
         }
         attrs[:reopened_at] = Time.current if ErrorLog.column_names.include?("reopened_at")
@@ -137,12 +207,32 @@ module RailsErrorDashboard
         error
       end
 
+      # Attributes for a brand-new group.
+      #
+      # The first occurrence IS the snapshot, so its provenance is stamped here
+      # rather than inferred later. Internal signalling keys (leading
+      # underscore) are carriers between commands, not columns -- passing them
+      # to create! raises UnknownAttributeError.
+      def new_record_attributes
+        attrs = @attributes.reject { |key, _| key.to_s.start_with?("_") }
+        attrs = attrs.reverse_merge(resolved: false)
+
+        if ErrorLog.column_names.include?("context_captured_at")
+          attrs[:context_captured_at] ||= @attributes[:occurred_at] || Time.current
+        end
+        if ErrorLog.column_names.include?("context_fidelity")
+          attrs[:context_fidelity] ||= @attributes[:_context_fidelity].presence || "full"
+        end
+
+        attrs
+      end
+
       def create_new_or_retry
         # Savepoint: on PostgreSQL a unique-violation poisons the enclosing
         # transaction, and the retry lookups below would fail with "current
         # transaction is aborted" instead of finding the winner's row.
         ErrorLog.transaction(requires_new: true) do
-          ErrorLog.create!(@attributes.reverse_merge(resolved: false))
+          ErrorLog.create!(new_record_attributes)
         end
       rescue ActiveRecord::RecordNotUnique
         # Race condition: another process created the same error
@@ -173,10 +263,51 @@ module RailsErrorDashboard
           if retry_resolved
             reopen_existing(retry_resolved)
           else
-            raise
+            # A RecordNotUnique means a row with this exact group identity
+            # exists right now, so the only way to get here is for the two
+            # lookups above to disagree with the index: the colliding row sits
+            # outside the 24 h window (its occurred_at was moved, or the clock
+            # skewed) yet still holds this identity. Raising here would abort a
+            # capture whose group demonstrably exists -- LogError's blanket
+            # rescue turns that into a silently dropped error. Match the row
+            # the index actually objected to and increment it instead.
+            claim_conflicting_row || raise
           end
         end
       end
+
+      # The unresolved row that owns this group identity, matched exactly as
+      # the unique index defines it (application, hash, environment and the
+      # immutable group_window bucket) with no time window of its own.
+      def claim_conflicting_row
+        return nil unless ErrorLog.column_names.include?("group_window")
+
+        scope = ErrorLog.unresolved
+          .where(error_hash: @error_hash)
+          .where(application_id: @attributes[:application_id])
+        scope = scope.where(environment: @attributes[:environment]) if ErrorLog.column_names.include?("environment")
+        scope = scope.where(group_window: window_for_attributes)
+
+        conflicting = scope.lock.first
+        return nil unless conflicting
+
+        conflicting.update!(
+          occurrence_count: conflicting.occurrence_count + 1,
+          last_seen_at: Time.current,
+          **latest_context,
+          **environment_adoption(conflicting)
+        )
+        conflicting
+      end
+
+      # The bucket this capture would have been stamped with -- the same value
+      # ErrorLog#set_group_window computes.
+      def window_for_attributes
+        basis = @attributes[:occurred_at] || Time.current
+        basis.utc.strftime("%Y-%m-%d")
+      rescue StandardError
+        nil
+      end
     end
   end
 end

data/lib/rails_error_dashboard/commands/flush_storm_counts.rb

@@ -16,35 +16,72 @@ module RailsErrorDashboard
     # Counts are exact. Notifications are NOT dispatched from here — during a
     # storm they're suppressed by design; the storm notification covers it.
     class FlushStormCounts
-      def self.call(entries:, overflow: 0, episode: nil)
-        new(entries: entries, overflow: overflow, episode: episode).call
+      def self.call(entries:, overflow: 0, episode: nil, batch_id: nil)
+        new(entries: entries, overflow: overflow, episode: episode, batch_id: batch_id).call
       end
 
-      def initialize(entries:, overflow: 0, episode: nil)
+      def initialize(entries:, overflow: 0, episode: nil, batch_id: nil)
         @entries = Array(entries)
         @overflow = overflow.to_i
         @episode = episode
+        @batch_id = batch_id
       end
 
       def call
         application = resolve_application
         counted = 0
+        failed = 0
 
-        @entries.each do |entry|
-          entry = entry.with_indifferent_access if entry.respond_to?(:with_indifferent_access)
-          counted += reconcile_entry(entry, application)
-        rescue => e
-          # A corrupt (non-Hash) entry must not abort the whole batch — and the
-          # log line itself must not assume `entry` is subscriptable (an Integer
-          # from a broken serializer would raise again here, escaping this rescue).
-          error_class = entry.is_a?(Hash) ? entry["error_class"] : entry.class
-          RailsErrorDashboard::Logger.error(
-            "[RailsErrorDashboard] Storm count reconcile failed for #{error_class}: #{e.class} - #{e.message}"
-          )
+        # Counts are applied additively (occurrence_count + N), which is not
+        # idempotent: delivering the same snapshot twice counted it twice. The
+        # realistic replay is this job's own retry -- StormFlushJob raises when
+        # a batch reconciles nothing, and ApplicationJob retries it three times
+        # with the identical payload -- and a queue that redelivers does the
+        # same.
+        #
+        # The batch digest is inserted in the SAME transaction as the
+        # increments, so either both land or neither does. A replay violates
+        # the unique index and is reported as already applied rather than
+        # counted again.
+        ledger = batch_ledger_entry
+        return already_applied_result if ledger == :already_applied
+
+        ErrorLog.transaction do
+          claim_batch!(ledger)
+
+          @entries.each do |entry|
+            entry = entry.with_indifferent_access if entry.respond_to?(:with_indifferent_access)
+            counted += reconcile_entry(entry, application)
+          rescue => e
+            # A corrupt (non-Hash) entry must not abort the whole batch — and the
+            # log line itself must not assume `entry` is subscriptable (an Integer
+            # from a broken serializer would raise again here, escaping this rescue).
+            failed += 1
+            error_class = entry.is_a?(Hash) ? entry["error_class"] : entry.class
+            RailsErrorDashboard::Logger.error(
+              "[RailsErrorDashboard] Storm count reconcile failed for #{error_class}: #{e.class} - #{e.message}"
+            )
+          end
+
+          # Nothing was written, so there is nothing to protect from a replay:
+          # roll the claim back and let the job retry the whole batch.
+          raise ActiveRecord::Rollback if failed.positive? && counted.zero?
+
+          finalize_batch!(ledger, counted)
+        end
+
+        # Every entry failed and none was written. Reporting success with
+        # reconciled: 0 made a total loss indistinguishable from an empty
+        # batch, so the job acknowledged counts that never reached the
+        # database. Partial success stays successful: the entries that were
+        # written are written, and replaying the batch would double them.
+        if failed.positive? && counted.zero?
+          return { success: false, reconciled: 0, failed: failed, overflow: @overflow,
+                   error: "all #{failed} entries failed to reconcile" }
         end
 
         upsert_storm_event(counted)
-        { success: true, reconciled: counted, overflow: @overflow }
+        { success: true, reconciled: counted, failed: failed, overflow: @overflow }
       rescue => e
         RailsErrorDashboard::Logger.error(
           "[RailsErrorDashboard] FlushStormCounts failed: #{e.class} - #{e.message}"
@@ -54,6 +91,60 @@ module RailsErrorDashboard
 
       private
 
+      # nil when the ledger is unavailable (table not migrated yet -- the
+      # command then behaves exactly as it did before), :already_applied when
+      # this exact batch is already recorded, otherwise the digest to claim.
+      def batch_ledger_entry
+        return nil unless ledger_available?
+
+        digest = StormFlushBatch.digest_for(
+          entries: @entries, overflow: @overflow, episode: @episode, batch_id: @batch_id
+        )
+        return :already_applied if StormFlushBatch.exists?(digest: digest)
+
+        digest
+      rescue => e
+        # The ledger is a safety net, not a gate: if it cannot be consulted,
+        # reconcile anyway rather than dropping counts that exist nowhere else.
+        RailsErrorDashboard::Logger.debug(
+          "[RailsErrorDashboard] Storm batch ledger unavailable: #{e.class} - #{e.message}"
+        )
+        nil
+      end
+
+      def claim_batch!(digest)
+        return unless digest.is_a?(String)
+
+        StormFlushBatch.create!(
+          digest: digest,
+          entry_count: @entries.size,
+          occurrences_applied: 0,
+          applied_at: Time.current
+        )
+      end
+
+      # Record what the batch actually applied, for an operator reading the
+      # ledger. The row already exists; this only fills in the total.
+      def finalize_batch!(digest, counted)
+        return unless digest.is_a?(String)
+
+        StormFlushBatch.where(digest: digest).update_all(occurrences_applied: counted)
+      end
+
+      def already_applied_result
+        RailsErrorDashboard::Logger.info(
+          "[RailsErrorDashboard] Storm flush batch already applied — skipping replay"
+        )
+        { success: true, reconciled: 0, failed: 0, overflow: @overflow, already_applied: true }
+      end
+
+      def ledger_available?
+        defined?(StormFlushBatch) && StormFlushBatch.table_exists?
+      rescue StandardError
+        false
+      end
+
+
       def reconcile_entry(entry, application)
         count = entry["count"].to_i
         return 0 if count <= 0
@@ -117,13 +208,13 @@ module RailsErrorDashboard
         # from the exemplar (no backtrace/context was captured; the next
         # occurrence after the storm fills in detail via the normal path).
         #
-        # The exemplar message is RAW — the gate stores what the exception
-        # said so the canonical hash (computed above, from the raw message,
-        # exactly as the full path does) still lands on the same row. It must
-        # therefore go through the same redaction as LogError before it is
-        # persisted; storm protection and sensitive filtering are both on by
-        # default, and an incident is exactly when a password in a message
-        # must not reach the database.
+        # The exemplar message arrives ALREADY REDACTED: the gate filters it
+        # before buffering, because the buffer is shipped to StormFlushJob over
+        # a durable queue. Grouping does not depend on the raw text -- the gate
+        # hashed the identity from it and sent the digest along
+        # (opaque_identity) -- so this row still lands where the full capture
+        # path would put it. The filter below stays as a second line of
+        # defence for entries from an older release still in flight.
         create_attrs = {
           environment: env,
           application_id: application.id,
@@ -138,6 +229,18 @@ module RailsErrorDashboard
           error_hash: error_hash,
           resolved: false
         }.compact
+
+        # This row is reconstructed from a counted-only exemplar: there is no
+        # backtrace beyond the first app frame and no context at all. Saying so
+        # is what lets the dashboard distinguish "nothing was captured" from
+        # "nothing happened", and what lets the next full capture upgrade the
+        # backtrace instead of leaving a bare path forever.
+        if ErrorLog.column_names.include?("context_fidelity")
+          create_attrs[:context_fidelity] = "minimal"
+        end
+        if ErrorLog.column_names.include?("context_captured_at")
+          create_attrs[:context_captured_at] = create_attrs[:occurred_at]
+        end
         ErrorLog.create!(**ErrorLog.clamp_string_attributes(Services::SensitiveDataFilter.filter_attributes(create_attrs)))
         count
       end
@@ -163,22 +266,24 @@ module RailsErrorDashboard
         Services::SensitiveDataFilter.filter_attributes({ message: message.to_s })[:message]
       end
 
-      # Mirrors ErrorHashGenerator.call exactly: same fields, same order,
-      # same normalization — so counts land on the same ErrorLog the full
-      # capture path would have used.
+      # Finishes the same two-stage fingerprint the full capture path uses, so
+      # counts land on the ErrorLog that path would have chosen.
       def canonical_hash(entry, application)
         return entry["custom_hash"] if entry["custom_hash"].present?
 
-        digest_input = [
-          entry["error_class"],
-          Services::ErrorHashGenerator.normalize_message(entry["message"]),
-          entry["first_app_frame"],
-          entry["controller_name"],
-          entry["action_name"],
-          application.id.to_s
-        ].compact.join("|")
+        # The gate hashed the identity from the RAW message and buffered only
+        # the digest; the buffered "message" is redacted, so recomputing from
+        # it here would produce a DIFFERENT fingerprint and storm counts would
+        # land on a different row than the full capture path.
+        opaque = entry["opaque_identity"].presence || Services::ErrorHashGenerator.opaque_identity(
+          error_class: entry["error_class"],
+          normalized_message: Services::ErrorHashGenerator.normalize_message(entry["message"]),
+          frames: entry["first_app_frame"],
+          controller_name: entry["controller_name"],
+          action_name: entry["action_name"]
+        )
 
-        Digest::SHA256.hexdigest(digest_input)[0..15]
+        Services::ErrorHashGenerator.complete(opaque, application.id)
       end
 
       # nil when the column is not migrated yet, so every environment clause

data/lib/rails_error_dashboard/commands/link_existing_issue.rb

@@ -36,11 +36,20 @@ module RailsErrorDashboard
         error = ErrorLog.find(@error_id)
         parsed = parse_issue_url(@issue_url)
 
-        error.update!(
+        attrs = {
           external_issue_url: @issue_url,
           external_issue_number: parsed[:number],
           external_issue_provider: parsed[:provider]&.to_s
-        )
+        }
+        # The repository is half the issue's identity: issue 42 in acme/api and
+        # issue 42 in acme/web are different issues. It was parsed here and
+        # thrown away, so a webhook from either repository resolved whichever
+        # error happened to match on provider + number alone.
+        if ErrorLog.column_names.include?("external_issue_repo")
+          attrs[:external_issue_repo] = normalize_repo(parsed[:provider], parsed[:repo])
+        end
+
+        error.update!(attrs)
 
         { success: true, issue_url: @issue_url, provider: parsed[:provider] }
       rescue ActiveRecord::RecordNotFound
@@ -51,6 +60,15 @@ module RailsErrorDashboard
 
       private
 
+      # Linear has no repository -- `repo` is the team key, and Linear renders
+      # it upper-case ("ENG-123"), so it is stored upper-case to match what a
+      # webhook reports. Git forge paths are case-sensitive and kept verbatim.
+      def normalize_repo(provider, repo)
+        return nil if repo.blank?
+
+        provider.to_s == "linear" ? repo.to_s.upcase : repo.to_s
+      end
+
       def parse_issue_url(url)
         PROVIDER_PATTERNS.each do |provider, pattern|
           match = url.match(pattern)