rails_error_dashboard 0.13.0 → 0.14.0

data/db/migrate/20260919000001_create_event_counts.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Time buckets for storm-shed event volume.
+#
+# Every time-window figure on the dashboard ("errors today", the daily trend,
+# the hourly chart, the error rate) used to filter error_logs by occurred_at
+# and then SUM(occurrence_count). occurred_at is FIRST-SEEN and is never
+# rewritten on recurrence, so that sum reports the lifetime volume of the
+# groups born inside the window -- not the events that happened in it. An
+# error first seen at 23:59 and recurring at 00:01 reported "0 errors today"
+# and put both events on yesterday.
+#
+# The obvious fix -- count error_occurrences rows in the window -- is wrong on
+# its own. Storm protection deliberately writes NO occurrence row while
+# shedding: it folds N events into occurrence_count and nothing else. Counting
+# occurrence rows would therefore erase storm volume from every time window,
+# which is exactly when the numbers matter most.
+#
+# So shed volume needs its own timestamp, and this is it: one row per
+# (error group, 15-minute bucket), holding how many shed events landed in it.
+# Window volume is then
+#
+#   occurrence rows in the window + shed bucket counts in the window
+#
+# which is correct for both ordinary and shed events.
+#
+# Small by construction: a row is written only while a storm is actually
+# shedding, at most one per group per quarter hour.
+#
+# Cleanup follows the GROUP, not the bucket's own age: RetentionCleanupJob
+# deletes the buckets of groups it expires, and ErrorLog has_many :event_counts
+# with dependent: :delete_all covers an explicit destroy. Buckets of a
+# still-active group are deliberately kept -- pruning them by bucket_at alone
+# would silently redistribute those events onto the group's first-seen date,
+# because EventVolume falls back to the group's lifetime count for any group
+# with no per-event record left.
+#
+# (An earlier version of this comment claimed pruning "on bucket_at" that was
+# never implemented. Both mechanisms above are now covered by
+# spec/models/rails_error_dashboard/event_count_cleanup_spec.rb.)
+class CreateEventCounts < ActiveRecord::Migration[7.0]
+  def change
+    # Guard against a squashed schema migration having already created this
+    # table -- without it, every later migration is silently cancelled.
+    return if table_exists?(:rails_error_dashboard_event_counts)
+
+    create_table :rails_error_dashboard_event_counts do |t|
+      t.bigint :error_log_id, null: false
+      # Truncated to a 15-MINUTE bucket, in UTC (EventCount::BUCKET_SECONDS,
+      # shared with the producer). Not the hour: every UTC offset in use
+      # divides into 15 minutes -- including +05:30 and +05:45 -- so a local
+      # midnight falls on a bucket EDGE and a day's total is exact. The row
+      # count stays bounded even through a long storm: at most one row per
+      # group per quarter hour, and only while shedding.
+      t.datetime :bucket_at, null: false
+      t.bigint :count, null: false, default: 0
+      t.timestamps
+    end
+
+    # The upsert target: one bucket per group per quarter hour. Named explicitly --
+    # an auto-generated name here would exceed PostgreSQL's 63-character limit
+    # and fail during the HOST app's deploy (see mailboxer#480).
+    add_index :rails_error_dashboard_event_counts, [ :error_log_id, :bucket_at ],
+              unique: true,
+              name: "index_red_event_counts_on_group_and_bucket"
+
+    # Window queries scan by time; retention prunes on the same column.
+    add_index :rails_error_dashboard_event_counts, :bucket_at,
+              name: "index_red_event_counts_on_bucket_at"
+  end
+end

data/db/migrate/20260920000001_add_buckets_incomplete_to_storm_events.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# Records that a storm episode lost its per-bucket TIMING evidence.
+#
+# When the rollup table is permanently unavailable (never migrated, or an
+# adapter that refuses the statement), FlushStormCounts deliberately degrades:
+# it keeps the authoritative lifetime occurrence_count and skips the bucket.
+# That is the right trade -- losing a time bucket beats losing a count -- but
+# it leaves every time-window figure for that episode resting on the group's
+# own occurred_at rather than on per-event evidence.
+#
+# The flag was returned in the flush result and then dropped on the floor: no
+# caller persisted it, no query read it, and nothing on the dashboard said so.
+# A replay lost it entirely. Persisting it on the episode is what lets the
+# Overview report an incomplete dimension instead of presenting a figure of
+# unknown completeness as fact -- exactly as affected_users_incomplete already
+# does for storm-shed occurrence rows.
+class AddBucketsIncompleteToStormEvents < ActiveRecord::Migration[7.0]
+  def change
+    return unless table_exists?(:rails_error_dashboard_storm_events)
+    return if column_exists?(:rails_error_dashboard_storm_events, :buckets_incomplete)
+
+    add_column :rails_error_dashboard_storm_events, :buckets_incomplete, :boolean, default: false
+  end
+end

data/db/migrate/20260920000002_create_event_timing_gaps.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# One row per interval whose per-event TIMING evidence was lost.
+#
+# When the rollup table is permanently unavailable, FlushStormCounts keeps the
+# authoritative lifetime occurrence_count and skips the bucket. The counts stay
+# exact; only their placement in time becomes approximate. The dashboard has to
+# say so, or it presents a figure of unknown completeness as fact.
+#
+# Three earlier attempts to carry that fact failed, each for a structural
+# reason, and this table exists to fix all three at once:
+#
+#   1. A boolean in the command's RETURN HASH. A background job's return value
+#      never reaches the dashboard.
+#   2. A flag on the storm EPISODE. The episode is OPTIONAL -- the gate can
+#      shed events with its breaker closed and pass episode: nil -- so the
+#      warning silently vanished exactly when no episode existed.
+#   3. Written by upsert_storm_event, which runs AFTER the counts transaction
+#      commits and rescues its own failures. A transient save failure lost the
+#      marker while the batch ledger recorded the batch as applied, so the
+#      replay was suppressed and the gap was never recorded.
+#
+# So: its own table, written INSIDE the same transaction as the counts (either
+# both land or neither does), keyed by the interval it describes rather than by
+# an episode that may not exist.
+#
+# Bounded by construction: one row per (application, bucket) per degraded
+# flush, and only while the rollup is actually unusable -- which is a
+# misconfiguration, not a steady state. Retention prunes it by covered_until.
+class CreateEventTimingGaps < ActiveRecord::Migration[7.0]
+  def change
+    return if table_exists?(:rails_error_dashboard_event_timing_gaps)
+
+    create_table :rails_error_dashboard_event_timing_gaps do |t|
+      # Nullable: a gap can predate application scoping, and a NULL here means
+      # "applies to every application" rather than "unknown".
+      t.bigint :application_id
+      # The interval whose timing is unreliable. covered_from is the earliest
+      # event in the degraded flush, covered_until the latest.
+      t.datetime :covered_from, null: false
+      t.datetime :covered_until, null: false
+      # How many events lost their timestamps, so the UI can say how much of
+      # the window is affected rather than only that something is.
+      t.bigint :events_affected, null: false, default: 0
+      t.timestamps
+    end
+
+    # The read is "does any gap overlap the window I am displaying", which
+    # scans on covered_until. Named explicitly -- an auto-generated name here
+    # would exceed PostgreSQL's 63-character limit and fail during the HOST
+    # app's deploy (see mailboxer#480).
+    add_index :rails_error_dashboard_event_timing_gaps, :covered_until,
+              name: "index_red_timing_gaps_on_covered_until"
+  end
+end

data/lib/rails_error_dashboard/commands/find_or_increment_error.rb

@@ -137,21 +137,78 @@ module RailsErrorDashboard
       # leaves both the snapshot and its provenance alone -- otherwise the row
       # would claim a fresh capture time for evidence from an older event,
       # which is precisely the confusion this exists to remove.
-      def context_provenance(refreshed)
+      def context_provenance(refreshed, error = nil)
         return {} unless refreshed.any? || refreshed_request_identity?
         return {} unless ErrorLog.column_names.include?("context_captured_at")
 
         provenance = { context_captured_at: @attributes[:occurred_at] || Time.current }
         if ErrorLog.column_names.include?("context_fidelity")
-          provenance[:context_fidelity] = @attributes[:_context_fidelity].presence || "full"
+          provenance[:context_fidelity] =
+            @attributes[:_context_fidelity].presence || snapshot_fidelity(error)
         end
         provenance
       end
 
+      # "full" means every displayed field came from THIS occurrence. When the
+      # `||` chain keeps an older value beside a newly refreshed one, the row
+      # is showing a mixture of two events, and calling that a fresh full
+      # capture is what made the snapshot unreadable: a new request URL sat
+      # beside a previous occurrence's user and locals under one timestamp.
+      #
+      # Keeping the older value is still the right behaviour -- a useful
+      # exemplar beats a blank one -- so only the LABEL changes.
+      def snapshot_fidelity(error)
+        return "full" if error.nil?
+
+        retains_older_value?(error) ? "partial" : "full"
+      end
+
+      # Every field whose stored value this capture could RETAIN from an
+      # earlier occurrence. Defined once, so a field cannot join the displayed
+      # snapshot without joining the provenance policy -- which is exactly how
+      # local variables came to be shown beside a "full" label and a newer
+      # timestamp while belonging to a different event.
+      #
+      # Request identity plus the context payloads: both are subject to the
+      # same `||` chain, and a reader cannot tell them apart on the page.
+      PROVENANCE_TRACKED = (REFRESHED_REQUEST_IDENTITY + REFRESHED_CONTEXT).uniq.freeze
+
+      # True when the row already holds a displayed value that this occurrence
+      # did NOT supply, so the `||` chain is about to keep it and the stored
+      # snapshot will describe two different events.
+      def retains_older_value?(error)
+        PROVENANCE_TRACKED.any? do |key|
+          next false unless ErrorLog.column_names.include?(key.to_s)
+
+          @attributes[key].nil? && previous_value(error, key).present?
+        end
+      end
+
+      # read_attribute, never public_send: :instance_variables would otherwise
+      # resolve to Ruby's own Object#instance_variables if the generated
+      # attribute method were ever absent (ignored_columns, load order), and
+      # silently compare an Array of symbols against captured context.
+      def previous_value(error, key)
+        error.read_attribute(key)
+      rescue StandardError
+        nil
+      end
+
       def refreshed_request_identity?
         REFRESHED_REQUEST_IDENTITY.any? { |key| !@attributes[key].nil? }
       end
 
+      # Inside the 24-hour matching window find_unresolved uses, so the group
+      # this creates can still be found by its own recurrences.
+      def clamped_group_time(time)
+        return Time.current if time.blank?
+
+        floor = 24.hours.ago + 1.minute
+        time < floor ? floor : time
+      rescue StandardError
+        Time.current
+      end
+
       # A group first seen during a storm has a MINIMAL exemplar: the flush job
       # could only record the first app frame, because a counted-only event
       # captures no backtrace. The comment there promises the next occurrence
@@ -205,7 +262,7 @@ module RailsErrorDashboard
           user_agent: @attributes[:user_agent] || error.user_agent,
           ip_address: @attributes[:ip_address] || error.ip_address,
           **refreshed,
-          **context_provenance(refreshed),
+          **context_provenance(refreshed, error),
           **backtrace_upgrade(error),
           **environment_adoption(error)
         )
@@ -225,7 +282,7 @@ module RailsErrorDashboard
           user_agent: @attributes[:user_agent] || error.user_agent,
           ip_address: @attributes[:ip_address] || error.ip_address,
           **(refreshed = latest_context),
-          **context_provenance(refreshed),
+          **context_provenance(refreshed, error),
           **backtrace_upgrade(error),
           **environment_adoption(error)
         }
@@ -245,6 +302,15 @@ module RailsErrorDashboard
         attrs = @attributes.reject { |key, _| key.to_s.start_with?("_") }
         attrs = attrs.reverse_merge(resolved: false)
 
+        # The GROUP's occurred_at is clamped into the matching window, even
+        # when the EVENT is older. find_unresolved matches on
+        # `occurred_at >= 24.hours.ago`, so a backdated report (a mobile client
+        # flushing a queue it collected offline) would otherwise create a row
+        # that can never be matched again -- every recurrence making yet
+        # another group. The event's true time is preserved on its occurrence
+        # row, which is what the time-window queries read.
+        attrs[:occurred_at] = clamped_group_time(attrs[:occurred_at])
+
         if ErrorLog.column_names.include?("context_captured_at")
           attrs[:context_captured_at] ||= @attributes[:occurred_at] || Time.current
         end

data/lib/rails_error_dashboard/commands/flush_storm_counts.rb

@@ -16,6 +16,12 @@ module RailsErrorDashboard
     # Counts are exact. Notifications are NOT dispatched from here — during a
     # storm they're suppressed by design; the storm notification covers it.
     class FlushStormCounts
+      # A bucket write failed in a way that may succeed on retry. Raised so the
+      # per-entry rescue in #call classifies it exactly as it classifies a
+      # transient COUNT failure: roll the batch back, leave the ledger
+      # unclaimed, let the job retry the whole batch intact.
+      class EventCountWriteFailed < StandardError; end
+
       def self.call(entries:, overflow: 0, episode: nil, batch_id: nil)
         new(entries: entries, overflow: overflow, episode: episode, batch_id: batch_id).call
       end
@@ -26,6 +32,10 @@ module RailsErrorDashboard
         # exemplar becomes an ErrorLog row, and its message is matched by regex.
         @entries = Array(Services::EncodingSanitizer.scrub_deep(entries))
         @overflow = overflow.to_i
+        # Set when a bucket write was permanently unavailable. The counts are
+        # still correct; only their placement in TIME is missing, and a caller
+        # reading a time window deserves to know that.
+        @buckets_incomplete = false
         @episode = episode
         @batch_id = batch_id
       end
@@ -34,6 +44,7 @@ module RailsErrorDashboard
         application = resolve_application
         counted = 0
         failed = 0
+        aborted = false
 
         # Counts are applied additively (occurrence_count + N), which is not
         # idempotent: delivering the same snapshot twice counted it twice. The
@@ -55,6 +66,20 @@ module RailsErrorDashboard
           @entries.each do |entry|
             entry = entry.with_indifferent_access if entry.respond_to?(:with_indifferent_access)
             counted += reconcile_entry(entry, application)
+          rescue EventCountWriteFailed, *Commands::LogError::RETRYABLE_STORE_ERRORS => e
+            # A transient store failure is NOT a bad entry. Claiming the batch
+            # here would commit the ledger row and strand every entry not yet
+            # applied: the retry is then suppressed as a replay and those events
+            # are lost for good. Re-raise so the whole transaction rolls back --
+            # nothing was committed, so nothing can double -- and let the job
+            # retry the batch intact. The generic rescue below still keeps a
+            # permanently malformed entry from poisoning its batch.
+            failed += 1
+            aborted = true
+            RailsErrorDashboard::Logger.error(
+              "[RailsErrorDashboard] Storm batch aborted by a transient store failure: #{e.class} - #{e.message}"
+            )
+            raise
           rescue => e
             # A corrupt (non-Hash) entry must not abort the whole batch — and the
             # log line itself must not assume `entry` is subscriptable (an Integer
@@ -70,26 +95,56 @@ module RailsErrorDashboard
           # roll the claim back and let the job retry the whole batch.
           raise ActiveRecord::Rollback if failed.positive? && counted.zero?
 
+          # INSIDE the transaction, deliberately.
+          #
+          # The counts and the record that their timing is unreliable have to
+          # land together or not at all. Writing this after the commit (as the
+          # storm-episode marker did) meant a transient failure lost the
+          # marker while the ledger had already recorded the batch as applied
+          # -- the replay was then suppressed and the gap was never recorded,
+          # so the dashboard reported completeness it could not vouch for.
+          #
+          # If this write fails, the whole batch rolls back and stays
+          # replayable. Counts whose unreliability we cannot record are worth
+          # retrying, not committing silently.
+          record_timing_gap!(counted) if @buckets_incomplete && counted.positive?
+
           finalize_batch!(ledger, counted)
         end
 
         # Every entry failed and none was written. Reporting success with
         # reconciled: 0 made a total loss indistinguishable from an empty
         # batch, so the job acknowledged counts that never reached the
-        # database. Partial success stays successful: the entries that were
-        # written are written, and replaying the batch would double them.
+        # database.
+        #
+        # Reaching here means every failure was PERMANENT -- a transient store
+        # failure re-raises above and rolls the whole batch back. Partial
+        # success over permanent failures stays successful: the entries that
+        # were written are written, replaying would double them, and retrying a
+        # corrupt payload only loops forever.
         if failed.positive? && counted.zero?
-          return { success: false, reconciled: 0, failed: failed, overflow: @overflow,
+          return { success: false, retryable: false, reconciled: 0, failed: failed, overflow: @overflow,
+                   buckets_incomplete: @buckets_incomplete,
                    error: "all #{failed} entries failed to reconcile" }
         end
 
         upsert_storm_event(counted)
-        { success: true, reconciled: counted, failed: failed, overflow: @overflow }
+        result = { success: true, reconciled: counted, failed: failed, overflow: @overflow }
+        result[:buckets_incomplete] = true if @buckets_incomplete
+        result
       rescue => e
         RailsErrorDashboard::Logger.error(
           "[RailsErrorDashboard] FlushStormCounts failed: #{e.class} - #{e.message}"
         )
-        { success: false, error: "#{e.class}: #{e.message}" }
+        # retryable: true says "the batch is intact, replay it" -- nothing was
+        # committed, so the job can retry without doubling. A permanent failure
+        # carries no such promise.
+        retryable = Commands::LogError::RETRYABLE_STORE_ERRORS.any? { |klass| e.is_a?(klass) }
+        result = { success: false, retryable: retryable, error: "#{e.class}: #{e.message}" }
+        # reconciled: 0 because the transaction rolled back -- whatever this
+        # batch had counted in memory never reached the database.
+        result.merge!(reconciled: 0, failed: failed, overflow: @overflow) if aborted
+        result
       end
 
       private
@@ -148,7 +203,68 @@ module RailsErrorDashboard
       end
 
 
+      # Reconcile one buffered entry and, on every path that adds counts, give
+      # those shed events a TIME BUCKET as well as a total.
+      #
+      # occurrence_count alone is a lifetime counter: it says how many events
+      # there were but not when, so no window query can place them. Ordinary
+      # captures carry their own ErrorOccurrence row; shed events write none by
+      # design, which is what made them invisible to "errors today". The bucket
+      # written here is what Queries::EventVolume adds to the occurrence rows.
       def reconcile_entry(entry, application)
+        error_log_id = nil
+        count = reconcile_entry_count(entry, application) { |id| error_log_id = id }
+        write_event_buckets(entry, error_log_id, count) if count.positive? && error_log_id
+        count
+      end
+
+      # Write one row per BUCKET the producer recorded, not one row for the
+      # whole entry.
+      #
+      # The buffer tallies events per 15-minute bucket precisely so this does
+      # not have to guess: assigning the entry's whole total to last_seen_at's
+      # bucket put an event from 23:59:59 and one from 00:00:01 on the same
+      # day. A payload from an older release carries no buckets, so it falls
+      # back to the old behaviour rather than losing the count.
+      def write_event_buckets(entry, error_log_id, count)
+        buckets = entry["buckets"]
+        buckets = nil unless buckets.is_a?(Hash) && buckets.any?
+
+        pairs =
+          if buckets
+            buckets.map { |at, n| [ Time.zone.at(at.to_i), n.to_i ] }
+          else
+            [ [ parse_time(entry["last_seen_at"]) || Time.current, count ] ]
+          end
+
+        pairs.each do |bucket_at, n|
+          next unless n.positive?
+
+          # Three outcomes, three responses.
+          #
+          # :written     -- done.
+          # raises       -- TRANSIENT. Handled by the caller's rescue, which
+          #                 aborts and replays the batch intact. Swallowing it
+          #                 finalized the ledger with the bucket missing, so
+          #                 the replay was suppressed as already-applied and
+          #                 the time window lost those events permanently
+          #                 while the lifetime count stayed correct.
+          # :unavailable -- PERMANENT. Degrade: the rollup is simply not
+          #                 usable on this host (never migrated, adapter
+          #                 refuses the statement). Raising here rolled back
+          #                 the surrounding transaction and destroyed the
+          #                 authoritative lifetime count along with it --
+          #                 turning a missing time bucket into a lost count,
+          #                 which is strictly worse. Record it instead, so the
+          #                 result can say the timing evidence is incomplete.
+          case EventCount.accumulate(error_log_id: error_log_id, bucket_at: bucket_at, count: n)
+          when :written then next
+          else @buckets_incomplete = true
+          end
+        end
+      end
+
+      def reconcile_entry_count(entry, application)
         count = entry["count"].to_i
         return 0 if count <= 0
 
@@ -185,6 +301,7 @@ module RailsErrorDashboard
               "occurrence_count = occurrence_count + ?, last_seen_at = ?", count, last_seen
             ])
           end
+          yield target.id if block_given?
           return count
         end
 
@@ -197,18 +314,43 @@ module RailsErrorDashboard
           resolved_scope = resolved_scope.where(environment: [ env, nil ])
             .order(Arel.sql("CASE WHEN environment IS NULL THEN 1 ELSE 0 END"))
         end
-        resolved = resolved_scope.order(last_seen_at: :desc).first
+        # .lock (SELECT ... FOR UPDATE) held to commit by the transaction opened
+        # in #call, exactly as FindOrIncrementError does for the same reopen.
+        # Without it this branch read occurrence_count into Ruby and wrote an
+        # ABSOLUTE value back, so two concurrent batches both read N and both
+        # wrote N+count -- one batch's events vanished while both reported
+        # success. The unresolved branch above is safe because it increments in
+        # SQL; this branch cannot use update_all because reopening is a state
+        # transition the dashboard must see, and update_all skips the
+        # after_update_commit broadcast.
+        resolved = resolved_scope.lock.order(last_seen_at: :desc).first
         if resolved
+          # The count is incremented in SQL, never read into Ruby and written
+          # back. This branch used to compute `resolved.occurrence_count + count`
+          # and write that ABSOLUTE value, so two concurrent batches both read N
+          # and both wrote N+count -- one batch's events vanished while both
+          # reported success. The .lock above serializes the pair on
+          # PostgreSQL/MySQL; the atomic increment below conserves the count on
+          # every adapter, including SQLite where FOR UPDATE is a no-op.
+          ErrorLog.where(id: resolved.id).update_all([
+            "occurrence_count = occurrence_count + ?", count
+          ])
+
+          # The reopen is a state transition the dashboard must see, so it stays
+          # an update! -- update_all would skip the after_update_commit
+          # broadcast. Reload first so this write does not clobber the increment
+          # just made with a stale in-memory occurrence_count.
+          resolved.reload
           attrs = {
             resolved: false,
             status: "new",
             resolved_at: nil,
-            occurrence_count: resolved.occurrence_count + count,
             last_seen_at: last_seen
           }
           attrs[:reopened_at] = Time.current if ErrorLog.column_names.include?("reopened_at")
           attrs[:environment] = env if env && resolved.environment.blank?
           resolved.update!(attrs)
+          yield resolved.id if block_given?
           return count
         end
 
@@ -249,10 +391,48 @@ module RailsErrorDashboard
         if ErrorLog.column_names.include?("context_captured_at")
           create_attrs[:context_captured_at] = create_attrs[:occurred_at]
         end
-        ErrorLog.create!(**ErrorLog.clamp_string_attributes(Services::SensitiveDataFilter.filter_attributes(create_attrs)))
+        begin
+          # requires_new opens a SAVEPOINT: on PostgreSQL a failed INSERT aborts
+          # its transaction, and every later statement -- including the recovery
+          # lookup below -- fails with InFailedSqlTransaction. The savepoint
+          # confines the damage to this INSERT so the batch can continue.
+          ErrorLog.transaction(requires_new: true) do
+            created = ErrorLog.create!(**ErrorLog.clamp_string_attributes(Services::SensitiveDataFilter.filter_attributes(create_attrs)))
+            yield created.id if block_given?
+          end
+        rescue ActiveRecord::RecordNotUnique
+          # Another flush created this group between our lookups and this
+          # INSERT -- the group-identity index objected. Two concurrent batches
+          # for the same fingerprint is the NORMAL storm shape (every process
+          # flushes its own batch), so this must not fail the entry: the counts
+          # exist nowhere but in this payload. Re-run the same lookups and add
+          # to the row that now exists, exactly as FindOrIncrementError does.
+          #
+          # Nested in its own transaction because the failed INSERT poisons the
+          # surrounding one on PostgreSQL.
+          raise unless (target = existing_target(error_hash, application, env))
+
+          ErrorLog.where(id: target.id).update_all([
+            "occurrence_count = occurrence_count + ?, last_seen_at = ?", count, last_seen
+          ])
+          # Yield on the RECOVERY path too: these counts are as real as the ones
+          # the winning INSERT wrote, so they need a time bucket as well, or a
+          # raced create silently loses its volume from every window query.
+          yield target.id if block_given?
+        end
         count
       end
 
+      # The row a retried INSERT should add to: any row holding this group
+      # identity, whatever its status. Deliberately wider than the
+      # priority-ordered lookups above -- the index has already proved a row
+      # with this identity exists, so refusing to match a resolved or wont_fix
+      # one would drop the counts instead.
+      def existing_target(error_hash, application, env)
+        scope = ErrorLog.where(error_hash: error_hash, application_id: application.id)
+        scope = scope.where(environment: [ env, nil ]) if env
+        scope.order(last_seen_at: :desc).select(:id).first
+      end
 
       # The unresolved row the full capture path would increment right now.
       # No time window: "won't fix" holds for as long as the row keeps the status.
@@ -326,6 +506,36 @@ module RailsErrorDashboard
         Application.find_or_create_by_name(app_name)
       end
 
+      # Record the interval whose per-event timing was lost.
+      #
+      # Keyed by the interval, NOT by a storm episode: the gate can shed events
+      # with its breaker closed and pass episode: nil, and a marker on the
+      # episode vanished in exactly that case. The events are just as
+      # untimed whether or not an episode object happens to exist.
+      #
+      # Bounds come from the entries themselves, so the gap describes when the
+      # events actually happened rather than when the worker got to them.
+      def record_timing_gap!(counted)
+        return unless EventTimingGap.table_exists?
+
+        times = @entries.filter_map do |entry|
+          entry = entry.with_indifferent_access if entry.respond_to?(:with_indifferent_access)
+          parse_time(entry["last_seen_at"]) || parse_time(entry["first_seen_at"])
+        end
+        first_seen = @entries.filter_map do |entry|
+          entry = entry.with_indifferent_access if entry.respond_to?(:with_indifferent_access)
+          parse_time(entry["first_seen_at"])
+        end
+
+        now = Time.current
+        EventTimingGap.create!(
+          application_id: resolve_application&.id,
+          covered_from: (first_seen + times).min || now,
+          covered_until: times.max || now,
+          events_affected: counted
+        )
+      end
+
       def upsert_storm_event(counted)
         return unless @episode.is_a?(Hash)
         return unless StormEvent.table_exists?
@@ -346,6 +556,13 @@ module RailsErrorDashboard
         event.fingerprints_affected = [ event.fingerprints_affected.to_i, @entries.size ].max
         event.peak_rate_per_minute = [ event.peak_rate_per_minute.to_i, @episode["peak_rate_per_minute"].to_i ].max
         event.reached_open ||= @episode["reached_open"] == true
+        # Sticky, like reached_open: once an episode has lost bucket timing it
+        # has lost it, and a later flush that happens to succeed does not make
+        # the earlier gap reappear. Guarded on the column so a host that has
+        # not run the migration yet keeps flushing normally.
+        if @buckets_incomplete && event.respond_to?(:buckets_incomplete)
+          event.buckets_incomplete = true
+        end
         event.top_fingerprints = top_fingerprints_json(event)
         event.ended_at = parse_time(@episode["ended_at"]) if @episode["ended_at"]
         event.save!