pgbus 0.17.1 → 0.17.2

data/lib/pgbus/web/data_source.rb

@@ -5,6 +5,18 @@ require "time"
 module Pgbus
   module Web
     class DataSource
+      include JobList
+      include DeadLetter
+      include ListCounts
+      include QueueSummary
+      include EventList
+      include ProcessList
+      include DashboardSignals
+
+      # pgbus_failed_events rows outside the EventBus handler queues; binds
+      # handler_queue_names_literal as $1.
+      NOT_HANDLER_QUEUE = "NOT (queue_name = ANY($1::text[]))"
+
       # Ceiling on how many parked jobs one dashboard release promotes, so a
       # key with thousands parked cannot hold the request open. The dispatcher
       # sweep picks up whatever is left on its next pass.
@@ -51,6 +63,9 @@ module Pgbus
       def reset_cache!
         @queues_with_metrics = nil
         @concurrency_summary = nil
+        @queue_health_stats = nil
+        @process_rows = nil
+        @stream_queue_names = nil
         self
       end
 
@@ -86,13 +101,18 @@ module Pgbus
       # archive msg_id index fingerprint (issue #366). Stream delivery is a
       # non-consuming peek, so stream messages sit visible with read_ct=0
       # forever — health verdicts must not read that as a wedge (issue #359).
-      # Loaded fresh per call (reset + registry + fingerprint queries):
-      # verdicts run on coarse intervals and a long-lived process must see
-      # streams registered since the last check. Degrades to an empty Set
-      # when the registry/pgmq schema is absent or unreadable.
+      # Loaded fresh per instance (reset + registry + fingerprint queries),
+      # then memoized until reset_cache!: the home page asks three times per
+      # render (issue #492). A long-lived holder (the MCP server, the AppSignal
+      # probe) calls reset_cache! per check and HealthApp builds a DataSource
+      # per check, so each verdict still sees streams registered since the
+      # last one. Degrades to an empty Set when the registry/pgmq schema is
+      # absent or unreadable.
       def stream_queue_names
-        StreamQueue.reset_cache!
-        StreamQueue.known_names
+        @stream_queue_names ||= begin
+          StreamQueue.reset_cache!
+          StreamQueue.known_names
+        end
       end
 
       # Fingerprint-matched stream queues missing from the registry (issue #366).
@@ -209,9 +229,12 @@ module Pgbus
         @client.archive_message(queue_name, msg_id.to_i, prefixed: false)
       end
 
+      # Job queues only: the Events page owns the EventBus handler queues
+      # (issue #494).
       def discard_all_enqueued
         dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
-        queues = queues_with_metrics.reject { |q| q[:name].end_with?(dlq_suffix) }
+        handlers = handler_queue_physical_names
+        queues = queues_with_metrics.reject { |q| q[:name].end_with?(dlq_suffix) || handlers.include?(q[:name]) }
         total = 0
 
         queues.each do |q|
@@ -307,11 +330,14 @@ module Pgbus
         false
       end
 
+      # Failed jobs only, like discard_all_failed: an event's message is still
+      # in its handler queue, so a re-sent copy would run the handler twice.
       def retry_all_failed
         count = 0
         loop do
           batch = connection.select_all(
-            "SELECT * FROM pgbus_failed_events ORDER BY id LIMIT 100", "Pgbus Retry Batch"
+            "SELECT * FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE} ORDER BY id LIMIT 100", "Pgbus Retry Batch",
+            [handler_queue_names_literal]
           ).to_a
           break if batch.empty?
 
@@ -334,129 +360,21 @@ module Pgbus
         count
       end
 
+      # Failed jobs only: failed EventBus handler rows belong to the Events
+      # page, whose list no Jobs-page action may reach (issue #494).
       def discard_all_failed
         release_locks_for_failed_events
         archive_all_failed_messages
 
-        result = connection.execute("DELETE FROM pgbus_failed_events")
-        result.cmd_tuples
+        connection.exec_delete(
+          "DELETE FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE}", "Pgbus Discard All Failed",
+          [handler_queue_names_literal]
+        )
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error discarding all failed events: #{e.message}" }
         0
       end
 
-      # Dead letter queue
-      # Note: DLQ queue names from queues_with_metrics are already fully qualified
-      # (e.g., "pgbus_default_dlq"), so we use them directly without re-prefixing.
-      def dlq_messages(page: 1, per_page: 25)
-        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
-        queues = queues_with_metrics.select { |q| q[:name].end_with?(dlq_suffix) }
-        offset = (page - 1) * per_page
-
-        paginated_queue_messages(queues.map { |q| q[:name] }, per_page, offset)
-      rescue StandardError => e
-        Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ messages: #{e.message}" }
-        []
-      end
-
-      def dlq_total_count
-        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
-        queues_with_metrics
-          .select { |q| q[:name].end_with?(dlq_suffix) }
-          .sum { |q| q[:queue_length] }
-      rescue StandardError => e
-        Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ count: #{e.message}" }
-        0
-      end
-
-      def dlq_message_detail(msg_id)
-        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
-        queues = queues_with_metrics.select { |q| q[:name].end_with?(dlq_suffix) }
-        queues.each do |q|
-          row = connection.select_one(
-            "SELECT * FROM pgmq.q_#{sanitize_name(q[:name])} WHERE msg_id = $1",
-            "Pgbus DLQ Detail",
-            [msg_id.to_i]
-          )
-          return format_message(row, q[:name]) if row
-        end
-        nil
-      rescue StandardError => e
-        Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ message #{msg_id}: #{e.message}" }
-        nil
-      end
-
-      def retry_dlq_message(queue_name, msg_id)
-        # queue_name here is the full DLQ name (already prefixed)
-        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
-        original_queue = queue_name.delete_suffix(dlq_suffix)
-
-        row = connection.select_one(
-          "SELECT * FROM pgmq.q_#{sanitize_name(queue_name)} WHERE msg_id = $1",
-          "Pgbus DLQ Read",
-          [msg_id.to_i]
-        )
-        return false unless row
-
-        @client.transaction do |txn|
-          txn.produce(original_queue, row["message"], headers: row["headers"])
-          txn.delete(queue_name, msg_id.to_i)
-        end
-        true
-      rescue StandardError => e
-        Pgbus.logger.debug { "[Pgbus::Web] Error retrying DLQ message #{msg_id}: #{e.message}" }
-        false
-      end
-
-      def discard_dlq_message(queue_name, msg_id)
-        # queue_name here is the full DLQ name (already prefixed)
-        release_lock_for_message(queue_name, msg_id)
-        @client.delete_message(queue_name, msg_id.to_i, prefixed: false)
-        true
-      rescue StandardError => e
-        Pgbus.logger.debug { "[Pgbus::Web] Error discarding DLQ message #{msg_id}: #{e.message}" }
-        false
-      end
-
-      def retry_all_dlq
-        messages = dlq_messages(page: 1, per_page: 1000)
-        count = 0
-        messages.each do |m|
-          retry_dlq_message(m[:queue_name], m[:msg_id]) && count += 1
-        rescue StandardError => e
-          Pgbus.logger.debug { "[Pgbus::Web] Error retrying DLQ message #{m[:msg_id]}: #{e.message}" }
-          next
-        end
-        count
-      end
-
-      def discard_all_dlq
-        messages = dlq_messages(page: 1, per_page: 1000)
-        return 0 if messages.empty?
-
-        release_locks_for_messages(messages)
-
-        # Group by queue for batch delete — one call per DLQ instead of N calls
-        messages.group_by { |m| m[:queue_name] }.sum do |queue_name, msgs|
-          ids = msgs.map { |m| m[:msg_id].to_i }
-          @client.delete_batch(queue_name, ids, prefixed: false).size
-        rescue StandardError => e
-          Pgbus.logger.debug { "[Pgbus::Web] Error batch-discarding DLQ messages from #{queue_name}: #{e.message}" }
-          0
-        end
-      end
-
-      # Processes
-      def processes
-        rows = connection.select_all(
-          "SELECT * FROM pgbus_processes ORDER BY kind, created_at"
-        )
-        rows.to_a.map { |r| format_process(r) }
-      rescue StandardError => e
-        Pgbus.logger.debug { "[Pgbus::Web] Error fetching processes: #{e.message}" }
-        []
-      end
-
       # Processed events (audit trail)
       def processed_events(page: 1, per_page: 25)
         offset = (page - 1) * per_page
@@ -490,21 +408,12 @@ module Pgbus
         0
       end
 
-      def replay_event(event)
-        # Re-publish the event payload to all matching subscribers
-        routing_key = event["routing_key"] || event["handler_class"]
-        return false unless routing_key
-
-        @client.publish_to_topic(routing_key, event["payload"] || "{}")
-        true
-      rescue StandardError => e
-        Pgbus.logger.debug { "[Pgbus::Web] Error replaying event: #{e.message}" }
-        false
-      end
-
       # Recurring tasks
-      def recurring_tasks
-        records = RecurringTask.order(:key).to_a
+      # page/per_page nil returns every task (the MCP tool's default).
+      def recurring_tasks(page: nil, per_page: nil)
+        scope = RecurringTask.order(:key)
+        scope = scope.limit(per_page).offset(((page || 1) - 1) * per_page) if per_page
+        records = scope.to_a
         last_runs = RecurringExecution
                     .where(task_key: records.map(&:key))
                     .select("task_key, MAX(run_at) AS run_at")
@@ -616,13 +525,6 @@ module Pgbus
         false
       end
 
-      def recurring_tasks_count
-        RecurringTask.count
-      rescue StandardError => e
-        Pgbus.logger.debug { "[Pgbus::Web] Error counting recurring tasks: #{e.message}" }
-        0
-      end
-
       # Outbox
       def outbox_stats
         {
@@ -668,8 +570,8 @@ module Pgbus
       end
 
       # Job uniqueness keys
-      def job_locks
-        UniquenessKey.order(created_at: :desc).limit(100).map do |key|
+      def job_locks(page: 1, per_page: 100)
+        UniquenessKey.order(created_at: :desc, lock_key: :asc).limit(per_page).offset((page - 1) * per_page).map do |key|
           {
             lock_key: key.lock_key,
             queue_name: key.queue_name,
@@ -684,12 +586,11 @@ module Pgbus
       end
 
       # Concurrency keys — the `limits_concurrency` slot table and the jobs
-      # parked behind it. Aggregate numbers plus up to 100 key rows, busiest
-      # first. The two tables are joined FULL OUTER, not LEFT: a key can have
-      # parked rows and no semaphore (its holder died and the sweep removed the
-      # row) or a semaphore and nothing parked — an operator needs to see both.
-      def concurrency_stats
-        concurrency_summary.merge(keys: concurrency_keys)
+      # parked behind it. Aggregate numbers plus one page of key rows (100 by
+      # default), busiest first. The two tables are joined FULL OUTER, not LEFT
+      # (ListCounts::CONCURRENCY_KEYS_FROM) — an operator needs to see both.
+      def concurrency_stats(page: 1, per_page: 100)
+        concurrency_summary.merge(keys: concurrency_keys(page: page, per_page: per_page))
       end
 
       # Drop a key's semaphore row and promote whatever can now run.
@@ -761,8 +662,8 @@ module Pgbus
       end
 
       # Batches
-      def batches(limit: 100)
-        records = BatchEntry.order(created_at: :desc).limit(limit).to_a
+      def batches(page: 1, per_page: 25)
+        records = BatchEntry.order(created_at: :desc, id: :desc).limit(per_page).offset((page - 1) * per_page).to_a
         pending = pending_jobs_by_batch(records.map(&:batch_id))
         records.map { |r| format_batch(r, pending_jobs: pending[r.batch_id]) }
       rescue StandardError => e
@@ -788,13 +689,6 @@ module Pgbus
         nil
       end
 
-      def batches_count
-        BatchEntry.count
-      rescue StandardError => e
-        Pgbus.logger.debug { "[Pgbus::Web] Error counting batches: #{e.message}" }
-        0
-      end
-
       def active_batches_count
         BatchEntry.where.not(status: "finished").count
       rescue StandardError => e
@@ -831,6 +725,11 @@ module Pgbus
         []
       end
 
+      # Whether job stats record enqueue latency (rails g pgbus:add_job_stats_latency).
+      def latency_available?
+        JobStat.latency_columns?
+      end
+
       def latency_trend(minutes: 60)
         JobStat.latency_trend(minutes: minutes)
       rescue StandardError => e
@@ -848,7 +747,13 @@ module Pgbus
       # Queue health — vacuum stats, dead tuples, bloat, MVCC horizon.
       # Returns aggregate health across all queue and archive tables, plus
       # the oldest open transaction age (MVCC horizon pinning risk).
+      # Memoized per instance like queues_with_metrics: the home page reads it
+      # for the stats, the Queue Health panel and the health verdict.
       def queue_health_stats
+        @queue_health_stats ||= fetch_queue_health_stats
+      end
+
+      def fetch_queue_health_stats
         tables = fetch_all_table_stats
 
         total_dead = tables.sum { |t| t[:dead_tuples] }
@@ -874,6 +779,7 @@ module Pgbus
           oldest_transaction_age_sec: nil, tables: []
         }
       end
+      private :fetch_queue_health_stats
 
       # Per-queue health stats for the queue detail view.
       def queue_health_detail(queue_name)
@@ -954,6 +860,14 @@ module Pgbus
         []
       end
 
+      # Logical and physical handler queue names as a text[] literal: failed
+      # rows carry the logical name, older ones the physical. Queue names are
+      # validated word characters, so the literal needs no quoting.
+      def handler_queue_names_literal
+        names = registered_subscribers.flat_map { |s| [s[:queue_name], s[:physical_queue_name]] }
+        "{#{names.compact.uniq.join(",")}}"
+      end
+
       # Physical queue names for all registered subscribers. Used for both
       # pending_events lookup and server-side validation of target queues
       # in reroute_event.
@@ -973,6 +887,7 @@ module Pgbus
       def discard_event(queue_name, msg_id)
         release_lock_for_message(queue_name, msg_id)
         @client.archive_message(queue_name, msg_id.to_i, prefixed: false)
+        clear_event_failure(queue_name, msg_id)
         true
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error discarding event #{msg_id}: #{e.message}" }
@@ -996,16 +911,14 @@ module Pgbus
         event_id = raw["event_id"]
         return false unless event_id
 
-        ProcessedEvent.insert(
-          { event_id: event_id, handler_class: handler_class, processed_at: Time.now.utc },
-          unique_by: %i[event_id handler_class]
-        )
+        record_event_handled(event_id, handler_class)
         # Release the uniqueness lock while we still hold the payload in
         # memory — otherwise the message is archived but the lock row stays
         # behind, blocking later publishes with the same key. Mirrors
         # discard_event.
         release_lock_for_payload(detail[:message])
         @client.archive_message(queue_name, msg_id.to_i, prefixed: false)
+        clear_event_failure(queue_name, msg_id)
         true
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error marking event #{msg_id} handled: #{e.message}" }
@@ -1030,6 +943,7 @@ module Pgbus
           txn.produce(queue_name, parsed.to_json, headers: detail[:headers])
           txn.delete(queue_name, msg_id.to_i)
         end
+        clear_event_failure(queue_name, msg_id)
         true
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error editing event #{msg_id}: #{e.message}" }
@@ -1047,6 +961,7 @@ module Pgbus
           txn.produce(target_queue, detail[:message], headers: detail[:headers])
           txn.delete(source_queue, msg_id.to_i)
         end
+        clear_event_failure(source_queue, msg_id)
         true
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error rerouting event #{msg_id}: #{e.message}" }
@@ -1067,6 +982,33 @@ module Pgbus
         count
       end
 
+      # A completed marker on a two-phase schema (completed_at): without it the
+      # audit reads the row as a claim that went silent, and a pending claim
+      # left by a failed attempt would stay pending (issue #494). The legacy
+      # schema keeps the single-phase insert.
+      def record_event_handled(event_id, handler_class)
+        now = Time.now.utc
+        row = { event_id: event_id, handler_class: handler_class, processed_at: now }
+        if ProcessedEvent.completion_column?
+          ProcessedEvent.upsert(row.merge(completed_at: now), unique_by: %i[event_id handler_class],
+                                                              update_only: [:completed_at])
+        else
+          ProcessedEvent.insert(row, unique_by: %i[event_id handler_class])
+        end
+      end
+
+      # The failed row of an event message an action just took out of its
+      # handler queue (issue #494). The consumer records the logical name;
+      # both spellings are cleared. A failure here never undoes the action.
+      def clear_event_failure(queue_name, msg_id)
+        connection.exec_delete(
+          "DELETE FROM pgbus_failed_events WHERE queue_name IN ($1, $2) AND msg_id = $3", "Pgbus Clear Event Failure",
+          [logical_queue_name(queue_name), queue_name, msg_id.to_i]
+        )
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus::Web] Error clearing the failed row of event #{msg_id}: #{e.class}: #{e.message}" }
+      end
+
       # Subscriber registry. `queue_name` is the logical name the subscriber
       # registered with; `physical_queue_name` is what the queue is actually
       # called in `pgmq.meta` (e.g. logical "task_completion_handler" ->
@@ -1152,7 +1094,7 @@ module Pgbus
       # `lease_fresh` is the one fact an operator needs before releasing a key:
       # a live lease means a holder is probably still running, so releasing lets
       # another job start beside it.
-      def concurrency_keys(limit: 100)
+      def concurrency_keys(page: 1, per_page: 100)
         rows = connection.select_all(<<~SQL, "Pgbus Concurrency Keys")
           SELECT COALESCE(s.key, b.concurrency_key) AS key,
                  s.value AS value,
@@ -1161,14 +1103,9 @@ module Pgbus
                  (s.value > 0 AND s.expires_at > now()) AS lease_fresh,
                  COALESCE(b.parked_count, 0) AS parked_count,
                  EXTRACT(EPOCH FROM (now() - b.oldest_parked_at))::bigint AS oldest_parked_age_sec
-          FROM pgbus_semaphores s
-          FULL OUTER JOIN (
-            SELECT concurrency_key, COUNT(*) AS parked_count, MIN(created_at) AS oldest_parked_at
-            FROM pgbus_blocked_executions
-            GROUP BY concurrency_key
-          ) b ON s.key = b.concurrency_key
-          ORDER BY COALESCE(b.parked_count, 0) DESC, s.expires_at ASC NULLS LAST
-          LIMIT #{limit.to_i}
+          #{CONCURRENCY_KEYS_FROM}
+          ORDER BY COALESCE(b.parked_count, 0) DESC, s.expires_at ASC NULLS LAST, COALESCE(s.key, b.concurrency_key)
+          LIMIT #{per_page.to_i} OFFSET #{(page.to_i - 1) * per_page.to_i}
         SQL
 
         rows.to_a.map { |row| format_concurrency_key(row) }
@@ -1353,28 +1290,34 @@ module Pgbus
       # name goes through sanitize_name (which calls QueueNameValidator)
       # so it's safe to interpolate into both the schema-qualified table
       # and the literal column. limit/offset are bound parameters.
-      def paginated_queue_messages(queue_names, limit, offset)
+      #
+      # where: an optional condition appended inside every fragment, with its
+      # values in binds (numbered from $3, after limit and offset). The
+      # default call builds exactly the unfiltered query.
+      def paginated_queue_messages(queue_names, limit, offset, where: nil, binds: [])
         return [] if queue_names.empty?
 
-        sanitized = queue_names.map { |name| [name, sanitize_name(name)] }
-        fragments = sanitized.map do |(name, qtable)|
-          <<~SQL.strip
-            SELECT msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers,
-                   '#{name}' AS queue_name
-            FROM pgmq.q_#{qtable}
-          SQL
-        end
-
         sql = <<~SQL
-          SELECT * FROM (#{fragments.join("\nUNION ALL\n")}) AS combined
+          SELECT * FROM (#{queue_message_fragments(queue_names, where)}) AS combined
           ORDER BY msg_id DESC
           LIMIT $1 OFFSET $2
         SQL
 
-        rows = connection.select_all(sql, "Pgbus Paginated Queue Messages", [limit, offset])
+        rows = connection.select_all(sql, "Pgbus Paginated Queue Messages", [limit, offset, *binds])
         rows.to_a.map { |r| format_message(r, r["queue_name"]) }
       end
 
+      def queue_message_fragments(queue_names, where = nil)
+        queue_names.map do |name|
+          fragment = <<~SQL.strip
+            SELECT msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers,
+                   '#{name}' AS queue_name
+            FROM pgmq.q_#{sanitize_name(name)}
+          SQL
+          where ? "#{fragment}\nWHERE #{where}" : fragment
+        end.join("\nUNION ALL\n")
+      end
+
       def batched_queue_metrics(queue_names)
         return [] if queue_names.empty?
 
@@ -1499,72 +1442,6 @@ module Pgbus
         }
       end
 
-      def format_process(row)
-        heartbeat = row["last_heartbeat_at"]
-        heartbeat_time = heartbeat.is_a?(String) ? Time.parse(heartbeat) : heartbeat
-        stale = heartbeat_time && (Time.now - heartbeat_time) > Process::Heartbeat::ALIVE_THRESHOLD
-        metadata = row["metadata"].is_a?(String) ? JSON.parse(row["metadata"]) : row["metadata"]
-        status = derive_process_status(stale, metadata, row["kind"])
-
-        {
-          id: row["id"].to_i,
-          kind: row["kind"],
-          hostname: row["hostname"],
-          pid: row["pid"].to_i,
-          metadata: metadata,
-          last_heartbeat_at: heartbeat_time,
-          healthy: status == :healthy,
-          status: status,
-          created_at: row["created_at"]
-        }
-      end
-
-      def derive_process_status(stale, metadata, kind)
-        return :stale if stale
-        return :healthy unless metadata.is_a?(Hash)
-
-        threshold = stall_threshold_for(kind)
-        return :healthy unless threshold
-
-        loop_tick = metadata["loop_tick_at"]
-        return :healthy unless loop_tick
-
-        age = Time.now.to_f - loop_tick.to_f
-        age > threshold ? :stalled : :healthy
-      end
-
-      # Kind-aware stall threshold. Workers run a tight loop, so the base
-      # stall_threshold applies directly. The dispatcher and scheduler sleep
-      # between iterations, so their beacon is naturally up to one sleep
-      # interval stale even when healthy — widen the window by that interval to
-      # avoid false stalls. Consumers idle on an empty-read wait that can reach
-      # NOTIFY_FALLBACK_POLL_SECONDS (15s) when a live listener drives wake-up,
-      # so widen by that max wait when notify is on, or by the poll interval
-      # otherwise. Kinds with no loop beacon (nil) are never stalled.
-      def stall_threshold_for(kind)
-        config = Pgbus.configuration
-        base = config.stall_threshold
-        return nil unless base&.positive?
-
-        case kind
-        when "worker" then base
-        when "dispatcher" then base + config.dispatch_interval
-        when "scheduler" then base + config.recurring_schedule_interval
-        when "consumer" then base + consumer_stall_slack(config)
-        end
-      end
-
-      # Widen the consumer's stall window by its maximum empty-read wait. With
-      # notify wake-up live, that wait rises to NOTIFY_FALLBACK_POLL_SECONDS as a
-      # safety net; without it, the consumer polls at polling_interval.
-      def consumer_stall_slack(config)
-        if config.worker_notify_wakeup?
-          [config.polling_interval, Pgbus::Process::Consumer::NOTIFY_FALLBACK_POLL_SECONDS].max
-        else
-          config.polling_interval
-        end
-      end
-
       def format_batch(record, pending_jobs: nil)
         total = record.total_jobs
         failed = failure_job_count(record)
@@ -1706,7 +1583,8 @@ module Pgbus
       # Collect uniqueness keys from all failed events and release their locks.
       def release_locks_for_failed_events
         rows = connection.select_all(
-          "SELECT payload FROM pgbus_failed_events", "Pgbus Collect Failed Keys"
+          "SELECT payload FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE}", "Pgbus Collect Failed Keys",
+          [handler_queue_names_literal]
         )
 
         keys = rows.to_a.filter_map { |row| extract_uniqueness_key_from_payload_str(row["payload"]) }
@@ -1750,8 +1628,8 @@ module Pgbus
       # so one bad queue can't block progress on the others.
       def archive_all_failed_messages
         rows = connection.select_all(
-          "SELECT id, queue_name, msg_id FROM pgbus_failed_events WHERE msg_id IS NOT NULL",
-          "Pgbus Collect Failed Messages"
+          "SELECT id, queue_name, msg_id FROM pgbus_failed_events WHERE msg_id IS NOT NULL AND #{NOT_HANDLER_QUEUE}",
+          "Pgbus Collect Failed Messages", [handler_queue_names_literal]
         )
 
         grouped = rows.to_a.group_by { |row| row["queue_name"] }

data/lib/pgbus/web/dead_letter_reason.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    # Turns one dead-letter row into the words an operator reads: what killed
+    # the message and how hard pgbus tried (issue #495). Reads the
+    # DeadLetterHeader block the mover wrote; a row without one predates it.
+    # No I/O, like JobState: the view helper formats the args.
+    module DeadLetterReason
+      CELL_MESSAGE_LIMIT = 160
+
+      # reason is the block's own reason string ("max_retries_exceeded");
+      # reason_key picks the sentence the dashboard shows for it.
+      Result = Data.define(:reason, :reason_key, :reason_args, :legacy, :source, :source_queue, :attempts, :max_retries,
+                           :error_class, :error_message, :backtrace, :error_attempt, :retried_before, :died_at) do
+        def legacy? = legacy
+
+        # The attempts after the recorded error that recorded nothing (a
+        # worker crashed mid-perform), or nil when the error is the last one.
+        def later_attempts
+          return unless error_attempt && attempts
+
+          last_run = attempts - 1
+          (error_attempt + 1)..last_run if error_attempt < last_run
+        end
+      end
+
+      module_function
+
+      def present(row)
+        block = DeadLetterHeader.parse(row[:headers])
+        return legacy(row) unless block
+
+        attempts = number(block["attempts"])
+        max = number(block["max_retries"])
+        key, args = reason_for(block, attempts, max)
+        Result.new(
+          reason: block["reason"], reason_key: key, reason_args: args, legacy: false, source: block["source"],
+          source_queue: block["source_queue"], attempts: attempts, max_retries: max,
+          error_class: block["error_class"], error_message: block["error_message"], backtrace: block["backtrace"],
+          error_attempt: number(block["error_attempt"]), retried_before: number(block["retries_from_dlq"]),
+          died_at: row[:enqueued_at]
+        )
+      end
+
+      def reason_for(block, attempts, max)
+        error_class = block["error_class"]
+        unless error_class
+          key = block["source"] == "consumer" ? "event_no_error" : "no_error_recorded"
+          return [key, { attempts: attempts, max: max }]
+        end
+
+        args = { error_class: error_class, error_message: cell_message(block["error_message"]),
+                 attempts: attempts, max: max }
+        error_attempt = number(block["error_attempt"])
+        return ["error", args] unless error_attempt && attempts && error_attempt < attempts - 1
+
+        ["error_earlier_attempt", args.merge(error_attempt: error_attempt)]
+      end
+
+      def legacy(row)
+        Result.new(
+          reason: nil, reason_key: "not_recorded", reason_args: { version: DeadLetterHeader::SINCE }, legacy: true,
+          source: nil, source_queue: row[:queue_name].to_s.delete_suffix(Pgbus::DEAD_LETTER_SUFFIX),
+          attempts: nil, max_retries: nil, error_class: nil, error_message: nil, backtrace: nil,
+          error_attempt: nil, retried_before: nil, died_at: row[:enqueued_at]
+        )
+      end
+
+      # Header values are untrusted: nil stays nil, a non-number counts as 0.
+      def number(value)
+        DeadLetterHeader.count(value) unless value.nil?
+      end
+
+      def cell_message(message)
+        text = message.to_s
+        text.length > CELL_MESSAGE_LIMIT ? "#{text[0, CELL_MESSAGE_LIMIT - 1]}…" : text
+      end
+    end
+  end
+end