pgbus 0.21.0 → 0.21.2

data/lib/pgbus/process/supervisor.rb

@@ -9,6 +9,9 @@ module Pgbus
 
       FORK_WAIT = 1 # seconds between fork checks
       WATCHDOG_INTERVAL = 10 # seconds between stall checks
+      # Seconds between asking a child for a thread dump (TTIN) and the
+      # watchdog's SIGKILL, so the dump can reach the log (issue #587).
+      WATCHDOG_DUMP_GRACE = 2
 
       # A child that crashes within this many seconds of forking is treated
       # as crash-looping and restarted with exponential backoff (base
@@ -46,7 +49,8 @@ module Pgbus
         @notify_hub = notify_hub
         @intended_children = 0
         @readiness = Concurrent::AtomicReference.new(
-          ReadinessSnapshot.new(booted: false, shutting_down: shutting_down, expected: 0, live: forks.size)
+          ReadinessSnapshot.new(booted: false, shutting_down: shutting_down, expected: 0,
+                                live: checked_in_count, forked: forks.size)
         )
       end
 
@@ -151,16 +155,22 @@ module Pgbus
       end
 
       # Publish a fresh snapshot; the swapped-in Data is immutable, so the
-      # health server's accept thread always reads a consistent state.
+      # health server's accept thread always reads a consistent state. A
+      # child counts as live only once it has checked in (issue #587): a fork
+      # wedged before its first liveness signal keeps the gate shut.
       def refresh_readiness
         @readiness.set(
           ReadinessSnapshot.new(
             booted: !!@booted, shutting_down: @shutting_down,
-            expected: @expected_children || 0, live: @forks.size
+            expected: @expected_children || 0, live: checked_in_count, forked: @forks.size
           )
         )
       end
 
+      def checked_in_count
+        @forks.count { |_, info| info[:checked_in] }
+      end
+
       # Log a single boot diagnostics banner: the settings that actually
       # determine whether this deployment works. One consecutive block of
       # "[Pgbus] boot:"-prefixed info lines so it reads cleanly under both the
@@ -327,6 +337,7 @@ module Pgbus
         # :supervisor scope (hub present).
         wake_reader, wake_writer = IO.pipe if @notify_hub
 
+        spawned_wall_at = Time.current
         pid = fork do
           # Child owns the liveness writer + wake reader; close this fork's
           # own copies of the parent-side ends. Sibling pipe ends and the
@@ -367,8 +378,8 @@ module Pgbus
         register_fork_with_hub(pid, wake_writer, queues)
         @forks[pid] = {
           type: :worker, config: worker_config, slot: slot, process_index: process_index, spawned_at: monotonic_now,
-          liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
-          wake_writer: wake_writer
+          spawned_wall_at: spawned_wall_at, liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
+          wake_writer: wake_writer, checked_in: false
         }
         Pgbus.logger.info { "[Pgbus] Forked worker pid=#{pid} queues=#{queues.join(",")} mode=#{exec_mode}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
@@ -406,6 +417,7 @@ module Pgbus
 
       def fork_dispatcher
         note_intended_child
+        spawned_wall_at = Time.current
         pid = fork do
           restore_signals
           setup_child_process
@@ -419,7 +431,7 @@ module Pgbus
           return
         end
 
-        @forks[pid] = { type: :dispatcher, spawned_at: monotonic_now }
+        @forks[pid] = { type: :dispatcher, spawned_at: monotonic_now, spawned_wall_at: spawned_wall_at, checked_in: false }
         Pgbus.logger.info { "[Pgbus] Forked dispatcher pid=#{pid}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
         ErrorReporter.report(e, { action: "fork_dispatcher" })
@@ -437,6 +449,7 @@ module Pgbus
 
       def fork_scheduler
         note_intended_child
+        spawned_wall_at = Time.current
         pid = fork do
           restore_signals
           setup_child_process
@@ -453,7 +466,7 @@ module Pgbus
           return
         end
 
-        @forks[pid] = { type: :scheduler, spawned_at: monotonic_now }
+        @forks[pid] = { type: :scheduler, spawned_at: monotonic_now, spawned_wall_at: spawned_wall_at, checked_in: false }
         Pgbus.logger.info { "[Pgbus] Forked scheduler pid=#{pid}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
         ErrorReporter.report(e, { action: "fork_scheduler" })
@@ -542,6 +555,7 @@ module Pgbus
         # Wake channel from the NotifyHub (issue #381), as in fork_worker.
         wake_reader, wake_writer = IO.pipe if @notify_hub
 
+        spawned_wall_at = Time.current
         pid = fork do
           # Child owns the liveness writer + wake reader; close this fork's
           # own copies of the parent-side ends (see fork_worker).
@@ -572,8 +586,8 @@ module Pgbus
         register_consumer_with_hub(pid, wake_writer, topics)
         @forks[pid] = {
           type: :consumer, config: consumer_config, slot: slot, spawned_at: monotonic_now,
-          liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
-          wake_writer: wake_writer
+          spawned_wall_at: spawned_wall_at, liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
+          wake_writer: wake_writer, checked_in: false
         }
         Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
@@ -608,6 +622,7 @@ module Pgbus
 
       def fork_outbox_poller
         note_intended_child
+        spawned_wall_at = Time.current
         pid = fork do
           restore_signals
           setup_child_process
@@ -621,7 +636,7 @@ module Pgbus
           return
         end
 
-        @forks[pid] = { type: :outbox_poller, spawned_at: monotonic_now }
+        @forks[pid] = { type: :outbox_poller, spawned_at: monotonic_now, spawned_wall_at: spawned_wall_at, checked_in: false }
         Pgbus.logger.info { "[Pgbus] Forked outbox poller pid=#{pid}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
         ErrorReporter.report(e, { action: "fork_outbox_poller" })
@@ -710,7 +725,11 @@ module Pgbus
         key = [info[:type], info[:slot]]
         uptime = info[:spawned_at] ? monotonic_now - info[:spawned_at] : nil
 
-        if status&.success? || uptime.nil? || uptime >= RESTART_STABLE_UPTIME
+        # A child killed for its boot deadline never booted, whatever its
+        # uptime: back it off like a crash so a permanently hung boot does not
+        # re-fork every boot_timeout (issue #587).
+        stable = uptime && uptime >= RESTART_STABLE_UPTIME && !info[:boot_timed_out]
+        if status&.success? || uptime.nil? || stable
           @crash_counts.delete(key)
           return restart_child(info)
         end
@@ -746,55 +765,124 @@ module Pgbus
         end
       end
 
+      # Runs every WATCHDOG_INTERVAL. One process-row read (this host only)
+      # serves three jobs: it checks in forks that registered a row (the only
+      # check-in channel for the pipeless dispatcher, scheduler and outbox
+      # poller, so it runs whatever the thresholds are — /readyz depends on
+      # it); it enforces boot_timeout on forks that have not checked in
+      # (issue #587); and it feeds the loop-tick ages of the stall check.
       def check_stalled_workers
         now = monotonic_now
         return if (now - @last_watchdog_at) < WATCHDOG_INTERVAL
 
         @last_watchdog_at = now
-        threshold = config.stall_threshold
-        return unless threshold&.positive?
+        stall_threshold = positive_limit(config.stall_threshold)
+        boot_timeout = positive_limit(config.boot_timeout)
+
+        pids = watchdog_pids(stall_threshold)
+        return if pids.empty?
+
+        rows = watchdog_rows(pids)
+        mark_checked_in(rows.keys) if rows
+
+        victims = []
+        victims.concat(boot_timeout_victims(now, boot_timeout)) if boot_timeout && rows
+        victims.concat(stalled_victims(now, stall_threshold, rows || {})) if stall_threshold
+        kill_unresponsive_children(victims.uniq(&:first))
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog check failed: #{e.message}" }
+      end
+
+      def positive_limit(value)
+        value if value.is_a?(Numeric) && value.positive?
+      end
+
+      # Forks still waiting to check in, plus every worker/consumer when the
+      # stall check is on. A checked-in dispatcher/scheduler/outbox poller has
+      # no loop beacon, so once it has a row it is never read again.
+      def watchdog_pids(stall_threshold)
+        @forks.select do |_, info|
+          !info[:checked_in] || (stall_threshold && %i[worker consumer].include?(info[:type]))
+        end.keys
+      end
 
-        # Workers AND consumers both stamp a loop beacon and carry a liveness
-        # pipe (issue #274), so both are watched for a wedged claim/consume loop.
-        watched_pids = @forks.select { |_, info| %i[worker consumer].include?(info[:type]) }.keys
-        return if watched_pids.empty?
+      # {pid => ProcessEntry} for the forks' rows on THIS host — pids repeat
+      # across containers sharing one database, so a sibling's row must never
+      # vouch for a local child. A row last beaten before the fork belongs to
+      # a dead predecessor with the same pid (a SIGKILLed child never deletes
+      # its row, and a container restarted in place reuses hostname and
+      # pids), so it is dropped too. nil when the read fails: the stall check
+      # then falls back to the liveness pipe, and the boot deadline waits for
+      # a pass that can see the rows (a DB blip at boot must not kill every
+      # pipeless child).
+      def watchdog_rows(pids)
+        rows = ProcessEntry.where(hostname: Socket.gethostname, pid: pids).to_a
+        rows.select { |entry| row_of_current_fork?(entry) }.to_h { |entry| [entry.pid, entry] }
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog DB read failed, using pipe fallback: #{e.message}" }
+        nil
+      end
+
+      def row_of_current_fork?(entry)
+        spawned_wall_at = @forks.dig(entry.pid, :spawned_wall_at)
+        return true unless spawned_wall_at
 
-        db_ages = db_loop_tick_ages(watched_pids)
+        # floor: a table created without sub-second precision stores a row
+        # registered in the fork's own second below the fork's timestamp.
+        entry.last_heartbeat_at.present? && entry.last_heartbeat_at >= spawned_wall_at.floor
+      end
 
-        watched_pids.each do |pid|
+      def mark_checked_in(pids)
+        pids.each do |pid|
           info = @forks[pid]
-          next unless info
+          info[:checked_in] = true if info
+        end
+      end
 
-          kill_stalled_worker(pid, threshold) if worker_stalled?(info, db_ages[pid], now, threshold)
+      # A fork with neither a row nor a liveness byte boot_timeout seconds
+      # after fork is wedged in boot (issue #587). Without this it is never
+      # reaped (it does not exit) and never stalled (no channel ever spoke).
+      def boot_timeout_victims(now, boot_timeout)
+        @forks.filter_map do |pid, info|
+          next if info[:checked_in] || info[:spawned_at].nil?
+
+          waited = now - info[:spawned_at]
+          next unless waited > boot_timeout
+
+          info[:boot_timed_out] = true
+          [pid, "#{info[:type]} pid=#{pid} boot timeout (no check-in #{waited.round}s after fork, " \
+                "boot_timeout=#{boot_timeout}s)"]
+        end
+      end
+
+      def stalled_victims(now, threshold, rows)
+        db_ages = loop_tick_ages(rows)
+        @forks.filter_map do |pid, info|
+          next unless %i[worker consumer].include?(info[:type])
+          next unless worker_stalled?(info, db_ages[pid], now, threshold)
+
+          [pid, "#{info[:type]} pid=#{pid} claim loop stalled (no liveness within threshold=#{threshold}s)"]
         end
-      rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog check failed: #{e.message}" }
       end
 
-      # Read each watched fork's loop_tick_at from the process table and return
-      # a {pid => wall-clock age in seconds} map. Isolated in its own rescue so a
-      # database outage degrades to the OS-pipe fallback instead of skipping
-      # the whole watchdog — the exact failure this pipe channel exists to fix.
-      def db_loop_tick_ages(worker_pids)
-        ages = {}
-        ProcessEntry.where(kind: %w[worker consumer], pid: worker_pids).to_a.each do |entry|
+      # {pid => wall-clock age in seconds} of each worker/consumer row's
+      # loop_tick_at.
+      def loop_tick_ages(rows)
+        rows.each_with_object({}) do |(pid, entry), ages|
+          next unless %w[worker consumer].include?(entry.kind)
+
           meta = entry.metadata
-          next unless meta.is_a?(Hash)
+          next unless meta.is_a?(Hash) && meta["loop_tick_at"]
 
-          loop_tick = meta["loop_tick_at"]
-          ages[entry.pid] = Time.current.to_f - loop_tick.to_f if loop_tick
+          ages[pid] = Time.current.to_f - meta["loop_tick_at"].to_f
         end
-        ages
-      rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog DB read failed, using pipe fallback: #{e.message}" }
-        ages
       end
 
       # A worker is stalled only when EVERY liveness channel that has spoken
       # agrees it is stale (min-age / fresh-wins): a fresh DB row OR a fresh
       # pipe tick proves the loop is advancing. If no channel has any signal
       # (a slow-booting worker with no DB row and an unarmed pipe), we do not
-      # kill — mirroring the pre-existing "no loop_tick_at → skip" tolerance.
+      # kill — the boot deadline above owns that case.
       def worker_stalled?(info, db_age, now, threshold)
         pipe_age = (now - info[:last_pipe_tick_at] if info[:pipe_seen] && info[:last_pipe_tick_at])
         ages = [db_age, pipe_age].compact
@@ -803,14 +891,33 @@ module Pgbus
         ages.min > threshold
       end
 
-      def kill_stalled_worker(pid, threshold)
-        Pgbus.logger.error do
-          "[Pgbus] Supervisor watchdog: worker pid=#{pid} claim loop stalled " \
-            "(no liveness within threshold=#{threshold}s), sending SIGKILL"
+      # Log each victim at ERROR, ask it for a thread dump (TTIN, trapped in
+      # setup_child_process), give every dump one shared grace, then SIGKILL.
+      # The dump is best effort: a child stuck in a native call holding the
+      # GVL never answers. No waitpid here — reap_children must see the exit
+      # so schedule_restart re-forks the child.
+      def kill_unresponsive_children(victims)
+        return if victims.empty?
+
+        signalled = victims.filter_map do |pid, reason|
+          Pgbus.logger.error { "[Pgbus] Supervisor watchdog: #{reason} — dumping threads (TTIN), then SIGKILL" }
+          signal_child("TTIN", pid) && pid
         end
-        ::Process.kill("KILL", pid)
+        return if signalled.empty?
+
+        dump_grace_sleep
+        signalled.each { |pid| signal_child("KILL", pid) }
+      end
+
+      def signal_child(sig, pid)
+        ::Process.kill(sig, pid)
+        true
       rescue Errno::ESRCH
-        # already gone
+        false # already gone; reap_children handles it
+      end
+
+      def dump_grace_sleep
+        sleep(WATCHDOG_DUMP_GRACE)
       end
 
       # Drain each worker's liveness pipe. Any readable byte means the worker's
@@ -839,10 +946,11 @@ module Pgbus
             next
           end
 
-          if read_any
-            info[:last_pipe_tick_at] = now
-            info[:pipe_seen] = true
-          end
+          next unless read_any
+
+          info[:last_pipe_tick_at] = now
+          info[:pipe_seen] = true
+          info[:checked_in] = true
         end
       end
 
@@ -855,6 +963,8 @@ module Pgbus
       end
 
       def setup_child_process
+        # First, so a hang anywhere in the rest of the boot can be dumped.
+        install_thread_dump_trap
         # Every child type (worker, consumer, dispatcher, scheduler, outbox
         # poller) releases its copies of the parent's per-fork resources —
         # a dispatcher child holding a sibling worker's wake WRITER would
@@ -871,6 +981,26 @@ module Pgbus
         end
       end
 
+      # `kill -TTIN <child pid>` logs every thread's backtrace at ERROR and
+      # leaves the child running; the watchdog sends it before a SIGKILL
+      # (issue #587). Logger takes a lock, which Ruby forbids in trap context,
+      # so the trap snapshots the backtraces and one short-lived thread logs.
+      def install_thread_dump_trap
+        trap("TTIN") do
+          dump = Thread.list.map { |thread| [thread.inspect, thread.backtrace] }
+          Thread.new { log_thread_dump(dump) } # rubocop:disable ThreadSafety/NewThread
+        end
+      end
+
+      def log_thread_dump(dump)
+        Pgbus.logger.error { "[Pgbus] Thread dump pid=#{::Process.pid} (#{dump.size} threads, TTIN)" }
+        dump.each do |label, backtrace|
+          Pgbus.logger.error do
+            "[Pgbus] Thread #{label}\n  #{Array(backtrace).join("\n  ")}"
+          end
+        end
+      end
+
       # Lenient bootstrap for the parent (supervisor.rb#run). A transiently
       # unavailable DB at boot must not kill the supervisor — children
       # crash-and-backoff until it recovers — so every StandardError (including

data/lib/pgbus/process/worker.rb

@@ -296,7 +296,7 @@ module Pgbus
         @rate_counter.increment(:dequeued, tagged_messages.size)
         client = Pgbus.client
         tagged_messages.each do |queue_name, message, source_queue|
-          detect_zombie(queue_name, message)
+          detect_zombie(queue_name, message, source_queue)
           @in_flight.increment
           @claim_buffer.push(queue_name, message, source_queue, client: client)
         end
@@ -603,11 +603,12 @@ module Pgbus
         [RESTORE_COOLDOWN_BASE * (2**@restore_streak), RESTORE_COOLDOWN_MAX].min
       end
 
-      def detect_zombie(queue_name, message)
+      def detect_zombie(queue_name, message, source_queue = nil)
         return unless config.zombie_detection
         return unless message.read_ct.to_i > 1
 
-        return if FailedEventRecorder.exists?(queue_name: queue_name, msg_id: message.msg_id.to_i)
+        return if FailedEventRecorder.exists?(queue_name: queue_name, msg_id: message.msg_id.to_i,
+                                              source_queue: source_queue || config.queue_name(queue_name))
 
         Pgbus.logger.warn do
           "[Pgbus] Zombie message redelivered: queue=#{queue_name} msg_id=#{message.msg_id} " \

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.21.0"
+  VERSION = "0.21.2"
 end

data/lib/pgbus/web/data_source/job_list.rb

@@ -10,7 +10,8 @@ module Pgbus
       # Sources, one UNION ALL fragment each:
       # - every non-DLQ queue table, LEFT JOINed to its pgbus_failed_events row
       #   (failed_events.queue_name is the LOGICAL name; the physical one is
-      #   matched too for rows older code wrote)
+      #   matched too for rows older code wrote; source_queue, when set, names
+      #   the one table the row's message was read from — issue #583)
       # - failed rows whose message is no longer in its queue ("orphans")
       # - pgbus_blocked_executions (jobs parked by limits_concurrency)
       #
@@ -187,7 +188,7 @@ module Pgbus
               SELECT f.id, f.error_class::text AS error_class, f.error_message,
                      f.failed_at::timestamptz AS failed_at
               FROM pgbus_failed_events f
-              WHERE f.queue_name IN ('#{logical}', '#{qtable}') AND f.msg_id = m.msg_id
+              WHERE f.queue_name IN ('#{logical}', '#{qtable}') AND f.msg_id = m.msg_id#{failed_in(qtable)}
               ORDER BY f.failed_at DESC LIMIT 1
             ) f ON true
           SQL
@@ -197,7 +198,9 @@ module Pgbus
         # queue, plus (unfiltered) one for rows whose queue no longer exists.
         def orphan_fragments(scope, fetch)
           fragments = scope[:by_logical].map do |logical, tables|
-            missing = tables.map { |(qtable, _)| "NOT EXISTS (SELECT 1 FROM pgmq.q_#{qtable} m WHERE m.msg_id = f.msg_id)" }
+            missing = tables.map do |(qtable, _)|
+              "NOT EXISTS (SELECT 1 FROM pgmq.q_#{qtable} m WHERE m.msg_id = f.msg_id#{failed_in(qtable)})"
+            end
             names = quoted_list([logical, *tables.map(&:first)])
             orphan_fragment("'#{logical}'::text", "f.queue_name IN (#{names}) AND #{missing.join(" AND ")}", fetch)
           end
@@ -209,6 +212,16 @@ module Pgbus
           fragments << orphan_fragment("f.queue_name::text", where, fetch)
         end
 
+        # Under priority routing each _pN sub-queue numbers its messages from
+        # its own sequence: a row that names its table belongs to that table's
+        # message only. A row written before source_queue (NULL), or any row
+        # before the column is migrated, matches every table of its queue.
+        def failed_in(qtable)
+          return "" unless FailedEventRecorder.source_queue_column?
+
+          " AND (f.source_queue IS NULL OR f.source_queue = '#{qtable}')"
+        end
+
         def orphan_fragment(logical_sql, where, fetch)
           <<~SQL.strip
             (SELECT 'failed'::text AS source, f.id, f.queue_name::text AS queue_name, #{logical_sql} AS logical_queue,

data/lib/pgbus/web/data_source.rb

@@ -299,9 +299,9 @@ module Pgbus
         # duplicate (the original is still in the queue waiting for retry).
         # Falls back to enqueueing a fresh copy only if the original is gone
         # (e.g., already moved to DLQ).
-        msg_id = event["msg_id"]
-        if msg_id && @client.message_exists?(event["queue_name"], msg_id: msg_id.to_i)
-          @client.set_visibility_timeout(event["queue_name"], msg_id.to_i, vt: 0)
+        physical = failed_message_queue(event)
+        if physical
+          @client.set_visibility_timeout(physical, event["msg_id"].to_i, vt: 0, prefixed: false)
         else
           payload = JSON.parse(event["payload"])
           headers = event["headers"]
@@ -1627,20 +1627,22 @@ module Pgbus
       # lock handed to a retry before the failure would count as the job's.
       # A message that is gone has no send time, so only a bound lock goes.
       def release_locks_held_by_failed_events(condition, binds)
+        source = FailedEventRecorder.source_queue_column? ? ", source_queue" : ""
         rows = connection.select_all(
-          "SELECT queue_name, msg_id, payload FROM pgbus_failed_events WHERE #{condition}",
+          "SELECT queue_name, msg_id, payload#{source} FROM pgbus_failed_events WHERE #{condition}",
           "Pgbus Collect Failed Keys", binds
         )
 
         # queue_name is the logical name the executor saw: the message is looked
-        # up under it, and the lock is released under its table spelling.
+        # up under it (or in the table source_queue names), and the lock is
+        # released under its table spelling.
         # A key proc may return a number; the queue lookup reads it as text.
         holders = rows.to_a.filter_map do |row|
           key = extract_uniqueness_key_from_payload_str(row["payload"])&.to_s
-          [key, row["queue_name"], row["msg_id"].to_i] if key && row["msg_id"]
+          [key, row["queue_name"], row["msg_id"].to_i, row["source_queue"]] if key && row["msg_id"]
         end
-        holders = holders.group_by { |_key, queue, _msg_id| queue }.flat_map do |queue, entries|
-          failed_queue_lock_holders(queue, entries)
+        holders = holders.group_by { |_key, queue, _msg_id, table| [queue, table] }.flat_map do |(queue, table), entries|
+          failed_queue_lock_holders(queue, table, entries)
         end
         UniquenessKey.release_all_held_by!(holders) if holders.any?
       rescue StandardError => e
@@ -1650,8 +1652,8 @@ module Pgbus
       # The release tuples for one queue's failed jobs. A queue that cannot
       # be resolved (a stored name no longer valid as a table name) is skipped
       # with a warning, so it does not stop the other queues' release.
-      def failed_queue_lock_holders(queue, entries)
-        sent = queued_send_times(queue, entries)
+      def failed_queue_lock_holders(queue, table, entries)
+        sent = queued_send_times(queue, table, entries)
         lock_queue = Uniqueness.lock_queue_name(queue)
         entries.map { |key, _queue, msg_id| [key, lock_queue, msg_id, sent[[msg_id, key]]] }
       rescue ActiveRecord::StatementInvalid
@@ -1662,13 +1664,15 @@ module Pgbus
       end
 
       # { [msg_id, uniqueness key] => enqueued_at } for the messages of
-      # +entries+ ([key, queue, msg_id]) still in the logical queue. Every
-      # priority sub-queue is searched: a failed event records the logical
-      # queue only, and msg_ids repeat across sub-queues, so the key must
+      # +entries+ ([key, queue, msg_id, table]) still queued. A row that names
+      # its table (issue #583) is that table's message. A row written before
+      # source_queue records the logical queue only, so every priority
+      # sub-queue is searched: msg_ids repeat across them, so the key must
       # match too, and the earliest send wins.
-      def queued_send_times(queue, entries)
+      def queued_send_times(queue, table, entries)
         msg_ids = "{#{entries.map { |_key, _queue, msg_id| msg_id }.uniq.join(",")}}"
-        @client.physical_queue_names(queue).each_with_object({}) do |physical, found|
+        tables = table ? [table] : @client.physical_queue_names(queue)
+        tables.each_with_object({}) do |physical, found|
           queued_rows(physical, msg_ids).each do |row|
             pair = [row["msg_id"].to_i, row["lock_key"]]
             found[pair] = [found[pair], row["enqueued_at"]].compact.min
@@ -1721,12 +1725,36 @@ module Pgbus
         nil
       end
 
+      # The physical queue holding a failed job's message, or nil when it is
+      # gone. The job's job_id must match too: under priority routing a msg_id
+      # repeats across the _pN sub-queues (issue #577). A row that names the
+      # table its message was read from is looked up there alone (issue
+      # #583); a row written before source_queue, under every table of its
+      # logical queue.
+      def failed_message_queue(event)
+        return unless event["queue_name"] && event["msg_id"]
+
+        name, prefixed = failed_message_scope(event)
+        @client.locate_message(name, msg_id: event["msg_id"].to_i, job_id: failed_job_id(event), prefixed: prefixed)
+      end
+
+      # [queue, prefixed] to look a failed row's message up under: the physical
+      # table the row names, else its logical queue.
+      def failed_message_scope(event)
+        event["source_queue"] ? [event["source_queue"], false] : [event["queue_name"], true]
+      end
+
+      def failed_job_id(event)
+        payload = event["payload"]
+        payload = JSON.parse(payload) if payload.is_a?(String)
+        payload["job_id"] if payload.is_a?(Hash)
+      end
+
       # Archive the queue message a failed_event row points to. Idempotent —
       # silently no-ops if the message no longer exists in the queue.
       def archive_failed_message(event)
-        return unless event["queue_name"] && event["msg_id"]
-
-        @client.archive_message(event["queue_name"], event["msg_id"].to_i)
+        physical = failed_message_queue(event)
+        @client.archive_message(physical, event["msg_id"].to_i, prefixed: false) if physical
       rescue StandardError => e
         Pgbus.logger.debug do
           "[Pgbus::Web] Error archiving message for failed event #{event["id"]}: #{e.message}"
@@ -1734,30 +1762,33 @@ module Pgbus
       end
 
       # Archive every queue message referenced by a failed_event row.
-      # Groups by queue and uses archive_batch so an N-event discard is
-      # one SQL statement per queue instead of N per-row roundtrips.
-      # Falls back to per-row archive inside a rescue if a batch fails,
-      # so one bad queue can't block progress on the others.
+      # Groups by queue, then locates and archive_batches per physical table,
+      # so an N-event discard is two SQL statements per table instead of N
+      # per-row roundtrips. Falls back to per-row archive inside a rescue if
+      # a batch fails, so one bad queue can't block progress on the others.
       def archive_all_failed_messages
+        source = FailedEventRecorder.source_queue_column? ? ", source_queue" : ""
         rows = connection.select_all(
-          "SELECT id, queue_name, msg_id FROM pgbus_failed_events WHERE msg_id IS NOT NULL AND #{NOT_HANDLER_QUEUE}",
+          "SELECT id, queue_name, msg_id, payload#{source} FROM pgbus_failed_events " \
+          "WHERE msg_id IS NOT NULL AND #{NOT_HANDLER_QUEUE}",
           "Pgbus Collect Failed Messages", [handler_queue_names_literal]
         )
 
-        grouped = rows.to_a.group_by { |row| row["queue_name"] }
-        grouped.each do |queue_name, events|
-          msg_ids = events.filter_map { |row| row["msg_id"]&.to_i }
-          next if msg_ids.empty?
-
-          begin
-            @client.archive_batch(queue_name, msg_ids)
-          rescue StandardError => e
-            Pgbus.logger.debug do
-              "[Pgbus::Web] archive_batch failed for #{queue_name} (#{e.message}); " \
-                "falling back to per-row archive"
-            end
-            events.each { |row| archive_failed_message(row) }
+        grouped = rows.to_a.group_by { |row| failed_message_scope(row) }
+        grouped.each do |(queue_name, prefixed), events|
+          # A priority queue's msg_ids repeat across its _pN sub-queues, so
+          # each message is located first and archived from its own table.
+          located = @client.locate_messages(queue_name, events.map { |row| [row["msg_id"].to_i, failed_job_id(row)] },
+                                            prefixed: prefixed)
+          located.group_by { |_msg_id, physical| physical }.each do |physical, pairs|
+            @client.archive_batch(physical, pairs.map(&:first), prefixed: false)
+          end
+        rescue StandardError => e
+          Pgbus.logger.debug do
+            "[Pgbus::Web] archive_batch failed for #{queue_name} (#{e.message}); " \
+              "falling back to per-row archive"
           end
+          events.each { |row| archive_failed_message(row) }
         end
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error archiving failed messages: #{e.message}" }

data/lib/pgbus/web/health_app.rb

@@ -98,7 +98,7 @@ module Pgbus
       def local_readyz
         snapshot = @local_readiness.call
         status = snapshot.ready? ? 200 : 503
-        body = { status: snapshot.status, expected: snapshot.expected, live: snapshot.live }
+        body = { status: snapshot.status, expected: snapshot.expected, live: snapshot.live, forked: snapshot.forked }
         [status, JSON_HEADERS.dup, [body.to_json]]
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus::Web::HealthApp] local readiness check failed: #{e.class}: #{e.message}" }