pgbus 0.21.1 → 0.21.3

data/lib/pgbus/generators/migration_detector.rb

@@ -97,6 +97,7 @@ module Pgbus
         add_workflows: "pgbus:add_workflows",
         add_workflow_signals: "pgbus:add_workflow_signals",
         add_failed_events_index: "pgbus:add_failed_events_index",
+        add_failed_events_source_queue: "pgbus:add_failed_events_source_queue",
         add_processed_event_completion: "pgbus:add_processed_event_completion",
         add_batch_executions: "pgbus:add_batch_executions",
         add_batch_callback_jobs: "pgbus:add_batch_callback_jobs",
@@ -122,6 +123,7 @@ module Pgbus
         add_workflows: "workflow tables (durable multi-step workflows: runs, step attempts, audit trail)",
         add_workflow_signals: "waiting workflow steps (fan_out, wait_for: signals, wait timeouts)",
         add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
+        add_failed_events_source_queue: "source_queue on pgbus_failed_events (one row per priority sub-queue's failure)",
         add_processed_event_completion: "completed_at on pgbus_processed_events (two-phase idempotency claim)",
         add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
         add_batch_callback_jobs: "batch callback-job columns (configured ActiveJob instances as batch callbacks)",
@@ -151,7 +153,7 @@ module Pgbus
           *recurring_migrations,
           *deploy_tasks_migrations,
           *workflows_migrations,
-          *failed_events_index_migrations,
+          *failed_events_migrations,
           *processed_event_completion_migrations,
           *batch_executions_migrations,
           *batch_callback_jobs_migrations,
@@ -266,16 +268,21 @@ module Pgbus
         migrations
       end
 
-      def failed_events_index_migrations
-        # pgbus_failed_events is created by pgbus:install, but the
-        # unique (queue_name, msg_id) index was added later via its own
-        # generator. If the table exists without the unique index,
-        # FailedEventRecorder's upsert silently swallows ON CONFLICT
-        # errors — so this is a real bug waiting to bite.
+      # FailedEventRecorder's upsert needs the unique index that matches its
+      # ON CONFLICT target, or it fails and the failure goes unrecorded. The
+      # source_queue migration (issue #583) adds the column, creates the
+      # (queue_name, msg_id, source_queue) index and drops the old
+      # (queue_name, msg_id) one, so it is the only migration queued here:
+      # add_failed_events_index would put back the index that collapses two
+      # priority sub-queues' failures into one row. That index still present
+      # (or put back by running the old generator by name) queues it again.
+      def failed_events_migrations
         return [] unless table_exists?("pgbus_failed_events")
-        return [] if index_exists?("pgbus_failed_events", "idx_pgbus_failed_events_queue_msg")
+        return [] if column_names("pgbus_failed_events").include?("source_queue") &&
+                     index_exists?("pgbus_failed_events", "idx_pgbus_failed_events_queue_msg_source") &&
+                     !index_exists?("pgbus_failed_events", "idx_pgbus_failed_events_queue_msg")
 
-        [:add_failed_events_index]
+        [:add_failed_events_source_queue]
       end
 
       # completed_at backs the two-phase idempotency claim (issue #385).

data/lib/pgbus/mcp/health_analyzer.rb

@@ -35,8 +35,12 @@ module Pgbus
       # a multi-KB log line (issue #367 minor).
       NAME_LIMIT = 10
 
-      def initialize(data_source)
+      # +grace+ overrides Pgbus.configuration.health_unclaimed_grace (issue #589):
+      # seconds a visible read_ct=0 message must have been claimable before it
+      # counts as "never claimed". nil reads the config when the verdict runs.
+      def initialize(data_source, grace: nil)
         @data_source = data_source
+        @grace = grace
       end
 
       # Returns a machine-readable verdict hash suitable for both interactive
@@ -81,7 +85,9 @@ module Pgbus
           status: status,
           reasons: stalled + degraded,
           checked_at: Time.now.utc.iso8601,
-          summary: build_summary(queues, non_dlq, processes, health)
+          summary: build_summary(queues, non_dlq, processes, health).merge(
+            oldest_unread_claimable_age_sec: oldest_unread_age(drained)
+          )
         }
       end
 
@@ -139,7 +145,8 @@ module Pgbus
                      "while #{active_backlog.size} queue(s) have visible backlog: #{backlog_names(active_backlog)}"
         elsif live_drainers.any? && drained_backlog.any? && any_unread?(drained_backlog)
           unread = drained_backlog.select { |q| unread?(q) }
-          reasons << "#{unread.size} queue(s) have visible messages with read_ct=0 (never claimed) " \
+          reasons << "#{unread.size} queue(s) have visible messages with read_ct=0 " \
+                     "(never claimed#{unread_age_clause(unread)}) " \
                      "while #{live_drainers.size} draining process(es) are alive: #{backlog_names(unread)}"
         end
         reasons
@@ -211,10 +218,39 @@ module Pgbus
       #
       # When metrics don't expose the count (an old hash / a stubbed test) we
       # conservatively treat the queue as unread so we never miss the wedge.
+      #
+      # Issue #589: the oldest such row must also have been claimable for at
+      # least the grace. A fresh enqueue, or a delayed job whose wait just
+      # expired, is visible with read_ct=0 until a worker's next poll, so
+      # without the grace a healthy fleet read STALLED (and /readyz 503) on a
+      # large share of checks. An unknown age (key missing or nil) stays
+      # conservative and counts.
       def unread?(queue)
         return true unless queue.key?(:visible_unread_length)
+        return false unless queue[:visible_unread_length].to_i.positive?
+
+        age = unread_age(queue)
+        age.nil? || age >= grace
+      end
+
+      def grace
+        @grace || Pgbus.configuration.health_unclaimed_grace
+      end
+
+      def unread_age(queue)
+        queue[:oldest_unread_claimable_age_sec]
+      end
+
+      # Oldest known unread claimable age over +queues+, nil when none is known.
+      def oldest_unread_age(queues)
+        queues.filter_map { |q| unread_age(q) }.max
+      end
 
-        queue[:visible_unread_length].to_i.positive?
+      # ", oldest claimable for 74s" so an operator can tell a wedge from a
+      # spike; empty when no flagged queue reports an age.
+      def unread_age_clause(queues)
+        age = oldest_unread_age(queues)
+        age ? ", oldest claimable for #{age}s" : ""
       end
 
       # Join backlog queue names for a reason string, truncated so a flagged

data/lib/pgbus/mcp/tools/health_tool.rb

@@ -13,8 +13,10 @@ module Pgbus
         description <<~DESC
           Top-level pgbus health verdict. Returns OK, DEGRADED, or STALLED with
           the specific reasons. STALLED means the silent-wedge condition:
-          queues have visible backlog while workers are heart-beating but not
-          claiming (or a worker's claim loop has stalled). DEGRADED covers
+          queues hold a visible, never-claimed backlog older than
+          health_unclaimed_grace (default 60s) while workers are heart-beating
+          but not claiming (or a worker's claim loop has stalled); the summary's
+          oldest_unread_claimable_age_sec tells a wedge from a spike. DEGRADED covers
           stale processes, paused queues holding a backlog, dead-letter
           buildup, or a long-running transaction pinning the MVCC horizon. Use
           this first when triaging — it summarizes the whole system in one call

data/lib/pgbus/process/consumer.rb

@@ -267,7 +267,10 @@ module Pgbus
         Pgbus.client.archive_message(queue_name, message.msg_id.to_i)
         # A redelivery that succeeds leaves no stale failure behind. First
         # deliveries never had one: no query on the hot path (#484's gate).
-        FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i) if message.read_ct.to_i > 1
+        if message.read_ct.to_i > 1
+          FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i,
+                                     source_queue: config.queue_name(queue_name))
+        end
         @circuit_breaker.record_success(queue_name)
         record_stat(message, queue_name, "success", execution_start)
       rescue StandardError => e
@@ -293,7 +296,8 @@ module Pgbus
       # failed-event row, then announce it. A move that raises keeps the row,
       # so the next attempt still carries the reason, and announces nothing.
       def dead_letter(message, queue_name)
-        last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i)
+        last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i,
+                                                    source_queue: config.queue_name(queue_name))
         headers = DeadLetterHeader.build(
           existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "consumer",
           source_queue: config.queue_name(queue_name), source_msg_id: message.msg_id.to_i,
@@ -301,7 +305,8 @@ module Pgbus
           max_retries: config.max_retries, error: last_error
         )
         Pgbus.client.move_to_dead_letter(queue_name, message, headers: headers)
-        FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i)
+        FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i,
+                                   source_queue: config.queue_name(queue_name))
         instrument_dead_lettered(message, queue_name, last_error)
       end
 
@@ -361,10 +366,12 @@ module Pgbus
       # failure, under the LOGICAL queue name, so the dashboard reads the event
       # as retrying with its error. No ErrorReporter call: Handler#process!
       # already instruments pgbus.event_failed, which feeds the reporters.
+      # A handler queue has no priority sub-queues: its one table is the source.
       def record_failed_event(message, queue_name, error)
         FailedEventRecorder.record!(
-          queue_name: queue_name, msg_id: message.msg_id.to_i, payload: message.message,
-          headers: message.headers, error: error, retry_count: [message.read_ct.to_i - 1, 0].max
+          queue_name: queue_name, msg_id: message.msg_id.to_i, source_queue: config.queue_name(queue_name),
+          payload: message.message, headers: message.headers, error: error,
+          retry_count: [message.read_ct.to_i - 1, 0].max
         )
       end
 

data/lib/pgbus/process/heartbeat.rb

@@ -32,6 +32,7 @@ module Pgbus
       end
 
       def start
+        @started = true
         register_process
         @timer = Concurrent::TimerTask.new(execution_interval: INTERVAL) { beat }
         @timer.execute
@@ -44,7 +45,8 @@ module Pgbus
       end
 
       def beat
-        return unless @process_id && !@stopped
+        return if @stopped
+        return retry_registration unless @process_id
 
         @on_beat&.call
         updates = { last_heartbeat_at: Time.current }
@@ -57,6 +59,15 @@ module Pgbus
 
       private
 
+      # The registration at start failed (a DB blip at boot). Retry it here so
+      # the process does not stay invisible: a pipeless child checks in with
+      # the supervisor's boot deadline only through this row (issue #587).
+      def retry_registration
+        return unless @started
+
+        @mutex.synchronize { register_process unless @stopped }
+      end
+
       # Build the metadata hash to persist on this beat. Returns nil when no
       # supplier is configured so heartbeats without dynamic metadata
       # (supervisor/dispatcher/scheduler) leave the column untouched, exactly

data/lib/pgbus/process/readiness_snapshot.rb

@@ -7,12 +7,20 @@ module Pgbus
     # server's accept thread — the immutability is what makes the cross-thread
     # handoff safe without a lock (issue #386).
     #
-    # `expected` is the child count forked by boot_processes; `live` is the
-    # current fork-table size. A child sitting in crash-restart backoff keeps
-    # `live < expected`, which is exactly the signal a rolling deploy's health
-    # gate needs to fail on: the replacement container never goes ready, and
-    # the orchestrator keeps the old container running.
-    ReadinessSnapshot = Data.define(:booted, :shutting_down, :expected, :live) do
+    # `expected` is the child count forked by boot_processes; `forked` is the
+    # current fork-table size; `live` counts the forks that have checked in
+    # (a process row on this host, or a liveness byte). A child sitting in
+    # crash-restart backoff, or one that forked and hangs before checking in
+    # (issue #587), keeps `live < expected`, which is exactly the signal a
+    # rolling deploy's health gate needs to fail on: the replacement
+    # container never goes ready, and the orchestrator keeps the old
+    # container running. `forked` defaults to `live` for callers that do not
+    # track check-ins.
+    ReadinessSnapshot = Data.define(:booted, :shutting_down, :expected, :live, :forked) do
+      def initialize(booted:, shutting_down:, expected:, live:, forked: live)
+        super
+      end
+
       def ready?
         booted && !shutting_down && live >= expected
       end

data/lib/pgbus/process/supervisor.rb

@@ -9,6 +9,9 @@ module Pgbus
 
       FORK_WAIT = 1 # seconds between fork checks
       WATCHDOG_INTERVAL = 10 # seconds between stall checks
+      # Seconds between asking a child for a thread dump (TTIN) and the
+      # watchdog's SIGKILL, so the dump can reach the log (issue #587).
+      WATCHDOG_DUMP_GRACE = 2
 
       # A child that crashes within this many seconds of forking is treated
       # as crash-looping and restarted with exponential backoff (base
@@ -46,7 +49,8 @@ module Pgbus
         @notify_hub = notify_hub
         @intended_children = 0
         @readiness = Concurrent::AtomicReference.new(
-          ReadinessSnapshot.new(booted: false, shutting_down: shutting_down, expected: 0, live: forks.size)
+          ReadinessSnapshot.new(booted: false, shutting_down: shutting_down, expected: 0,
+                                live: checked_in_count, forked: forks.size)
         )
       end
 
@@ -151,16 +155,22 @@ module Pgbus
       end
 
       # Publish a fresh snapshot; the swapped-in Data is immutable, so the
-      # health server's accept thread always reads a consistent state.
+      # health server's accept thread always reads a consistent state. A
+      # child counts as live only once it has checked in (issue #587): a fork
+      # wedged before its first liveness signal keeps the gate shut.
       def refresh_readiness
         @readiness.set(
           ReadinessSnapshot.new(
             booted: !!@booted, shutting_down: @shutting_down,
-            expected: @expected_children || 0, live: @forks.size
+            expected: @expected_children || 0, live: checked_in_count, forked: @forks.size
           )
         )
       end
 
+      def checked_in_count
+        @forks.count { |_, info| info[:checked_in] }
+      end
+
       # Log a single boot diagnostics banner: the settings that actually
       # determine whether this deployment works. One consecutive block of
       # "[Pgbus] boot:"-prefixed info lines so it reads cleanly under both the
@@ -327,6 +337,7 @@ module Pgbus
         # :supervisor scope (hub present).
         wake_reader, wake_writer = IO.pipe if @notify_hub
 
+        spawned_wall_at = Time.current
         pid = fork do
           # Child owns the liveness writer + wake reader; close this fork's
           # own copies of the parent-side ends. Sibling pipe ends and the
@@ -367,8 +378,8 @@ module Pgbus
         register_fork_with_hub(pid, wake_writer, queues)
         @forks[pid] = {
           type: :worker, config: worker_config, slot: slot, process_index: process_index, spawned_at: monotonic_now,
-          liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
-          wake_writer: wake_writer
+          spawned_wall_at: spawned_wall_at, liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
+          wake_writer: wake_writer, checked_in: false
         }
         Pgbus.logger.info { "[Pgbus] Forked worker pid=#{pid} queues=#{queues.join(",")} mode=#{exec_mode}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
@@ -406,6 +417,7 @@ module Pgbus
 
       def fork_dispatcher
         note_intended_child
+        spawned_wall_at = Time.current
         pid = fork do
           restore_signals
           setup_child_process
@@ -419,7 +431,7 @@ module Pgbus
           return
         end
 
-        @forks[pid] = { type: :dispatcher, spawned_at: monotonic_now }
+        @forks[pid] = { type: :dispatcher, spawned_at: monotonic_now, spawned_wall_at: spawned_wall_at, checked_in: false }
         Pgbus.logger.info { "[Pgbus] Forked dispatcher pid=#{pid}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
         ErrorReporter.report(e, { action: "fork_dispatcher" })
@@ -437,6 +449,7 @@ module Pgbus
 
       def fork_scheduler
         note_intended_child
+        spawned_wall_at = Time.current
         pid = fork do
           restore_signals
           setup_child_process
@@ -453,7 +466,7 @@ module Pgbus
           return
         end
 
-        @forks[pid] = { type: :scheduler, spawned_at: monotonic_now }
+        @forks[pid] = { type: :scheduler, spawned_at: monotonic_now, spawned_wall_at: spawned_wall_at, checked_in: false }
         Pgbus.logger.info { "[Pgbus] Forked scheduler pid=#{pid}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
         ErrorReporter.report(e, { action: "fork_scheduler" })
@@ -542,6 +555,7 @@ module Pgbus
         # Wake channel from the NotifyHub (issue #381), as in fork_worker.
         wake_reader, wake_writer = IO.pipe if @notify_hub
 
+        spawned_wall_at = Time.current
         pid = fork do
           # Child owns the liveness writer + wake reader; close this fork's
           # own copies of the parent-side ends (see fork_worker).
@@ -572,8 +586,8 @@ module Pgbus
         register_consumer_with_hub(pid, wake_writer, topics)
         @forks[pid] = {
           type: :consumer, config: consumer_config, slot: slot, spawned_at: monotonic_now,
-          liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
-          wake_writer: wake_writer
+          spawned_wall_at: spawned_wall_at, liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
+          wake_writer: wake_writer, checked_in: false
         }
         Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
@@ -608,6 +622,7 @@ module Pgbus
 
       def fork_outbox_poller
         note_intended_child
+        spawned_wall_at = Time.current
         pid = fork do
           restore_signals
           setup_child_process
@@ -621,7 +636,7 @@ module Pgbus
           return
         end
 
-        @forks[pid] = { type: :outbox_poller, spawned_at: monotonic_now }
+        @forks[pid] = { type: :outbox_poller, spawned_at: monotonic_now, spawned_wall_at: spawned_wall_at, checked_in: false }
         Pgbus.logger.info { "[Pgbus] Forked outbox poller pid=#{pid}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
         ErrorReporter.report(e, { action: "fork_outbox_poller" })
@@ -710,7 +725,11 @@ module Pgbus
         key = [info[:type], info[:slot]]
         uptime = info[:spawned_at] ? monotonic_now - info[:spawned_at] : nil
 
-        if status&.success? || uptime.nil? || uptime >= RESTART_STABLE_UPTIME
+        # A child killed for its boot deadline never booted, whatever its
+        # uptime: back it off like a crash so a permanently hung boot does not
+        # re-fork every boot_timeout (issue #587).
+        stable = uptime && uptime >= RESTART_STABLE_UPTIME && !info[:boot_timed_out]
+        if status&.success? || uptime.nil? || stable
           @crash_counts.delete(key)
           return restart_child(info)
         end
@@ -746,55 +765,124 @@ module Pgbus
         end
       end
 
+      # Runs every WATCHDOG_INTERVAL. One process-row read (this host only)
+      # serves three jobs: it checks in forks that registered a row (the only
+      # check-in channel for the pipeless dispatcher, scheduler and outbox
+      # poller, so it runs whatever the thresholds are — /readyz depends on
+      # it); it enforces boot_timeout on forks that have not checked in
+      # (issue #587); and it feeds the loop-tick ages of the stall check.
       def check_stalled_workers
         now = monotonic_now
         return if (now - @last_watchdog_at) < WATCHDOG_INTERVAL
 
         @last_watchdog_at = now
-        threshold = config.stall_threshold
-        return unless threshold&.positive?
+        stall_threshold = positive_limit(config.stall_threshold)
+        boot_timeout = positive_limit(config.boot_timeout)
+
+        pids = watchdog_pids(stall_threshold)
+        return if pids.empty?
+
+        rows = watchdog_rows(pids)
+        mark_checked_in(rows.keys) if rows
+
+        victims = []
+        victims.concat(boot_timeout_victims(now, boot_timeout)) if boot_timeout && rows
+        victims.concat(stalled_victims(now, stall_threshold, rows || {})) if stall_threshold
+        kill_unresponsive_children(victims.uniq(&:first))
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog check failed: #{e.message}" }
+      end
+
+      def positive_limit(value)
+        value if value.is_a?(Numeric) && value.positive?
+      end
+
+      # Forks still waiting to check in, plus every worker/consumer when the
+      # stall check is on. A checked-in dispatcher/scheduler/outbox poller has
+      # no loop beacon, so once it has a row it is never read again.
+      def watchdog_pids(stall_threshold)
+        @forks.select do |_, info|
+          !info[:checked_in] || (stall_threshold && %i[worker consumer].include?(info[:type]))
+        end.keys
+      end
 
-        # Workers AND consumers both stamp a loop beacon and carry a liveness
-        # pipe (issue #274), so both are watched for a wedged claim/consume loop.
-        watched_pids = @forks.select { |_, info| %i[worker consumer].include?(info[:type]) }.keys
-        return if watched_pids.empty?
+      # {pid => ProcessEntry} for the forks' rows on THIS host — pids repeat
+      # across containers sharing one database, so a sibling's row must never
+      # vouch for a local child. A row last beaten before the fork belongs to
+      # a dead predecessor with the same pid (a SIGKILLed child never deletes
+      # its row, and a container restarted in place reuses hostname and
+      # pids), so it is dropped too. nil when the read fails: the stall check
+      # then falls back to the liveness pipe, and the boot deadline waits for
+      # a pass that can see the rows (a DB blip at boot must not kill every
+      # pipeless child).
+      def watchdog_rows(pids)
+        rows = ProcessEntry.where(hostname: Socket.gethostname, pid: pids).to_a
+        rows.select { |entry| row_of_current_fork?(entry) }.to_h { |entry| [entry.pid, entry] }
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog DB read failed, using pipe fallback: #{e.message}" }
+        nil
+      end
+
+      def row_of_current_fork?(entry)
+        spawned_wall_at = @forks.dig(entry.pid, :spawned_wall_at)
+        return true unless spawned_wall_at
 
-        db_ages = db_loop_tick_ages(watched_pids)
+        # floor: a table created without sub-second precision stores a row
+        # registered in the fork's own second below the fork's timestamp.
+        entry.last_heartbeat_at.present? && entry.last_heartbeat_at >= spawned_wall_at.floor
+      end
 
-        watched_pids.each do |pid|
+      def mark_checked_in(pids)
+        pids.each do |pid|
           info = @forks[pid]
-          next unless info
+          info[:checked_in] = true if info
+        end
+      end
 
-          kill_stalled_worker(pid, threshold) if worker_stalled?(info, db_ages[pid], now, threshold)
+      # A fork with neither a row nor a liveness byte boot_timeout seconds
+      # after fork is wedged in boot (issue #587). Without this it is never
+      # reaped (it does not exit) and never stalled (no channel ever spoke).
+      def boot_timeout_victims(now, boot_timeout)
+        @forks.filter_map do |pid, info|
+          next if info[:checked_in] || info[:spawned_at].nil?
+
+          waited = now - info[:spawned_at]
+          next unless waited > boot_timeout
+
+          info[:boot_timed_out] = true
+          [pid, "#{info[:type]} pid=#{pid} boot timeout (no check-in #{waited.round}s after fork, " \
+                "boot_timeout=#{boot_timeout}s)"]
+        end
+      end
+
+      def stalled_victims(now, threshold, rows)
+        db_ages = loop_tick_ages(rows)
+        @forks.filter_map do |pid, info|
+          next unless %i[worker consumer].include?(info[:type])
+          next unless worker_stalled?(info, db_ages[pid], now, threshold)
+
+          [pid, "#{info[:type]} pid=#{pid} claim loop stalled (no liveness within threshold=#{threshold}s)"]
         end
-      rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog check failed: #{e.message}" }
       end
 
-      # Read each watched fork's loop_tick_at from the process table and return
-      # a {pid => wall-clock age in seconds} map. Isolated in its own rescue so a
-      # database outage degrades to the OS-pipe fallback instead of skipping
-      # the whole watchdog — the exact failure this pipe channel exists to fix.
-      def db_loop_tick_ages(worker_pids)
-        ages = {}
-        ProcessEntry.where(kind: %w[worker consumer], pid: worker_pids).to_a.each do |entry|
+      # {pid => wall-clock age in seconds} of each worker/consumer row's
+      # loop_tick_at.
+      def loop_tick_ages(rows)
+        rows.each_with_object({}) do |(pid, entry), ages|
+          next unless %w[worker consumer].include?(entry.kind)
+
           meta = entry.metadata
-          next unless meta.is_a?(Hash)
+          next unless meta.is_a?(Hash) && meta["loop_tick_at"]
 
-          loop_tick = meta["loop_tick_at"]
-          ages[entry.pid] = Time.current.to_f - loop_tick.to_f if loop_tick
+          ages[pid] = Time.current.to_f - meta["loop_tick_at"].to_f
         end
-        ages
-      rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog DB read failed, using pipe fallback: #{e.message}" }
-        ages
       end
 
       # A worker is stalled only when EVERY liveness channel that has spoken
       # agrees it is stale (min-age / fresh-wins): a fresh DB row OR a fresh
       # pipe tick proves the loop is advancing. If no channel has any signal
       # (a slow-booting worker with no DB row and an unarmed pipe), we do not
-      # kill — mirroring the pre-existing "no loop_tick_at → skip" tolerance.
+      # kill — the boot deadline above owns that case.
       def worker_stalled?(info, db_age, now, threshold)
         pipe_age = (now - info[:last_pipe_tick_at] if info[:pipe_seen] && info[:last_pipe_tick_at])
         ages = [db_age, pipe_age].compact
@@ -803,14 +891,33 @@ module Pgbus
         ages.min > threshold
       end
 
-      def kill_stalled_worker(pid, threshold)
-        Pgbus.logger.error do
-          "[Pgbus] Supervisor watchdog: worker pid=#{pid} claim loop stalled " \
-            "(no liveness within threshold=#{threshold}s), sending SIGKILL"
+      # Log each victim at ERROR, ask it for a thread dump (TTIN, trapped in
+      # setup_child_process), give every dump one shared grace, then SIGKILL.
+      # The dump is best effort: a child stuck in a native call holding the
+      # GVL never answers. No waitpid here — reap_children must see the exit
+      # so schedule_restart re-forks the child.
+      def kill_unresponsive_children(victims)
+        return if victims.empty?
+
+        signalled = victims.filter_map do |pid, reason|
+          Pgbus.logger.error { "[Pgbus] Supervisor watchdog: #{reason} — dumping threads (TTIN), then SIGKILL" }
+          signal_child("TTIN", pid) && pid
         end
-        ::Process.kill("KILL", pid)
+        return if signalled.empty?
+
+        dump_grace_sleep
+        signalled.each { |pid| signal_child("KILL", pid) }
+      end
+
+      def signal_child(sig, pid)
+        ::Process.kill(sig, pid)
+        true
       rescue Errno::ESRCH
-        # already gone
+        false # already gone; reap_children handles it
+      end
+
+      def dump_grace_sleep
+        sleep(WATCHDOG_DUMP_GRACE)
       end
 
       # Drain each worker's liveness pipe. Any readable byte means the worker's
@@ -839,10 +946,11 @@ module Pgbus
             next
           end
 
-          if read_any
-            info[:last_pipe_tick_at] = now
-            info[:pipe_seen] = true
-          end
+          next unless read_any
+
+          info[:last_pipe_tick_at] = now
+          info[:pipe_seen] = true
+          info[:checked_in] = true
         end
       end
 
@@ -855,6 +963,8 @@ module Pgbus
       end
 
       def setup_child_process
+        # First, so a hang anywhere in the rest of the boot can be dumped.
+        install_thread_dump_trap
         # Every child type (worker, consumer, dispatcher, scheduler, outbox
         # poller) releases its copies of the parent's per-fork resources —
         # a dispatcher child holding a sibling worker's wake WRITER would
@@ -871,6 +981,26 @@ module Pgbus
         end
       end
 
+      # `kill -TTIN <child pid>` logs every thread's backtrace at ERROR and
+      # leaves the child running; the watchdog sends it before a SIGKILL
+      # (issue #587). Logger takes a lock, which Ruby forbids in trap context,
+      # so the trap snapshots the backtraces and one short-lived thread logs.
+      def install_thread_dump_trap
+        trap("TTIN") do
+          dump = Thread.list.map { |thread| [thread.inspect, thread.backtrace] }
+          Thread.new { log_thread_dump(dump) } # rubocop:disable ThreadSafety/NewThread
+        end
+      end
+
+      def log_thread_dump(dump)
+        Pgbus.logger.error { "[Pgbus] Thread dump pid=#{::Process.pid} (#{dump.size} threads, TTIN)" }
+        dump.each do |label, backtrace|
+          Pgbus.logger.error do
+            "[Pgbus] Thread #{label}\n  #{Array(backtrace).join("\n  ")}"
+          end
+        end
+      end
+
       # Lenient bootstrap for the parent (supervisor.rb#run). A transiently
       # unavailable DB at boot must not kill the supervisor — children
       # crash-and-backoff until it recovers — so every StandardError (including