pgbus 0.9.6 → 0.9.8

data/lib/pgbus/process/worker.rb

@@ -7,11 +7,30 @@ module Pgbus
     class Worker
       include SignalHandler
 
-      attr_reader :queues, :threads, :config, :execution_mode
-
+      attr_reader :queues, :threads, :config, :execution_mode,
+                  :rate_counter, :wake_signal, :restore_streak, :lifecycle
+      # stat_buffer is writable so a test can swap in a buffer double after
+      # construction to assert graceful_shutdown / check_recycle flush it. The
+      # executor captured the original buffer at construction, but these paths
+      # flush @stat_buffer directly, so the swap is observable.
+      attr_accessor :stat_buffer
+      # notify_listener / notify_retry_at / notify_retry_backoff are writable so
+      # tests can seed the self-healing listener state, re-arm the backoff window
+      # between calls, and simulate start_notify_listener assigning the listener
+      # from inside a stub (production mutates all three in the run loop).
+      attr_accessor :notify_listener, :notify_retry_at, :notify_retry_backoff
+
+      # The collaborators below (rate_counter, wake_signal, stat_buffer) and the
+      # recycle clock (started_at_monotonic) accept injected seeds so tests can
+      # observe or stub them without poking private ivars. All default to the
+      # exact values production constructs, so behavior is unchanged.
       def initialize(queues:, threads: 5, config: Pgbus.configuration,
                      single_active_consumer: false, consumer_priority: 0,
-                     execution_mode: :threads, group_mode: nil)
+                     execution_mode: :threads, group_mode: nil, liveness_pipe: nil,
+                     rate_counter: nil, wake_signal: nil, stat_buffer: :default,
+                     notify_listener: nil, notify_retry_at: 0.0,
+                     notify_retry_backoff: NOTIFY_RETRY_BASE_SECONDS,
+                     started_at_monotonic: nil)
         @queues = Array(queues)
         @initial_queues = @queues.dup.freeze
         @wildcard = @queues.include?("*")
@@ -38,20 +57,47 @@ module Pgbus
         @jobs_failed = Concurrent::AtomicFixnum.new(0)
         @in_flight = Concurrent::AtomicFixnum.new(0)
         @loop_tick_at = Concurrent::AtomicReference.new(nil)
-        @rate_counter = RateCounter.new(:processed, :failed, :dequeued)
+        @rate_counter = rate_counter || RateCounter.new(:processed, :failed, :dequeued)
         @started_at = Time.current
-        @started_at_monotonic = monotonic_now
-        @stat_buffer = config.stats_enabled ? Pgbus::StatBuffer.new : nil
+        @started_at_monotonic = started_at_monotonic || monotonic_now
+        # stat_buffer: :default means "build one iff config.stats_enabled";
+        # passing an explicit value (including nil) overrides that for tests.
+        @stat_buffer =
+          if stat_buffer == :default
+            if config.stats_enabled
+              Pgbus::StatBuffer.new(
+                flush_size: config.stats_flush_size,
+                flush_interval: config.stats_flush_interval
+              )
+            end
+          else
+            stat_buffer
+          end
         @executor = Pgbus::ActiveJob::Executor.new(stat_buffer: @stat_buffer)
-        @wake_signal = WakeSignal.new
+        @wake_signal = wake_signal || WakeSignal.new
         @pool = ExecutionPools.build(
           mode: @execution_mode,
           capacity: threads,
           on_state_change: -> { @wake_signal.notify! }
         )
         @circuit_breaker = Pgbus::CircuitBreaker.new(config: config)
+        @drain_started_at = nil
+        # Evict/restore cooldown state (issue #209). When a permanently-deleted
+        # queue is evicted down to an empty list, restoring the initial queues
+        # immediately re-triggers the same undefined-table error every loop tick.
+        # These back off the restore attempt on an exponential schedule instead.
+        @restore_streak = 0
+        @last_evicted_at = nil
+        @deferral_warned = false
         @queue_lock = QueueLock.new if @single_active_consumer
-        @notify_listener = nil
+        @notify_listener = notify_listener
+        @notify_retry_at = notify_retry_at
+        @notify_retry_backoff = notify_retry_backoff
+        # OS-level liveness channel to the supervisor watchdog. Optional: nil
+        # unless the supervisor forked us with one. Written from stamp_loop_tick
+        # so the watchdog can detect a wedged worker even when the database (and
+        # thus the Heartbeat's loop_tick_at) is unavailable.
+        @liveness_pipe = liveness_pipe
       end
 
       def stats
@@ -69,8 +115,47 @@ module Pgbus
         }.merge(@pool.metadata)
       end
 
+      # Test seams for the atomic counters recycle logic and prefetch
+      # flow-control consult. Production only increments these during message
+      # handling; seeding them lets a test cross a threshold without running
+      # thousands of real jobs.
+      def jobs_processed=(count)
+        @jobs_processed.value = count
+      end
+
+      def in_flight=(count)
+        @in_flight.value = count
+      end
+
+      # The last wall-clock loop-tick stamp (Time.now.to_f) fed to the
+      # heartbeat's loop_tick_supplier. Wall-clock — NOT monotonic — so it stays
+      # comparable across the process boundary the supervisor watchdog reads it
+      # over. nil until the first stamp_loop_tick.
+      def last_loop_tick
+        @loop_tick_at.get
+      end
+
       NOTIFY_FALLBACK_POLL_SECONDS = 15
 
+      # NotifyListener startup can fail on a transient boot-time condition (DB
+      # restarting, failover, DNS blip) or its thread can die mid-run. Rather
+      # than downgrade to blind polling until process restart, the loop retries
+      # from ensure_notify_listener on an exponential backoff: NOTIFY_RETRY_BASE
+      # doubling up to NOTIFY_RETRY_MAX. Constant-tuned, matching DRAIN_TIMEOUT
+      # and CircuitBreaker/Dispatcher precedent.
+      NOTIFY_RETRY_BASE_SECONDS = 5
+      NOTIFY_RETRY_MAX_SECONDS = 300
+
+      # Upper bound on the drain phase. Waiting for quiesced? must be
+      # bounded: a permanently-stuck job would otherwise hold the loop open
+      # forever — recycling never completes, TERM shutdown hangs the whole
+      # process tree, and the loop keeps stamping loop_tick so the
+      # supervisor watchdog never intervenes. After this many seconds the
+      # loop falls through to shutdown, whose wait_for_termination(30)
+      # bounds the remaining in-flight work. Tuned via constant rather than
+      # configuration, matching CircuitBreaker/Dispatcher precedent.
+      DRAIN_TIMEOUT = 30
+
       def run
         setup_signals
         start_heartbeat
@@ -87,9 +172,14 @@ module Pgbus
           process_signals
           check_recycle
           refresh_wildcard_queues
+          ensure_notify_listener
 
           break if @lifecycle.stopped?
-          break if @lifecycle.draining? && @pool.idle?
+          # quiesced? (all slots free), not idle? (any slot free) — exiting
+          # with work still in flight abandons those jobs to the 30s
+          # wait_for_termination timeout in shutdown. Bounded by
+          # DRAIN_TIMEOUT so a stuck job can't wedge the loop forever.
+          break if @lifecycle.draining? && (@pool.quiesced? || drain_deadline_exceeded?)
 
           claim_and_execute if @lifecycle.can_process?
           @stat_buffer&.flush_if_due
@@ -103,6 +193,13 @@ module Pgbus
         Pgbus.logger.info { "[Pgbus] Worker shutting down gracefully..." }
         Pgbus.stopping = true
         @lifecycle.transition_to(:draining)
+        # Flush buffered stats at drain entry so the ≤ stats_flush_interval /
+        # stats_flush_size window isn't lost if the supervisor watchdog SIGKILLs
+        # a stalled worker before the drain-loop shutdown flush runs. Runs on the
+        # main loop thread (signals are dispatched via process_signals, not in
+        # trap context), so the DB write is safe. flush is thread-safe and
+        # no-ops when the buffer is empty.
+        @stat_buffer&.flush
         @wake_signal.notify!
       end
 
@@ -121,6 +218,15 @@ module Pgbus
       # regex on every queue-missing error in hot fetch/read paths.
       MISSING_QUEUE_REGEX = /pgmq\.q_(\w+)/
 
+      # Exponential backoff bounds for restoring evicted queues (issue #209).
+      # After the worker's queues are fully evicted (queue table permanently
+      # deleted), the first restore waits RESTORE_COOLDOWN_BASE seconds; each
+      # consecutive failed restore doubles the wait up to RESTORE_COOLDOWN_MAX.
+      # A successful fetch resets the streak so a recreated queue restores
+      # promptly. Constant-tuned, matching NOTIFY_RETRY/DRAIN_TIMEOUT precedent.
+      RESTORE_COOLDOWN_BASE = 30
+      RESTORE_COOLDOWN_MAX = 300
+
       private
 
       def claim_and_execute
@@ -168,17 +274,33 @@ module Pgbus
           return []
         end
 
-        if priority_enabled?
-          fetch_prioritized(active_queues, qty)
-        elsif @group_mode
-          fetch_grouped(active_queues, qty)
-        elsif active_queues.size == 1
-          queue = active_queues.first
-          messages = Pgbus.client.read_batch(queue, qty: qty) || []
-          messages.map { |m| [queue, m] }
-        else
-          fetch_multi(active_queues, qty)
-        end
+        results =
+          if priority_enabled?
+            fetch_prioritized(active_queues, qty)
+          elsif @group_mode
+            fetch_grouped(active_queues, qty)
+          elsif active_queues.size == 1
+            queue = active_queues.first
+            messages = Pgbus.client.read_batch(queue, qty: qty) || []
+            messages.map { |m| [queue, m] }
+          else
+            fetch_multi(active_queues, qty)
+          end
+
+        # A read that reached here without an undefined-queue error means the
+        # queue tables exist again; drop the restore backoff so a recreated
+        # queue is reinstated promptly after the next eviction (issue #209).
+        @restore_streak = 0
+        results
+      rescue Pgbus::ConnectionCircuitOpenError
+        # The client-level connection breaker is open: the database has failed
+        # enough consecutive connection attempts that reads now fail fast. Idle
+        # this poll without an ErrorReporter call — the whole point of the
+        # breaker is to stop every worker flooding the error tracker for the
+        # duration of a database outage. The open/close transitions are logged
+        # once by the client (Client#log_circuit_open / #log_circuit_close),
+        # not per poll here.
+        []
       rescue StandardError => e
         if undefined_queue_table_error?(e)
           evict_missing_queues(e)
@@ -339,17 +461,56 @@ module Pgbus
           end
         end
         Pgbus.logger.error { "[Pgbus] Queue table missing: #{error.message}" }
-        restore_evicted_queues if @queues.empty? && !@wildcard
-      end
-
+        return unless @queues.empty? && !@wildcard
+
+        # Open (or re-arm) the restore cooldown window: stamp the eviction time so
+        # restore_evicted_queues can measure the backoff, and clear the
+        # per-window deferral-warned flag so the next deferral logs exactly once.
+        @last_evicted_at = monotonic_now
+        @deferral_warned = false
+        restore_evicted_queues
+      end
+
+      # Restore the worker's initial queues after a full eviction — but only once
+      # the exponential cooldown has elapsed (issue #209). While the cooldown is
+      # pending, leaves @queues empty (the caller idles via the empty-active_queues
+      # path) and logs a single deferral warn per window rather than one error pair
+      # per loop tick. Each actual restore escalates the streak so a permanently
+      # deleted queue backs off toward RESTORE_COOLDOWN_MAX instead of spinning.
       def restore_evicted_queues
+        return unless restore_cooldown_elapsed?
+
         @queues = @initial_queues.dup
+        @restore_streak += 1
         Pgbus.logger.warn do
           "[Pgbus] Worker queue list was empty after eviction — " \
             "restoring initial queues: #{@queues.join(", ")}"
         end
       end
 
+      # True once RESTORE_COOLDOWN_BASE * 2**streak seconds (capped at
+      # RESTORE_COOLDOWN_MAX) have passed since the window opened. When still
+      # pending, emits at most one deferral warn per window naming the wait.
+      def restore_cooldown_elapsed?
+        return true if @last_evicted_at.nil?
+
+        wait = restore_cooldown_seconds
+        return true if monotonic_now - @last_evicted_at >= wait
+
+        unless @deferral_warned
+          @deferral_warned = true
+          Pgbus.logger.warn do
+            "[Pgbus] All queues evicted; deferring restore — next restore attempt in #{wait}s " \
+              "(streak=#{@restore_streak})"
+          end
+        end
+        false
+      end
+
+      def restore_cooldown_seconds
+        [RESTORE_COOLDOWN_BASE * (2**@restore_streak), RESTORE_COOLDOWN_MAX].min
+      end
+
       def detect_zombie(queue_name, message)
         return unless config.zombie_detection
         return unless message.read_ct.to_i > 1
@@ -365,6 +526,20 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus] Zombie detection failed: #{e.class}: #{e.message}" }
       end
 
+      # Lazily stamps the drain start on first call — the predicate is only
+      # reached while draining, so this covers every path into the drain
+      # state (graceful_shutdown, recycling) without hooking each one.
+      def drain_deadline_exceeded?
+        @drain_started_at ||= monotonic_now
+        return false unless monotonic_now - @drain_started_at > DRAIN_TIMEOUT
+
+        Pgbus.logger.warn do
+          "[Pgbus] Worker drain deadline (#{DRAIN_TIMEOUT}s) reached with #{@in_flight.value} job(s) " \
+            "still in flight — proceeding to shutdown"
+        end
+        true
+      end
+
       def check_recycle
         return unless @lifecycle.running?
 
@@ -373,6 +548,9 @@ module Pgbus
 
         Pgbus.stopping = true
         @lifecycle.transition_to(:draining)
+        # Flush buffered stats on recycle-triggered drain for the same reason as
+        # graceful_shutdown: shrink the SIGKILL loss window. Same-thread, safe.
+        @stat_buffer&.flush
         Pgbus::Instrumentation.instrument(
           "pgbus.worker.recycle",
           reason: reason,
@@ -416,16 +594,9 @@ module Pgbus
         true
       end
 
+      # Instrumentation payload may report a value up to MEMORY_CHECK_TTL seconds old.
       def current_memory_mb
-        if RUBY_PLATFORM.include?("darwin")
-          `ps -o rss= -p #{::Process.pid}`.to_i / 1024
-        else
-          begin
-            File.read("/proc/#{::Process.pid}/statm").split[1].to_i * 4096 / (1024 * 1024)
-          rescue Errno::ENOENT
-            0
-          end
-        end
+        MemoryUsage.current_mb
       end
 
       def notify_wakeup?
@@ -433,7 +604,11 @@ module Pgbus
       end
 
       def wake_timeout
-        return effective_polling_interval unless notify_wakeup? && @notify_listener
+        # A dead listener (running? false) will never wake the loop, so treat
+        # it as absent and keep polling at the short interval until
+        # ensure_notify_listener restarts it. Only a live listener earns the
+        # long NOTIFY-mode ceiling.
+        return effective_polling_interval unless notify_wakeup? && @notify_listener&.running?
 
         [effective_polling_interval, config.polling_interval, NOTIFY_FALLBACK_POLL_SECONDS].max
       end
@@ -467,6 +642,42 @@ module Pgbus
         end
       end
 
+      # Self-heal a NotifyListener that never started or whose thread died.
+      # Called each loop iteration but gated by a monotonic backoff timestamp
+      # so a persistent outage retries on 5s→…→300s intervals, not every tick
+      # (mirrors refresh_wildcard_queues' throttle). A restarted listener has
+      # its queue subscription reconciled (wildcard workers) and the backoff
+      # reset; a still-failing restart doubles the backoff up to the cap.
+      def ensure_notify_listener
+        return unless notify_wakeup?
+        return if @notify_listener&.running?
+        return if monotonic_now < @notify_retry_at
+
+        stop_dead_notify_listener
+        start_notify_listener
+
+        if @notify_listener&.running?
+          sync_notify_listener_queues
+          @notify_retry_backoff = NOTIFY_RETRY_BASE_SECONDS
+        else
+          @notify_retry_backoff = [@notify_retry_backoff * 2, NOTIFY_RETRY_MAX_SECONDS].min
+        end
+        @notify_retry_at = monotonic_now + @notify_retry_backoff
+      end
+
+      # Stop a listener whose thread died so its dedicated PG connection is
+      # released before start_notify_listener allocates a fresh one. A nil or
+      # still-running listener is left alone (the caller already gated on it).
+      def stop_dead_notify_listener
+        return if @notify_listener.nil?
+
+        @notify_listener.stop
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Failed to stop dead NotifyListener: #{e.class}: #{e.message}" }
+      ensure
+        @notify_listener = nil
+      end
+
       def sync_notify_listener_queues
         return unless @notify_listener
 
@@ -494,12 +705,49 @@ module Pgbus
             queues: queues, threads: threads, pid: ::Process.pid,
             execution_mode: @execution_mode, consumer_priority: @consumer_priority
           },
-          on_beat: -> { @rate_counter.snapshot! },
-          loop_tick_supplier: -> { @loop_tick_at.get }
+          on_beat: -> { on_heartbeat },
+          loop_tick_supplier: -> { @loop_tick_at.get },
+          metadata_supplier: -> { throughput_metadata }
         )
         @heartbeat.start
       end
 
+      # Runs once per heartbeat interval (not per job), so it's the right place
+      # to snapshot the per-beat rate counters and emit connection-pool
+      # observability without touching any per-job hot path. Pool utilization
+      # goes out as a `pgbus.client.pool` event carrying {size:, available:,
+      # pool_timeout:}. Reading the pool must never crash the beat — pool_stats
+      # already rescues to {}, and this whole method is guarded so a listener or
+      # unexpected error can't take down the heartbeat thread.
+      def on_heartbeat
+        @rate_counter.snapshot!
+        emit_pool_stats
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] Worker heartbeat hook error: #{e.class}: #{e.message}" }
+      end
+
+      def emit_pool_stats
+        stats = Pgbus.client.pool_stats
+        return if stats.empty?
+
+        Pgbus::Instrumentation.instrument("pgbus.client.pool", stats)
+      end
+
+      # Per-worker throughput persisted into pgbus_processes.metadata on every
+      # heartbeat so the dashboard can show cluster-wide live rates. Called by
+      # the Heartbeat's metadata_supplier after on_beat (snapshot!) has
+      # refreshed the rate counter, so these rates are current. Keys are
+      # stringified because the value round-trips through JSON in the metadata
+      # column.
+      def throughput_metadata
+        {
+          "rates" => @rate_counter.rates.transform_keys(&:to_s),
+          "jobs_processed" => @jobs_processed.value,
+          "jobs_failed" => @jobs_failed.value,
+          "in_flight" => @in_flight.value
+        }
+      end
+
       def shutdown
         Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
         @notify_listener&.stop
@@ -520,8 +768,23 @@ module Pgbus
       # Wall clock is required because the supervisor watchdog reads
       # this value from a different process (cross-fork) and the
       # dashboard reads it from a different host.
+      #
+      # Also pokes the OS-level liveness pipe (when the supervisor forked us
+      # with one) so the watchdog has a database-independent signal. The write
+      # is non-blocking and never raises in the hot path: exception: false
+      # returns :wait_writable on a full pipe (which itself proves recent,
+      # undrained ticks — liveness — so a dropped write is fine), and the
+      # rescue covers the reader-gone / fd-closed cases so a dead pipe can
+      # never crash the worker loop. The payload is a content-free byte: the
+      # supervisor treats "any bytes readable" as liveness and stamps arrival
+      # time on its own monotonic clock, so no timestamp crosses the fork.
       def stamp_loop_tick
         @loop_tick_at.set(Time.now.to_f)
+        return unless @liveness_pipe
+
+        @liveness_pipe.write_nonblock("\0", exception: false)
+      rescue Errno::EPIPE, IOError, Errno::EBADF
+        nil
       end
     end
   end

data/lib/pgbus/recurring/scheduler.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "concurrent"
+
 module Pgbus
   module Recurring
     class Scheduler
@@ -12,6 +14,7 @@ module Pgbus
         @schedule = Schedule.new(config: config)
         @shutting_down = false
         @last_runs = {}
+        @loop_tick_at = Concurrent::AtomicReference.new(nil)
       end
 
       def run
@@ -25,6 +28,7 @@ module Pgbus
         end
 
         loop do
+          stamp_loop_tick
           break if @shutting_down
 
           process_signals
@@ -100,10 +104,20 @@ module Pgbus
         Pgbus.logger.error { "[Pgbus] Failed to sync recurring tasks: #{e.class}: #{e.message}" }
       end
 
+      # Wall clock (not monotonic) so the dashboard can compare the beacon
+      # against Time.now across processes; the heartbeat timer thread reads it
+      # via loop_tick_supplier. A scheduler wedged inside tick stops advancing
+      # this while heartbeats keep firing, so the beacon ages and the processes
+      # page can surface the stall.
+      def stamp_loop_tick
+        @loop_tick_at.set(Time.now.to_f)
+      end
+
       def start_heartbeat
         @heartbeat = Process::Heartbeat.new(
           kind: "scheduler",
-          metadata: { pid: ::Process.pid, tasks: schedule.tasks.size }
+          metadata: { pid: ::Process.pid, tasks: schedule.tasks.size },
+          loop_tick_supplier: -> { @loop_tick_at.get }
         )
         @heartbeat.start
       end

data/lib/pgbus/streams/turbo_broadcastable.rb

@@ -37,11 +37,13 @@ module Pgbus
       def broadcast_stream_to(*streamables, content:)
         name = stream_name_from(streamables)
         override = Thread.current[:pgbus_broadcast_durable]
-        durable = if override.nil?
-                    Pgbus.configuration.streams_default_broadcast_mode == :durable
-                  else
-                    override
-                  end
+        # When no explicit thread-local override is present, let the config
+        # resolver decide: it checks `streams_durable_patterns` first (exact
+        # string or regex match), then falls back to
+        # `streams_default_broadcast_mode`. Passing this through keeps
+        # pattern-based routing alive for the whole Turbo::Broadcastable and
+        # phlex-reactive broadcast path (see issue #267).
+        durable = override.nil? ? Pgbus.configuration.stream_durable?(name) : override
         Pgbus.stream(name, durable: durable).broadcast(
           content,
           exclude: Thread.current[:pgbus_broadcast_exclude],

data/lib/pgbus/table_maintenance.rb

@@ -80,12 +80,23 @@ module Pgbus
         "REINDEX TABLE CONCURRENTLY \"#{schema}\".\"#{relname}\""
       end
 
-      def run_maintenance(conn, threshold: BLOAT_THRESHOLD, reindex: true)
+      # stop_check is polled before each candidate; when it returns true the
+      # loop stops before touching the next table so a mid-pass shutdown does
+      # not block on VACUUM/REINDEX of the remaining tables. The default never
+      # stops, so callers that don't pass it are unaffected.
+      def run_maintenance(conn, threshold: BLOAT_THRESHOLD, reindex: true, stop_check: -> { false })
         candidates = vacuum_candidates(conn, threshold: threshold)
         return 0 if candidates.empty?
 
         maintained = 0
-        candidates.each do |candidate|
+        candidates.each_with_index do |candidate, index|
+          if stop_check.call
+            Pgbus.logger.info do
+              "[Pgbus::TableMaintenance] Maintenance interrupted by shutdown after #{index} of #{candidates.size}"
+            end
+            break
+          end
+
           table = candidate[:table]
           Pgbus.logger.info do
             "[Pgbus::TableMaintenance] Vacuuming #{table} " \

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.9.6"
+  VERSION = "0.9.8"
 end

data/lib/pgbus/web/data_source.rb

@@ -1234,18 +1234,34 @@ module Pgbus
 
       def derive_process_status(stale, metadata, kind)
         return :stale if stale
-        return :healthy unless kind == "worker" && metadata.is_a?(Hash)
+        return :healthy unless metadata.is_a?(Hash)
+
+        threshold = stall_threshold_for(kind)
+        return :healthy unless threshold
 
         loop_tick = metadata["loop_tick_at"]
         return :healthy unless loop_tick
 
-        threshold = Pgbus.configuration.stall_threshold
-        return :healthy unless threshold&.positive?
-
         age = Time.now.to_f - loop_tick.to_f
         age > threshold ? :stalled : :healthy
       end
 
+      # Kind-aware stall threshold. Workers run a tight loop, so the base
+      # stall_threshold applies directly. The dispatcher and scheduler sleep
+      # between iterations, so their beacon is naturally up to one sleep
+      # interval stale even when healthy — widen the window by that interval to
+      # avoid false stalls. Kinds with no loop beacon (nil) are never stalled.
+      def stall_threshold_for(kind)
+        base = Pgbus.configuration.stall_threshold
+        return nil unless base&.positive?
+
+        case kind
+        when "worker" then base
+        when "dispatcher" then base + Pgbus.configuration.dispatch_interval
+        when "scheduler" then base + Pgbus.configuration.recurring_schedule_interval
+        end
+      end
+
       def format_batch(record)
         total = record.total_jobs
         done = record.completed_jobs + record.discarded_jobs