pgbus 0.9.7 → 0.9.9

data/lib/pgbus/process/supervisor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "uri"
+
 module Pgbus
   module Process
     class Supervisor
@@ -8,26 +10,69 @@ module Pgbus
       FORK_WAIT = 1 # seconds between fork checks
       WATCHDOG_INTERVAL = 10 # seconds between stall checks
 
-      attr_reader :config
+      # A child that crashes within this many seconds of forking is treated
+      # as crash-looping and restarted with exponential backoff (base
+      # RESTART_BACKOFF_BASE, doubling per consecutive crash, capped at
+      # RESTART_BACKOFF_MAX). A child that ran at least this long — or that
+      # exited cleanly, like a recycling worker — restarts immediately and
+      # resets its crash streak.
+      RESTART_STABLE_UPTIME = 30
+      RESTART_BACKOFF_BASE = 1
+      RESTART_BACKOFF_MAX = 60
 
-      def initialize(config: Pgbus.configuration)
+      attr_reader :config
+      # forks is readable everywhere; the writer exists only so tests can seed a
+      # known set of children before exercising the reap/watchdog paths (in
+      # production forks is populated by fork_* as children spawn).
+      attr_accessor :forks
+
+      # The child-fork bookkeeping (`forks`, `pending_restarts`) and the
+      # `shutting_down` flag accept injected seeds so tests can drive the
+      # monitor/reap loops from a known state without poking private ivars. All
+      # default to the empty/false values production always starts from.
+      def initialize(config: Pgbus.configuration, forks: {}, shutting_down: false,
+                     pending_restarts: [], last_watchdog_at: nil)
         @config = config
-        @forks = {}
-        @shutting_down = false
-        @last_watchdog_at = monotonic_now
+        @forks = forks
+        @shutting_down = shutting_down
+        @last_watchdog_at = last_watchdog_at || monotonic_now
+        @pending_restarts = pending_restarts
+        @crash_counts = Hash.new(0)
       end
 
+      def shutting_down?
+        @shutting_down
+      end
+
+      # Test seam: reset the watchdog clock so check_stalled_workers runs on the
+      # next call instead of waiting out WATCHDOG_INTERVAL. Production advances
+      # this internally after each watchdog pass.
+      attr_writer :last_watchdog_at
+
       def run
         setup_signals
         start_heartbeat
+        start_health_server
 
         Pgbus.logger.info { "[Pgbus] Supervisor starting pid=#{::Process.pid}" }
 
+        # Emit a one-block boot diagnostics banner so operators can read the
+        # resolved deployment config (connection target, pool, notify flags,
+        # roles, capsules) straight from the log instead of attaching a console.
+        log_boot_banner
+
+        # Fail fast on a bad database_url / connection_params. PGMQ's pool is
+        # lazy, so an unreachable DB would otherwise only surface once forked
+        # children crash-loop against it. verify_connection! raises
+        # Pgbus::ConfigurationError with an actionable message; we let it
+        # propagate so the supervisor exits instead of forking anything.
+        Pgbus.client.verify_connection!
+
         # Bootstrap queues once in the parent process before forking children.
         # This avoids the deadlock that occurs when multiple forked children
         # race to call enable_notify_insert (DROP TRIGGER + CREATE TRIGGER)
         # concurrently on the same queue tables. Children still call
-        # bootstrap_queues post-fork but the idempotent check in
+        # bootstrap_queues! post-fork but the idempotent check in
         # notify_trigger_current? makes those calls cheap no-ops.
         bootstrap_queues
 
@@ -51,13 +96,133 @@ module Pgbus
 
       private
 
+      # Log a single boot diagnostics banner: the settings that actually
+      # determine whether this deployment works. One consecutive block of
+      # "[Pgbus] boot:"-prefixed info lines so it reads cleanly under both the
+      # text and JSON log formatters. Every DB-dependent field is wrapped so a
+      # transient failure degrades that field to "unknown" — the banner must
+      # never abort boot.
+      ROLE_FLAGS = %i[workers dispatcher scheduler consumers outbox].freeze
+      private_constant :ROLE_FLAGS
+
+      def log_boot_banner
+        Pgbus.logger.info { "[Pgbus] boot: pgbus #{Pgbus::VERSION} pid=#{::Process.pid}" }
+        Pgbus.logger.info do
+          "[Pgbus] boot: connection=#{redacted_connection_target} pool=#{banner_field { config.resolved_pool_size }}"
+        end
+        Pgbus.logger.info do
+          "[Pgbus] boot: pgmq_schema_mode=#{config.pgmq_schema_mode} pgmq_version=#{installed_pgmq_version}"
+        end
+        Pgbus.logger.info do
+          "[Pgbus] boot: listen_notify=#{config.listen_notify} worker_notify_wakeup=#{config.worker_notify_wakeup?}"
+        end
+        Pgbus.logger.info { "[Pgbus] boot: roles=#{enabled_roles.join(",")}" }
+        log_capsule_banner
+        log_consumer_banner
+      end
+
+      def log_capsule_banner
+        return unless config.role_enabled?(:workers)
+
+        Array(config.workers).each do |worker_config|
+          name = worker_config[:name] || worker_config["name"] || "anonymous"
+          queues = worker_config[:queues] || worker_config["queues"] || [config.default_queue]
+          threads = worker_config[:threads] || worker_config["threads"] || 5
+          mode = banner_field { config.execution_mode_for(worker_config) }
+          Pgbus.logger.info do
+            "[Pgbus] boot: capsule=#{name} queues=#{Array(queues).join(",")} threads=#{threads} mode=#{mode}"
+          end
+        end
+      end
+
+      def log_consumer_banner
+        return unless config.role_enabled?(:consumers)
+
+        Array(config.event_consumers).each do |consumer_config|
+          topics = consumer_config[:topics] || consumer_config["topics"] || []
+          threads = consumer_config[:threads] || consumer_config["threads"] || 3
+          Pgbus.logger.info do
+            "[Pgbus] boot: consumer topics=#{Array(topics).join(",")} threads=#{threads}"
+          end
+        end
+      end
+
+      # The set of roles that will actually boot, honoring config.roles.
+      def enabled_roles
+        ROLE_FLAGS.select { |role| config.role_enabled?(role) }
+      end
+
+      # Best-effort PGMQ version from the tracking table. "unknown" on any error
+      # (missing table, DB down) so a boot log never fails on diagnostics.
+      def installed_pgmq_version
+        Pgbus.client.pgmq_schema_version || "unknown"
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] boot: pgmq_schema_version lookup failed: #{e.class}: #{e.message}" }
+        "unknown"
+      end
+
+      # Reduce the configured connection to "host/dbname" with the password
+      # never printed, across all three connection_options forms: a URL string,
+      # a libpq keyword hash, or the AR-derived hash. Falls back to "unknown"
+      # rather than leaking anything if the shape is unexpected.
+      def redacted_connection_target
+        target = config.database_url ? parse_url_target(config.database_url) : parse_hash_target(banner_connection_hash)
+        target || "unknown"
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] boot: connection target redaction failed: #{e.class}: #{e.message}" }
+        "unknown"
+      end
+
+      # connection_options returns the URL string when database_url is set, a
+      # Hash for connection_params or AR-derived config, or a Proc fallback. We
+      # only reach here for the non-URL cases, so a non-Hash (Proc) yields nil.
+      def banner_connection_hash
+        opts = config.connection_options
+        opts.is_a?(Hash) ? opts : nil
+      end
+
+      def parse_url_target(url)
+        uri = URI.parse(url)
+        host = uri.host
+        dbname = uri.path.to_s.sub(%r{\A/}, "")
+        return nil if host.nil? && dbname.empty?
+
+        [host, dbname].reject { |s| s.nil? || s.empty? }.join("/")
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def parse_hash_target(hash)
+        return nil unless hash.is_a?(Hash)
+
+        host = hash[:host] || hash["host"]
+        dbname = hash[:dbname] || hash["dbname"] || hash[:database] || hash["database"]
+        return nil if host.nil? && dbname.nil?
+
+        [host, dbname].compact.join("/")
+      end
+
+      # Evaluate a banner field that touches config/DB; any failure degrades the
+      # single field to "unknown" so one broken resolver can't blank the banner.
+      def banner_field
+        yield
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] boot: banner field failed: #{e.class}: #{e.message}" }
+        "unknown"
+      end
+
       def boot_processes
         # Boot workers (workers may be nil for scheduler-only or
         # dispatcher-only deployments via --workers-only / --scheduler-only /
         # --dispatcher-only CLI flags). Each role is gated by
         # config.role_enabled?, which returns true unless +config.roles+ has
         # been narrowed.
-        Array(config.workers).each { |worker_config| fork_worker(worker_config) } if config.role_enabled?(:workers)
+        if config.role_enabled?(:workers)
+          # slot is the child's position in the config array — it keys the
+          # crash-streak tracking so identically-configured siblings don't
+          # share (and reset) each other's restart backoff.
+          Array(config.workers).each_with_index { |worker_config, slot| fork_worker(worker_config, slot: slot) }
+        end
 
         fork_dispatcher if config.role_enabled?(:dispatcher)
         boot_scheduler if config.role_enabled?(:scheduler)
@@ -65,35 +230,60 @@ module Pgbus
         boot_outbox_poller if config.role_enabled?(:outbox)
       end
 
-      def fork_worker(worker_config)
-        queues = worker_config[:queues] || worker_config["queues"] || [config.default_queue]
-        threads = worker_config[:threads] || worker_config["threads"] || 5
-        single_active = worker_config[:single_active_consumer] || worker_config["single_active_consumer"] || false
-        priority = worker_config[:consumer_priority] || worker_config["consumer_priority"] || 0
+      def fork_worker(worker_config, slot: nil)
+        queues = worker_config[:queues] || [config.default_queue]
+        threads = worker_config[:threads] || 5
+        single_active = worker_config[:single_active_consumer] || false
+        priority = worker_config[:consumer_priority] || 0
         exec_mode = config.execution_mode_for(worker_config)
-        grp_mode = worker_config[:group_mode] || worker_config["group_mode"] || config.group_mode
+        grp_mode = worker_config[:group_mode] || config.group_mode
+
+        # OS-level liveness channel: the child writes a byte each loop
+        # iteration, the parent drains the reader in monitor_loop. This lets
+        # the watchdog detect a wedged worker without the database.
+        liveness_reader, liveness_writer = IO.pipe
 
         pid = fork do
+          # Child owns the writer end. A fork copies the whole parent FD
+          # table, so this child also inherits every earlier sibling's
+          # liveness reader (they live in @forks). Close them — otherwise each
+          # new worker accumulates N-1 stale reader FDs. (Sibling *writers*
+          # are never inherited: the parent closes each writer below before
+          # forking the next worker, so a dead sibling's pipe still reaches
+          # EOF correctly.) Finally close this fork's own reader copy.
+          close_inherited_liveness_readers
+          liveness_reader.close
           restore_signals
           setup_child_process
           load_rails_app
-          bootstrap_queues
+          bootstrap_queues!
           worker = Worker.new(
             queues: queues, threads: threads, config: config,
             single_active_consumer: single_active, consumer_priority: priority,
-            execution_mode: exec_mode, group_mode: grp_mode
+            execution_mode: exec_mode, group_mode: grp_mode,
+            liveness_pipe: liveness_writer
           )
           worker.run
         end
 
         unless pid
+          close_pipe(liveness_reader)
+          close_pipe(liveness_writer)
           Pgbus.logger.error { "[Pgbus] Failed to fork worker for queues=#{queues.join(",")}" }
           return
         end
 
-        @forks[pid] = { type: :worker, config: worker_config }
+        # Parent keeps the reader, discards its writer copy so the pipe reaches
+        # EOF once the (sole remaining) child writer closes.
+        close_pipe(liveness_writer)
+        @forks[pid] = {
+          type: :worker, config: worker_config, slot: slot, spawned_at: monotonic_now,
+          liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false
+        }
         Pgbus.logger.info { "[Pgbus] Forked worker pid=#{pid} queues=#{queues.join(",")} mode=#{exec_mode}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
+        close_pipe(liveness_reader)
+        close_pipe(liveness_writer)
         ErrorReporter.report(e, { action: "fork_worker", queues: queues })
       end
 
@@ -111,14 +301,14 @@ module Pgbus
           return
         end
 
-        @forks[pid] = { type: :dispatcher }
+        @forks[pid] = { type: :dispatcher, spawned_at: monotonic_now }
         Pgbus.logger.info { "[Pgbus] Forked dispatcher pid=#{pid}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
         ErrorReporter.report(e, { action: "fork_dispatcher" })
       end
 
       def boot_scheduler
-        return if config.skip_recurring
+        return unless config.recurring_enabled
         return unless recurring_tasks_configured?
 
         fork_scheduler
@@ -130,7 +320,7 @@ module Pgbus
           setup_child_process
           load_rails_app
           load_recurring_config
-          bootstrap_queues
+          bootstrap_queues!
           scheduler = Recurring::Scheduler.new(config: config)
           scheduler.run
         end
@@ -140,7 +330,7 @@ module Pgbus
           return
         end
 
-        @forks[pid] = { type: :scheduler }
+        @forks[pid] = { type: :scheduler, spawned_at: monotonic_now }
         Pgbus.logger.info { "[Pgbus] Forked scheduler pid=#{pid}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
         ErrorReporter.report(e, { action: "fork_scheduler" })
@@ -182,31 +372,52 @@ module Pgbus
       def boot_consumers
         return unless config.event_consumers
 
-        config.event_consumers.each do |consumer_config|
-          fork_consumer(consumer_config)
+        config.event_consumers.each_with_index do |consumer_config, slot|
+          fork_consumer(consumer_config, slot: slot)
         end
       end
 
-      def fork_consumer(consumer_config)
-        topics = consumer_config[:topics] || consumer_config["topics"]
-        threads = consumer_config[:threads] || consumer_config["threads"] || 3
+      def fork_consumer(consumer_config, slot: nil)
+        topics = consumer_config[:topics]
+        threads = consumer_config[:threads] || 3
+
+        # OS-level liveness channel: the consumer writes a byte each loop
+        # iteration, the parent drains the reader in monitor_loop. This lets the
+        # watchdog detect a wedged consumer without the database (issue #274),
+        # exactly as fork_worker does for workers.
+        liveness_reader, liveness_writer = IO.pipe
 
         pid = fork do
+          # Child owns the writer end. Close every earlier sibling's inherited
+          # liveness reader and this fork's own reader copy before becoming a
+          # Consumer (see fork_worker for the full rationale).
+          close_inherited_liveness_readers
+          liveness_reader.close
           restore_signals
           setup_child_process
           load_rails_app
-          consumer = Consumer.new(topics: topics, threads: threads, config: config)
+          consumer = Consumer.new(topics: topics, threads: threads, config: config, liveness_pipe: liveness_writer)
           consumer.run
         end
 
         unless pid
+          close_pipe(liveness_reader)
+          close_pipe(liveness_writer)
           Pgbus.logger.error { "[Pgbus] Failed to fork consumer for topics=#{topics.join(",")}" }
           return
         end
 
-        @forks[pid] = { type: :consumer, config: consumer_config }
+        # Parent keeps the reader, discards its writer copy so the pipe reaches
+        # EOF once the (sole remaining) child writer closes.
+        close_pipe(liveness_writer)
+        @forks[pid] = {
+          type: :consumer, config: consumer_config, slot: slot, spawned_at: monotonic_now,
+          liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false
+        }
         Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
+        close_pipe(liveness_reader)
+        close_pipe(liveness_writer)
         ErrorReporter.report(e, { action: "fork_consumer", topics: topics })
       end
 
@@ -230,7 +441,7 @@ module Pgbus
           return
         end
 
-        @forks[pid] = { type: :outbox_poller }
+        @forks[pid] = { type: :outbox_poller, spawned_at: monotonic_now }
         Pgbus.logger.info { "[Pgbus] Forked outbox poller pid=#{pid}" }
       rescue Errno::EAGAIN, Errno::ENOMEM => e
         ErrorReporter.report(e, { action: "fork_outbox_poller" })
@@ -242,7 +453,11 @@ module Pgbus
 
           process_signals
           reap_children
-          check_stalled_workers unless @shutting_down
+          drain_liveness_pipes
+          unless @shutting_down
+            process_pending_restarts
+            check_stalled_workers
+          end
           interruptible_sleep(FORK_WAIT)
         end
       end
@@ -255,29 +470,69 @@ module Pgbus
           info = @forks.delete(pid)
           next unless info
 
+          # Close the liveness reader as the fork leaves @forks so a crash-loop
+          # (restart deferred up to RESTART_BACKOFF_MAX) can't leak an FD per
+          # crash. Scrub the key so the closed IO never rides into a restart.
+          close_pipe(info.delete(:liveness_reader))
+
           if @shutting_down
             Pgbus.logger.info { "[Pgbus] Child #{info[:type]} pid=#{pid} exited (status=#{status.exitstatus})" }
           else
             Pgbus.logger.warn do
-              "[Pgbus] Child #{info[:type]} pid=#{pid} exited unexpectedly (status=#{status&.exitstatus}), restarting..."
+              "[Pgbus] Child #{info[:type]} pid=#{pid} exited unexpectedly (status=#{status&.exitstatus})"
             end
-            restart_child(info)
+            schedule_restart(info, status)
           end
         rescue Errno::ECHILD
           break
         end
       end
 
+      # Restart policy: a clean exit (worker recycling) or a crash after a
+      # stable run restarts immediately with a fresh crash streak. A crash
+      # within RESTART_STABLE_UPTIME of forking is a crash loop — the child
+      # is dying on boot (bad config, unreachable DB, raising initializer) —
+      # so back off exponentially instead of fork-crash-forking at full speed.
+      # A child with no spawned_at (never set in practice) restarts
+      # immediately, preserving the pre-backoff behavior.
+      def schedule_restart(info, status)
+        # Keyed on [type, slot], NOT the config hash — identically-configured
+        # sibling workers would otherwise share one streak, letting one
+        # sibling's clean recycle reset another sibling's crash backoff.
+        key = [info[:type], info[:slot]]
+        uptime = info[:spawned_at] ? monotonic_now - info[:spawned_at] : nil
+
+        if status&.success? || uptime.nil? || uptime >= RESTART_STABLE_UPTIME
+          @crash_counts.delete(key)
+          return restart_child(info)
+        end
+
+        crashes = @crash_counts[key] += 1
+        backoff = [RESTART_BACKOFF_BASE * (2**(crashes - 1)), RESTART_BACKOFF_MAX].min
+        Pgbus.logger.warn do
+          "[Pgbus] Child #{info[:type]} crashed after #{uptime.round(1)}s uptime " \
+            "(crash ##{crashes}) — restarting in #{backoff}s"
+        end
+        @pending_restarts << { info: info, at: monotonic_now + backoff }
+      end
+
+      def process_pending_restarts
+        now = monotonic_now
+        due, pending = @pending_restarts.partition { |r| r[:at] <= now }
+        @pending_restarts = pending
+        due.each { |r| restart_child(r[:info]) }
+      end
+
       def restart_child(info)
         case info[:type]
         when :worker
-          fork_worker(info[:config])
+          fork_worker(info[:config], slot: info[:slot])
         when :dispatcher
           fork_dispatcher
         when :scheduler
           fork_scheduler
         when :consumer
-          fork_consumer(info[:config])
+          fork_consumer(info[:config], slot: info[:slot])
         when :outbox_poller
           fork_outbox_poller
         end
@@ -291,33 +546,96 @@ module Pgbus
         threshold = config.stall_threshold
         return unless threshold&.positive?
 
-        worker_pids = @forks.select { |_, info| info[:type] == :worker }.keys
-        return if worker_pids.empty?
+        # Workers AND consumers both stamp a loop beacon and carry a liveness
+        # pipe (issue #274), so both are watched for a wedged claim/consume loop.
+        watched_pids = @forks.select { |_, info| %i[worker consumer].include?(info[:type]) }.keys
+        return if watched_pids.empty?
+
+        db_ages = db_loop_tick_ages(watched_pids)
 
-        rows = ProcessEntry.where(kind: "worker", pid: worker_pids).to_a
-        rows.each do |entry|
+        watched_pids.each do |pid|
+          info = @forks[pid]
+          next unless info
+
+          kill_stalled_worker(pid, threshold) if worker_stalled?(info, db_ages[pid], now, threshold)
+        end
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog check failed: #{e.message}" }
+      end
+
+      # Read each watched fork's loop_tick_at from the process table and return
+      # a {pid => wall-clock age in seconds} map. Isolated in its own rescue so a
+      # database outage degrades to the OS-pipe fallback instead of skipping
+      # the whole watchdog — the exact failure this pipe channel exists to fix.
+      def db_loop_tick_ages(worker_pids)
+        ages = {}
+        ProcessEntry.where(kind: %w[worker consumer], pid: worker_pids).to_a.each do |entry|
           meta = entry.metadata
           next unless meta.is_a?(Hash)
 
           loop_tick = meta["loop_tick_at"]
-          next unless loop_tick
+          ages[entry.pid] = Time.current.to_f - loop_tick.to_f if loop_tick
+        end
+        ages
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog DB read failed, using pipe fallback: #{e.message}" }
+        ages
+      end
 
-          age = Time.current.to_f - loop_tick.to_f
-          next unless age > threshold
+      # A worker is stalled only when EVERY liveness channel that has spoken
+      # agrees it is stale (min-age / fresh-wins): a fresh DB row OR a fresh
+      # pipe tick proves the loop is advancing. If no channel has any signal
+      # (a slow-booting worker with no DB row and an unarmed pipe), we do not
+      # kill — mirroring the pre-existing "no loop_tick_at → skip" tolerance.
+      def worker_stalled?(info, db_age, now, threshold)
+        pipe_age = (now - info[:last_pipe_tick_at] if info[:pipe_seen] && info[:last_pipe_tick_at])
+        ages = [db_age, pipe_age].compact
+        return false if ages.empty?
+
+        ages.min > threshold
+      end
 
-          pid = entry.pid
-          Pgbus.logger.error do
-            "[Pgbus] Supervisor watchdog: worker pid=#{pid} claim loop stalled " \
-              "(loop_tick_at age=#{age.round(1)}s > threshold=#{threshold}s), sending SIGKILL"
-          end
+      def kill_stalled_worker(pid, threshold)
+        Pgbus.logger.error do
+          "[Pgbus] Supervisor watchdog: worker pid=#{pid} claim loop stalled " \
+            "(no liveness within threshold=#{threshold}s), sending SIGKILL"
+        end
+        ::Process.kill("KILL", pid)
+      rescue Errno::ESRCH
+        # already gone
+      end
+
+      # Drain each worker's liveness pipe. Any readable byte means the worker's
+      # loop advanced since the last drain, so we stamp arrival on the parent's
+      # own monotonic clock (never a worker timestamp — that would cross the
+      # fork's incomparable CLOCK_MONOTONIC) and arm pipe_seen. Bounded to a
+      # few reads per reader so a fast-writing worker can't wedge the 1s loop;
+      # "any bytes ⇒ alive" needs no full drain. Per-reader rescue so one
+      # closed reader (racing reap/shutdown) can't skip the rest.
+      def drain_liveness_pipes
+        now = monotonic_now
+        @forks.each_value do |info|
+          reader = info[:liveness_reader]
+          next unless reader
+
+          read_any = false
           begin
-            ::Process.kill("KILL", pid)
-          rescue Errno::ESRCH
-            # already gone
+            2.times do
+              reader.read_nonblock(4096)
+              read_any = true
+            end
+          rescue IO::WaitReadable, EOFError
+            # empty / drained, or writer closed (worker exiting — reap handles it)
+          rescue IOError, Errno::EBADF
+            # reader closed by a racing reap/shutdown this tick
+            next
+          end
+
+          if read_any
+            info[:last_pipe_tick_at] = now
+            info[:pipe_seen] = true
           end
         end
-      rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Supervisor watchdog check failed: #{e.message}" }
       end
 
       def signal_children(sig)
@@ -338,12 +656,33 @@ module Pgbus
         end
       end
 
+      # Lenient bootstrap for the parent (supervisor.rb#run). A transiently
+      # unavailable DB at boot must not kill the supervisor — children
+      # crash-and-backoff until it recovers — so every StandardError (including
+      # SchemaNotReady) is reported and swallowed.
       def bootstrap_queues
         Pgbus.client.ensure_all_queues
       rescue StandardError => e
         ErrorReporter.report(e, { action: "bootstrap_queues" })
       end
 
+      # Strict bootstrap for forked children (fork_worker, fork_scheduler). A
+      # genuinely missing schema (database absent, migrations not run) surfaces
+      # as Pgbus::SchemaNotReady — log its already-actionable message once and
+      # re-raise so the child exits non-zero. schedule_restart then paces the
+      # crash loop with exponential backoff (up to RESTART_BACKOFF_MAX) instead
+      # of letting children boot and drown in downstream "relation does not
+      # exist" errors. Other StandardErrors stay reported-and-swallowed, exactly
+      # as the lenient variant handles them.
+      def bootstrap_queues!
+        Pgbus.client.ensure_all_queues
+      rescue Pgbus::SchemaNotReady => e
+        Pgbus.logger.error { "[Pgbus] #{e.message}" }
+        raise
+      rescue StandardError => e
+        ErrorReporter.report(e, { action: "bootstrap_queues" })
+      end
+
       def load_rails_app
         return unless defined?(Rails) && Rails.respond_to?(:application) && Rails.application
 
@@ -358,10 +697,39 @@ module Pgbus
         @heartbeat.start
       end
 
+      # Serve /livez and /readyz over a plain TCP server when health_port is
+      # configured. This gives orchestrators (Kubernetes) an HTTP probe surface
+      # on the supervisor itself — the process that forks and watches workers —
+      # without booting Rails or the dashboard. Disabled (nil) by default;
+      # host apps that already run Puma can mount Pgbus::Web::HealthApp instead.
+      def start_health_server
+        return unless config.health_port
+
+        @health_server = Pgbus::Web::HealthServer.new(port: config.health_port, bind: config.health_bind)
+        @health_server.start
+      end
+
       def monotonic_now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
 
+      # Close a pipe IO idempotently. nil (non-worker forks, already-scrubbed
+      # entries) and already-closed IOs are no-ops; a rare EBADF/IOError from a
+      # racing close is swallowed. FD management is single-threaded (main loop
+      # only), so the closed? check-then-close needs no lock.
+      def close_pipe(io)
+        io.close if io && !io.closed?
+      rescue IOError, Errno::EBADF
+        nil
+      end
+
+      # Close every sibling liveness reader this fork inherited from the
+      # parent's FD table. Called only inside a just-forked child, before it
+      # becomes a Worker, so the worker never holds its siblings' pipe ends.
+      def close_inherited_liveness_readers
+        @forks.each_value { |info| close_pipe(info[:liveness_reader]) }
+      end
+
       def shutdown
         # Wait for all children with timeout
         deadline = Time.now + 30
@@ -374,6 +742,11 @@ module Pgbus
         # Force kill any remaining
         signal_children("KILL") unless @forks.empty?
 
+        # Close any liveness readers still open on un-reaped children so the
+        # supervisor never leaks FDs across a restart of itself.
+        @forks.each_value { |info| close_pipe(info[:liveness_reader]) }
+
+        @health_server&.stop
         @heartbeat&.stop
         restore_signals
         Pgbus.logger.info { "[Pgbus] Supervisor stopped" }