pgbus 0.13.0 → 0.13.2

data/lib/pgbus/health_probe.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module Pgbus
+  # Dependency-free readiness probe for container HEALTHCHECKs (issue #386).
+  #
+  # exe/pgbus-health loads this file via require_relative and nothing else:
+  # a docker HEALTHCHECK runs the probe every few seconds, so it must never
+  # drag in Bundler, Zeitwerk, Rails, or the rest of the gem. Only Ruby's
+  # bundled socket library is allowed here.
+  #
+  #   healthcheck:
+  #     cmd: bin/pgbus-health            # port from PGBUS_HEALTH_PORT
+  #     cmd: bin/pgbus-health --port 9394 --path /livez
+  #
+  # Exit codes: 0 healthy (HTTP 2xx), 1 unhealthy (non-2xx, refused, timeout),
+  # 2 usage error (no/invalid port).
+  class HealthProbe
+    EXIT_OK = 0
+    EXIT_UNHEALTHY = 1
+    EXIT_USAGE = 2
+
+    DEFAULT_PATH = "/readyz"
+    DEFAULT_TIMEOUT = 2.0
+    HOST = "127.0.0.1"
+
+    USAGE = "usage: pgbus-health [--port PORT] [--path PATH] [--timeout SECONDS]\n       " \
+            "port falls back to the PGBUS_HEALTH_PORT environment variable\n"
+
+    def self.run(argv, env: ENV, out: $stdout, err: $stderr)
+      new(argv, env: env, out: out, err: err).run
+    end
+
+    def initialize(argv, env: ENV, out: $stdout, err: $stderr)
+      @out = out
+      @err = err
+      @path = DEFAULT_PATH
+      @timeout = DEFAULT_TIMEOUT
+      @port = env["PGBUS_HEALTH_PORT"]
+      @usage_error = false
+      parse(argv)
+    end
+
+    def run
+      return usage_failure if @usage_error
+
+      port = Integer(@port, exception: false)
+      # Out-of-range ports would reach Socket.tcp and raise SocketError — a
+      # backtrace where a HEALTHCHECK needs a deterministic exit code.
+      return usage_failure unless port&.between?(1, 65_535)
+
+      probe(port)
+    end
+
+    private
+
+    # Hand-rolled flag parsing: three flags do not justify optparse in a
+    # script whose reason to exist is loading nothing.
+    def parse(argv)
+      args = argv.dup
+      until args.empty?
+        flag = args.shift
+        value = args.shift
+        return @usage_error = true if value.nil?
+
+        case flag
+        when "--port" then @port = value
+        when "--path" then @path = value
+        when "--timeout"
+          # A typo'd timeout must be a usage error, not `to_f`'s silent 0.0 —
+          # a zero deadline reports the container unhealthy on every probe.
+          timeout = Float(value, exception: false)
+          return @usage_error = true unless timeout&.positive?
+
+          @timeout = timeout
+        else
+          return @usage_error = true
+        end
+      end
+    end
+
+    def usage_failure
+      @err.write(USAGE)
+      EXIT_USAGE
+    end
+
+    def probe(port)
+      status = http_status(port)
+      healthy = status&.between?(200, 299)
+      @out.write("pgbus-health: #{@path} -> #{status || "no response"}\n")
+      healthy ? EXIT_OK : EXIT_UNHEALTHY
+    rescue SystemCallError, IOError, SocketError => e
+      @err.write("pgbus-health: #{@path} -> #{e.class}: #{e.message}\n")
+      EXIT_UNHEALTHY
+    end
+
+    # Minimal HTTP/1.0 exchange: send the request, read just the status line.
+    # The deadline covers connect and read together.
+    def http_status(port)
+      deadline = monotonic_now + @timeout
+      Socket.tcp(HOST, port, connect_timeout: @timeout) do |sock|
+        sock.write("GET #{@path} HTTP/1.0\r\nHost: #{HOST}\r\nConnection: close\r\n\r\n")
+        line = read_status_line(sock, deadline)
+        code = line&.split(" ", 3)&.fetch(1, nil)
+        Integer(code, exception: false)
+      end
+    end
+
+    def read_status_line(sock, deadline)
+      buffer = +""
+      until buffer.include?("\n")
+        remaining = deadline - monotonic_now
+        return nil if remaining <= 0 || !sock.wait_readable(remaining)
+
+        chunk = sock.read_nonblock(1024, exception: false)
+        return nil if chunk.nil? # EOF before a full status line
+        next if chunk == :wait_readable # spurious wakeup — re-wait on the deadline
+
+        buffer << chunk
+      end
+      buffer[/\A[^\r\n]*/]
+    end
+
+    # ::Process, not Process — inside the Pgbus namespace the bare constant
+    # resolves to Pgbus::Process (the process model), which is also why this
+    # file must never be renamed into that namespace.
+    def monotonic_now
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/pgbus/process/consumer.rb

@@ -480,7 +480,10 @@ module Pgbus
       def shutdown
         stop_wake_source
         @pool.shutdown
-        @pool.wait_for_termination(30)
+        # The consumer has no quiesce-gated drain loop like Worker's, so this
+        # wait IS its drain window — bound it by the same knob workers use
+        # instead of a hardcoded 30s (issue #386).
+        @pool.wait_for_termination(config.drain_timeout)
         @stat_buffer&.stop
         @heartbeat&.stop
         restore_signals

data/lib/pgbus/process/readiness_snapshot.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    # Immutable container-local readiness state, published by the supervisor
+    # (one atomic swap per monitor pass) and read by the standalone health
+    # server's accept thread — the immutability is what makes the cross-thread
+    # handoff safe without a lock (issue #386).
+    #
+    # `expected` is the child count forked by boot_processes; `live` is the
+    # current fork-table size. A child sitting in crash-restart backoff keeps
+    # `live < expected`, which is exactly the signal a rolling deploy's health
+    # gate needs to fail on: the replacement container never goes ready, and
+    # the orchestrator keeps the old container running.
+    ReadinessSnapshot = Data.define(:booted, :shutting_down, :expected, :live) do
+      def ready?
+        booted && !shutting_down && live >= expected
+      end
+
+      # DRAINING wins over BOOTING: a supervisor told to stop mid-boot is
+      # leaving, not arriving, and must never look like it will become ready.
+      def status
+        return "DRAINING" if shutting_down
+        return "BOOTING" unless booted
+
+        ready? ? "OK" : "DEGRADED"
+      end
+    end
+  end
+end

data/lib/pgbus/process/supervisor.rb

@@ -44,6 +44,16 @@ module Pgbus
         @pending_restarts = pending_restarts
         @crash_counts = Hash.new(0)
         @notify_hub = notify_hub
+        @intended_children = 0
+        @readiness = Concurrent::AtomicReference.new(
+          ReadinessSnapshot.new(booted: false, shutting_down: shutting_down, expected: 0, live: forks.size)
+        )
+      end
+
+      # The current container-local readiness state. Safe to call from any
+      # thread (the health server's accept thread reads it per probe).
+      def readiness_snapshot
+        @readiness.get
       end
 
       def shutting_down?
@@ -97,6 +107,7 @@ module Pgbus
         start_notify_hub
 
         boot_processes
+        mark_booted
         monitor_loop
       ensure
         shutdown
@@ -105,17 +116,51 @@ module Pgbus
       def graceful_shutdown
         Pgbus.logger.info { "[Pgbus] Supervisor: graceful shutdown requested" }
         @shutting_down = true
+        refresh_readiness
         signal_children("TERM")
       end
 
       def immediate_shutdown
         Pgbus.logger.warn { "[Pgbus] Supervisor: immediate shutdown requested" }
         @shutting_down = true
+        refresh_readiness
         signal_children("QUIT")
       end
 
       private
 
+      # Boot is complete: connection verified, queues bootstrapped, every
+      # configured child fork ATTEMPTED. The baseline is the larger of the
+      # intended-attempt count and the fork-table size: a boot-time fork
+      # failure (EAGAIN/ENOMEM, logged-and-swallowed in fork_*) leaves
+      # intended > live, so the readiness gate reports DEGRADED instead of
+      # blessing a container that is missing workers. Roles that legitimately
+      # declined to boot (scheduler with no recurring tasks) never reach a
+      # fork_* method and are counted by neither side.
+      def mark_booted
+        @booted = true
+        @expected_children = [@intended_children, @forks.size].max
+        refresh_readiness
+      end
+
+      # Count a child the configuration intends this boot to run. Called at
+      # the top of every fork_* method — before the fork can fail — and only
+      # pre-boot, so restart_child's re-forks never inflate the baseline.
+      def note_intended_child
+        @intended_children += 1 unless @booted
+      end
+
+      # Publish a fresh snapshot; the swapped-in Data is immutable, so the
+      # health server's accept thread always reads a consistent state.
+      def refresh_readiness
+        @readiness.set(
+          ReadinessSnapshot.new(
+            booted: !!@booted, shutting_down: @shutting_down,
+            expected: @expected_children || 0, live: @forks.size
+          )
+        )
+      end
+
       # Log a single boot diagnostics banner: the settings that actually
       # determine whether this deployment works. One consecutive block of
       # "[Pgbus] boot:"-prefixed info lines so it reads cleanly under both the
@@ -253,6 +298,7 @@ module Pgbus
       end
 
       def fork_worker(worker_config, slot: nil)
+        note_intended_child
         queues = worker_config[:queues] || [config.default_queue]
         threads = worker_config[:threads] || 5
         single_active = worker_config[:single_active_consumer] || false
@@ -337,6 +383,7 @@ module Pgbus
       end
 
       def fork_dispatcher
+        note_intended_child
         pid = fork do
           restore_signals
           setup_child_process
@@ -364,6 +411,7 @@ module Pgbus
       end
 
       def fork_scheduler
+        note_intended_child
         pid = fork do
           restore_signals
           setup_child_process
@@ -427,6 +475,7 @@ module Pgbus
       end
 
       def fork_consumer(consumer_config, slot: nil)
+        note_intended_child
         # Array() so a consumer entry without :topics can't NoMethodError the
         # supervisor on the topics.join log lines below.
         topics = Array(consumer_config[:topics])
@@ -504,6 +553,7 @@ module Pgbus
       end
 
       def fork_outbox_poller
+        note_intended_child
         pid = fork do
           restore_signals
           setup_child_process
@@ -537,6 +587,9 @@ module Pgbus
             # refresh, and fork status broadcast (issue #381).
             @notify_hub&.tick
           end
+          # After reap + restarts so a clean recycle (reaped and re-forked in
+          # the same pass) never dips the published live count (issue #386).
+          refresh_readiness
           interruptible_sleep(FORK_WAIT)
         end
       end
@@ -821,7 +874,12 @@ module Pgbus
       def start_health_server
         return unless config.health_port
 
-        @health_server = Pgbus::Web::HealthServer.new(port: config.health_port, bind: config.health_bind)
+        # The standalone server answers /readyz from THIS supervisor's
+        # container-local snapshot — a rolling deploy's health gate must
+        # measure the new container, not the fleet-wide verdict a sibling
+        # container's workers can satisfy (issue #386).
+        app = Pgbus::Web::HealthApp.new(local_readiness: -> { readiness_snapshot })
+        @health_server = Pgbus::Web::HealthServer.new(port: config.health_port, bind: config.health_bind, app: app)
         @health_server.start
       end
 
@@ -876,8 +934,11 @@ module Pgbus
       end
 
       def shutdown
-        # Wait for all children with timeout
-        deadline = Time.now + 30
+        # Wait for children to drain and exit, bounded by config.shutdown_timeout
+        # (default drain_timeout + 5) so raising the drain window can never
+        # mean SIGKILLing workers mid-drain. An orchestrator's stop grace
+        # period should exceed this value (issue #386).
+        deadline = Time.now + config.shutdown_timeout
 
         until @forks.empty? || Time.now > deadline
           reap_children

data/lib/pgbus/process/worker.rb

@@ -155,6 +155,12 @@ module Pgbus
       NOTIFY_RETRY_BASE_SECONDS = 5
       NOTIFY_RETRY_MAX_SECONDS = 300
 
+      # Residual pool wait in #shutdown, AFTER the drain loop already spent up
+      # to config.drain_timeout on in-flight jobs. Short by design: a job still
+      # running has proven it won't finish, and this wait competes with the
+      # supervisor's shutdown_timeout deadline (issue #386).
+      POOL_TERMINATION_WAIT = 5
+
       def run
         setup_signals
         start_heartbeat
@@ -175,9 +181,9 @@ module Pgbus
 
           break if @lifecycle.stopped?
           # quiesced? (all slots free), not idle? (any slot free) — exiting
-          # with work still in flight abandons those jobs to the 30s
-          # wait_for_termination timeout in shutdown. Bounded by
-          # config.drain_timeout so a stuck job can't wedge the loop forever.
+          # with work still in flight abandons those jobs to shutdown's short
+          # POOL_TERMINATION_WAIT residual. Bounded by config.drain_timeout so
+          # a stuck job can't wedge the loop forever.
           break if @lifecycle.draining? && (@pool.quiesced? || drain_deadline_exceeded?)
 
           claim_and_execute if @lifecycle.can_process?
@@ -792,7 +798,11 @@ module Pgbus
         Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
         stop_wake_source
         @pool.shutdown
-        @pool.wait_for_termination(30)
+        # Residual wait only: the drain loop already waited up to
+        # config.drain_timeout for in-flight jobs. A job still running here has
+        # proven it won't finish; waiting another full window would push the
+        # worker past the supervisor's shutdown_timeout deadline (issue #386).
+        @pool.wait_for_termination(POOL_TERMINATION_WAIT)
         @stat_buffer&.stop
         @queue_lock&.unlock_all
         @heartbeat&.stop

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.13.0"
+  VERSION = "0.13.2"
 end

data/lib/pgbus/web/health_app.rb

@@ -49,8 +49,15 @@ module Pgbus
       # @param data_source [Pgbus::Web::DataSource, nil] read layer for /readyz.
       #   nil (the default) builds a fresh DataSource per readiness check, which
       #   avoids serving stale metrics from a long-lived app's memoized instance.
-      def initialize(data_source: nil)
+      # @param local_readiness [#call, nil] when set, /readyz answers from this
+      #   callable's {Process::ReadinessSnapshot} instead of the cluster-wide
+      #   analyzer — the supervisor's standalone HealthServer passes its own
+      #   snapshot so a rolling deploy's health gate measures THIS container,
+      #   not the fleet (issue #386). The Rails-mounted app leaves it nil and
+      #   keeps the cluster verdict.
+      def initialize(data_source: nil, local_readiness: nil)
         @data_source = data_source
+        @local_readiness = local_readiness
       end
 
       def call(env)
@@ -69,6 +76,8 @@ module Pgbus
       end
 
       def readyz
+        return local_readyz if @local_readiness
+
         # HealthAnalyzer lives in the MCP namespace, which is excluded from
         # Zeitwerk (its *tools* subclass the optional `mcp` gem). The analyzer
         # itself has no gem dependency, so require just that one file — the
@@ -83,6 +92,19 @@ module Pgbus
         [503, JSON_HEADERS.dup, [{ status: "ERROR", error: e.message }.to_json]]
       end
 
+      # Container-local readiness: no database, no analyzer — just the
+      # supervisor's published snapshot. The error path mirrors the cluster
+      # readyz: 503 ERROR, logged, never swallowed.
+      def local_readyz
+        snapshot = @local_readiness.call
+        status = snapshot.ready? ? 200 : 503
+        body = { status: snapshot.status, expected: snapshot.expected, live: snapshot.live }
+        [status, JSON_HEADERS.dup, [body.to_json]]
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus::Web::HealthApp] local readiness check failed: #{e.class}: #{e.message}" }
+        [503, JSON_HEADERS.dup, [{ status: "ERROR", error: e.message }.to_json]]
+      end
+
       # Reuse an injected DataSource (tests, an app that wants one shared
       # instance); otherwise build a fresh one each check so per-instance
       # memoization can never serve stale queue/process metrics.

data/lib/pgbus/web/streamer/failover_listener.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    module Streamer
+      # The worker-side seam between the two listening modes (issue #382):
+      # starts on the master hub (HubClient) and fails over — once, one-way —
+      # to a per-worker Listener when the hub transport dies (master gone,
+      # ack deadline, eviction). The Dispatcher/Instance consume the same
+      # ensure_listening/remove_listening/stop surface either way and never
+      # learn which mode is active.
+      #
+      # Fallback direction is settled on #382: per-worker listener, not
+      # poll-only — ephemeral broadcasts have no polling equivalent (their
+      # payload exists only in the NOTIFY), so an outage trades connections
+      # for unchanged semantics. Once fallen back, the worker stays local
+      # until it recycles; no flap-back.
+      #
+      # The subscription set is recorded here so failover can rebuild the
+      # exact LISTEN set on the fresh local connection before anything else
+      # relies on it. ensure_listening NEVER raises to the dispatcher: on a
+      # double failure (hub dead AND local build failing — e.g. DB down) it
+      # logs and returns nil, matching the Listener's own ack-timeout
+      # contract, which the dispatcher already tolerates.
+      class FailoverListener
+        def initialize(hub_client:, local_listener_factory:, logger: Pgbus.logger)
+          @hub_client = hub_client
+          @local_listener_factory = local_listener_factory
+          @logger = logger
+          # @state_mutex guards the cheap shared state (@subscriptions, @impl,
+          # @failed_over) and is only ever held for constant-time work — the
+          # dispatcher's ensure/remove path must never wait behind a failover
+          # build. @failover_mutex serializes the (blocking) build + replay:
+          # a fresh PG connect + N re-LISTEN acks can stall for seconds when
+          # the trigger IS a database problem (review on #384).
+          @state_mutex = Mutex.new
+          @failover_mutex = Mutex.new
+          @subscriptions = Set.new
+          @impl = hub_client
+          @failed_over = false
+        end
+
+        # Interface parity with Listener for Instance#start: the hub client
+        # connected at construction and the fallback starts itself on swap.
+        def start
+          self
+        end
+
+        def ensure_listening(queue)
+          @state_mutex.synchronize { @subscriptions.add(queue) }
+          current_impl.ensure_listening(queue)
+        rescue HubClient::HubUnavailableError
+          fail_over!
+          begin
+            current_impl.ensure_listening(queue)
+          rescue HubClient::HubUnavailableError
+            # fail_over! itself failed (factory raised) and @impl is still the
+            # dead client — reported there; honor the nil-on-timeout contract.
+            nil
+          end
+        end
+
+        def remove_listening(queue)
+          @state_mutex.synchronize { @subscriptions.delete(queue) }
+          current_impl.remove_listening(queue)
+        rescue HubClient::HubUnavailableError => e
+          @logger.debug do
+            "[Pgbus::Streamer::FailoverListener] remove_listening on a dead hub client " \
+              "(#{e.message}) — ignoring, unlisten GC is best-effort"
+          end
+          nil
+        end
+
+        # Idempotent, callable from the client's on_failure (reader thread)
+        # and from a synchronous ensure failure (dispatcher thread).
+        # @failover_mutex serializes concurrent callers — the second blocks
+        # until the first finishes and then no-ops, so a synchronous retry
+        # after fail_over! always lands on the swapped-in local listener.
+        # The blocking build + replay runs OUTSIDE @state_mutex so concurrent
+        # ensure/remove/stop calls never stall behind it.
+        def fail_over!
+          local = nil
+          @failover_mutex.synchronize do
+            return if @state_mutex.synchronize { @failed_over }
+
+            local = @local_listener_factory.call
+            @state_mutex.synchronize { @subscriptions.dup }.each { |q| local.ensure_listening(q) }
+            # Subscriptions recorded between the snapshot and this swap arrive
+            # via their own retried ensure_listening call on the new impl.
+            @state_mutex.synchronize do
+              @impl = local
+              @failed_over = true
+            end
+            # Ownership transferred to @impl — the rescue must not stop it.
+            local = nil
+          end
+        rescue StandardError => e
+          # A listener the factory STARTED but that never swapped in (the
+          # replay raised) would otherwise leak its thread and LISTEN
+          # connection alongside the dead hub client.
+          begin
+            local&.stop
+          rescue StandardError
+            nil
+          end
+          # Hub dead AND the local listener can't be built (DB down, config
+          # broken). Mark failed-over so callers stop rebuilding; @impl stays
+          # on the dead client — every ensure_listening resolves nil and the
+          # dispatcher rides its existing timeout tolerance until the worker
+          # recycles.
+          @state_mutex.synchronize { @failed_over = true }
+          @logger.error do
+            "[Pgbus::Streamer::FailoverListener] fallback listener failed to build " \
+              "(#{e.class}: #{e.message}) — streams degraded until this worker recycles"
+          end
+        end
+
+        def stop
+          current_impl.stop
+        end
+
+        private
+
+        def current_impl
+          @state_mutex.synchronize { @impl }
+        end
+      end
+    end
+  end
+end