pgbus 0.12.4 → 0.13.1

data/lib/pgbus/process/wake_pipe.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    # Fork-side receiver for supervisor-mediated wake-ups (issue #381,
+    # worker_notify_scope: :supervisor). The supervisor's NotifyHub holds the
+    # write end of a per-fork pipe; this class owns the inherited read end and
+    # a single watcher thread that translates the byte protocol into the
+    # fork's existing primitives:
+    #
+    #   W — a NOTIFY arrived for a queue this fork reads → wake_signal.notify!
+    #   H — the shared listener is healthy (connected + delivering) → the
+    #       fork may sleep up to the NOTIFY poll ceiling
+    #   P — the shared listener is degraded (dead thread, mid-reconnect,
+    #       pooler-deaf) → the fork falls back to fast polling
+    #
+    # Status starts optimistic (mirrors NotifyListener's @delivering default)
+    # so a just-forked worker isn't pinned to fast polling before the hub's
+    # first broadcast. EOF on the pipe means the supervisor is gone: mark
+    # not-delivering and let the fork run on plain polling.
+    #
+    # A nil reader (scope :fork, or the supervisor never armed the pipe)
+    # yields an inert instance: start is a no-op and delivering? is false, so
+    # wake_timeout math treats it exactly like an absent listener.
+    class WakePipe
+      WAKE = "W"
+      HEALTHY = "H"
+      DEGRADED = "P"
+
+      def initialize(reader, wake_signal:, logger: Pgbus.logger)
+        @reader = reader
+        @wake_signal = wake_signal
+        @logger = logger
+        @state_mutex = Mutex.new
+        @running = false
+        @thread = nil
+        @delivering = !reader.nil?
+      end
+
+      def delivering?
+        @state_mutex.synchronize { @delivering }
+      end
+
+      def running?
+        @state_mutex.synchronize { @running }
+      end
+
+      def start
+        return self if @reader.nil?
+
+        @state_mutex.synchronize do
+          return self if @running
+
+          @running = true
+        end
+        @thread = Thread.new { run_loop }
+        self
+      end
+
+      def stop
+        @state_mutex.synchronize do
+          return self unless @running
+
+          @running = false
+        end
+        # Closing the pipe FD interrupts the watcher's blocking readpartial —
+        # Ruby raises IOError in the blocked thread at the interpreter level.
+        # (Safe for a plain IO, unlike PG::Connection#close, which is PQfinish
+        # under a concurrent libpq call — issue #375. That constraint is about
+        # libpq, not IO.)
+        close_reader_quietly
+        @thread&.join(2)
+        @thread = nil
+        self
+      end
+
+      private
+
+      def run_loop
+        loop do
+          break unless running?
+
+          handle_bytes(@reader.readpartial(4096))
+        end
+      rescue EOFError
+        # Supervisor exited: no more wakes will ever arrive on this pipe.
+        mark_not_delivering("supervisor wake pipe reached EOF — falling back to polling")
+      rescue IOError, Errno::EBADF
+        # Reader closed under us. Expected during #stop (running? already
+        # false); anything else degrades to polling like EOF.
+        mark_not_delivering("supervisor wake pipe closed — falling back to polling") if running?
+      rescue StandardError => e
+        mark_not_delivering("supervisor wake pipe failed (#{e.class}: #{e.message}) — falling back to polling")
+      ensure
+        @state_mutex.synchronize { @running = false }
+      end
+
+      # One coalesced read may carry several bytes. Status transitions apply
+      # in order; any number of W bytes collapses into one notify! (WakeSignal
+      # coalesces concurrent notifies anyway).
+      def handle_bytes(data)
+        wake = false
+        data.each_char do |byte|
+          case byte
+          when WAKE then wake = true
+          when HEALTHY then update_delivering(true)
+          when DEGRADED then update_delivering(false)
+          end
+        end
+        @wake_signal.notify! if wake
+      end
+
+      def update_delivering(value)
+        @state_mutex.synchronize { @delivering = value }
+      end
+
+      def mark_not_delivering(message)
+        update_delivering(false)
+        @logger.warn { "[Pgbus::WakePipe] #{message}" }
+      end
+
+      def close_reader_quietly
+        @reader.close if @reader && !@reader.closed?
+      rescue IOError, Errno::EBADF
+        nil
+      end
+    end
+  end
+end

data/lib/pgbus/process/wildcard_queue_resolver.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    # Resolves a wildcard ("*") queue list to the concrete logical queue names
+    # a job worker may adopt: every pgmq.meta queue minus dead-letter queues,
+    # stream queues (issue #309/#366 — a job worker adopting one would claim
+    # durable broadcasts and DLQ-move them out of replay history), and
+    # event-subscriber queues (issue #333 — event payloads, not ActiveJob
+    # jobs), with the configured prefix stripped.
+    #
+    # Shared by Worker#resolve_wildcard_queues (per-fork adoption) and the
+    # supervisor-owned NotifyHub (issue #381 — the LISTEN union for wildcard
+    # capsules), so both sides of the wake path agree on what "*" means.
+    module WildcardQueueResolver
+      module_function
+
+      def resolve(config: Pgbus.configuration)
+        prefix = "#{config.queue_prefix}_"
+
+        # Reset first so a stream created since the last resolve is excluded.
+        Pgbus::StreamQueue.reset_cache!
+        stream_names = Pgbus::StreamQueue.known_names
+        event_names = Pgbus::EventBus::Registry.instance.event_queue_names
+
+        conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
+        conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
+            .reject { |q| q.end_with?(Pgbus::DEAD_LETTER_SUFFIX) }
+            .reject { |q| stream_names.include?(q) }
+            .reject { |q| event_names.include?(q) }
+            .map { |q| q.delete_prefix(prefix) }
+      end
+    end
+  end
+end

data/lib/pgbus/process/worker.rb

@@ -19,6 +19,11 @@ module Pgbus
       # between calls, and simulate start_notify_listener assigning the listener
       # from inside a stub (production mutates all three in the run loop).
       attr_accessor :notify_listener, :notify_retry_at, :notify_retry_backoff
+      # Supervisor-mediated wake source (issue #381, worker_notify_scope:
+      # :supervisor). Non-nil iff the supervisor forked us with a wake pipe;
+      # its presence switches the whole notify lifecycle from a fork-local
+      # NotifyListener to the shared hub. Readable as a test seam.
+      attr_reader :wake_pipe
 
       # The collaborators below (rate_counter, wake_signal, stat_buffer) and the
       # recycle clock (started_at_monotonic) accept injected seeds so tests can
@@ -30,7 +35,7 @@ module Pgbus
                      rate_counter: nil, wake_signal: nil, stat_buffer: :default,
                      notify_listener: nil, notify_retry_at: 0.0,
                      notify_retry_backoff: NOTIFY_RETRY_BASE_SECONDS,
-                     started_at_monotonic: nil)
+                     started_at_monotonic: nil, wake_pipe: nil)
         @queues = Array(queues)
         @initial_queues = @queues.dup.freeze
         @wildcard = @queues.include?("*")
@@ -98,6 +103,10 @@ module Pgbus
         # so the watchdog can detect a wedged worker even when the database (and
         # thus the Heartbeat's loop_tick_at) is unavailable.
         @liveness_pipe = liveness_pipe
+        # Supervisor wake pipe (read end, inherited across fork). When present
+        # the fork owns NO LISTEN connection: wakes and listener-health status
+        # arrive as bytes from the supervisor's NotifyHub.
+        @wake_pipe = wake_pipe ? WakePipe.new(wake_pipe, wake_signal: @wake_signal) : nil
       end
 
       def stats
@@ -150,7 +159,7 @@ module Pgbus
         setup_signals
         start_heartbeat
         resolve_wildcard_queues
-        start_notify_listener
+        start_wake_source
         @lifecycle.transition_to!(:running)
         Pgbus.logger.info do
           "[Pgbus] Worker started: queues=#{queues.join(",")} threads=#{threads} " \
@@ -404,34 +413,10 @@ module Pgbus
       def resolve_wildcard_queues
         return unless @wildcard
 
-        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
-        prefix = "#{config.queue_prefix}_"
-
-        # Stream queues share the job namespace (pgbus_<name>) but must never
-        # be adopted by a wildcard worker: a worker would claim durable
-        # broadcasts, fail to deserialize them, and DLQ-move them out of the
-        # stream's replay history. known_names includes fingerprint-matched
-        # dormant pre-registry streams (issue #366) so they are excluded even
-        # before backfill. Reset first so a stream created since the last
-        # resolve is excluded.
-        Pgbus::StreamQueue.reset_cache!
-        stream_names = Pgbus::StreamQueue.known_names
-
-        # Event-bus subscriber queues also share the job namespace (pgbus_<handler>)
-        # but carry event payloads, not ActiveJob jobs. A wildcard worker that
-        # adopts one hands the event to the executor, which fails to deserialize
-        # it and DLQ-moves it out of the consumer's reach — so an app running the
-        # event bus had to hand-maintain an explicit queue list. Exclude them,
-        # like stream queues (issue #333).
-        event_names = Pgbus::EventBus::Registry.instance.event_queue_names
-
-        conn = Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
-        all_queues = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
-        resolved = all_queues
-                   .reject { |q| q.end_with?(dlq_suffix) }
-                   .reject { |q| stream_names.include?(q) }
-                   .reject { |q| event_names.include?(q) }
-                   .map { |q| q.delete_prefix(prefix) }
+        # Exclusion rationale (streams, event queues, DLQs) lives with the
+        # shared resolver — the NotifyHub uses the same one, so the LISTEN
+        # union and the fork's adopted set can't drift (issue #381).
+        resolved = WildcardQueueResolver.resolve(config: config)
 
         if resolved.empty?
           Pgbus.logger.warn { "[Pgbus] Wildcard queue '*' resolved to no queues — falling back to default" }
@@ -627,6 +612,10 @@ module Pgbus
       end
 
       def listener_delivering?
+        # Supervisor scope: the hub's H/P broadcasts (via WakePipe) stand in
+        # for the local listener's health — same fast-poll fallback rules.
+        return @wake_pipe.delivering? if @wake_pipe
+
         @notify_listener&.running? && @notify_listener.delivering?
       end
 
@@ -642,12 +631,35 @@ module Pgbus
         config.polling_interval
       end
 
+      # :supervisor scope: the fork opens NO LISTEN connection; the WakePipe
+      # watcher is the wake source. A missing pipe under that scope means the
+      # hub failed to start — plain polling, NEVER a local listener, or a hub
+      # outage would balloon the host back to one direct connection per fork
+      # (the exact budget the scope exists to protect). :fork scope: the
+      # fork-local NotifyListener, exactly as before.
+      def start_wake_source
+        return @wake_pipe.start if @wake_pipe
+
+        start_notify_listener if local_listener_scope?
+      end
+
+      # Local NotifyListener lifecycle (start + self-heal) is allowed only
+      # under :fork scope.
+      def local_listener_scope?
+        config.worker_notify_scope == :fork
+      end
+
+      def stop_wake_source
+        @wake_pipe&.stop
+        @notify_listener&.stop
+      end
+
       def start_notify_listener
         return unless notify_wakeup?
 
         @notify_listener = NotifyListener.new(
           physical_queues: physical_queue_names,
-          on_wake: -> { @wake_signal.notify! },
+          on_wake: ->(_channel) { @wake_signal.notify! },
           connection_options: config.worker_notify_connection_options,
           health_check_ms: (config.polling_interval * 1000).to_i.clamp(250, 5_000),
           logger: Pgbus.logger
@@ -666,6 +678,12 @@ module Pgbus
       # its queue subscription reconciled (wildcard workers) and the backoff
       # reset; a still-failing restart doubles the backoff up to the cap.
       def ensure_notify_listener
+        # Supervisor scope: listener self-healing is the hub's job (it runs
+        # once per host in the supervisor's monitor tick), and a pipe-less
+        # fork under that scope must stay on plain polling — self-healing a
+        # listener it was never allowed to start would leak a connection.
+        return if @wake_pipe
+        return unless local_listener_scope?
         return unless notify_wakeup?
         return if @notify_listener&.running?
         return if monotonic_now < @notify_retry_at
@@ -706,13 +724,18 @@ module Pgbus
         Pgbus.logger.warn { "[Pgbus] NotifyListener queue sync failed: #{e.class}: #{e.message}" }
       end
 
+      # Through config.queue_name (not raw concatenation) so a logical name
+      # needing normalization (e.g. "orders-handler" → "orders_handler")
+      # yields the SAME physical name the queue table was created with — the
+      # NOTIFY channel derives from the table name, so a raw concat would
+      # LISTEN on a channel that never fires. Keeps the fork-local (:fork
+      # scope) channels identical to the NotifyHub's (issue #381 review).
       def physical_queue_names
-        prefix = "#{config.queue_prefix}_"
-        queues.map { |q| "#{prefix}#{q}" }
+        queues.map { |q| config.queue_name(q) }
       end
 
       def channel_to_physical(channel)
-        channel.delete_prefix(NotifyListener::CHANNEL_PREFIX).delete_suffix(NotifyListener::CHANNEL_SUFFIX)
+        NotifyListener.physical_for(channel)
       end
 
       def start_heartbeat
@@ -767,7 +790,7 @@ module Pgbus
 
       def shutdown
         Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
-        @notify_listener&.stop
+        stop_wake_source
         @pool.shutdown
         @pool.wait_for_termination(30)
         @stat_buffer&.stop

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.12.4"
+  VERSION = "0.13.1"
 end

data/lib/pgbus/web/streamer/failover_listener.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    module Streamer
+      # The worker-side seam between the two listening modes (issue #382):
+      # starts on the master hub (HubClient) and fails over — once, one-way —
+      # to a per-worker Listener when the hub transport dies (master gone,
+      # ack deadline, eviction). The Dispatcher/Instance consume the same
+      # ensure_listening/remove_listening/stop surface either way and never
+      # learn which mode is active.
+      #
+      # Fallback direction is settled on #382: per-worker listener, not
+      # poll-only — ephemeral broadcasts have no polling equivalent (their
+      # payload exists only in the NOTIFY), so an outage trades connections
+      # for unchanged semantics. Once fallen back, the worker stays local
+      # until it recycles; no flap-back.
+      #
+      # The subscription set is recorded here so failover can rebuild the
+      # exact LISTEN set on the fresh local connection before anything else
+      # relies on it. ensure_listening NEVER raises to the dispatcher: on a
+      # double failure (hub dead AND local build failing — e.g. DB down) it
+      # logs and returns nil, matching the Listener's own ack-timeout
+      # contract, which the dispatcher already tolerates.
+      class FailoverListener
+        def initialize(hub_client:, local_listener_factory:, logger: Pgbus.logger)
+          @hub_client = hub_client
+          @local_listener_factory = local_listener_factory
+          @logger = logger
+          # @state_mutex guards the cheap shared state (@subscriptions, @impl,
+          # @failed_over) and is only ever held for constant-time work — the
+          # dispatcher's ensure/remove path must never wait behind a failover
+          # build. @failover_mutex serializes the (blocking) build + replay:
+          # a fresh PG connect + N re-LISTEN acks can stall for seconds when
+          # the trigger IS a database problem (review on #384).
+          @state_mutex = Mutex.new
+          @failover_mutex = Mutex.new
+          @subscriptions = Set.new
+          @impl = hub_client
+          @failed_over = false
+        end
+
+        # Interface parity with Listener for Instance#start: the hub client
+        # connected at construction and the fallback starts itself on swap.
+        def start
+          self
+        end
+
+        def ensure_listening(queue)
+          @state_mutex.synchronize { @subscriptions.add(queue) }
+          current_impl.ensure_listening(queue)
+        rescue HubClient::HubUnavailableError
+          fail_over!
+          begin
+            current_impl.ensure_listening(queue)
+          rescue HubClient::HubUnavailableError
+            # fail_over! itself failed (factory raised) and @impl is still the
+            # dead client — reported there; honor the nil-on-timeout contract.
+            nil
+          end
+        end
+
+        def remove_listening(queue)
+          @state_mutex.synchronize { @subscriptions.delete(queue) }
+          current_impl.remove_listening(queue)
+        rescue HubClient::HubUnavailableError => e
+          @logger.debug do
+            "[Pgbus::Streamer::FailoverListener] remove_listening on a dead hub client " \
+              "(#{e.message}) — ignoring, unlisten GC is best-effort"
+          end
+          nil
+        end
+
+        # Idempotent, callable from the client's on_failure (reader thread)
+        # and from a synchronous ensure failure (dispatcher thread).
+        # @failover_mutex serializes concurrent callers — the second blocks
+        # until the first finishes and then no-ops, so a synchronous retry
+        # after fail_over! always lands on the swapped-in local listener.
+        # The blocking build + replay runs OUTSIDE @state_mutex so concurrent
+        # ensure/remove/stop calls never stall behind it.
+        def fail_over!
+          local = nil
+          @failover_mutex.synchronize do
+            return if @state_mutex.synchronize { @failed_over }
+
+            local = @local_listener_factory.call
+            @state_mutex.synchronize { @subscriptions.dup }.each { |q| local.ensure_listening(q) }
+            # Subscriptions recorded between the snapshot and this swap arrive
+            # via their own retried ensure_listening call on the new impl.
+            @state_mutex.synchronize do
+              @impl = local
+              @failed_over = true
+            end
+            # Ownership transferred to @impl — the rescue must not stop it.
+            local = nil
+          end
+        rescue StandardError => e
+          # A listener the factory STARTED but that never swapped in (the
+          # replay raised) would otherwise leak its thread and LISTEN
+          # connection alongside the dead hub client.
+          begin
+            local&.stop
+          rescue StandardError
+            nil
+          end
+          # Hub dead AND the local listener can't be built (DB down, config
+          # broken). Mark failed-over so callers stop rebuilding; @impl stays
+          # on the dead client — every ensure_listening resolves nil and the
+          # dispatcher rides its existing timeout tolerance until the worker
+          # recycles.
+          @state_mutex.synchronize { @failed_over = true }
+          @logger.error do
+            "[Pgbus::Streamer::FailoverListener] fallback listener failed to build " \
+              "(#{e.class}: #{e.message}) — streams degraded until this worker recycles"
+          end
+        end
+
+        def stop
+          current_impl.stop
+        end
+
+        private
+
+        def current_impl
+          @state_mutex.synchronize { @impl }
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/streamer/hub_client.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module Pgbus
+  module Web
+    module Streamer
+      # Worker-side client for the MasterHub (issue #382). Presents the same
+      # surface the Dispatcher consumes from a Listener — synchronous
+      # `ensure_listening` (the no-lost-broadcast ack contract, now crossing
+      # the process boundary), async `remove_listening` — while wakes arrive
+      # as HubProtocol frames and are re-materialized into the worker's
+      # dispatch queue as WakeMessages.
+      #
+      # Failure model: this class never retries. Connect refusal, an ack
+      # deadline, or transport EOF (master died / eviction) marks the client
+      # dead, fails every pending sub, and fires +on_failure+ exactly once —
+      # the FailoverListener's cue to swap in a per-worker Listener. One-way:
+      # once a worker has fallen back it stays local until it recycles
+      # (settled on #382 — no flap-back complexity).
+      class HubClient
+        class HubUnavailableError < StandardError; end
+
+        # Optimistic before the first status broadcast, mirroring WakePipe /
+        # NotifyListener: a just-connected worker isn't treated as degraded
+        # before the hub has said anything.
+        def initialize(socket_path:, dispatch_queue:, ack_timeout: 2.0,
+                       on_failure: nil, logger: Pgbus.logger)
+          @socket_path = socket_path
+          @dispatch_queue = dispatch_queue
+          @ack_timeout = ack_timeout
+          @on_failure = on_failure
+          @logger = logger
+          @write_mutex = Mutex.new
+          @ack_mutex = Mutex.new
+          @pending_acks = Hash.new { |h, k| h[k] = [] }
+          @hub_healthy = true
+          @dead = false
+          @stopping = false
+          @sock = nil
+          @reader = nil
+        end
+
+        def connect
+          @sock = UNIXSocket.new(@socket_path)
+          @reader = Thread.new { reader_loop }
+          self
+        rescue SystemCallError, IOError, ArgumentError, ThreadError => e
+          # ArgumentError: a socket path over the platform sun_path limit;
+          # IOError: a path that exists but is not a socket; ThreadError: the
+          # reader thread could not spawn. All must fall back exactly like a
+          # refused connect, never abort worker boot — and never leak the
+          # half-opened socket.
+          close_quietly(@sock)
+          @sock = nil
+          raise HubUnavailableError, "cannot reach master hub at #{@socket_path}: #{e.class}: #{e.message}"
+        end
+
+        def hub_healthy?
+          @hub_healthy
+        end
+
+        def dead?
+          @dead
+        end
+
+        # Synchronous, bounded: returns :done once the master has confirmed
+        # LISTEN is active for +queue+. Raises HubUnavailableError on a dead
+        # transport or an expired ack deadline (which also kills the
+        # transport — a hub that can't ack in time can't be trusted with the
+        # no-lost-broadcast contract either).
+        def ensure_listening(queue)
+          raise HubUnavailableError, "master hub transport is dead" if @dead
+
+          waiter = Queue.new
+          @ack_mutex.synchronize { @pending_acks[queue] << waiter }
+          write_frame({ "t" => "sub", "q" => queue })
+
+          result = waiter.pop(timeout: @ack_timeout)
+          if result.nil?
+            discard_waiter(queue, waiter)
+            mark_dead("sub ack for #{queue} not received within #{@ack_timeout}s")
+            raise HubUnavailableError, "master hub ack timeout for #{queue}"
+          end
+          raise HubUnavailableError, "master hub died while awaiting ack for #{queue}" if result == :dead
+
+          :done
+        end
+
+        # Lazy GC, fire-and-forget — no correctness path waits on UNLISTEN
+        # (mirrors Listener#remove_listening). A dead transport is a no-op:
+        # the master's EOF cleanup already released this worker's refs.
+        def remove_listening(queue)
+          return if @dead
+
+          write_frame({ "t" => "unsub", "q" => queue })
+        rescue HubUnavailableError
+          nil
+        end
+
+        def stop
+          @stopping = true
+          close_quietly(@sock)
+          @reader&.join(2)
+          @reader = nil
+          self
+        end
+
+        private
+
+        def reader_loop
+          loop do
+            frame = HubProtocol.read_frame(@sock)
+            break if frame.nil?
+
+            handle_frame(frame)
+          end
+          mark_dead("master hub closed the transport") unless @stopping
+        rescue HubProtocol::ProtocolError => e
+          mark_dead("master hub protocol error: #{e.message}") unless @stopping
+        rescue IOError, Errno::EBADF, Errno::ECONNRESET
+          mark_dead("master hub transport error") unless @stopping
+        rescue StandardError => e
+          # The reader thread is the ONLY detector of hub death — an
+          # unexpected error must not let it exit with the client still
+          # reporting healthy, or the worker goes silently deaf.
+          mark_dead("master hub reader crashed: #{e.class}: #{e.message}") unless @stopping
+        end
+
+        def handle_frame(frame)
+          case frame["t"]
+          when "wake"
+            @dispatch_queue << Listener::WakeMessage.new(queue_name: frame["q"], payload: frame["p"])
+          when "ack"
+            @ack_mutex.synchronize { @pending_acks[frame["q"]].shift }&.push(:ack)
+          when "status"
+            @hub_healthy = frame["healthy"]
+          else
+            @logger.warn { "[Pgbus::Streamer::HubClient] unknown frame from master: #{frame["t"].inspect}" }
+          end
+        end
+
+        # Frames must never interleave — all writes go through one mutex
+        # (writers: dispatcher thread via ensure/remove; no writer thread
+        # needed client-side, sub/unsub frames are tiny). Bounded: a master
+        # that stopped draining its input would otherwise block this write
+        # forever, and the ack deadline only starts ticking AFTER the write
+        # returns — so a stalled write is itself a failover trigger.
+        def write_frame(message)
+          data = HubProtocol.encode(message)
+          deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @ack_timeout
+          @write_mutex.synchronize do
+            until data.empty?
+              begin
+                written = @sock.write_nonblock(data)
+                data = data.byteslice(written..)
+              rescue IO::WaitWritable
+                remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+                raise Errno::ETIMEDOUT, "write stalled" if remaining <= 0 || !@sock.wait_writable(remaining)
+              end
+            end
+          end
+        rescue IOError, Errno::EPIPE, Errno::EBADF, Errno::ECONNRESET, Errno::ETIMEDOUT => e
+          mark_dead("write to master hub failed: #{e.class}")
+          raise HubUnavailableError, "master hub transport is dead"
+        end
+
+        # Idempotent: first caller flips @dead, fails every waiter, fires
+        # on_failure once. Reachable from the reader (EOF/protocol error) and
+        # from ack timeouts / failed writes on caller threads.
+        def mark_dead(reason)
+          waiters = @ack_mutex.synchronize do
+            return if @dead
+
+            @dead = true
+            drained = @pending_acks.values.flatten
+            @pending_acks.clear
+            drained
+          end
+          @hub_healthy = false
+          waiters.each { |w| w << :dead }
+          close_quietly(@sock)
+          @logger.warn { "[Pgbus::Streamer::HubClient] #{reason} — falling back to a per-worker listener" }
+          @on_failure&.call
+        end
+
+        def discard_waiter(queue, waiter)
+          @ack_mutex.synchronize { @pending_acks[queue].delete(waiter) }
+        end
+
+        def close_quietly(io)
+          io.close if io && !io.closed?
+        rescue IOError, Errno::EBADF
+          nil
+        end
+      end
+    end
+  end
+end