pgbus 0.13.0 → 0.13.2

data/lib/pgbus/web/streamer/hub_client.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module Pgbus
+  module Web
+    module Streamer
+      # Worker-side client for the MasterHub (issue #382). Presents the same
+      # surface the Dispatcher consumes from a Listener — synchronous
+      # `ensure_listening` (the no-lost-broadcast ack contract, now crossing
+      # the process boundary), async `remove_listening` — while wakes arrive
+      # as HubProtocol frames and are re-materialized into the worker's
+      # dispatch queue as WakeMessages.
+      #
+      # Failure model: this class never retries. Connect refusal, an ack
+      # deadline, or transport EOF (master died / eviction) marks the client
+      # dead, fails every pending sub, and fires +on_failure+ exactly once —
+      # the FailoverListener's cue to swap in a per-worker Listener. One-way:
+      # once a worker has fallen back it stays local until it recycles
+      # (settled on #382 — no flap-back complexity).
+      class HubClient
+        class HubUnavailableError < StandardError; end
+
+        # Optimistic before the first status broadcast, mirroring WakePipe /
+        # NotifyListener: a just-connected worker isn't treated as degraded
+        # before the hub has said anything.
+        def initialize(socket_path:, dispatch_queue:, ack_timeout: 2.0,
+                       on_failure: nil, logger: Pgbus.logger)
+          @socket_path = socket_path
+          @dispatch_queue = dispatch_queue
+          @ack_timeout = ack_timeout
+          @on_failure = on_failure
+          @logger = logger
+          @write_mutex = Mutex.new
+          @ack_mutex = Mutex.new
+          @pending_acks = Hash.new { |h, k| h[k] = [] }
+          @hub_healthy = true
+          @dead = false
+          @stopping = false
+          @sock = nil
+          @reader = nil
+        end
+
+        def connect
+          @sock = UNIXSocket.new(@socket_path)
+          @reader = Thread.new { reader_loop }
+          self
+        rescue SystemCallError, IOError, ArgumentError, ThreadError => e
+          # ArgumentError: a socket path over the platform sun_path limit;
+          # IOError: a path that exists but is not a socket; ThreadError: the
+          # reader thread could not spawn. All must fall back exactly like a
+          # refused connect, never abort worker boot — and never leak the
+          # half-opened socket.
+          close_quietly(@sock)
+          @sock = nil
+          raise HubUnavailableError, "cannot reach master hub at #{@socket_path}: #{e.class}: #{e.message}"
+        end
+
+        def hub_healthy?
+          @hub_healthy
+        end
+
+        def dead?
+          @dead
+        end
+
+        # Synchronous, bounded: returns :done once the master has confirmed
+        # LISTEN is active for +queue+. Raises HubUnavailableError on a dead
+        # transport or an expired ack deadline (which also kills the
+        # transport — a hub that can't ack in time can't be trusted with the
+        # no-lost-broadcast contract either).
+        def ensure_listening(queue)
+          raise HubUnavailableError, "master hub transport is dead" if @dead
+
+          waiter = Queue.new
+          @ack_mutex.synchronize { @pending_acks[queue] << waiter }
+          write_frame({ "t" => "sub", "q" => queue })
+
+          result = waiter.pop(timeout: @ack_timeout)
+          if result.nil?
+            discard_waiter(queue, waiter)
+            mark_dead("sub ack for #{queue} not received within #{@ack_timeout}s")
+            raise HubUnavailableError, "master hub ack timeout for #{queue}"
+          end
+          raise HubUnavailableError, "master hub died while awaiting ack for #{queue}" if result == :dead
+
+          :done
+        end
+
+        # Lazy GC, fire-and-forget — no correctness path waits on UNLISTEN
+        # (mirrors Listener#remove_listening). A dead transport is a no-op:
+        # the master's EOF cleanup already released this worker's refs.
+        def remove_listening(queue)
+          return if @dead
+
+          write_frame({ "t" => "unsub", "q" => queue })
+        rescue HubUnavailableError
+          nil
+        end
+
+        def stop
+          @stopping = true
+          close_quietly(@sock)
+          @reader&.join(2)
+          @reader = nil
+          self
+        end
+
+        private
+
+        def reader_loop
+          loop do
+            frame = HubProtocol.read_frame(@sock)
+            break if frame.nil?
+
+            handle_frame(frame)
+          end
+          mark_dead("master hub closed the transport") unless @stopping
+        rescue HubProtocol::ProtocolError => e
+          mark_dead("master hub protocol error: #{e.message}") unless @stopping
+        rescue IOError, Errno::EBADF, Errno::ECONNRESET
+          mark_dead("master hub transport error") unless @stopping
+        rescue StandardError => e
+          # The reader thread is the ONLY detector of hub death — an
+          # unexpected error must not let it exit with the client still
+          # reporting healthy, or the worker goes silently deaf.
+          mark_dead("master hub reader crashed: #{e.class}: #{e.message}") unless @stopping
+        end
+
+        def handle_frame(frame)
+          case frame["t"]
+          when "wake"
+            @dispatch_queue << Listener::WakeMessage.new(queue_name: frame["q"], payload: frame["p"])
+          when "ack"
+            @ack_mutex.synchronize { @pending_acks[frame["q"]].shift }&.push(:ack)
+          when "status"
+            @hub_healthy = frame["healthy"]
+          else
+            @logger.warn { "[Pgbus::Streamer::HubClient] unknown frame from master: #{frame["t"].inspect}" }
+          end
+        end
+
+        # Frames must never interleave — all writes go through one mutex
+        # (writers: dispatcher thread via ensure/remove; no writer thread
+        # needed client-side, sub/unsub frames are tiny). Bounded: a master
+        # that stopped draining its input would otherwise block this write
+        # forever, and the ack deadline only starts ticking AFTER the write
+        # returns — so a stalled write is itself a failover trigger.
+        def write_frame(message)
+          data = HubProtocol.encode(message)
+          deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @ack_timeout
+          @write_mutex.synchronize do
+            until data.empty?
+              begin
+                written = @sock.write_nonblock(data)
+                data = data.byteslice(written..)
+              rescue IO::WaitWritable
+                remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+                raise Errno::ETIMEDOUT, "write stalled" if remaining <= 0 || !@sock.wait_writable(remaining)
+              end
+            end
+          end
+        rescue IOError, Errno::EPIPE, Errno::EBADF, Errno::ECONNRESET, Errno::ETIMEDOUT => e
+          mark_dead("write to master hub failed: #{e.class}")
+          raise HubUnavailableError, "master hub transport is dead"
+        end
+
+        # Idempotent: first caller flips @dead, fails every waiter, fires
+        # on_failure once. Reachable from the reader (EOF/protocol error) and
+        # from ack timeouts / failed writes on caller threads.
+        def mark_dead(reason)
+          waiters = @ack_mutex.synchronize do
+            return if @dead
+
+            @dead = true
+            drained = @pending_acks.values.flatten
+            @pending_acks.clear
+            drained
+          end
+          @hub_healthy = false
+          waiters.each { |w| w << :dead }
+          close_quietly(@sock)
+          @logger.warn { "[Pgbus::Streamer::HubClient] #{reason} — falling back to a per-worker listener" }
+          @on_failure&.call
+        end
+
+        def discard_waiter(queue, waiter)
+          @ack_mutex.synchronize { @pending_acks[queue].delete(waiter) }
+        end
+
+        def close_quietly(io)
+          io.close if io && !io.closed?
+        rescue IOError, Errno::EBADF
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/streamer/hub_protocol.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  module Web
+    module Streamer
+      # Framing for the master-hub Unix socket (issue #382): 4-byte big-endian
+      # payload length + UTF-8 JSON. Unlike the job-side wake pipes (1-byte,
+      # lossy-by-design — Process::WakePipe), stream frames can carry an
+      # ephemeral broadcast's ONLY copy of its HTML, so the transport is
+      # length-prefixed and lossless; drop decisions are made per-message by
+      # the MasterHub, never by the wire format.
+      #
+      # Message shapes (JSON objects; "t" is the discriminator):
+      #   worker → master: {t:"sub", q:}   subscribe, synchronous — master acks
+      #                    {t:"unsub", q:} unsubscribe, fire-and-forget
+      #   master → worker: {t:"ack", q:}                 sub acknowledged (LISTEN active)
+      #                    {t:"wake", q:, p: <String|nil>} durable (p:nil) or ephemeral wake
+      #                    {t:"status", healthy: <bool>}   listener health broadcast
+      #
+      # Reads are blocking (each side owns a dedicated reader thread); a short
+      # read means the peer died mid-frame and is reported as EOF (nil), never
+      # as a truncated message.
+      module HubProtocol
+        class ProtocolError < StandardError; end
+
+        HEADER_BYTES = 4
+        # Generous ceiling for ephemeral HTML payloads; a frame announcing
+        # more than this is a corrupt stream or a runaway producer — sever
+        # rather than allocate.
+        MAX_FRAME_BYTES = 4 * 1024 * 1024
+
+        module_function
+
+        def encode(message)
+          json = JSON.generate(message)
+          bytes = json.b
+          raise ProtocolError, "frame too large: #{bytes.bytesize} bytes (max #{MAX_FRAME_BYTES})" if
+            bytes.bytesize > MAX_FRAME_BYTES
+
+          [bytes.bytesize].pack("N") + bytes
+        end
+
+        # Returns the decoded Hash, or nil on EOF — clean close, peer death
+        # mid-frame, OR a connection reset: an abrupt close can surface as
+        # ECONNRESET instead of orderly EOF depending on unread data and
+        # platform (Ruby 4.0 reports it deterministically where 3.x saw EOF),
+        # and both mean the same thing here: the peer is gone. Raises
+        # ProtocolError on an oversized announcement or malformed JSON.
+        def read_frame(io)
+          header = read_exactly(io, HEADER_BYTES)
+          return nil unless header
+
+          length = header.unpack1("N")
+          raise ProtocolError, "frame too large: #{length} bytes (max #{MAX_FRAME_BYTES})" if length > MAX_FRAME_BYTES
+
+          body = read_exactly(io, length)
+          return nil unless body
+
+          body = body.force_encoding(Encoding::UTF_8)
+          raise ProtocolError, "malformed frame: invalid UTF-8" unless body.valid_encoding?
+
+          decoded = JSON.parse(body)
+          raise ProtocolError, "malformed frame: expected a JSON object, got #{decoded.class}" unless decoded.is_a?(Hash)
+
+          decoded
+        rescue JSON::ParserError => e
+          raise ProtocolError, "malformed frame: #{e.message}"
+        rescue Errno::ECONNRESET
+          nil
+        end
+
+        # Blocking read of exactly +count+ bytes; nil on EOF (including EOF
+        # partway through — IO#read returns the short tail once, then nil).
+        def read_exactly(io, count)
+          data = io.read(count)
+          return nil if data.nil? || data.bytesize < count
+
+          data
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/streamer/instance.rb

@@ -39,7 +39,6 @@ module Pgbus
           @dispatch_queue = dispatch_queue || Queue.new
 
           @stream_counter = StreamCounter.new
-          @pg_connection = pg_connection || build_pg_connection
           # Self-tuning streams-pool autoscaler (issue #323). Opt-in; nil unless
           # enabled AND on the dedicated connection path (the shared-AR streams
           # pool aliases the non-thread-safe job pool and resize is a no-op there).
@@ -50,25 +49,7 @@ module Pgbus
             if @config.streams_pool_autoscale && !@client.shared_connection?
               Pgbus::Streams::PoolAutoscaler.new(client: @client, config: @config, logger: @logger)
             end
-          @listener = Listener.new(
-            pg_connection: @pg_connection,
-            dispatch_queue: @dispatch_queue,
-            health_check_ms: @config.streams_listen_health_check_ms,
-            # Opt-in dispatch-queue backpressure (issue #315 item 3). 0 =
-            # unbounded (default). The queue itself stays an unbounded
-            # Queue.new so the request-thread Connect push and the dispatcher's
-            # own prune_dead self-post never block.
-            dispatch_queue_limit: @config.streams_dispatch_queue_limit,
-            maintenance: build_autoscale_maintenance,
-            logger: @logger,
-            # On reconnect the Listener rebuilds its OWN connection via this
-            # factory (fresh connect re-resolves DNS, converges on the promoted
-            # primary after a failover) instead of resetting a possibly-dead
-            # socket. Always provided — even when an initial pg_connection: is
-            # injected, the reconnect path builds a fresh raw connection. A test
-            # can inject its own factory to avoid touching real configuration.
-            connection_factory: connection_factory || -> { build_raw_pg_connection }
-          )
+          @listener = build_listener(pg_connection, connection_factory)
           # Off-thread durable fanout writer (issue #321). Built only when
           # streams_writer_threads > 0; nil means fanout writes stay inline on
           # the dispatcher thread (the default, pre-#321 behavior). The pump
@@ -169,6 +150,74 @@ module Pgbus
 
         private
 
+        # Selects the wake source by streams_listen_scope (issue #382).
+        # :master with a reachable hub socket → FailoverListener over a
+        # HubClient (NO per-worker LISTEN connection is opened). Anything
+        # else — scope :process, no socket exported (single mode,
+        # non-preforking server, hub failed to start), or a refused connect —
+        # keeps today's per-worker Listener.
+        def build_listener(pg_connection, connection_factory)
+          hub = build_hub_listener(connection_factory)
+          return hub if hub
+
+          build_local_listener(pg_connection || build_pg_connection, connection_factory)
+        end
+
+        def build_hub_listener(connection_factory)
+          return nil unless @config.streams_listen_scope == :master
+
+          socket_path = ENV.fetch("PGBUS_STREAMS_HUB_SOCKET", nil)
+          return nil if socket_path.nil? || socket_path.empty?
+
+          # The worker's ack deadline must exceed the master's own internal
+          # ensure_listening budget (its listener's health-check cycle + 1s).
+          failover = nil
+          client = HubClient.new(
+            socket_path: socket_path,
+            dispatch_queue: @dispatch_queue,
+            ack_timeout: (@config.streams_listen_health_check_ms / 1000.0) + 2.0,
+            # failover is assigned right below; a transport death in the gap
+            # is caught by the FailoverListener's synchronous ensure path.
+            on_failure: -> { failover&.fail_over! },
+            logger: @logger
+          )
+          client.connect
+          failover = FailoverListener.new(
+            hub_client: client,
+            local_listener_factory: lambda do
+              build_local_listener(build_pg_connection, connection_factory).tap(&:start)
+            end,
+            logger: @logger
+          )
+        rescue HubClient::HubUnavailableError => e
+          @logger.info do
+            "[Pgbus::Streamer] master hub not reachable (#{e.message}) — using a per-worker listener"
+          end
+          nil
+        end
+
+        def build_local_listener(pg_connection, connection_factory)
+          Listener.new(
+            pg_connection: pg_connection,
+            dispatch_queue: @dispatch_queue,
+            health_check_ms: @config.streams_listen_health_check_ms,
+            # Opt-in dispatch-queue backpressure (issue #315 item 3). 0 =
+            # unbounded (default). The queue itself stays an unbounded
+            # Queue.new so the request-thread Connect push and the dispatcher's
+            # own prune_dead self-post never block.
+            dispatch_queue_limit: @config.streams_dispatch_queue_limit,
+            maintenance: build_autoscale_maintenance,
+            logger: @logger,
+            # On reconnect the Listener rebuilds its OWN connection via this
+            # factory (fresh connect re-resolves DNS, converges on the promoted
+            # primary after a failover) instead of resetting a possibly-dead
+            # socket. Always provided — even when an initial pg_connection: is
+            # injected, the reconnect path builds a fresh raw connection. A test
+            # can inject its own factory to avoid touching real configuration.
+            connection_factory: connection_factory || -> { build_raw_pg_connection }
+          )
+        end
+
         def safely
           yield
         rescue StandardError => e

data/lib/pgbus/web/streamer/listener.rb

@@ -41,8 +41,11 @@ module Pgbus
           end
         end
 
-        CHANNEL_PREFIX = "pgmq.q_"
-        CHANNEL_SUFFIX = ".INSERT"
+        # Single-sourced from NotifyListener, which owns the pgmq channel
+        # format (issue #381 review — the two copies had already drifted apart
+        # once in spirit if not in bytes).
+        CHANNEL_PREFIX = Pgbus::Process::NotifyListener::CHANNEL_PREFIX
+        CHANNEL_SUFFIX = Pgbus::Process::NotifyListener::CHANNEL_SUFFIX
 
         RECONNECT_BACKOFF_SECONDS = 0.5
 
@@ -102,6 +105,18 @@ module Pgbus
           self
         end
 
+        # Health signals for the MasterHub's status broadcasts (issue #382).
+        # Read cross-thread without synchronization: ivar assignment is atomic
+        # in MRI and a momentarily stale value only delays one status tick —
+        # these must never touch the connection itself (single-owner, #375).
+        def alive?
+          !!@thread&.alive?
+        end
+
+        def connected?
+          !@conn.nil?
+        end
+
         def stop
           return unless @running