pgbus 0.12.3 → 0.13.0

data/lib/pgbus/process/notify_hub.rb

@@ -0,0 +1,273 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    # Supervisor-owned LISTEN hub (issue #381, worker_notify_scope:
+    # :supervisor). Owns ONE NotifyListener — one direct PG connection for the
+    # whole host — on the union of every capsule's and consumer's queue
+    # channels, and fans wakes out to worker/consumer forks over per-fork
+    # pipes (see WakePipe for the fork side and the W/H/P byte protocol).
+    #
+    # Union: explicit capsule queues from config.workers, wildcard capsules
+    # via the shared WildcardQueueResolver, consumer queues via
+    # Registry#queue_names_for_topics — each gated by config.role_enabled?.
+    # Refreshed every REFRESH_INTERVAL_SECONDS from the supervisor's monitor
+    # tick so wildcard churn and late registry subscriptions converge; a
+    # transient gap is bounded by the forks' NOTIFY poll ceiling (15s), so
+    # under-listening degrades latency, never liveness.
+    #
+    # Routing: a NOTIFY for queue Q wakes every fork whose registered set
+    # contains Q, plus every wildcard fork. Over-waking costs one empty read;
+    # under-waking is the only failure mode that matters, so wildcard routing
+    # is unconditional.
+    #
+    # Status: healthy? = listener running + connected + delivering. Broadcast
+    # to all pipes on change and every STATUS_REBROADCAST_SECONDS (idempotent
+    # 1-byte writes), so a lost byte or a freshly full pipe can't wedge a fork
+    # in the wrong mode.
+    #
+    # Threading: register/deregister/tick/stop run on the supervisor's main
+    # loop; the wake callback runs on the listener's thread. The fork table is
+    # the only shared state — guarded by @table_mutex. Pipe writes are 1-byte
+    # write_nonblock calls (atomic well under PIPE_BUF) and never block: a
+    # full pipe already guarantees a pending readable byte, so the wake is
+    # skipped, and status repair rides the periodic rebroadcast.
+    class NotifyHub
+      REFRESH_INTERVAL_SECONDS = 30
+      STATUS_REBROADCAST_SECONDS = 5
+      RETRY_BASE_SECONDS = 5
+      RETRY_MAX_SECONDS = 300
+
+      def initialize(config:, listener_factory: nil, clock: nil, logger: Pgbus.logger)
+        @config = config
+        @logger = logger
+        @clock = clock || -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) }
+        @listener_factory = listener_factory || default_listener_factory
+        @table_mutex = Mutex.new
+        @forks = {}
+        @listener = nil
+        @last_status = nil
+        @last_broadcast_at = nil
+        @last_refresh_at = nil
+        @retry_at = 0.0
+        @retry_backoff = RETRY_BASE_SECONDS
+      end
+
+      def start
+        build_listener
+        @last_refresh_at = now
+        self
+      end
+
+      def stop
+        stop_listener_quietly
+        @table_mutex.synchronize do
+          @forks.each_value { |entry| close_quietly(entry[:pipe]) }
+          @forks.clear
+        end
+        self
+      end
+
+      # Register a fork's wake pipe. +queues+ are PHYSICAL queue names (already
+      # prefixed); wildcard forks pass wildcard: true and are woken for every
+      # channel. The fork's current status byte is sent immediately so a
+      # freshly forked worker starts in the right polling mode.
+      def register_fork(pid:, queues:, pipe:, wildcard: false)
+        entry = { pipe: pipe, queues: Set.new(queues), wildcard: wildcard }
+        @table_mutex.synchronize { @forks[pid] = entry }
+        byte = status_byte
+        write_byte(entry, byte)
+        # The registration write is the broadcast baseline: without stamping
+        # it, the next tick would treat @last_status as unknown and re-send
+        # the same byte to every fork immediately.
+        @last_status = byte
+        @last_broadcast_at = now
+        self
+      end
+
+      def deregister_fork(pid)
+        entry = @table_mutex.synchronize { @forks.delete(pid) }
+        close_quietly(entry[:pipe]) if entry
+        self
+      end
+
+      # Called ONLY inside a just-forked child: release the child's copies of
+      # the parent's hub resources — the LISTEN socket fd (without a libpq
+      # Terminate, see NotifyListener#close_inherited_socket!) and the fork
+      # table's pipe write ends. Never touches parent state (fork copied it).
+      def close_inherited!
+        @listener&.close_inherited_socket!
+        @listener = nil
+        @table_mutex.synchronize do
+          @forks.each_value { |entry| close_quietly(entry[:pipe]) }
+          @forks.clear
+        end
+        self
+      end
+
+      # One supervisor monitor-loop beat: self-heal the listener, refresh the
+      # LISTEN union, keep fork status fresh.
+      def tick
+        ensure_listener
+        refresh_union_if_due
+        broadcast_status
+        self
+      end
+
+      def healthy?
+        listener = @listener
+        !!(listener&.running? && listener.connected? && listener.delivering?)
+      end
+
+      private
+
+      attr_reader :config
+
+      def now
+        @clock.call
+      end
+
+      def default_listener_factory
+        lambda do |physical_queues:, on_wake:|
+          NotifyListener.new(
+            physical_queues: physical_queues,
+            on_wake: on_wake,
+            connection_options: config.worker_notify_connection_options,
+            health_check_ms: (config.polling_interval * 1000).to_i.clamp(250, 5_000),
+            logger: @logger
+          )
+        end
+      end
+
+      def build_listener
+        @listener = @listener_factory.call(
+          physical_queues: desired_physical_queues,
+          on_wake: method(:route)
+        )
+        @listener.start
+      rescue StandardError => e
+        @listener = nil
+        @logger.error { "[Pgbus::NotifyHub] listener failed to start: #{e.class}: #{e.message}" }
+      end
+
+      # Mirrors Worker#ensure_notify_listener: restart a dead listener thread
+      # on an exponential backoff so a persistent outage doesn't thrash a
+      # reconnect per supervisor tick.
+      def ensure_listener
+        return if @listener&.running?
+        return if now < @retry_at
+
+        stop_listener_quietly
+        build_listener
+
+        @retry_backoff = if @listener&.running?
+                           RETRY_BASE_SECONDS
+                         else
+                           [@retry_backoff * 2, RETRY_MAX_SECONDS].min
+                         end
+        @retry_at = now + @retry_backoff
+      end
+
+      def stop_listener_quietly
+        @listener&.stop
+      rescue StandardError => e
+        @logger.warn { "[Pgbus::NotifyHub] failed to stop listener: #{e.class}: #{e.message}" }
+      ensure
+        @listener = nil
+      end
+
+      def refresh_union_if_due
+        return unless @listener
+        return if @last_refresh_at && (now - @last_refresh_at) < REFRESH_INTERVAL_SECONDS
+
+        @last_refresh_at = now
+        desired = desired_physical_queues.to_set
+        current = @listener.listening_to.to_set { |channel| channel_to_physical(channel) }
+        (desired - current).each { |q| @listener.add_queue(q) }
+        (current - desired).each { |q| @listener.remove_queue(q) }
+      rescue StandardError => e
+        @logger.warn { "[Pgbus::NotifyHub] union refresh failed: #{e.class}: #{e.message}" }
+      end
+
+      def broadcast_status(status = status_byte)
+        rebroadcast_due = @last_broadcast_at.nil? || (now - @last_broadcast_at) >= STATUS_REBROADCAST_SECONDS
+        return if status == @last_status && !rebroadcast_due
+
+        entries = @table_mutex.synchronize { @forks.values.dup }
+        entries.each { |entry| write_byte(entry, status) }
+        @last_status = status
+        @last_broadcast_at = now
+      end
+
+      def status_byte
+        healthy? ? WakePipe::HEALTHY : WakePipe::DEGRADED
+      end
+
+      # Runs on the LISTENER thread (NotifyListener on_wake callback).
+      def route(channel)
+        physical = channel_to_physical(channel)
+        entries = @table_mutex.synchronize { @forks.values.dup }
+        entries.each do |entry|
+          write_byte(entry, WakePipe::WAKE) if entry[:wildcard] || entry[:queues].include?(physical)
+        end
+      end
+
+      # Never block, never raise: a full pipe already guarantees a pending
+      # readable byte (wake semantics are level-triggered), and a dead pipe's
+      # fork is on its way through reap → deregister_fork.
+      def write_byte(entry, byte)
+        entry[:pipe].write_nonblock(byte)
+      rescue IO::WaitWritable, Errno::EPIPE, IOError, Errno::EBADF
+        nil
+      end
+
+      def desired_physical_queues
+        names = []
+        names.concat(capsule_queue_names) if config.role_enabled?(:workers)
+        names.concat(consumer_queue_names) if config.role_enabled?(:consumers)
+        names.uniq.map { |q| config.queue_name(q) }
+      end
+
+      def capsule_queue_names
+        names = []
+        wildcard = false
+        Array(config.workers).each do |worker_config|
+          queues = worker_config[:queues] || worker_config["queues"] || [config.default_queue]
+          Array(queues).each do |q|
+            q.to_s == "*" ? wildcard = true : names << q.to_s
+          end
+        end
+        names.concat(resolve_wildcard) if wildcard
+        names
+      end
+
+      # A resolver failure (DB blip at boot) degrades to the explicit union;
+      # wildcard queues rejoin on the next successful refresh, and the forks'
+      # poll ceiling keeps them processing meanwhile.
+      def resolve_wildcard
+        WildcardQueueResolver.resolve(config: config)
+      rescue StandardError => e
+        @logger.warn { "[Pgbus::NotifyHub] wildcard resolution failed: #{e.class}: #{e.message}" }
+        []
+      end
+
+      def consumer_queue_names
+        registry = Pgbus::EventBus::Registry.instance
+        Array(config.event_consumers).flat_map do |consumer_config|
+          topics = consumer_config[:topics] || consumer_config["topics"] || []
+          registry.queue_names_for_topics(Array(topics))
+        end
+      end
+
+      def channel_to_physical(channel)
+        NotifyListener.physical_for(channel)
+      end
+
+      def close_quietly(pipe)
+        pipe.close if pipe && !pipe.closed?
+      rescue IOError, Errno::EBADF
+        nil
+      end
+    end
+  end
+end

data/lib/pgbus/process/notify_listener.rb

@@ -30,12 +30,32 @@ module Pgbus
     # blocking IO call where the mutex MUST NOT be held), so wait_once reads
     # the connection out of the mutex first and operates on a local. Reconnect
     # publishes the new connection + channel set under the mutex.
+    #
+    # The mutex only makes the ivar READ safe. PG::Connection itself is not
+    # thread-safe, so the connection is single-owner: the listener thread is
+    # the ONLY thread that may exec, wait, or close on it, from build through
+    # teardown. #stop signals by clearing @running and joining — it never
+    # touches the connection, because #close is PQfinish and freeing the
+    # PGconn under a concurrent libpq call is a process-killing SEGV, not a
+    # rescuable PG::Error (issue #375).
     class NotifyListener
       CHANNEL_PREFIX = "pgmq.q_"
       CHANNEL_SUFFIX = ".INSERT"
 
+      # Inverse of #channel_for: map a NOTIFY channel back to the physical
+      # queue name. Class-level because the channel format is owned here —
+      # NotifyHub (wake routing + union refresh) and Worker (queue-set sync)
+      # both consume it.
+      def self.physical_for(channel)
+        channel.delete_prefix(CHANNEL_PREFIX).delete_suffix(CHANNEL_SUFFIX)
+      end
+
       RECONNECT_BACKOFF_SECONDS = 0.5
 
+      # Grace added to one health-check cycle when #stop joins the listener
+      # thread. See #stop_join_timeout.
+      STOP_JOIN_GRACE_SECONDS = 5
+
       def initialize(physical_queues:, on_wake:, connection_options:,
                      health_check_ms: 1000, logger: Pgbus.logger)
         @physical_queues = Array(physical_queues)
@@ -58,6 +78,16 @@ module Pgbus
         @state_mutex.synchronize { @listening_to.dup }
       end
 
+      # Whether a live PG connection is currently published. running? stays
+      # true during a reconnect (the thread is alive, looping in reconnect!),
+      # so this is the signal that distinguishes "parked in wait_for_notify"
+      # from "between connections". The supervisor NotifyHub (issue #381)
+      # consults it to broadcast degraded status to forks the moment the
+      # shared connection drops, and healthy again once it is rebuilt.
+      def connected?
+        @state_mutex.synchronize { !@conn.nil? }
+      end
+
       # Whether the start-time self-probe confirmed this connection can actually
       # receive a NOTIFY. False when a transaction-mode pooler or replica
       # silently drops LISTEN: the thread is still alive (running? == true) but
@@ -80,22 +110,20 @@ module Pgbus
       end
 
       def stop
-        conn_to_close = nil
         @state_mutex.synchronize do
           return self unless @running
 
           @running = false
-          conn_to_close = @conn
         end
         @commands << [:stop]
-        # Interrupt the blocking wait by closing the socket; the rescue in
-        # wait_once sees @running == false and exits cleanly.
-        begin
-          conn_to_close&.close if conn_to_close.respond_to?(:close)
-        rescue StandardError
-          nil
-        end
-        @thread&.join(5)
+        # Deliberately does NOT touch @conn. PG::Connection is not thread-safe
+        # and #close is PQfinish: it frees the PGconn and its OpenSSL objects
+        # out from under whatever libpq call the listener thread is making on
+        # the same connection. That is a use-after-free — a process-killing
+        # SEGV, not a rescuable PG::Error (issue #375). The listener thread is
+        # the sole owner of @conn for its entire life and closes it in
+        # run_loop's ensure; clearing @running above is the whole stop signal.
+        @thread&.join(stop_join_timeout)
         @thread = nil
         self
       end
@@ -115,6 +143,30 @@ module Pgbus
         @state_mutex.synchronize { @running }
       end
 
+      # Called ONLY inside a just-forked child (issue #381 hub hygiene): drop
+      # this process's copy of the LISTEN socket fd WITHOUT PQfinish — #close
+      # would send a libpq Terminate over the socket shared with the parent,
+      # killing the parent's LISTEN session. Closing the IO wrapper just
+      # closes the child's fd. The listener thread does not exist in the
+      # child (fork copies only the calling thread), so there is no
+      # concurrent owner and the single-owner rule (#375) does not apply.
+      def close_inherited_socket!
+        conn = @state_mutex.synchronize do
+          c = @conn
+          @conn = nil
+          @running = false
+          c
+        end
+        conn&.socket_io&.close
+      rescue StandardError => e
+        # Best-effort (a lingering fd copy is benign until the parent dies),
+        # but never silent: the child keeps booting either way.
+        @logger.warn do
+          "[Pgbus::NotifyListener] inherited socket cleanup failed: #{e.class}: #{e.message}"
+        end
+        nil
+      end
+
       private
 
       def run_loop
@@ -153,9 +205,26 @@ module Pgbus
         # Clear @running so #start can spawn a fresh thread after a fatal exit
         # (e.g. build_connection raising at boot). Without this, the dead
         # thread's @running stays true and #start returns early forever.
-        @state_mutex.synchronize { @running = false }
-        safe_unlisten_all
-        safe_close
+        #
+        # The LISTEN set is dropped as bookkeeping only — no UNLISTEN
+        # round-trip. We close on the next line and closing the session
+        # deregisters every LISTEN server-side, so the exec bought nothing
+        # while being the statement that raced #stop into a SEGV (issue #375).
+        #
+        # Capturing @conn in the SAME critical section that clears @running is
+        # what makes restart safe. #start is public and guarded only by
+        # @running, so a caller watching running? may spawn a fresh thread the
+        # instant it flips. If teardown read @conn in a later critical section
+        # it could pick up the NEW thread's connection and PQfinish it mid-use
+        # — the same use-after-free, reached through restart instead of #stop.
+        conn = @state_mutex.synchronize do
+          @running = false
+          @listening_to.clear
+          c = @conn
+          @conn = nil
+          c
+        end
+        close_quietly(conn)
       end
 
       def wait_once
@@ -163,10 +232,16 @@ module Pgbus
         return reconnect! unless conn
 
         timeout_s = @health_check_ms / 1000.0
-        got_notify = conn.wait_for_notify(timeout_s) do |_channel, _pid, _payload|
-          @on_wake.call
+        got_notify = conn.wait_for_notify(timeout_s) do |channel, _pid, _payload|
+          # The channel rides along so a hub caller (issue #381) can route the
+          # wake to the fork(s) reading that queue; fork-owned listeners take
+          # ->(_channel) and ignore it.
+          @on_wake.call(channel)
         end
-        run_health_check(conn) unless got_notify
+        # Skip the keepalive when a stop landed during the wait: the loop is
+        # about to exit and close this connection anyway, so the round-trip
+        # would only add latency to shutdown.
+        run_health_check(conn) if !got_notify && running?
       rescue IOError, PG::Error => e
         return unless running?
 
@@ -270,14 +345,14 @@ module Pgbus
         end
       end
 
-      def safe_unlisten_all
-        channels, conn = @state_mutex.synchronize { [@listening_to.to_a, @conn] }
-        channels.each do |channel|
-          conn&.exec(%(UNLISTEN "#{channel}"))
-        rescue PG::Error
-          nil
-        end
-        @state_mutex.synchronize { @listening_to.clear }
+      # How long #stop waits for the listener thread to notice the cleared
+      # @running and finish teardown. The thread can be parked in
+      # wait_for_notify for one full health-check cycle before it re-checks the
+      # flag, so the budget is that cycle plus grace — a flat timeout would
+      # expire before a listener with a large health_check_ms had even one
+      # chance to observe the stop.
+      def stop_join_timeout
+        (@health_check_ms / 1000.0) + STOP_JOIN_GRACE_SECONDS
       end
 
       def safe_close
@@ -289,8 +364,10 @@ module Pgbus
         close_quietly(conn)
       end
 
-      # Close an unpublished PG::Connection (one that never made it into @conn,
-      # e.g. a half-built reconnect attempt where LISTEN raised). Best-effort.
+      # Close a PG::Connection we are done with — a half-built reconnect
+      # attempt that never made it into @conn, or the connection captured out
+      # of @conn during teardown. Always called on the listener thread, which
+      # owns the connection. Best-effort.
       def close_quietly(conn)
         conn&.close if conn.respond_to?(:close)
       rescue StandardError