pgbus 0.9.10 → 0.9.11

data/lib/pgbus/process/worker.rb

@@ -407,10 +407,19 @@ module Pgbus
         dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         prefix = "#{config.queue_prefix}_"
 
+        # Stream queues share the job namespace (pgbus_<name>) but must never
+        # be adopted by a wildcard worker: a worker would claim durable
+        # broadcasts, fail to deserialize them, and DLQ-move them out of the
+        # stream's replay history. The registry is what tells them apart.
+        # Reset first so a stream created since the last resolve is excluded.
+        Pgbus::StreamQueue.reset_cache!
+        stream_names = Pgbus::StreamQueue.all_names
+
         conn = Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
         all_queues = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
         resolved = all_queues
                    .reject { |q| q.end_with?(dlq_suffix) }
+                   .reject { |q| stream_names.include?(q) }
                    .map { |q| q.delete_prefix(prefix) }
 
         if resolved.empty?

data/lib/pgbus/streams/pool_autoscaler.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Streams
+    # Self-tuning decision logic for the dedicated streams DB pool (issue #323).
+    #
+    # A PURE DECISION OBJECT — it owns no thread and no connection. Two callers
+    # drive it, both feeding it a live-headroom reading:
+    #   * the streamer's Listener, as a periodic maintenance check on its idle
+    #     LISTEN connection (see Maintenance below); and
+    #   * a publisher (Client#send_stream_message), throttled, on the job pool.
+    # #evaluate reads the streams pool's busy ratio, decides, and resizes via the
+    # #325 hot-swap primitive.
+    #
+    # Cadence is slow (streams_pool_autoscale_interval, default 5 min), so the
+    # interval itself is the debounce: it acts on each check with no multi-sample
+    # hysteresis and no cooldown. A sustained burst converges over a few checks
+    # (one grow step each); a swap's transient (a freshly-swapped connection_pool
+    # is lazy → reads busy ≈ 0) simply means "no shrink this check" and is gone by
+    # the next check minutes later.
+    #
+    # Decision priority per check:
+    #   1. EMERGENCY SHRINK — DB free connections critically low → resize to
+    #      baseline now (protect the DB; overrides the busy signal).
+    #   2. GROW — pool saturated AND a fair share of real headroom exists.
+    #   3. SHRINK — pool idle → step toward baseline.
+    #   4. HOLD.
+    #
+    # SAFETY (proven in the #323 design; independent of cadence):
+    #   * No multi-process exhaustion: four stacked grow guards (GROW_RESERVE
+    #     gate, SAFETY peer inflation, STEP_MAX, per-process floor(free/2)).
+    #   * No grow↔emergency limit cycle: GROW_RESERVE (0.20·maxc) ≥ 4×
+    #     EMERGENCY_MARGIN (0.05·maxc) → a ~15% dead-zone.
+    class PoolAutoscaler
+      GROW_THRESHOLD   = 0.85 # busy_ratio to grow
+      SHRINK_THRESHOLD = 0.30 # busy_ratio to shrink
+      FAIR_FRACTION    = 0.25 # claim only ¼ of the computed fair share per grow
+      SAFETY           = 1.5  # inflate peer count → deflate fair share (undercount guard)
+      STEP_MAX         = 4    # hard cap on connections added per single grow
+
+      # SQL a caller runs on a connection to gather headroom. Public so the
+      # Listener/publisher can issue it; $1 is the application_name LIKE pattern.
+      HEADROOM_SQL = <<~SQL
+        SELECT current_setting('max_connections')::int AS maxc,
+               count(*) AS used,
+               count(DISTINCT application_name)
+                 FILTER (WHERE application_name LIKE $1) AS peers
+        FROM pg_stat_activity
+      SQL
+
+      def initialize(client:, config:, logger: Pgbus.logger)
+        @client = client
+        @config = config
+        @logger = logger
+        @baseline = config.streams_pool_size
+      end
+
+      # One maintenance decision. `headroom` is {maxc:, used:, peers:} from a
+      # caller's query, or nil if it failed (→ HOLD). Returns the action taken
+      # (:emergency_shrink / :grow / :shrink / :hold) — handy for tests + logging.
+      #
+      # `allow_shrink:` gates the NORMAL idle shrink (priority 3). The streamer
+      # passes true (it sees the sustained idle picture across replay reads). A
+      # publisher passes false: it only ever reacts to its OWN publish pressure
+      # (grow), and leaves idle-shrink to the streamer/consumer. Emergency shrink
+      # is NEVER gated — a DB running out of connections must always be relieved,
+      # whoever notices first.
+      def evaluate(headroom, allow_shrink: true)
+        return :hold if headroom.nil?
+
+        free = headroom[:maxc] - headroom[:used]
+
+        # PRIORITY 1 — EMERGENCY SHRINK (keys off DB `free`, an external fact).
+        return emergency_shrink(free, headroom[:maxc]) if free < emergency_margin(headroom[:maxc])
+
+        size, busy_ratio = pool_busy
+        return :hold if size.nil?
+
+        if busy_ratio >= GROW_THRESHOLD
+          maybe_grow(size, free, headroom)
+        elsif allow_shrink && busy_ratio < SHRINK_THRESHOLD
+          maybe_shrink(size)
+        else
+          :hold
+        end
+      end
+
+      private
+
+      # PRIORITY 2 — GROW into a bounded fair share of live headroom.
+      def maybe_grow(size, free, headroom)
+        return :hold if free <= grow_reserve(headroom[:maxc]) # not enough headroom
+        return :hold if size <= 1 # a telemetry checkout would starve publishing
+
+        delta = grow_delta(free, headroom[:peers])
+        return :hold if delta < 1
+
+        target = size + delta
+        cap = @config.streams_pool_max
+        target = [target, cap].min if cap
+        return :hold if target <= size
+
+        act(size, target, :grow)
+      end
+
+      # PRIORITY 3 — SHRINK one step toward the baseline.
+      def maybe_shrink(size)
+        return :hold if size <= @baseline
+
+        act(size, [@baseline, size - STEP_MAX].max, :shrink)
+      end
+
+      def emergency_shrink(free, maxc)
+        return :hold if pool_current_size <= @baseline
+
+        if swapped?(@client.resize_streams_pool(@baseline))
+          @logger.warn do
+            "[Pgbus::Streams::PoolAutoscaler] EMERGENCY streams-pool shrink to " \
+              "#{@baseline} (free=#{free}/#{maxc})"
+          end
+          :emergency_shrink
+        else
+          # The DB is critically low on connections but the shrink was a no-op
+          # (unchanged / shared-AR). This is the one case where a failed resize
+          # matters most — leave a trace instead of silently reporting success.
+          @logger.warn do
+            "[Pgbus::Streams::PoolAutoscaler] EMERGENCY streams-pool shrink to " \
+              "#{@baseline} did NOT apply (free=#{free}/#{maxc}) — resize was a no-op"
+          end
+          :hold
+        end
+      end
+
+      # fair_share = free / (peers · SAFETY); claim FAIR_FRACTION, never more than
+      # STEP_MAX or half the remaining headroom (leave ≥half for unknown peers →
+      # geometric contraction → N processes can't collectively exhaust the DB).
+      def grow_delta(free, peers)
+        peer_count = [peers.to_i, 1].max
+        fair_share = free.to_f / (peer_count * SAFETY)
+        [(FAIR_FRACTION * fair_share).floor, STEP_MAX, (free / 2)].min
+      end
+
+      def act(from_size, target_size, kind)
+        if swapped?(@client.resize_streams_pool(target_size))
+          @logger.info { "[Pgbus::Streams::PoolAutoscaler] streams pool #{from_size}->#{target_size} (#{kind})" }
+          kind
+        else
+          :hold # unchanged / shared-AR no-op
+        end
+      end
+
+      # Margins self-derive from live max_connections; the 4× gap forbids a
+      # grow↔emergency limit cycle.
+      def emergency_margin(maxc) = [5, (0.05 * maxc).ceil].max
+      def grow_reserve(maxc)     = [20, (0.20 * maxc).ceil].max
+
+      def pool_busy
+        stats = @client.streams_pool_stats
+        return [nil, nil] if stats.nil? || stats.empty?
+
+        size = stats[:size]
+        available = stats[:available]
+        return [nil, nil] if size.nil? || available.nil? || size <= 0
+
+        [size, (size - available).to_f / size]
+      end
+
+      def pool_current_size
+        stats = @client.streams_pool_stats
+        stats.is_a?(Hash) ? stats[:size].to_i : 0
+      end
+
+      def swapped?(result) = result.is_a?(Pgbus::Client::ResizablePool::SwapStats)
+
+      # Wraps an autoscaler as a Listener maintenance task: on each throttled idle
+      # window the Listener calls #run(conn); this queries live headroom on THAT
+      # connection (the streamer's existing LISTEN connection — no extra
+      # connection) and hands it to the autoscaler. `interval` is the throttle
+      # (seconds) the Listener honors between runs.
+      class Maintenance
+        attr_reader :interval
+
+        def initialize(autoscaler:, interval:, application_name_prefix:)
+          @autoscaler = autoscaler
+          @interval = interval
+          @like = "#{application_name_prefix}_%"
+        end
+
+        def run(conn)
+          row = conn.exec_params(HEADROOM_SQL, [@like]).first
+          # Explicit positional hash (braces) — evaluate now takes an optional
+          # allow_shrink: keyword, so a bare hash would be parsed as keywords.
+          # The streamer allows shrink (it sees the sustained idle picture).
+          @autoscaler.evaluate(
+            { maxc: row["maxc"].to_i, used: row["used"].to_i, peers: row["peers"].to_i },
+            allow_shrink: true
+          )
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/streams/pool_trigger.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  module Streams
+    # Publisher-side throttled autoscale trigger (issue #323 follow-up).
+    #
+    # The streamer autoscales via a periodic check on its idle LISTEN connection,
+    # but a pure-*publisher* process (a worker that fans out broadcasts via
+    # Client#send_stream_message but serves no SSE) has no streamer — so its
+    # streams pool would never autoscale. This closes that gap: each publish
+    # opportunistically triggers a headroom check, throttled to at most once per
+    # `interval` seconds across all publisher threads.
+    #
+    # OFF THE HOT PATH: the publishing thread only does a lock-free
+    # compare-and-set to claim the throttle window and, if it wins, posts the
+    # actual work to a background single-thread executor and returns immediately.
+    # It never runs the query or the resize inline. A quiet (non-publishing)
+    # process does zero work — this object is only ever built by the first publish
+    # (Client#streams_pool_trigger), and only when autoscale is on and the pool is
+    # dedicated, so a process that never publishes a stream never spawns the
+    # executor thread. There is no always-on timer.
+    #
+    # AT MOST ONE CHECK AT A TIME is enforced primarily by the single-thread
+    # executor (it serializes run_check calls end-to-end) plus the throttle CAS
+    # (one post per interval). The @running flag is a belt-and-suspenders
+    # invariant that becomes load-bearing only if a MULTI-threaded executor is
+    # injected (the executor: seam) — then it is the sole guard against two
+    # overlapping check bodies. It is cheap (one AtomicBoolean per check), so it
+    # stays.
+    #
+    # The query runs through the JOB pool, NOT the streams pool: pg_stat_activity
+    # is global, so any connection to the same database reads the same headroom,
+    # and using the job pool means the check can never starve on a saturated
+    # streams pool (the very pool it's trying to grow). The decision reuses
+    # PoolAutoscaler#evaluate with allow_shrink: false — a publisher only ever
+    # reacts to its own publish pressure (grow) and to DB exhaustion (emergency
+    # shrink); the streamer/consumer handles normal idle shrink.
+    class PoolTrigger
+      def initialize(autoscaler:, job_pool:, interval:, application_name_prefix:, clock: nil,
+                     executor: nil, logger: Pgbus.logger)
+        @autoscaler = autoscaler
+        @job_pool = job_pool
+        @interval = interval
+        @like = "#{application_name_prefix}_%"
+        @clock = clock || -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) }
+        @logger = logger
+        # A single background worker with a GENUINELY bounded 1-slot queue that
+        # DISCARDS overflow. NOTE: Concurrent::SingleThreadExecutor silently drops
+        # max_queue: (it hardcodes an unbounded queue), so build the base
+        # ThreadPoolExecutor directly to actually honor the bound. Built eagerly
+        # (assigned once, safely published with the object) — the quiet-process
+        # "no thread" guarantee is owned by Client#streams_pool_trigger (this
+        # object only exists after a real publish with autoscale on), so there's
+        # no reason to lazy-build inside. Injectable for tests.
+        @executor = executor || Concurrent::ThreadPoolExecutor.new(
+          min_threads: 1, max_threads: 1, max_queue: 1, fallback_policy: :discard
+        )
+        # nil = never checked. AtomicReference gives a lock-free compare-and-set
+        # so exactly one concurrent publisher claims each throttle window.
+        @last_check = Concurrent::AtomicReference.new(nil)
+        # At-most-one-check guard — load-bearing only under an injected
+        # multi-threaded executor (see class comment).
+        @running = Concurrent::AtomicBoolean.new(false)
+      end
+
+      # Called from the publish path. Claims the throttle window with a single CAS
+      # and, if it wins, defers the actual check to the background executor. Does
+      # NOT run the query or resize on the calling thread. Never raises.
+      def maybe_check
+        return unless claim_window
+
+        @executor.post { run_check }
+      rescue StandardError => e
+        # A post after shutdown raises RejectedExecutionError; swallow it so a
+        # publish never fails. The check body itself is guarded inside run_check.
+        @logger.debug { "[Pgbus::Streams::PoolTrigger] dispatch failed: #{e.class}: #{e.message}" }
+      end
+
+      # Stop the background executor (idempotent). For a clean Client#close.
+      def shutdown
+        @executor.shutdown
+        @executor.wait_for_termination(5)
+      end
+
+      private
+
+      # True for exactly one caller per `interval` window; false for the rest.
+      # A losing CAS (another thread advanced @last_check first) also returns
+      # false, so only one thread posts the check per window.
+      def claim_window
+        now = @clock.call
+        last = @last_check.get
+        return false if last && (now - last) < @interval
+
+        @last_check.compare_and_set(last, now)
+      end
+
+      # Runs on the executor thread. Skips if a prior check is still in flight so
+      # a slow query can't queue up work. Fully guarded — a failure here never
+      # escapes the executor.
+      def run_check
+        return unless @running.make_true # false if already running
+
+        begin
+          headroom = read_headroom
+          @autoscaler.evaluate(headroom, allow_shrink: false) if headroom
+        ensure
+          @running.make_false
+        end
+      rescue StandardError => e
+        @logger.debug { "[Pgbus::Streams::PoolTrigger] check failed: #{e.class}: #{e.message}" }
+      end
+
+      def read_headroom
+        @job_pool.with_connection do |conn|
+          row = conn.exec_params(PoolAutoscaler::HEADROOM_SQL, [@like]).first
+          { maxc: row["maxc"].to_i, used: row["used"].to_i, peers: row["peers"].to_i }
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/streams/turbo_broadcastable.rb

@@ -62,5 +62,28 @@ module Pgbus
 
       ::Turbo::StreamsChannel.singleton_class.prepend(TurboBroadcastable)
     end
+
+    # turbo-rails' async broadcast helpers (`broadcast_*_later_to`, the
+    # default for `broadcasts_to`/`broadcasts_refreshes`) enqueue these three
+    # ActiveJobs, which ship with no `queue_as` and so land on the default
+    # queue — where a render+broadcast can wait behind long-running jobs
+    # before the browser sees the update (#311). When the operator sets
+    # `config.streams_broadcast_queue`, route them to that dedicated queue so
+    # a `realtime:`-style worker capsule can isolate broadcast latency from job
+    # throughput. Called from the engine's turbo_broadcastable initializer.
+    # No-op when the queue is nil or turbo-rails is not loaded.
+    def self.install_broadcast_queue!(queue_name)
+      return if queue_name.nil?
+
+      %w[
+        Turbo::Streams::ActionBroadcastJob
+        Turbo::Streams::BroadcastJob
+        Turbo::Streams::BroadcastStreamJob
+      ].each do |const_name|
+        next unless Object.const_defined?(const_name)
+
+        Object.const_get(const_name).queue_as(queue_name)
+      end
+    end
   end
 end

data/lib/pgbus/streams.rb

@@ -157,10 +157,10 @@ module Pgbus
         }
         Instrumentation.instrument("pgbus.stream.broadcast", instrument_payload) do
           if transaction
-            transaction.after_commit { @client.send_message(@name, wrapped) }
+            transaction.after_commit { @client.send_stream_message(@name, wrapped) }
             nil
           else
-            @client.send_message(@name, wrapped)
+            @client.send_stream_message(@name, wrapped)
           end
         end
       end

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.9.10"
+  VERSION = "0.9.11"
 end

data/lib/pgbus/web/streamer/connection.rb

@@ -13,17 +13,29 @@ module Pgbus
       # cursor only for envelopes that actually wrote successfully. This is
       # the client-side leg of the replay-race fix (§6.5 of the design doc).
       class Connection
-        attr_reader :id, :stream_name, :io, :mutex, :last_msg_id_sent, :context
+        attr_reader :id, :stream_name, :io, :mutex, :context
         # The presence member id this connection auto-joined as, or nil for
         # non-presence streams / anonymous connections. Set by the
         # Dispatcher on connect; read on disconnect and heartbeat touch.
         attr_accessor :presence_member
 
+        # last_msg_id_sent is the ONE field whose writer can cross threads: with
+        # streams_writer_threads > 0 (issue #321) the OutboundPump worker thread
+        # advances it inside #enqueue, while the dispatcher thread reads it via
+        # cursor_for / handle_connect. A Concurrent::AtomicReference gives a real
+        # happens-before on every Ruby engine (not just MRI's GVL) at ~ns cost —
+        # one worker only ever writes a given connection's field (stable
+        # partition), so there's no writer-writer contention. Inline mode
+        # (default) reads/writes it single-threaded, semantically identical.
+        def last_msg_id_sent
+          @last_msg_id_sent.get
+        end
+
         def initialize(id:, stream_name:, io:, since_id:, writer:, write_deadline_ms:, context: nil)
           @id = id
           @stream_name = stream_name
           @io = io
-          @last_msg_id_sent = since_id.to_i
+          @last_msg_id_sent = Concurrent::AtomicReference.new(since_id.to_i)
           @writer = writer
           @write_deadline_ms = write_deadline_ms
           @mutex = Mutex.new
@@ -40,11 +52,15 @@ module Pgbus
           @context = context
         end
 
-        def enqueue(envelopes)
+        # deadline_ms defaults to the connection's own write deadline so every
+        # existing caller is unchanged. The Dispatcher overrides it with the
+        # short streams_fanout_write_deadline_ms for hot-loop fanout writes,
+        # bounding head-of-line blocking on a slow client (issue #315 item 3).
+        def enqueue(envelopes, deadline_ms: @write_deadline_ms)
           written = []
           envelopes.each do |envelope|
             ephemeral = envelope.msg_id.negative?
-            next if !ephemeral && envelope.msg_id <= @last_msg_id_sent
+            next if !ephemeral && envelope.msg_id <= @last_msg_id_sent.get
 
             bytes = Pgbus::Streams::Envelope.message(
               id: envelope.msg_id,
@@ -52,9 +68,9 @@ module Pgbus
               data: envelope.payload
             )
 
-            result = @writer.write(self, bytes, deadline_ms: @write_deadline_ms)
+            result = @writer.write(self, bytes, deadline_ms: deadline_ms)
             if result == :ok
-              @last_msg_id_sent = envelope.msg_id unless ephemeral
+              @last_msg_id_sent.set(envelope.msg_id) unless ephemeral
               @last_write_at = monotonic
               written << envelope
             else

data/lib/pgbus/web/streamer/falcon_connection.rb

@@ -12,15 +12,23 @@ module Pgbus
       # directly to body.write which is backed by Thread::Queue and
       # is fiber-safe under Falcon's scheduler.
       class FalconConnection
-        attr_reader :id, :stream_name, :io, :mutex, :last_msg_id_sent, :context
+        attr_reader :id, :stream_name, :io, :mutex, :context
         attr_accessor :presence_member
 
+        # last_msg_id_sent mirrors Connection's cross-thread cursor (issue #321):
+        # a Concurrent::AtomicReference so the OutboundPump worker's advance and
+        # the dispatcher's read have a happens-before on every Ruby engine. See
+        # the note on Connection#last_msg_id_sent.
+        def last_msg_id_sent
+          @last_msg_id_sent.get
+        end
+
         def initialize(id:, stream_name:, body:, since_id:, write_deadline_ms:, context: nil)
           @id = id
           @stream_name = stream_name
           @body = body
           @io = body
-          @last_msg_id_sent = since_id.to_i
+          @last_msg_id_sent = Concurrent::AtomicReference.new(since_id.to_i)
           @write_deadline_ms = write_deadline_ms
           @mutex = Mutex.new
           @dead = false
@@ -31,10 +39,14 @@ module Pgbus
           @context = context
         end
 
-        def enqueue(envelopes)
+        # deadline_ms is accepted for duck-type parity with Connection#enqueue
+        # (so the Dispatcher's safe_enqueue can call either class) but ignored:
+        # write_to_body is non-blocking under Falcon's fiber reactor, so there
+        # is no head-of-line blocking to bound here (issue #315 item 3).
+        def enqueue(envelopes, deadline_ms: @write_deadline_ms) # rubocop:disable Lint/UnusedMethodArgument
           written = []
           envelopes.each do |envelope|
-            next if envelope.msg_id <= @last_msg_id_sent
+            next if envelope.msg_id <= @last_msg_id_sent.get
 
             bytes = Pgbus::Streams::Envelope.message(
               id: envelope.msg_id,
@@ -44,7 +56,7 @@ module Pgbus
 
             result = write_to_body(bytes)
             if result == :ok
-              @last_msg_id_sent = envelope.msg_id
+              @last_msg_id_sent.set(envelope.msg_id)
               @last_write_at = monotonic
               written << envelope
             else

data/lib/pgbus/web/streamer/instance.rb

@@ -20,7 +20,8 @@ module Pgbus
       # production the module-level Streamer.current(...) builds all of the
       # defaults from the configuration.
       class Instance
-        attr_reader :registry, :listener, :dispatcher, :heartbeat, :dispatch_queue, :stream_counter
+        attr_reader :registry, :listener, :dispatcher, :heartbeat, :dispatch_queue, :stream_counter, :pump,
+                    :autoscaler
 
         def initialize(
           client: Pgbus.client,
@@ -39,10 +40,26 @@ module Pgbus
 
           @stream_counter = StreamCounter.new
           @pg_connection = pg_connection || build_pg_connection
+          # Self-tuning streams-pool autoscaler (issue #323). Opt-in; nil unless
+          # enabled AND on the dedicated connection path (the shared-AR streams
+          # pool aliases the non-thread-safe job pool and resize is a no-op there).
+          # It's a pure decision object — no thread, no connection. It runs as a
+          # throttled maintenance task on the Listener's idle LISTEN connection
+          # (zero extra connections), querying live headroom there.
+          @autoscaler =
+            if @config.streams_pool_autoscale && !@client.shared_connection?
+              Pgbus::Streams::PoolAutoscaler.new(client: @client, config: @config, logger: @logger)
+            end
           @listener = Listener.new(
             pg_connection: @pg_connection,
             dispatch_queue: @dispatch_queue,
             health_check_ms: @config.streams_listen_health_check_ms,
+            # Opt-in dispatch-queue backpressure (issue #315 item 3). 0 =
+            # unbounded (default). The queue itself stays an unbounded
+            # Queue.new so the request-thread Connect push and the dispatcher's
+            # own prune_dead self-post never block.
+            dispatch_queue_limit: @config.streams_dispatch_queue_limit,
+            maintenance: build_autoscale_maintenance,
             logger: @logger,
             # On reconnect the Listener rebuilds its OWN connection via this
             # factory (fresh connect re-resolves DNS, converges on the promoted
@@ -52,6 +69,14 @@ module Pgbus
             # can inject its own factory to avoid touching real configuration.
             connection_factory: connection_factory || -> { build_raw_pg_connection }
           )
+          # Off-thread durable fanout writer (issue #321). Built only when
+          # streams_writer_threads > 0; nil means fanout writes stay inline on
+          # the dispatcher thread (the default, pre-#321 behavior). The pump
+          # reports write acks on @ack_queue (drained by the dispatcher) and
+          # posts a DisconnectMessage via on_dead when a write fails, so the
+          # dispatcher owns all cursor + registry cleanup on its own thread.
+          @ack_queue = Queue.new
+          @pump = build_pump
           @dispatcher = StreamEventDispatcher.new(
             client: @client,
             registry: @registry,
@@ -59,7 +84,9 @@ module Pgbus
             dispatch_queue: @dispatch_queue,
             logger: @logger,
             config: @config,
-            stream_counter: @stream_counter
+            stream_counter: @stream_counter,
+            pump: @pump,
+            ack_queue: @ack_queue
           )
           @heartbeat = Heartbeat.new(
             registry: @registry,
@@ -76,6 +103,9 @@ module Pgbus
           return if @started
 
           @started = true
+          # Pump first: it must be ready to accept writes before the dispatcher
+          # can post any (issue #321). No-op when offload is off (@pump nil).
+          @pump&.start
           @listener.start
           @dispatcher.start
           @heartbeat.start
@@ -112,9 +142,15 @@ module Pgbus
         #   2. Listener next (stop accepting new NOTIFYs)
         #   3. Dispatcher next (drain the queue; it's now finite because
         #      nothing else writes into it)
-        #   4. Send pgbus:shutdown sentinel to every connection and close
-        #      their sockets. We do this AFTER stopping the dispatcher so
-        #      no one else is writing to these IOs concurrently.
+        #   4. Writer pump next (issue #321): once the dispatcher — the pump's
+        #      only producer — is stopped, each partition is finite, so the
+        #      pump drains every accepted-but-unflushed durable frame and joins
+        #      its workers before we touch the sockets. No-op when offload is
+        #      off. Must come BEFORE close_all_connections so buffered frames
+        #      flush before their sockets close.
+        #   5. Send pgbus:shutdown sentinel to every connection and close
+        #      their sockets. We do this AFTER stopping the dispatcher AND the
+        #      pump so nothing else is writing to these IOs concurrently.
         #
         # Bounded by the configured write deadline per connection; a dead
         # client drops instantly, a slow one stalls for at most write_deadline_ms.
@@ -126,6 +162,7 @@ module Pgbus
             safely { @heartbeat.stop }
             safely { @listener.stop }
             safely { @dispatcher.stop }
+            safely { @pump&.stop }
             close_all_connections
           end
         end
@@ -138,6 +175,39 @@ module Pgbus
           @logger.warn { "[Pgbus::Streamer::Instance] component stop raised: #{e.class}: #{e.message}" }
         end
 
+        # Off-thread durable fanout writer (issue #321). nil (the default) keeps
+        # fanout writes inline on the dispatcher thread. When on_dead fires
+        # (a write failed), the pump posts a DisconnectMessage onto the shared
+        # dispatch queue — the SAME explicit protocol prune_dead and the
+        # heartbeat use — so the dispatcher thread owns the lockless cursor +
+        # registry cleanup and the pump never touches dispatcher state.
+        # Wrap the autoscaler as a throttled Listener maintenance task (issue
+        # #323), so headroom is read on the Listener's existing idle LISTEN
+        # connection every streams_pool_autoscale_interval seconds. nil (no
+        # autoscaler) means the Listener runs no maintenance.
+        def build_autoscale_maintenance
+          return nil unless @autoscaler
+
+          Pgbus::Streams::PoolAutoscaler::Maintenance.new(
+            autoscaler: @autoscaler,
+            interval: @config.streams_pool_autoscale_interval,
+            application_name_prefix: @config.streams_application_name
+          )
+        end
+
+        def build_pump
+          threads = @config.streams_writer_threads
+          return nil unless threads.positive?
+
+          OutboundPump.new(
+            threads: threads,
+            ack_queue: @ack_queue,
+            on_dead: ->(conn) { @dispatch_queue << StreamEventDispatcher::DisconnectMessage.new(connection: conn) },
+            buffer_limit: @config.streams_writer_buffer_limit,
+            logger: @logger
+          )
+        end
+
         def close_all_connections
           sentinel_bytes = Pgbus::Streams::Envelope.message(
             id: 0,

data/lib/pgbus/web/streamer/io_writer.rb

@@ -11,11 +11,17 @@ module Pgbus
       # pattern.
       #
       # The write loop uses write_nonblock + IO.select so a slow client at most
-      # stalls *its own* mutex-protected write for `deadline_ms`, never the
-      # dispatcher or heartbeat thread. When the deadline expires with bytes
-      # still pending, we return :blocked; the caller (Connection#enqueue or
-      # Connection#write_comment) translates that into mark_dead!, and the
-      # heartbeat sweep eventually unregisters the connection.
+      # stalls the CALLING thread's mutex-protected write for `deadline_ms`.
+      # During fanout the dispatcher IS that caller, writing to each connection
+      # serially, so K slow-but-not-yet-dead clients stack the deadline
+      # (~K * deadline_ms) before each is marked dead — the head-of-line block.
+      # Fanout writes therefore pass the SHORT streams_fanout_write_deadline_ms
+      # (not streams_write_deadline_ms) to bound that stall (issue #315 item 3).
+      # When the deadline expires with bytes still pending, we return :blocked;
+      # the caller (Connection#enqueue or Connection#write_comment) translates
+      # that into mark_dead!, and the heartbeat sweep unregisters the
+      # connection — the client then reconnects and replays the gap from the
+      # durable archive.
       #
       # Returns:
       #   :ok       — all bytes written