pgbus 0.9.10 → 0.9.11

data/lib/pgbus/web/streamer/listener.rb

@@ -48,8 +48,20 @@ module Pgbus
         #   Required — the reconnect loop always rebuilds rather than resetting a
         #   possibly-dead socket, so every caller (production and tests) must pass
         #   one.
+        # dispatch_queue_limit (issue #315 item 3): 0 = unbounded (default). A
+        # positive value makes handle_notify DROP a durable wake when the
+        # dispatch queue is at/over the cap — safe because the next durable
+        # wake for that stream re-reads from the min cursor. Ephemeral wakes
+        # and Connect/Disconnect messages are never dropped.
+        # `maintenance:` is an optional periodic callback (issue #323 pool
+        # autoscaler). On the listener's idle health-check window — the only place
+        # the non-thread-safe LISTEN connection is safely idle — it is invoked at
+        # most once per `maintenance.interval` seconds with THIS connection, so a
+        # maintenance query (e.g. pg_stat_activity headroom) reuses the existing
+        # idle connection instead of opening its own. nil disables it.
         def initialize(pg_connection:, dispatch_queue:, health_check_ms:,
-                       connection_factory:, logger: Pgbus.logger)
+                       connection_factory:, dispatch_queue_limit: 0, maintenance: nil,
+                       logger: Pgbus.logger, clock: nil)
           raise ArgumentError, "connection_factory is required" unless connection_factory.respond_to?(:call)
 
           @conn = pg_connection
@@ -57,12 +69,22 @@ module Pgbus
           @health_check_ms = health_check_ms
           @logger = logger
           @connection_factory = connection_factory
+          @dispatch_queue_limit = dispatch_queue_limit
+          @maintenance = maintenance
+          @clock = clock || -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) }
+          @maintenance_last = nil
+          @dropped_wakes = 0
           @listening_to = Set.new
           @commands = Queue.new
           @running = false
           @thread = nil
         end
 
+        # Count of durable wakes dropped due to the dispatch-queue cap. Read by
+        # tests and available for operator introspection. Only mutated on the
+        # Listener thread (in handle_notify), so no synchronization needed.
+        attr_reader :dropped_wakes
+
         def start
           return if @running
 
@@ -202,11 +224,49 @@ module Pgbus
           queue_name = queue_name_from(channel)
           return unless queue_name
 
+          # Backpressure applies ONLY to durable wakes (payload nil): they
+          # self-heal because the next durable wake for the stream re-reads
+          # from the min cursor. Ephemeral wakes (payload present) carry the
+          # only copy of their HTML with no archive to replay, so they are
+          # never dropped (issue #315 item 3). @dispatch_queue.size on a stdlib
+          # Queue is internally synchronized; this runs only on the Listener
+          # thread and touches no dispatcher-owned state.
+          if payload.nil? && @dispatch_queue_limit.positive? &&
+             @dispatch_queue.size >= @dispatch_queue_limit
+            @dropped_wakes += 1
+            if (@dropped_wakes % 100) == 1
+              @logger.warn do
+                "[Pgbus::Streamer::Listener] dispatch queue at limit " \
+                  "(#{@dispatch_queue_limit}); dropped durable wake for " \
+                  "#{queue_name} (total dropped: #{@dropped_wakes})"
+              end
+            end
+            return
+          end
+
           @dispatch_queue << WakeMessage.new(queue_name: queue_name, payload: payload)
         end
 
         def run_health_check
           @conn.exec("SELECT 1")
+          run_maintenance
+        end
+
+        # Periodic maintenance on the idle LISTEN connection, throttled to
+        # @maintenance.interval (issue #323). Runs on the listener thread, so it
+        # is the only place the non-thread-safe @conn may be queried outside
+        # wait_for_notify. Fail-soft: a raising maintenance callback is logged and
+        # never disturbs the listen loop.
+        def run_maintenance
+          return unless @maintenance
+
+          now = @clock.call
+          return if @maintenance_last && (now - @maintenance_last) < @maintenance.interval
+
+          @maintenance_last = now
+          @maintenance.run(@conn)
+        rescue StandardError => e
+          @logger.debug { "[Pgbus::Streamer::Listener] maintenance raised: #{e.class}: #{e.message}" }
         end
 
         # Retry reconnect until we succeed (fresh conn + every channel

data/lib/pgbus/web/streamer/outbound_pump.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    module Streamer
+      # Off-thread durable-stream fanout writer (issue #321).
+      #
+      # The dispatcher must never block on a slow client's socket write. When
+      # streams_writer_threads > 0, StreamEventDispatcher#handle_durable_wake
+      # hands each (connection, filtered envelopes, batch_max) to this pump
+      # instead of writing inline. The pump owns N worker threads; each
+      # connection is pinned to ONE worker by `id.hash % N`, so a connection's
+      # frames stay strictly ordered and its per-io mutex is never contended
+      # across workers.
+      #
+      # The pump calls the UNCHANGED Connection#enqueue on a worker thread (the
+      # blocking write, last_msg_id_sent advance, and mark_dead! all move off
+      # the dispatcher), then reports back:
+      #   - success  → a WriteAckMessage(connection, accepted_max) onto ack_queue.
+      #                accepted_max is the highest msg_id actually written (or
+      #                the batch_max for a fully-filtered empty batch, so the
+      #                dispatcher's scan cursor still advances past the hidden
+      #                window). The dispatcher — the SOLE owner of @scanned_cursor
+      #                — applies it on its own thread. This keeps the lockless
+      #                single-owner invariant intact: the writer only reports a
+      #                value, it never mutates dispatcher state.
+      #   - failure  → the injected on_dead callback (which posts a
+      #                DisconnectMessage), NOT an ack. A failed write must never
+      #                advance the cursor past a frame that never reached the
+      #                socket (issue #321 B2). The dispatcher then scrubs the
+      #                connection's state deterministically (B4), even on an
+      #                otherwise-quiet stream.
+      #
+      # EPHEMERAL frames are NEVER routed here — they have no archive to replay,
+      # so an async drop would be unrecoverable (issue #321 B1). #post raises
+      # ArgumentError on a negative msg_id as a defense-in-depth guard against a
+      # future refactor accidentally offloading one.
+      class OutboundPump
+        WriteJob = Data.define(:connection, :envelopes, :batch_max, :deadline_ms)
+        private_constant :WriteJob
+
+        DRAIN = :__drain__
+
+        def initialize(threads:, ack_queue:, on_dead:, buffer_limit: 0, logger: Pgbus.logger)
+          raise ArgumentError, "threads must be positive" unless threads.positive?
+
+          @ack_queue = ack_queue
+          @on_dead = on_dead
+          @buffer_limit = buffer_limit
+          @logger = logger
+          # One partition per worker. Each is a Partition wrapping a bounded
+          # per-connection buffer so the drop-oldest policy is per connection,
+          # not per partition (a fast connection can't be starved by a slow one
+          # sharing its worker beyond ordering).
+          @partitions = Array.new(threads) { Partition.new(buffer_limit, @logger) }
+          @threads = []
+          @started = false
+        end
+
+        def start
+          return self if @started
+
+          @started = true
+          @partitions.each do |partition|
+            @threads << Thread.new { run_worker(partition) }
+          end
+          self
+        end
+
+        # True while any writer thread is still alive. Lets callers assert the
+        # pump's OWN threads stopped after #stop without inspecting the global
+        # Thread.list (which is noisy and can't distinguish a pump leak from
+        # unrelated thread churn).
+        def alive?
+          @threads.any?(&:alive?)
+        end
+
+        # Snapshot of the pump's live writer threads — for test introspection.
+        def worker_threads
+          @threads.dup
+        end
+
+        # Hand a durable fanout write to the pump. Returns immediately — the
+        # dispatcher does not block. Raises on a negative (ephemeral) msg_id.
+        def post(connection, envelopes, batch_max, deadline_ms:)
+          if envelopes.any? { |e| e.msg_id.negative? }
+            raise ArgumentError,
+                  "OutboundPump received an ephemeral (negative msg_id) envelope; " \
+                  "ephemeral fanout must stay inline on the dispatcher thread (issue #321 B1)"
+          end
+
+          partition_for(connection).push(
+            WriteJob.new(connection: connection, envelopes: envelopes,
+                         batch_max: batch_max, deadline_ms: deadline_ms)
+          )
+        end
+
+        # Graceful drain: signal every partition to flush what it holds, then
+        # join each worker bounded by the write deadline. Never Thread#kill — a
+        # kill mid write_nonblock corrupts IO state (mirrors
+        # StreamEventDispatcher#stop). Idempotent.
+        def stop
+          return self unless @started
+
+          @started = false
+          @partitions.each { |p| p.push(DRAIN) }
+          @threads.each do |t|
+            next if t.join(join_timeout_seconds)
+
+            @logger.warn { "[Pgbus::Streamer::OutboundPump] writer thread did not drain within #{join_timeout_seconds}s" }
+          end
+          @threads.clear
+          self
+        end
+
+        private
+
+        def partition_for(connection)
+          @partitions[connection.id.hash % @partitions.size]
+        end
+
+        def join_timeout_seconds
+          5
+        end
+
+        def run_worker(partition)
+          loop do
+            job = partition.pop
+            break if job == DRAIN
+
+            process(job)
+          end
+        rescue StandardError => e
+          @logger.error { "[Pgbus::Streamer::OutboundPump] worker crashed: #{e.class}: #{e.message}" }
+          raise
+        end
+
+        def process(job)
+          conn = job.connection
+          return if conn.dead?
+
+          written = conn.enqueue(job.envelopes, deadline_ms: job.deadline_ms)
+
+          if conn.dead?
+            @on_dead.call(conn)
+          else
+            @ack_queue << StreamEventDispatcher::WriteAckMessage.new(
+              connection: conn,
+              accepted_max: accepted_max_for(job, written)
+            )
+          end
+        rescue StandardError => e
+          @logger.error { "[Pgbus::Streamer::OutboundPump] write failed for #{job.connection.id}: #{e.class}: #{e.message}" }
+          safe_mark_dead(job.connection)
+        end
+
+        # The scan cursor may advance to:
+        #   - the highest msg_id actually written, or
+        #   - the batch_max when the filtered batch was empty (advance past the
+        #     audience-hidden window so read_after moves forward), or
+        #   - the connection's current cursor when every frame was a dedup
+        #     no-op (a max-guarded no-op on the dispatcher side).
+        def accepted_max_for(job, written)
+          return job.batch_max if job.envelopes.empty?
+
+          written.map(&:msg_id).max || job.connection.last_msg_id_sent
+        end
+
+        def safe_mark_dead(connection)
+          connection.mark_dead!
+          @on_dead.call(connection)
+        rescue StandardError => e
+          @logger.debug { "[Pgbus::Streamer::OutboundPump] on_dead failed: #{e.class}: #{e.message}" }
+        end
+
+        # A worker's inbox: a control channel (for the DRAIN sentinel, always
+        # delivered) plus a bounded per-connection buffer. When buffer_limit is
+        # positive and a connection's pending frames exceed it, the OLDEST
+        # durable frame for that connection is dropped — safe because durable
+        # frames are archive-recoverable on reconnect (issue #321). A drop is
+        # logged (throttled) so it isn't silent.
+        class Partition
+          # Total durable frames this partition has dropped under buffer_limit.
+          # Monotonic; read by tests and folded into a throttled operator log.
+          attr_reader :dropped
+
+          def initialize(buffer_limit, logger)
+            @buffer_limit = buffer_limit
+            @logger = logger
+            @jobs = []                 # FIFO of WriteJob (and the DRAIN sentinel)
+            @pending = Hash.new(0)     # connection.id → buffered frame count
+            @mutex = Mutex.new
+            @cond = ConditionVariable.new
+            @dropped = 0
+          end
+
+          def push(job)
+            @mutex.synchronize do
+              if job == DRAIN
+                @jobs << job
+              else
+                enforce_limit(job)
+                @jobs << job
+                @pending[job.connection.id] += job.envelopes.size
+              end
+              @cond.signal
+            end
+          end
+
+          def pop
+            @mutex.synchronize do
+              @cond.wait(@mutex) while @jobs.empty?
+              job = @jobs.shift
+              @pending[job.connection.id] -= job.envelopes.size unless job == DRAIN
+              job
+            end
+          end
+
+          private
+
+          # Caller holds @mutex. Drop the oldest still-buffered frames for this
+          # connection until adding `job`'s frames keeps it at/under the cap.
+          def enforce_limit(job)
+            return unless @buffer_limit.positive?
+
+            cid = job.connection.id
+            while @pending[cid] + job.envelopes.size > @buffer_limit
+              dropped = drop_oldest_for(cid)
+              break unless dropped # nothing left to drop → let it through
+            end
+          end
+
+          # Caller holds @mutex. Remove the oldest queued WriteJob for cid,
+          # decrementing the pending count. Returns the dropped job or nil.
+          # A drop is logged (throttled to powers of two so a sustained
+          # overflow can't flood the log) so lost frames aren't silent.
+          def drop_oldest_for(cid)
+            idx = @jobs.index { |j| j != DRAIN && j.connection.id == cid }
+            return nil unless idx
+
+            dropped = @jobs.delete_at(idx)
+            @pending[cid] -= dropped.envelopes.size
+            @dropped += 1
+            log_drop(cid) if @dropped.nobits?(@dropped - 1) # power-of-two throttle
+            dropped
+          end
+
+          def log_drop(cid)
+            @logger.warn do
+              "[Pgbus::Streamer::OutboundPump] dropped #{@dropped} durable frame(s) " \
+                "under buffer_limit=#{@buffer_limit} (connection #{cid}) — the client " \
+                "replays them from the archive on reconnect"
+            end
+          end
+        end
+        private_constant :Partition
+      end
+    end
+  end
+end

data/lib/pgbus/web/streamer/stream_event_dispatcher.rb

@@ -35,6 +35,13 @@ module Pgbus
         WakeMessage          = Listener::WakeMessage
         ConnectMessage       = Data.define(:connection)
         DisconnectMessage    = Data.define(:connection)
+        # Posted by the OutboundPump (issue #321) after a successful off-thread
+        # durable fanout write, carrying the highest msg_id the writer actually
+        # committed for the connection. The dispatcher — the sole owner of
+        # @scanned_cursor — drains these via apply_acks and advances the scan
+        # cursor by accepted_max, so a failed/partial write never advances the
+        # cursor past a frame that didn't reach the socket.
+        WriteAckMessage      = Data.define(:connection, :accepted_max)
         # Posted by the Heartbeat once per tick with the current presence
         # connections, so the touch (a last_seen_at refresh) runs on the
         # dispatcher thread where AR connections are released each pass.
@@ -68,13 +75,21 @@ module Pgbus
         def initialize(client:, registry:, listener:, dispatch_queue:,
                        logger: Pgbus.logger, read_limit: DEFAULT_READ_LIMIT,
                        filters: nil, config: nil, stream_counter: nil,
-                       presence_provider: nil)
+                       presence_provider: nil, pump: nil, ack_queue: nil)
           @client = client
           @registry = registry
           @listener = listener
           @queue = dispatch_queue
           @logger = logger
           @read_limit = read_limit
+          # Off-thread durable fanout (issue #321). When @pump is non-nil
+          # (streams_writer_threads > 0), handle_durable_wake hands each
+          # registered-connection write to the pump instead of writing inline;
+          # the pump reports back an accepted-max on @ack_queue, which the
+          # dispatcher drains in apply_acks to advance @scanned_cursor. When
+          # nil (default), fanout writes stay inline — the pre-#321 behavior.
+          @pump = pump
+          @ack_queue = ack_queue
           # Vends a presence handle for a logical stream name. Injected so
           # tests can record join/leave/touch without a DB. Production
           # defaults to the real per-stream Presence via Pgbus.stream.
@@ -90,6 +105,14 @@ module Pgbus
           # process-wide setting. Falls back to the global config
           # for production call sites that don't specify one.
           @config = config || Pgbus.configuration
+          # Two write deadlines (issue #315 item 3). Fanout writes run serially
+          # on this single thread, so a slow client stalls the connections
+          # queued behind it — the short fanout deadline bounds that stall.
+          # Connect-replay writes run once per subscribe (not in the hot loop),
+          # so a fresh client catching up from the archive keeps the full
+          # deadline and isn't spuriously evicted.
+          @fanout_write_deadline_ms  = @config.streams_fanout_write_deadline_ms
+          @connect_write_deadline_ms = @config.streams_write_deadline_ms
           @stream_counter = stream_counter || StreamCounter.new
           # stream_name → Array<[connection, Array<Envelope>]>
           @in_flight = Hash.new { |h, k| h[k] = [] }
@@ -149,6 +172,12 @@ module Pgbus
 
         def run_loop
           while @running
+            # Apply any writer acks before doing anything else, so the scan
+            # cursor is current for the wake we're about to process — without
+            # this top-of-loop drain a wake burst would compute minimum_cursor
+            # from a stale floor and re-read/re-filter the whole un-acked
+            # window every wake (the re-read storm, issue #321).
+            apply_acks
             msg = @queue.pop
             break if msg == :__stop__
 
@@ -227,6 +256,9 @@ module Pgbus
         end
 
         def handle_durable_wake(stream, registered, in_flight_pairs, started_at)
+          # Drain writer acks before computing the read floor so a burst of
+          # wakes doesn't re-read the un-acked window (issue #321).
+          apply_acks
           min_seen = minimum_cursor(registered, in_flight_pairs)
           raw_envelopes = @client.read_after(stream, after_id: min_seen, limit: @read_limit)
           return if raw_envelopes.empty?
@@ -235,8 +267,18 @@ module Pgbus
           max_msg_id = envelopes.map(&:msg_id).max
 
           registered.each do |conn|
-            safe_enqueue(conn, visible_envelopes_for(envelopes, conn))
-            advance_scanned_cursor(conn, max_msg_id)
+            visible = visible_envelopes_for(envelopes, conn)
+            if @pump
+              # Off-thread: hand the write to the pump and DEFER the scan-cursor
+              # advance to the ack (which carries the writer's accepted max).
+              # Post even when `visible` is empty, with batch max, so an
+              # audience-filtered-away batch still advances the cursor past the
+              # hidden window (issue #321 B2). The pump raises on ephemerals.
+              @pump.post(conn, visible, max_msg_id, deadline_ms: @fanout_write_deadline_ms)
+            else
+              safe_enqueue(conn, visible)
+              advance_scanned_cursor(conn, max_msg_id)
+            end
           end
           in_flight_pairs.each do |(conn, buffer)|
             buffer.concat(visible_envelopes_for(envelopes, conn))
@@ -321,13 +363,18 @@ module Pgbus
             limit: @read_limit
           )
           initial = raw_initial.map { |e| unwrap_stream_envelope(e) }
-          safe_enqueue(connection, visible_envelopes_for(initial, connection))
+          # Connect-replay (steps 3 & 4) uses the FULL write deadline, not the
+          # short fanout one: this runs once per subscribe, outside the serial
+          # hot loop, and a new client catching up a large archive backlog must
+          # not be killed by the 250ms fanout budget (issue #315 item 3).
+          safe_enqueue(connection, visible_envelopes_for(initial, connection),
+                       deadline_ms: @connect_write_deadline_ms)
 
           # Step 4: drain the in-flight buffer (anything published between
           # step 2 and now). Connection#enqueue dedupes by cursor, so
           # overlap with step 3 is safe. The buffer entries were already
           # filtered when enqueued by handle_wake, so no re-filter here.
-          safe_enqueue(connection, buffer)
+          safe_enqueue(connection, buffer, deadline_ms: @connect_write_deadline_ms)
 
           # Step 5: promote to the main registry. From this point the
           # regular WakeMessage path handles the connection. If the
@@ -440,11 +487,46 @@ module Pgbus
           @scanned_cursor[connection] = msg_id if msg_id > current
         end
 
-        def safe_enqueue(connection, envelopes_or_buffer)
+        # Drain every pending writer ack and advance the scan cursor by the
+        # writer's accepted max (issue #321). Runs ONLY on the dispatcher
+        # thread, so @scanned_cursor keeps its single-owner invariant — the
+        # pump only reports a value via @ack_queue, it never mutates cursor
+        # state. Non-blocking: stops when the queue is empty. A no-op when
+        # offload is off (@ack_queue nil). advance_scanned_cursor is
+        # monotonic-max-guarded, so a re-applied or cross-connection-reordered
+        # ack is idempotent.
+        #
+        # Drops a stale ack for a connection that is no longer registered: the
+        # ack queue is drained independently of DisconnectMessages, so a
+        # successful-write ack can arrive AFTER handle_disconnect has already
+        # deleted @scanned_cursor[conn] (e.g. the heartbeat swept an idle client
+        # mid-write). Advancing the cursor then would resurrect the connection
+        # in @scanned_cursor and leak that entry forever. The equal? check also
+        # rejects an ack for a since-replaced connection object sharing the same
+        # id.
+        def apply_acks
+          return unless @ack_queue
+
+          loop do
+            ack = @ack_queue.pop(true)
+            next unless @registry.lookup(ack.connection.id).equal?(ack.connection)
+
+            advance_scanned_cursor(ack.connection, ack.accepted_max)
+          rescue ThreadError
+            break # queue drained
+          end
+        end
+
+        # deadline_ms defaults to the SHORT fanout deadline: every hot-loop
+        # fanout call site (handle_durable_wake, handle_ephemeral_wake) uses
+        # it implicitly. handle_connect passes @connect_write_deadline_ms
+        # explicitly so a new client's initial replay keeps the full window
+        # (issue #315 item 3).
+        def safe_enqueue(connection, envelopes_or_buffer, deadline_ms: @fanout_write_deadline_ms)
           return if connection.dead?
           return if envelopes_or_buffer.empty?
 
-          connection.enqueue(envelopes_or_buffer)
+          connection.enqueue(envelopes_or_buffer, deadline_ms: deadline_ms)
         end
 
         def prune_dead(connections)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.9.10
+  version: 0.9.11
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -158,6 +158,7 @@ files:
 - app/models/pgbus/recurring_execution.rb
 - app/models/pgbus/recurring_task.rb
 - app/models/pgbus/semaphore.rb
+- app/models/pgbus/stream_queue.rb
 - app/models/pgbus/stream_stat.rb
 - app/models/pgbus/uniqueness_key.rb
 - app/views/layouts/pgbus/application.html.erb
@@ -215,6 +216,7 @@ files:
 - lib/generators/pgbus/add_presence_generator.rb
 - lib/generators/pgbus/add_queue_states_generator.rb
 - lib/generators/pgbus/add_recurring_generator.rb
+- lib/generators/pgbus/add_stream_queues_generator.rb
 - lib/generators/pgbus/add_stream_stats_generator.rb
 - lib/generators/pgbus/add_uniqueness_keys_generator.rb
 - lib/generators/pgbus/install_generator.rb
@@ -228,11 +230,12 @@ files:
 - lib/generators/pgbus/templates/add_presence.rb.erb
 - lib/generators/pgbus/templates/add_queue_states.rb.erb
 - lib/generators/pgbus/templates/add_recurring_tables.rb.erb
+- lib/generators/pgbus/templates/add_stream_queues.rb.erb
 - lib/generators/pgbus/templates/add_stream_stats.rb.erb
 - lib/generators/pgbus/templates/add_uniqueness_keys.rb.erb
+- lib/generators/pgbus/templates/initializer.rb.erb
 - lib/generators/pgbus/templates/migrate_job_locks_to_uniqueness_keys.rb.erb
 - lib/generators/pgbus/templates/migration.rb.erb
-- lib/generators/pgbus/templates/pgbus.yml.erb
 - lib/generators/pgbus/templates/pgbus_binstub.erb
 - lib/generators/pgbus/templates/recurring.yml.erb
 - lib/generators/pgbus/templates/tune_autovacuum.rb.erb
@@ -256,10 +259,10 @@ files:
 - lib/pgbus/client/ensure_stream_queue.rb
 - lib/pgbus/client/notify_stream.rb
 - lib/pgbus/client/read_after.rb
+- lib/pgbus/client/resizable_pool.rb
 - lib/pgbus/concurrency.rb
 - lib/pgbus/concurrency/blocked_execution.rb
 - lib/pgbus/concurrency/semaphore.rb
-- lib/pgbus/config_loader.rb
 - lib/pgbus/configuration.rb
 - lib/pgbus/configuration/capsule_dsl.rb
 - lib/pgbus/dedup_cache.rb
@@ -275,7 +278,6 @@ files:
 - lib/pgbus/execution_pools/async_pool.rb
 - lib/pgbus/execution_pools/thread_pool.rb
 - lib/pgbus/failed_event_recorder.rb
-- lib/pgbus/generators/config_converter.rb
 - lib/pgbus/generators/database_target_detector.rb
 - lib/pgbus/generators/migration_detector.rb
 - lib/pgbus/instrumentation.rb
@@ -350,6 +352,8 @@ files:
 - lib/pgbus/streams/envelope.rb
 - lib/pgbus/streams/filters.rb
 - lib/pgbus/streams/key.rb
+- lib/pgbus/streams/pool_autoscaler.rb
+- lib/pgbus/streams/pool_trigger.rb
 - lib/pgbus/streams/presence.rb
 - lib/pgbus/streams/renderer.rb
 - lib/pgbus/streams/signed_name.rb
@@ -378,6 +382,7 @@ files:
 - lib/pgbus/web/streamer/instance.rb
 - lib/pgbus/web/streamer/io_writer.rb
 - lib/pgbus/web/streamer/listener.rb
+- lib/pgbus/web/streamer/outbound_pump.rb
 - lib/pgbus/web/streamer/registry.rb
 - lib/pgbus/web/streamer/stream_counter.rb
 - lib/pgbus/web/streamer/stream_event_dispatcher.rb

data/lib/generators/pgbus/templates/pgbus.yml.erb

@@ -1,76 +0,0 @@
-# Pgbus configuration
-# https://github.com/mhenrixon/pgbus
-
-default: &default
-  # Queue prefix for all PGMQ queues
-  queue_prefix: pgbus
-
-  # Default queue name (without prefix)
-  default_queue: default
-
-  # Connection pool for PGMQ client.
-  # pool_size auto-tunes from your worker thread counts:
-  #   sum(workers.threads) + sum(event_consumers.threads) + 2
-  # Set explicitly only if you need a tighter or looser pool than that.
-  # pool_size: 5
-  pool_timeout: 5
-
-  # Use PostgreSQL LISTEN/NOTIFY for instant job wake-up
-  listen_notify: true
-
-  # Visibility timeout in seconds (how long a message is invisible after being read)
-  visibility_timeout: 30
-
-  # Dead letter queue
-  max_retries: 5
-
-  # Idempotency deduplication TTL (seconds)
-  idempotency_ttl: 604800  # 7 days
-
-  # Worker definitions
-  workers:
-    - queues:
-        - critical
-        - default
-      threads: 5
-    - queues:
-        - low
-      threads: 2
-
-  # Worker recycling (prevents memory bloat)
-  max_jobs_per_worker: 10000
-  max_memory_mb: 512
-  # max_worker_lifetime: 3600  # seconds
-
-  # Dispatcher settings (maintenance tasks)
-  dispatch_interval: 1.0
-
-  # Event consumers (uncomment to enable event bus)
-  # event_consumers:
-  #   - topics:
-  #       - "orders.#"
-  #     threads: 3
-  #   - topics:
-  #       - "notifications.#"
-  #     threads: 1
-
-development:
-  <<: *default
-  workers:
-    - queues:
-        - default
-      threads: 2
-  max_jobs_per_worker: ~
-  max_memory_mb: ~
-
-test:
-  <<: *default
-  workers:
-    - queues:
-        - default
-      threads: 1
-  polling_interval: 0.01
-  visibility_timeout: 5
-
-production:
-  <<: *default