pgbus 0.9.10 → 0.9.11

data/lib/pgbus/configuration.rb

@@ -130,9 +130,14 @@ module Pgbus
                   :streams_write_deadline_ms, :streams_falcon_streaming_body,
                   :streams_stats_enabled, :streams_test_mode,
                   :streams_orphan_sweep_interval, :streams_orphan_threshold,
-                  :streams_durable_patterns,
+                  :streams_durable_patterns, :streams_broadcast_queue,
                   :streams_presence_patterns, :streams_presence_member,
-                  :streams_host, :streams_port, :streams_database_url
+                  :streams_host, :streams_port, :streams_database_url,
+                  :streams_pool_size, :streams_pool_timeout,
+                  :streams_fanout_write_deadline_ms, :streams_dispatch_queue_limit,
+                  :streams_writer_threads, :streams_writer_buffer_limit,
+                  :streams_pool_autoscale, :streams_pool_max,
+                  :streams_pool_autoscale_interval, :streams_application_name
     attr_reader :streams_default_broadcast_mode # rubocop:disable Style/AccessorGrouping
 
     # NOTIFY-gated worker wakeups. When true, each Worker fork owns a
@@ -155,8 +160,8 @@ module Pgbus
     #   a Pgbus::Metrics::Backend instance — a custom backend
     attr_accessor :metrics_backend, :statsd_host, :statsd_port
 
-    # When true (default), Pgbus.configure and ConfigLoader.apply run
-    # validate! after applying settings, so an invalid value fails loud at
+    # When true (default), Pgbus.configure runs validate! after the block
+    # yields, so an invalid value fails loud at
     # boot instead of dormant until a worker path consumes it. Set false to
     # opt out (exotic setups that intentionally hold a transiently-invalid
     # config); explicit validate! still works.
@@ -263,6 +268,11 @@ module Pgbus
 
       @streams_enabled = true
       @streams_path = nil
+      # Retained for backward compatibility only — NO LONGER USED for queue
+      # naming or stream detection. Stream queues are named like job queues
+      # (`#{queue_prefix}_<name>`, see #queue_name) and are identified via the
+      # `pgbus_stream_queues` registry (Pgbus::StreamQueue), not by this prefix.
+      # Setting it has no effect. See issue #308.
       @streams_queue_prefix = "pgbus_stream"
       # Streamer-only connection overrides. The Streamer's Listener owns a
       # dedicated long-lived `wait_for_notify` PG connection that can't go
@@ -280,6 +290,42 @@ module Pgbus
       @streams_host = nil
       @streams_port = nil
       @streams_database_url = nil
+      # Dedicated DB connection pool for the streamer's publish + replay hot
+      # paths (Client#send_stream_message and read_after/stream_current_msg_id).
+      # Isolated from the job pool (`pool_size`) so a saturated worker pool
+      # can't delay broadcast INSERTs and the dispatcher's per-wake read_after
+      # reuses a persistent connection instead of a fresh PG.connect per call
+      # (issue #315). A small fixed default — streams fan out from one
+      # dispatcher thread per Puma worker, so a handful of connections is
+      # plenty; raise it only for very high broadcast concurrency. Ignored on
+      # the shared-ActiveRecord (Proc) connection path, where libpq isn't
+      # thread-safe and streams keep using the single serialized connection.
+      @streams_pool_size = 5
+      @streams_pool_timeout = 5
+      # Self-tuning streams-pool autoscaling (issue #323). Opt-in, default off.
+      # When true, a per-web-process control loop grows the dedicated streams
+      # pool into a FAIR SHARE of live Postgres connection headroom under
+      # saturation and shrinks it back to streams_pool_size (the baseline/floor)
+      # when idle — and emergency-shrinks immediately if the DB runs critically
+      # low on connections. No connection-count config: every threshold derives
+      # from live max_connections. streams_pool_max is an OPTIONAL hard ceiling
+      # (nil = the dynamic fair share is the effective cap).
+      @streams_pool_autoscale = false
+      @streams_pool_max = nil
+      # Slow maintenance cadence (seconds) — the autoscaler runs as a periodic
+      # check on the streamer's idle LISTEN connection (pghero-style), not a busy
+      # loop. 5 minutes: the interval itself debounces, so there is no per-sample
+      # hysteresis; a sustained burst converges over a few checks.
+      @streams_pool_autoscale_interval = 300.0
+      # Distinctive application_name prefix stamped on streams-pool connections
+      # so the autoscaler can count peer processes (DISTINCT application_name
+      # LIKE '<prefix>_%') from pg_stat_activity. Per-process suffix is appended
+      # at build time. Namespace it per deployment if several pgbus fleets share
+      # one database. NOTE: a transaction-mode PgBouncer strips application_name;
+      # peer inference then degrades to "assume alone" (the safety margin +
+      # emergency-shrink cushion this) — connect the streamer DIRECTLY for
+      # accurate autoscaling.
+      @streams_application_name = "pgbus_streams"
       @streams_signed_name_secret = nil
       @streams_default_retention = 5 * 60 # 5 minutes
       @streams_retention = {}
@@ -296,6 +342,49 @@ module Pgbus
       # PG keepalive interval.
       @streams_listen_health_check_ms = 250
       @streams_write_deadline_ms = 5_000
+      # Deadline (ms) for a socket write made INSIDE the dispatcher's
+      # serial fanout loop (handle_durable_wake / handle_ephemeral_wake).
+      # Kept far below streams_write_deadline_ms (5s) because these writes
+      # run one after another on the single dispatcher thread: K
+      # slow-but-not-yet-dead clients stack the stall, so the whole-thread
+      # worst case is ~K * this_value before each is marked dead
+      # (issue #315 item 3). A slow-but-alive client that can't absorb a
+      # fanout frame in this window is marked dead and reconnects — its
+      # EventSource Last-Event-ID then replays the missed frames from the
+      # durable archive (or triggers a fresh re-render for ephemeral
+      # streams, which have no archive). Connect-replay writes keep the
+      # full streams_write_deadline_ms. Set this equal to
+      # streams_write_deadline_ms to restore the pre-#315 timing.
+      @streams_fanout_write_deadline_ms = 250
+      # Optional cap on dispatch-queue depth checked by the Listener before
+      # it pushes a durable WakeMessage. 0 (default) = unbounded — today's
+      # behavior. A positive value makes the Listener DROP a durable wake
+      # when the queue is at/over the cap; this is safe because the next
+      # durable wake for that stream re-reads from the min cursor. It
+      # bounds DISTINCT-STREAM backlog only — a single hot stream's wakes
+      # coalesce (drain_wakes_for) so its depth stays low regardless.
+      # Ephemeral wakes (which carry the only copy of their HTML) and
+      # Connect/Disconnect messages are NEVER dropped (issue #315 item 3).
+      @streams_dispatch_queue_limit = 0
+      # Number of writer threads that flush durable-stream fanout socket writes
+      # OFF the single dispatcher thread (issue #321). 0 (default) = inline —
+      # byte-for-byte the pre-#321 behavior, no pump, no extra allocation. A
+      # positive value spawns N writer threads; each durable broadcast fanout
+      # write is handed to a writer (partitioned by connection.id.hash % N, so
+      # a connection's frames stay ordered and its per-io mutex is respected),
+      # freeing the dispatcher to service the next wake/connect. Fast clients
+      # then stop waiting behind slow ones. EPHEMERAL fanout and connect-replay
+      # writes always stay inline regardless of this value: ephemerals have no
+      # archive to replay, so they must not risk an async drop (see #321 B1).
+      @streams_writer_threads = 0
+      # Per-connection cap on the writer's outbound buffer when
+      # streams_writer_threads > 0. 0 (default) = unbounded. A positive value
+      # drops the OLDEST buffered durable frame for a connection whose writer
+      # can't keep up — safe because durable frames live in the a_<stream>
+      # archive and are re-read on reconnect via Last-Event-ID; it's a memory
+      # guard against a pathologically slow-but-alive client, not a delivery
+      # guarantee. Ignored when streams_writer_threads == 0 (issue #321).
+      @streams_writer_buffer_limit = 0
       # EXPERIMENTAL — exempt from the 1.0 stability promise.
       @streams_falcon_streaming_body = false
       # Opt-in: when true, the Dispatcher writes one row to
@@ -311,6 +400,14 @@ module Pgbus
       @streams_orphan_sweep_interval = 3600    # 1 hour
       @streams_orphan_threshold = 86_400       # 24 hours
       @streams_durable_patterns = []
+      # Dedicated queue for turbo-rails' async broadcast jobs
+      # (Turbo::Streams::ActionBroadcastJob and siblings). nil (default) leaves
+      # them on the default queue, where a `broadcasts_to`/`broadcasts_refreshes`
+      # render+broadcast can wait behind long-running jobs before the browser
+      # sees the update. Set to a queue name (e.g. "realtime") and back it with
+      # a dedicated worker capsule to isolate broadcast latency from job
+      # throughput. Applied at engine boot when turbo-rails is loaded. See #311.
+      @streams_broadcast_queue = nil
       # EXPERIMENTAL (both presence settings) — exempt from the 1.0 stability
       # promise. Streams matching these patterns get connection-driven presence:
       # auto-join on SSE connect, auto-leave on disconnect, touch on the
@@ -528,8 +625,43 @@ module Pgbus
         raise Pgbus::ConfigurationError, "streams_write_deadline_ms must be a positive integer"
       end
 
+      unless streams_fanout_write_deadline_ms.is_a?(Integer) && streams_fanout_write_deadline_ms.positive?
+        raise Pgbus::ConfigurationError, "streams_fanout_write_deadline_ms must be a positive integer"
+      end
+
+      # >= 0, NOT .positive? — 0 is the valid "unbounded" sentinel.
+      unless streams_dispatch_queue_limit.is_a?(Integer) && streams_dispatch_queue_limit >= 0
+        raise Pgbus::ConfigurationError, "streams_dispatch_queue_limit must be a non-negative integer (0 = unbounded)"
+      end
+
+      # >= 0, NOT .positive? — 0 is the valid "inline" sentinel.
+      unless streams_writer_threads.is_a?(Integer) && streams_writer_threads >= 0
+        raise Pgbus::ConfigurationError, "streams_writer_threads must be a non-negative integer (0 = inline)"
+      end
+
+      # >= 0, NOT .positive? — 0 is the valid "unbounded" sentinel.
+      unless streams_writer_buffer_limit.is_a?(Integer) && streams_writer_buffer_limit >= 0
+        raise Pgbus::ConfigurationError, "streams_writer_buffer_limit must be a non-negative integer (0 = unbounded)"
+      end
+
+      unless streams_pool_size.is_a?(Integer) && streams_pool_size.positive?
+        raise Pgbus::ConfigurationError, "streams_pool_size must be a positive integer"
+      end
+
+      unless streams_pool_timeout.is_a?(Numeric) && streams_pool_timeout.positive?
+        raise Pgbus::ConfigurationError, "streams_pool_timeout must be a positive number"
+      end
+
+      validate_streams_autoscale!
+
       raise Pgbus::ConfigurationError, "streams_retention must be a Hash" unless streams_retention.is_a?(Hash)
 
+      unless streams_broadcast_queue.nil? ||
+             (streams_broadcast_queue.is_a?(String) && !streams_broadcast_queue.empty?)
+        raise Pgbus::ConfigurationError,
+              "streams_broadcast_queue must be a non-empty String (a queue name) or nil to disable"
+      end
+
       if streams_orphan_sweep_interval && !(streams_orphan_sweep_interval.is_a?(Numeric) && streams_orphan_sweep_interval.positive?)
         raise Pgbus::ConfigurationError, "streams_orphan_sweep_interval must be a positive number or nil to disable"
       end
@@ -554,6 +686,26 @@ module Pgbus
       raise Pgbus::ConfigurationError, "streams_orphan_threshold must be a positive number or nil to disable"
     end
 
+    def validate_streams_autoscale!
+      raise Pgbus::ConfigurationError, "streams_pool_autoscale must be true or false" unless [true, false].include?(streams_pool_autoscale)
+
+      # nil = no hard cap (dynamic fair-share is the ceiling). A positive value
+      # below the baseline is an inverted ceiling — fail loud at boot.
+      unless streams_pool_max.nil? ||
+             (streams_pool_max.is_a?(Integer) && streams_pool_max >= streams_pool_size)
+        raise Pgbus::ConfigurationError,
+              "streams_pool_max must be an Integer >= streams_pool_size " \
+              "(#{streams_pool_size}) or nil to leave growth uncapped"
+      end
+
+      unless streams_pool_autoscale_interval.is_a?(Numeric) && streams_pool_autoscale_interval.positive?
+        raise Pgbus::ConfigurationError, "streams_pool_autoscale_interval must be a positive number"
+      end
+
+      raise Pgbus::ConfigurationError, "streams_application_name must be a non-empty String" \
+        unless streams_application_name.is_a?(String) && !streams_application_name.empty?
+    end
+
     def validate_metrics_backend!
       value = metrics_backend
       return if value.nil?
@@ -661,9 +813,9 @@ module Pgbus
                  end
     end
 
-    # Event consumer configs arrive with symbol keys (Ruby DSL) or string keys
-    # (YAML via ConfigLoader). Normalize once here so every read site uses
-    # symbol access only. nil passes through (no consumers configured).
+    # Event consumer configs arrive with symbol keys (Ruby DSL) or, from an
+    # explicit Array literal, string keys. Normalize once here so every read
+    # site uses symbol access only. nil passes through (no consumers configured).
     def event_consumers=(value)
       @event_consumers =
         case value
@@ -854,6 +1006,15 @@ module Pgbus
     # timeouts under load). Setting pool_size explicitly is still supported
     # for advanced cases where you need a tighter or looser pool than the
     # default formula provides.
+    #
+    # NOTE: this covers the JOB pool only. On the dedicated-connection path
+    # (database_url / connection_params) every Client also opens a separate
+    # streams pool of +streams_pool_size+ (default 5) for durable-stream
+    # publish + replay (issue #315). Both pools are lazy, so an idle process
+    # holds few real connections, but when budgeting Postgres/PgBouncer
+    # max_connections, size for +resolved_pool_size + streams_pool_size+ per
+    # process — every forked worker/dispatcher/scheduler/consumer builds its
+    # own streams pool even if it rarely touches streams.
     POOL_SIZE_WARN_THRESHOLD = 50
 
     # Connections needed per async worker: one for the reactor's serial
@@ -1057,8 +1218,8 @@ module Pgbus
     end
 
     # Symbolize the top-level keys of a worker/consumer config entry so every
-    # downstream read site can use symbol access only. YAML (via ConfigLoader)
-    # yields string keys; the Ruby DSL and +capsule+ builder yield symbols.
+    # downstream read site can use symbol access only. An explicit Array literal
+    # may carry string keys; the Ruby DSL and +capsule+ builder yield symbols.
     # Entries are flat hashes with array/scalar values — no deep recursion.
     def normalize_entry(entry, group:)
       raise Pgbus::ConfigurationError, "#{group} entry must be a Hash, got #{entry.class}: #{entry.inspect}" unless entry.is_a?(Hash)

data/lib/pgbus/doctor.rb

@@ -5,7 +5,7 @@ require "pgbus/mcp/health_analyzer"
 
 module Pgbus
   # Preflight diagnostics for a pgbus deployment — the single command that
-  # answers "is this environment healthy enough to run?". Runs seven checks and
+  # answers "is this environment healthy enough to run?". Runs eight checks and
   # returns a machine-readable result plus a human report, so `pgbus doctor`
   # and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
   # 1 on any failure).
@@ -41,7 +41,8 @@ module Pgbus
         check_queues,
         check_notify,
         check_processes,
-        check_allowed_global_id_models
+        check_allowed_global_id_models,
+        check_broadcast_queue
       ].map(&:to_h)
     end
 
@@ -210,6 +211,52 @@ module Pgbus
       Check.new(name: "GlobalID allowlist", status: :fail, detail: "#{e.class}: #{e.message}")
     end
 
+    # 8. Broadcast-queue isolation — latency + correctness. With turbo-rails
+    # loaded, the default broadcasts_to/broadcasts_refreshes path enqueues
+    # render+broadcast ActiveJobs on the DEFAULT queue, so a browser SSE update
+    # can wait behind long-running jobs (#311). Two failure modes:
+    #
+    #   (a) streams_broadcast_queue is nil — broadcasts share the default queue
+    #       and can wait behind long jobs. A latency concern, so warn only in
+    #       production (where the blast radius is real).
+    #   (b) streams_broadcast_queue is set but NO worker capsule drains it —
+    #       broadcasts route there and pile up unread, so the browser never
+    #       updates. A correctness bug, so warn everywhere, not just production.
+    #
+    # Both are warnings, never failures. Only relevant when Turbo is loaded.
+    def check_broadcast_queue
+      broadcast_queue = @config.streams_broadcast_queue
+      turbo = @config.streams_enabled && defined?(::Turbo::Broadcastable)
+
+      if turbo && broadcast_queue && !worker_drains?(broadcast_queue)
+        return Check.new(name: "Broadcast queue", status: :warn,
+                         detail: "streams_broadcast_queue is \"#{broadcast_queue}\" but no worker capsule " \
+                                 "drains it — broadcasts will pile up unread and never reach the browser; " \
+                                 "add a capsule for the \"#{broadcast_queue}\" queue (or a wildcard worker)")
+      end
+
+      if turbo && production? && broadcast_queue.nil?
+        return Check.new(name: "Broadcast queue", status: :warn,
+                         detail: "streams_broadcast_queue is nil — turbo-rails broadcast jobs share the " \
+                                 "default queue and can wait behind long-running jobs; set a dedicated queue " \
+                                 "(e.g. \"realtime\") and back it with a worker capsule")
+      end
+
+      Check.new(name: "Broadcast queue", status: :ok,
+                detail: broadcast_queue ? "dedicated: #{broadcast_queue}" : "n/a")
+    rescue StandardError => e
+      Check.new(name: "Broadcast queue", status: :fail, detail: "#{e.class}: #{e.message}")
+    end
+
+    # True when some configured worker capsule drains the given queue — either
+    # by naming it explicitly or via a "*" wildcard (which drains every queue).
+    def worker_drains?(queue)
+      Array(@config.workers).any? do |w|
+        queues = w[:queues] || w["queues"] || []
+        queues.include?("*") || queues.include?(queue)
+      end
+    end
+
     # Physical queue names known to PGMQ. list_queues rows may be Hashes (with a
     # :queue_name key) or value objects responding to #queue_name.
     def existing_queue_names

data/lib/pgbus/engine.rb

@@ -12,9 +12,30 @@ module Pgbus
       Pgbus.teardown_models_loader!
     end
 
-    initializer "pgbus.configure" do |app|
-      config_path = app.root.join("config", "pgbus.yml")
-      Pgbus::ConfigLoader.load(config_path) if config_path.exist?
+    # config/pgbus.yml is no longer loaded (removed in 1.0). If an app still
+    # ships one, it's now silently inert — warn once at boot so the leftover
+    # file doesn't quietly diverge from the real (Ruby initializer) config.
+    initializer "pgbus.legacy_yaml_warning" do |app|
+      ActiveSupport.on_load(:after_initialize) do
+        Pgbus::Engine.warn_if_legacy_yaml_config(app.root, Pgbus.logger)
+      end
+    end
+
+    # Emits the legacy-YAML deprecation warning when +root+/config/pgbus.yml
+    # exists. Extracted so it can be unit-tested without booting Rails. A nil
+    # logger is a no-op (logger may be unset in a bare/embedded boot).
+    def self.warn_if_legacy_yaml_config(root, logger)
+      return if logger.nil?
+
+      config_path = root.join("config", "pgbus.yml")
+      return unless config_path.exist?
+
+      logger.warn do
+        "[Pgbus] config/pgbus.yml is no longer loaded (YAML config was removed in 1.0). " \
+          "Port its settings into config/initializers/pgbus.rb as a Pgbus.configure block " \
+          "and delete the YAML. See https://pgbus.zoolutions.llc/docs/configuration " \
+          "(the pgbus MCP server can also help port it)."
+      end
     end
 
     initializer "pgbus.recurring" do |app|
@@ -154,6 +175,12 @@ module Pgbus
           if defined?(::Turbo::Broadcastable)
             _autoload_trigger_broadcastable = Pgbus::Streams::BroadcastableOverride
             Pgbus::Streams::BroadcastableOverride.install!(::Turbo::Broadcastable)
+
+            # Route turbo-rails' async broadcast jobs to a dedicated queue when
+            # configured, so a broadcasts_to render+broadcast doesn't wait
+            # behind long-running jobs on the default queue (#311). No-op when
+            # streams_broadcast_queue is nil.
+            Pgbus::Streams.install_broadcast_queue!(Pgbus.configuration.streams_broadcast_queue)
           end
 
           # Subscribe-side patch: override turbo_stream_from to render

data/lib/pgbus/generators/migration_detector.rb

@@ -71,6 +71,7 @@ module Pgbus
         add_job_stats_latency: "pgbus:add_job_stats_latency",
         add_job_stats_queue_index: "pgbus:add_job_stats_queue_index",
         add_stream_stats: "pgbus:add_stream_stats",
+        add_stream_queues: "pgbus:add_stream_queues",
         add_presence: "pgbus:add_presence",
         add_queue_states: "pgbus:add_queue_states",
         add_outbox: "pgbus:add_outbox",
@@ -88,6 +89,7 @@ module Pgbus
         add_job_stats_latency: "job stats latency columns (enqueue_latency_ms, retry_count)",
         add_job_stats_queue_index: "job stats (queue_name, created_at) index",
         add_stream_stats: "stream stats table (opt-in real-time Insights)",
+        add_stream_queues: "stream queue registry (per-stream retention, orphan sweep, wildcard-worker safety)",
         add_presence: "presence members table (Turbo Streams presence)",
         add_queue_states: "queue states table (pause/resume)",
         add_outbox: "outbox entries table (transactional outbox)",
@@ -112,6 +114,7 @@ module Pgbus
           *uniqueness_key_migrations,
           *job_stats_migrations,
           *stream_stats_migrations,
+          *stream_queues_migrations,
           *presence_migrations,
           *queue_states_migrations,
           *outbox_migrations,
@@ -166,6 +169,10 @@ module Pgbus
         table_exists?("pgbus_stream_stats") ? [] : [:add_stream_stats]
       end
 
+      def stream_queues_migrations
+        table_exists?("pgbus_stream_queues") ? [] : [:add_stream_queues]
+      end
+
       def presence_migrations
         table_exists?("pgbus_presence_members") ? [] : [:add_presence]
       end

data/lib/pgbus/process/dispatcher.rb

@@ -37,6 +37,16 @@ module Pgbus
       MAINTENANCE_BACKOFF_BASE = 30    # seconds; first backoff window
       MAINTENANCE_BACKOFF_MAX = 600    # seconds; window ceiling (10 minutes)
 
+      # A pooled backend the server terminated while the dispatcher sat idle
+      # between cycles surfaces with this exact PostgreSQL text on the next
+      # query. It is expected and self-healing — release_maintenance_connections
+      # returns the dead socket to the pool and the next cycle reconnects — so a
+      # task that hits it logs a calm INFO rather than an alarming WARN. Matched
+      # case-insensitively as a substring of the exception message. Common in
+      # local development (Postgres restart, `rails db:migrate` terminating idle
+      # backends, foreman/overmind restarting the DB), rare in production.
+      TERMINATED_CONNECTION_SIGNAL = "terminating connection due to administrator command"
+
       # The per-task monotonic timestamps run_maintenance consults via run_if_due.
       # Enumerated so set_maintenance_timestamp can validate its argument.
       MAINTENANCE_TIMESTAMPS = %i[
@@ -166,6 +176,51 @@ module Pgbus
 
         results = run_maintenance_tasks(now)
         update_maintenance_backoff(now, results)
+      ensure
+        release_maintenance_connections
+      end
+
+      # Return every AR connection this cycle leased back to the pool. The
+      # dispatcher is a long-lived loop that queries the pool on coarse
+      # intervals (concurrency cleanup every 300s, most tasks hourly). If it
+      # kept its pooled connection leased across the idle sleep, a backend the
+      # server terminates while it sits idle — routine in local development
+      # (a Postgres restart, `rails db:migrate` calling pg_terminate_backend,
+      # foreman/overmind restarting the DB) — would be reused dead on the next
+      # cycle, and every task would raise PG::ConnectionBad
+      # ("terminating connection due to administrator command"). Releasing here
+      # means the next cycle checks out afresh, and AR's verify!-on-checkout
+      # transparently discards a dead socket and reconnects. Best-effort: a
+      # failure to release must never crash the loop (mirrors the Streamer's
+      # release_ar_connections).
+      def release_maintenance_connections
+        return unless defined?(::ActiveRecord::Base)
+
+        Pgbus::BusRecord.connection_handler.clear_active_connections!
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] Maintenance connection release failed: #{e.class}: #{e.message}" }
+      end
+
+      # True when the error is a pooled backend the server terminated while the
+      # dispatcher was idle between cycles. Self-healing (see
+      # TERMINATED_CONNECTION_SIGNAL / release_maintenance_connections), so
+      # callers downgrade the log from WARN to INFO.
+      def terminated_connection_error?(error)
+        error.message.to_s.downcase.include?(TERMINATED_CONNECTION_SIGNAL)
+      end
+
+      # Log a maintenance task failure at the right volume: a self-healing
+      # terminated-idle-connection is a calm INFO ("will reconnect next cycle"),
+      # anything else stays a WARN operators should look at.
+      def log_maintenance_failure(label, error)
+        if terminated_connection_error?(error)
+          Pgbus.logger.info do
+            "[Pgbus] #{label} skipped: database connection was terminated while idle; " \
+              "reconnecting on the next cycle (#{error.message.lines.first&.strip})"
+          end
+        else
+          Pgbus.logger.warn { "[Pgbus] #{label} failed: #{error.message}" }
+        end
       end
 
       # Runs every due maintenance task and returns their results. Each entry
@@ -284,7 +339,7 @@ module Pgbus
         deleted = ProcessedEvent.expired(Time.current - ttl).delete_all
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} expired processed events" } if deleted.positive?
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Idempotency cleanup failed: #{e.message}" }
+        log_maintenance_failure("Idempotency cleanup", e)
         e
       end
 
@@ -293,7 +348,7 @@ module Pgbus
         deleted = ProcessEntry.stale(Time.current - threshold).delete_all
         Pgbus.logger.info { "[Pgbus] Reaped #{deleted} stale processes" } if deleted.positive?
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Stale process reaping failed: #{e.message}" }
+        log_maintenance_failure("Stale process reaping", e)
         e
       end
 
@@ -306,7 +361,7 @@ module Pgbus
         orphaned = Concurrency::BlockedExecution.expire_stale
         Pgbus.logger.debug { "[Pgbus] Expired #{orphaned} orphaned blocked executions" } if orphaned.positive?
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Concurrency cleanup failed: #{e.message}" }
+        log_maintenance_failure("Concurrency cleanup", e)
         e
       end
 
@@ -314,14 +369,14 @@ module Pgbus
         promoted = Concurrency::BlockedExecution.promote_next(key, client: Pgbus.client)
         Pgbus.logger.debug { "[Pgbus] Released blocked execution for key: #{key}" } if promoted
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Failed to release blocked execution for #{key}: #{e.message}" }
+        log_maintenance_failure("Releasing blocked execution for #{key}", e)
       end
 
       def cleanup_batches
         deleted = Batch.cleanup(older_than: Time.current - (7 * 24 * 3600)) # 7 days
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} finished batches" } if deleted.positive?
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Batch cleanup failed: #{e.message}" }
+        log_maintenance_failure("Batch cleanup", e)
         e
       end
 
@@ -353,7 +408,7 @@ module Pgbus
         )
         Pgbus.logger.info { "[Pgbus] Table maintenance completed: #{maintained} table(s) vacuumed" } if maintained.positive?
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Table maintenance failed: #{e.message}" }
+        log_maintenance_failure("Table maintenance", e)
         e
       end
 
@@ -378,7 +433,7 @@ module Pgbus
         reaped = reap_orphaned_uniqueness_keys
         Pgbus.logger.info { "[Pgbus] Reaped #{reaped} orphaned uniqueness keys" } if reaped.positive?
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Job lock cleanup failed: #{e.message}" }
+        log_maintenance_failure("Job lock cleanup", e)
         e
       end
 
@@ -434,7 +489,7 @@ module Pgbus
         deleted = OutboxEntry.published_before(Time.current - retention).delete_all
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} published outbox entries" } if deleted.positive?
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Outbox cleanup failed: #{e.message}" }
+        log_maintenance_failure("Outbox cleanup", e)
         e
       end
 
@@ -445,6 +500,10 @@ module Pgbus
         cutoff = Time.current - retention
         batch_size = ARCHIVE_COMPACTION_BATCH_SIZE
         prefix = config.queue_prefix
+        # Stream queues share the job prefix but are pruned separately by
+        # prune_stream_archives (per-stream retention). Skip them here so a
+        # stream archive is compacted by exactly one path, not both.
+        stream_names = current_stream_queue_names
 
         conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
         queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
@@ -453,6 +512,7 @@ module Pgbus
         queue_names.each_with_index do |full_name, index|
           break if interrupted?("Archive compaction", index, queue_names.size)
           next unless full_name.start_with?("#{prefix}_")
+          next if stream_names.include?(full_name)
 
           stripped = full_name.delete_prefix("#{prefix}_")
           deleted = Pgbus.client.purge_archive(stripped, older_than: cutoff, batch_size: batch_size)
@@ -464,7 +524,7 @@ module Pgbus
         end
         outcome.result
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Archive compaction failed: #{e.message}" }
+        log_maintenance_failure("Archive compaction", e)
         e
       end
 
@@ -475,8 +535,8 @@ module Pgbus
       # Lookup order for a given queue: exact string match, then regex
       # match, then streams_default_retention (5 minutes by default).
       def prune_stream_archives
-        prefix = config.streams_queue_prefix
-        return if prefix.nil? || prefix.empty?
+        stream_names = current_stream_queue_names
+        return if stream_names.empty?
 
         batch_size = ARCHIVE_COMPACTION_BATCH_SIZE
         conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
@@ -485,7 +545,7 @@ module Pgbus
         outcome = LoopOutcome.new
         queue_names.each_with_index do |full_name, index|
           break if interrupted?("Stream archive compaction", index, queue_names.size)
-          next unless full_name.start_with?("#{prefix}_")
+          next unless stream_names.include?(full_name)
 
           retention = retention_for_stream_queue(full_name)
           next unless retention.positive?
@@ -505,7 +565,7 @@ module Pgbus
         end
         outcome.result
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Stream archive compaction failed: #{e.message}" }
+        log_maintenance_failure("Stream archive compaction", e)
         e
       end
 
@@ -524,12 +584,12 @@ module Pgbus
       end
 
       def sweep_orphan_streams
-        prefix = config.streams_queue_prefix
-        return if prefix.nil? || prefix.empty?
-
         threshold = config.streams_orphan_threshold
         return unless threshold
 
+        stream_names = current_stream_queue_names
+        return if stream_names.empty?
+
         conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
         queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
 
@@ -537,7 +597,7 @@ module Pgbus
         outcome = LoopOutcome.new
         queue_names.each_with_index do |full_name, index|
           break if interrupted?("Orphan stream sweep", index, queue_names.size)
-          next unless full_name.start_with?("#{prefix}_")
+          next unless stream_names.include?(full_name)
 
           safe_name = QueueNameValidator.sanitize!(full_name)
           row = conn.select_one(<<~SQL, "Pgbus Orphan Check")
@@ -571,10 +631,20 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus] Orphan stream sweep complete: dropped #{dropped} queue(s)" } if dropped.positive?
         outcome.result
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Orphan stream sweep failed: #{e.message}" }
+        log_maintenance_failure("Orphan stream sweep", e)
         e
       end
 
+      # Fresh set of physical stream-queue names for this maintenance pass.
+      # Reset first so a stream created since the last hourly pass is picked
+      # up without a process restart. Empty on unmigrated installs (registry
+      # table absent) — callers then treat every queue as a job queue, which
+      # is the pre-registry behavior.
+      def current_stream_queue_names
+        Pgbus::StreamQueue.reset_cache!
+        Pgbus::StreamQueue.all_names
+      end
+
       def cleanup_recurring_executions
         retention = config.recurring_execution_retention
         return unless retention&.positive?
@@ -582,7 +652,7 @@ module Pgbus
         deleted = RecurringExecution.older_than(Time.current - retention).delete_all
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old recurring executions" } if deleted.positive?
       rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Recurring execution cleanup failed: #{e.message}" }
+        log_maintenance_failure("Recurring execution cleanup", e)
         e
       end