pgbus 0.9.9 → 0.9.11

data/lib/pgbus/configuration.rb

@@ -130,9 +130,14 @@ module Pgbus
                   :streams_write_deadline_ms, :streams_falcon_streaming_body,
                   :streams_stats_enabled, :streams_test_mode,
                   :streams_orphan_sweep_interval, :streams_orphan_threshold,
-                  :streams_durable_patterns,
+                  :streams_durable_patterns, :streams_broadcast_queue,
                   :streams_presence_patterns, :streams_presence_member,
-                  :streams_host, :streams_port, :streams_database_url
+                  :streams_host, :streams_port, :streams_database_url,
+                  :streams_pool_size, :streams_pool_timeout,
+                  :streams_fanout_write_deadline_ms, :streams_dispatch_queue_limit,
+                  :streams_writer_threads, :streams_writer_buffer_limit,
+                  :streams_pool_autoscale, :streams_pool_max,
+                  :streams_pool_autoscale_interval, :streams_application_name
     attr_reader :streams_default_broadcast_mode # rubocop:disable Style/AccessorGrouping
 
     # NOTIFY-gated worker wakeups. When true, each Worker fork owns a
@@ -155,8 +160,8 @@ module Pgbus
     #   a Pgbus::Metrics::Backend instance — a custom backend
     attr_accessor :metrics_backend, :statsd_host, :statsd_port
 
-    # When true (default), Pgbus.configure and ConfigLoader.apply run
-    # validate! after applying settings, so an invalid value fails loud at
+    # When true (default), Pgbus.configure runs validate! after the block
+    # yields, so an invalid value fails loud at
     # boot instead of dormant until a worker path consumes it. Set false to
     # opt out (exotic setups that intentionally hold a transiently-invalid
     # config); explicit validate! still works.
@@ -263,6 +268,11 @@ module Pgbus
 
       @streams_enabled = true
       @streams_path = nil
+      # Retained for backward compatibility only — NO LONGER USED for queue
+      # naming or stream detection. Stream queues are named like job queues
+      # (`#{queue_prefix}_<name>`, see #queue_name) and are identified via the
+      # `pgbus_stream_queues` registry (Pgbus::StreamQueue), not by this prefix.
+      # Setting it has no effect. See issue #308.
       @streams_queue_prefix = "pgbus_stream"
       # Streamer-only connection overrides. The Streamer's Listener owns a
       # dedicated long-lived `wait_for_notify` PG connection that can't go
@@ -280,6 +290,42 @@ module Pgbus
       @streams_host = nil
       @streams_port = nil
       @streams_database_url = nil
+      # Dedicated DB connection pool for the streamer's publish + replay hot
+      # paths (Client#send_stream_message and read_after/stream_current_msg_id).
+      # Isolated from the job pool (`pool_size`) so a saturated worker pool
+      # can't delay broadcast INSERTs and the dispatcher's per-wake read_after
+      # reuses a persistent connection instead of a fresh PG.connect per call
+      # (issue #315). A small fixed default — streams fan out from one
+      # dispatcher thread per Puma worker, so a handful of connections is
+      # plenty; raise it only for very high broadcast concurrency. Ignored on
+      # the shared-ActiveRecord (Proc) connection path, where libpq isn't
+      # thread-safe and streams keep using the single serialized connection.
+      @streams_pool_size = 5
+      @streams_pool_timeout = 5
+      # Self-tuning streams-pool autoscaling (issue #323). Opt-in, default off.
+      # When true, a per-web-process control loop grows the dedicated streams
+      # pool into a FAIR SHARE of live Postgres connection headroom under
+      # saturation and shrinks it back to streams_pool_size (the baseline/floor)
+      # when idle — and emergency-shrinks immediately if the DB runs critically
+      # low on connections. No connection-count config: every threshold derives
+      # from live max_connections. streams_pool_max is an OPTIONAL hard ceiling
+      # (nil = the dynamic fair share is the effective cap).
+      @streams_pool_autoscale = false
+      @streams_pool_max = nil
+      # Slow maintenance cadence (seconds) — the autoscaler runs as a periodic
+      # check on the streamer's idle LISTEN connection (pghero-style), not a busy
+      # loop. 5 minutes: the interval itself debounces, so there is no per-sample
+      # hysteresis; a sustained burst converges over a few checks.
+      @streams_pool_autoscale_interval = 300.0
+      # Distinctive application_name prefix stamped on streams-pool connections
+      # so the autoscaler can count peer processes (DISTINCT application_name
+      # LIKE '<prefix>_%') from pg_stat_activity. Per-process suffix is appended
+      # at build time. Namespace it per deployment if several pgbus fleets share
+      # one database. NOTE: a transaction-mode PgBouncer strips application_name;
+      # peer inference then degrades to "assume alone" (the safety margin +
+      # emergency-shrink cushion this) — connect the streamer DIRECTLY for
+      # accurate autoscaling.
+      @streams_application_name = "pgbus_streams"
       @streams_signed_name_secret = nil
       @streams_default_retention = 5 * 60 # 5 minutes
       @streams_retention = {}
@@ -296,6 +342,49 @@ module Pgbus
       # PG keepalive interval.
       @streams_listen_health_check_ms = 250
       @streams_write_deadline_ms = 5_000
+      # Deadline (ms) for a socket write made INSIDE the dispatcher's
+      # serial fanout loop (handle_durable_wake / handle_ephemeral_wake).
+      # Kept far below streams_write_deadline_ms (5s) because these writes
+      # run one after another on the single dispatcher thread: K
+      # slow-but-not-yet-dead clients stack the stall, so the whole-thread
+      # worst case is ~K * this_value before each is marked dead
+      # (issue #315 item 3). A slow-but-alive client that can't absorb a
+      # fanout frame in this window is marked dead and reconnects — its
+      # EventSource Last-Event-ID then replays the missed frames from the
+      # durable archive (or triggers a fresh re-render for ephemeral
+      # streams, which have no archive). Connect-replay writes keep the
+      # full streams_write_deadline_ms. Set this equal to
+      # streams_write_deadline_ms to restore the pre-#315 timing.
+      @streams_fanout_write_deadline_ms = 250
+      # Optional cap on dispatch-queue depth checked by the Listener before
+      # it pushes a durable WakeMessage. 0 (default) = unbounded — today's
+      # behavior. A positive value makes the Listener DROP a durable wake
+      # when the queue is at/over the cap; this is safe because the next
+      # durable wake for that stream re-reads from the min cursor. It
+      # bounds DISTINCT-STREAM backlog only — a single hot stream's wakes
+      # coalesce (drain_wakes_for) so its depth stays low regardless.
+      # Ephemeral wakes (which carry the only copy of their HTML) and
+      # Connect/Disconnect messages are NEVER dropped (issue #315 item 3).
+      @streams_dispatch_queue_limit = 0
+      # Number of writer threads that flush durable-stream fanout socket writes
+      # OFF the single dispatcher thread (issue #321). 0 (default) = inline —
+      # byte-for-byte the pre-#321 behavior, no pump, no extra allocation. A
+      # positive value spawns N writer threads; each durable broadcast fanout
+      # write is handed to a writer (partitioned by connection.id.hash % N, so
+      # a connection's frames stay ordered and its per-io mutex is respected),
+      # freeing the dispatcher to service the next wake/connect. Fast clients
+      # then stop waiting behind slow ones. EPHEMERAL fanout and connect-replay
+      # writes always stay inline regardless of this value: ephemerals have no
+      # archive to replay, so they must not risk an async drop (see #321 B1).
+      @streams_writer_threads = 0
+      # Per-connection cap on the writer's outbound buffer when
+      # streams_writer_threads > 0. 0 (default) = unbounded. A positive value
+      # drops the OLDEST buffered durable frame for a connection whose writer
+      # can't keep up — safe because durable frames live in the a_<stream>
+      # archive and are re-read on reconnect via Last-Event-ID; it's a memory
+      # guard against a pathologically slow-but-alive client, not a delivery
+      # guarantee. Ignored when streams_writer_threads == 0 (issue #321).
+      @streams_writer_buffer_limit = 0
       # EXPERIMENTAL — exempt from the 1.0 stability promise.
       @streams_falcon_streaming_body = false
       # Opt-in: when true, the Dispatcher writes one row to
@@ -311,6 +400,14 @@ module Pgbus
       @streams_orphan_sweep_interval = 3600    # 1 hour
       @streams_orphan_threshold = 86_400       # 24 hours
       @streams_durable_patterns = []
+      # Dedicated queue for turbo-rails' async broadcast jobs
+      # (Turbo::Streams::ActionBroadcastJob and siblings). nil (default) leaves
+      # them on the default queue, where a `broadcasts_to`/`broadcasts_refreshes`
+      # render+broadcast can wait behind long-running jobs before the browser
+      # sees the update. Set to a queue name (e.g. "realtime") and back it with
+      # a dedicated worker capsule to isolate broadcast latency from job
+      # throughput. Applied at engine boot when turbo-rails is loaded. See #311.
+      @streams_broadcast_queue = nil
       # EXPERIMENTAL (both presence settings) — exempt from the 1.0 stability
       # promise. Streams matching these patterns get connection-driven presence:
       # auto-join on SSE connect, auto-leave on disconnect, touch on the
@@ -528,8 +625,43 @@ module Pgbus
         raise Pgbus::ConfigurationError, "streams_write_deadline_ms must be a positive integer"
       end
 
+      unless streams_fanout_write_deadline_ms.is_a?(Integer) && streams_fanout_write_deadline_ms.positive?
+        raise Pgbus::ConfigurationError, "streams_fanout_write_deadline_ms must be a positive integer"
+      end
+
+      # >= 0, NOT .positive? — 0 is the valid "unbounded" sentinel.
+      unless streams_dispatch_queue_limit.is_a?(Integer) && streams_dispatch_queue_limit >= 0
+        raise Pgbus::ConfigurationError, "streams_dispatch_queue_limit must be a non-negative integer (0 = unbounded)"
+      end
+
+      # >= 0, NOT .positive? — 0 is the valid "inline" sentinel.
+      unless streams_writer_threads.is_a?(Integer) && streams_writer_threads >= 0
+        raise Pgbus::ConfigurationError, "streams_writer_threads must be a non-negative integer (0 = inline)"
+      end
+
+      # >= 0, NOT .positive? — 0 is the valid "unbounded" sentinel.
+      unless streams_writer_buffer_limit.is_a?(Integer) && streams_writer_buffer_limit >= 0
+        raise Pgbus::ConfigurationError, "streams_writer_buffer_limit must be a non-negative integer (0 = unbounded)"
+      end
+
+      unless streams_pool_size.is_a?(Integer) && streams_pool_size.positive?
+        raise Pgbus::ConfigurationError, "streams_pool_size must be a positive integer"
+      end
+
+      unless streams_pool_timeout.is_a?(Numeric) && streams_pool_timeout.positive?
+        raise Pgbus::ConfigurationError, "streams_pool_timeout must be a positive number"
+      end
+
+      validate_streams_autoscale!
+
       raise Pgbus::ConfigurationError, "streams_retention must be a Hash" unless streams_retention.is_a?(Hash)
 
+      unless streams_broadcast_queue.nil? ||
+             (streams_broadcast_queue.is_a?(String) && !streams_broadcast_queue.empty?)
+        raise Pgbus::ConfigurationError,
+              "streams_broadcast_queue must be a non-empty String (a queue name) or nil to disable"
+      end
+
       if streams_orphan_sweep_interval && !(streams_orphan_sweep_interval.is_a?(Numeric) && streams_orphan_sweep_interval.positive?)
         raise Pgbus::ConfigurationError, "streams_orphan_sweep_interval must be a positive number or nil to disable"
       end
@@ -554,6 +686,26 @@ module Pgbus
       raise Pgbus::ConfigurationError, "streams_orphan_threshold must be a positive number or nil to disable"
     end
 
+    def validate_streams_autoscale!
+      raise Pgbus::ConfigurationError, "streams_pool_autoscale must be true or false" unless [true, false].include?(streams_pool_autoscale)
+
+      # nil = no hard cap (dynamic fair-share is the ceiling). A positive value
+      # below the baseline is an inverted ceiling — fail loud at boot.
+      unless streams_pool_max.nil? ||
+             (streams_pool_max.is_a?(Integer) && streams_pool_max >= streams_pool_size)
+        raise Pgbus::ConfigurationError,
+              "streams_pool_max must be an Integer >= streams_pool_size " \
+              "(#{streams_pool_size}) or nil to leave growth uncapped"
+      end
+
+      unless streams_pool_autoscale_interval.is_a?(Numeric) && streams_pool_autoscale_interval.positive?
+        raise Pgbus::ConfigurationError, "streams_pool_autoscale_interval must be a positive number"
+      end
+
+      raise Pgbus::ConfigurationError, "streams_application_name must be a non-empty String" \
+        unless streams_application_name.is_a?(String) && !streams_application_name.empty?
+    end
+
     def validate_metrics_backend!
       value = metrics_backend
       return if value.nil?
@@ -661,9 +813,9 @@ module Pgbus
                  end
     end
 
-    # Event consumer configs arrive with symbol keys (Ruby DSL) or string keys
-    # (YAML via ConfigLoader). Normalize once here so every read site uses
-    # symbol access only. nil passes through (no consumers configured).
+    # Event consumer configs arrive with symbol keys (Ruby DSL) or, from an
+    # explicit Array literal, string keys. Normalize once here so every read
+    # site uses symbol access only. nil passes through (no consumers configured).
     def event_consumers=(value)
       @event_consumers =
         case value
@@ -854,6 +1006,15 @@ module Pgbus
     # timeouts under load). Setting pool_size explicitly is still supported
     # for advanced cases where you need a tighter or looser pool than the
     # default formula provides.
+    #
+    # NOTE: this covers the JOB pool only. On the dedicated-connection path
+    # (database_url / connection_params) every Client also opens a separate
+    # streams pool of +streams_pool_size+ (default 5) for durable-stream
+    # publish + replay (issue #315). Both pools are lazy, so an idle process
+    # holds few real connections, but when budgeting Postgres/PgBouncer
+    # max_connections, size for +resolved_pool_size + streams_pool_size+ per
+    # process — every forked worker/dispatcher/scheduler/consumer builds its
+    # own streams pool even if it rarely touches streams.
     POOL_SIZE_WARN_THRESHOLD = 50
 
     # Connections needed per async worker: one for the reactor's serial
@@ -1057,8 +1218,8 @@ module Pgbus
     end
 
     # Symbolize the top-level keys of a worker/consumer config entry so every
-    # downstream read site can use symbol access only. YAML (via ConfigLoader)
-    # yields string keys; the Ruby DSL and +capsule+ builder yield symbols.
+    # downstream read site can use symbol access only. An explicit Array literal
+    # may carry string keys; the Ruby DSL and +capsule+ builder yield symbols.
     # Entries are flat hashes with array/scalar values — no deep recursion.
     def normalize_entry(entry, group:)
       raise Pgbus::ConfigurationError, "#{group} entry must be a Hash, got #{entry.class}: #{entry.inspect}" unless entry.is_a?(Hash)

data/lib/pgbus/doctor.rb

@@ -5,7 +5,7 @@ require "pgbus/mcp/health_analyzer"
 
 module Pgbus
   # Preflight diagnostics for a pgbus deployment — the single command that
-  # answers "is this environment healthy enough to run?". Runs seven checks and
+  # answers "is this environment healthy enough to run?". Runs eight checks and
   # returns a machine-readable result plus a human report, so `pgbus doctor`
   # and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
   # 1 on any failure).
@@ -41,7 +41,8 @@ module Pgbus
         check_queues,
         check_notify,
         check_processes,
-        check_allowed_global_id_models
+        check_allowed_global_id_models,
+        check_broadcast_queue
       ].map(&:to_h)
     end
 
@@ -210,6 +211,52 @@ module Pgbus
       Check.new(name: "GlobalID allowlist", status: :fail, detail: "#{e.class}: #{e.message}")
     end
 
+    # 8. Broadcast-queue isolation — latency + correctness. With turbo-rails
+    # loaded, the default broadcasts_to/broadcasts_refreshes path enqueues
+    # render+broadcast ActiveJobs on the DEFAULT queue, so a browser SSE update
+    # can wait behind long-running jobs (#311). Two failure modes:
+    #
+    #   (a) streams_broadcast_queue is nil — broadcasts share the default queue
+    #       and can wait behind long jobs. A latency concern, so warn only in
+    #       production (where the blast radius is real).
+    #   (b) streams_broadcast_queue is set but NO worker capsule drains it —
+    #       broadcasts route there and pile up unread, so the browser never
+    #       updates. A correctness bug, so warn everywhere, not just production.
+    #
+    # Both are warnings, never failures. Only relevant when Turbo is loaded.
+    def check_broadcast_queue
+      broadcast_queue = @config.streams_broadcast_queue
+      turbo = @config.streams_enabled && defined?(::Turbo::Broadcastable)
+
+      if turbo && broadcast_queue && !worker_drains?(broadcast_queue)
+        return Check.new(name: "Broadcast queue", status: :warn,
+                         detail: "streams_broadcast_queue is \"#{broadcast_queue}\" but no worker capsule " \
+                                 "drains it — broadcasts will pile up unread and never reach the browser; " \
+                                 "add a capsule for the \"#{broadcast_queue}\" queue (or a wildcard worker)")
+      end
+
+      if turbo && production? && broadcast_queue.nil?
+        return Check.new(name: "Broadcast queue", status: :warn,
+                         detail: "streams_broadcast_queue is nil — turbo-rails broadcast jobs share the " \
+                                 "default queue and can wait behind long-running jobs; set a dedicated queue " \
+                                 "(e.g. \"realtime\") and back it with a worker capsule")
+      end
+
+      Check.new(name: "Broadcast queue", status: :ok,
+                detail: broadcast_queue ? "dedicated: #{broadcast_queue}" : "n/a")
+    rescue StandardError => e
+      Check.new(name: "Broadcast queue", status: :fail, detail: "#{e.class}: #{e.message}")
+    end
+
+    # True when some configured worker capsule drains the given queue — either
+    # by naming it explicitly or via a "*" wildcard (which drains every queue).
+    def worker_drains?(queue)
+      Array(@config.workers).any? do |w|
+        queues = w[:queues] || w["queues"] || []
+        queues.include?("*") || queues.include?(queue)
+      end
+    end
+
     # Physical queue names known to PGMQ. list_queues rows may be Hashes (with a
     # :queue_name key) or value objects responding to #queue_name.
     def existing_queue_names

data/lib/pgbus/engine.rb

@@ -12,9 +12,30 @@ module Pgbus
       Pgbus.teardown_models_loader!
     end
 
-    initializer "pgbus.configure" do |app|
-      config_path = app.root.join("config", "pgbus.yml")
-      Pgbus::ConfigLoader.load(config_path) if config_path.exist?
+    # config/pgbus.yml is no longer loaded (removed in 1.0). If an app still
+    # ships one, it's now silently inert — warn once at boot so the leftover
+    # file doesn't quietly diverge from the real (Ruby initializer) config.
+    initializer "pgbus.legacy_yaml_warning" do |app|
+      ActiveSupport.on_load(:after_initialize) do
+        Pgbus::Engine.warn_if_legacy_yaml_config(app.root, Pgbus.logger)
+      end
+    end
+
+    # Emits the legacy-YAML deprecation warning when +root+/config/pgbus.yml
+    # exists. Extracted so it can be unit-tested without booting Rails. A nil
+    # logger is a no-op (logger may be unset in a bare/embedded boot).
+    def self.warn_if_legacy_yaml_config(root, logger)
+      return if logger.nil?
+
+      config_path = root.join("config", "pgbus.yml")
+      return unless config_path.exist?
+
+      logger.warn do
+        "[Pgbus] config/pgbus.yml is no longer loaded (YAML config was removed in 1.0). " \
+          "Port its settings into config/initializers/pgbus.rb as a Pgbus.configure block " \
+          "and delete the YAML. See https://pgbus.zoolutions.llc/docs/configuration " \
+          "(the pgbus MCP server can also help port it)."
+      end
     end
 
     initializer "pgbus.recurring" do |app|
@@ -154,6 +175,12 @@ module Pgbus
           if defined?(::Turbo::Broadcastable)
             _autoload_trigger_broadcastable = Pgbus::Streams::BroadcastableOverride
             Pgbus::Streams::BroadcastableOverride.install!(::Turbo::Broadcastable)
+
+            # Route turbo-rails' async broadcast jobs to a dedicated queue when
+            # configured, so a broadcasts_to render+broadcast doesn't wait
+            # behind long-running jobs on the default queue (#311). No-op when
+            # streams_broadcast_queue is nil.
+            Pgbus::Streams.install_broadcast_queue!(Pgbus.configuration.streams_broadcast_queue)
           end
 
           # Subscribe-side patch: override turbo_stream_from to render

data/lib/pgbus/generators/migration_detector.rb

@@ -71,6 +71,7 @@ module Pgbus
         add_job_stats_latency: "pgbus:add_job_stats_latency",
         add_job_stats_queue_index: "pgbus:add_job_stats_queue_index",
         add_stream_stats: "pgbus:add_stream_stats",
+        add_stream_queues: "pgbus:add_stream_queues",
         add_presence: "pgbus:add_presence",
         add_queue_states: "pgbus:add_queue_states",
         add_outbox: "pgbus:add_outbox",
@@ -88,6 +89,7 @@ module Pgbus
         add_job_stats_latency: "job stats latency columns (enqueue_latency_ms, retry_count)",
         add_job_stats_queue_index: "job stats (queue_name, created_at) index",
         add_stream_stats: "stream stats table (opt-in real-time Insights)",
+        add_stream_queues: "stream queue registry (per-stream retention, orphan sweep, wildcard-worker safety)",
         add_presence: "presence members table (Turbo Streams presence)",
         add_queue_states: "queue states table (pause/resume)",
         add_outbox: "outbox entries table (transactional outbox)",
@@ -112,6 +114,7 @@ module Pgbus
           *uniqueness_key_migrations,
           *job_stats_migrations,
           *stream_stats_migrations,
+          *stream_queues_migrations,
           *presence_migrations,
           *queue_states_migrations,
           *outbox_migrations,
@@ -166,6 +169,10 @@ module Pgbus
         table_exists?("pgbus_stream_stats") ? [] : [:add_stream_stats]
       end
 
+      def stream_queues_migrations
+        table_exists?("pgbus_stream_queues") ? [] : [:add_stream_queues]
+      end
+
       def presence_migrations
         table_exists?("pgbus_presence_members") ? [] : [:add_presence]
       end

data/lib/pgbus/mcp/rack_app.rb

@@ -22,11 +22,12 @@ module Pgbus
     # / behind your VPN, never internet-exposed.
     class RackApp
       BEARER_PREFIX = "Bearer "
-      UNAUTHORIZED = [
-        401,
-        { "Content-Type" => "application/json", "WWW-Authenticate" => "Bearer" },
-        [{ jsonrpc: "2.0", id: nil, error: { code: -32_001, message: "Unauthorized" } }.to_json]
-      ].freeze
+      # Only the JSON body string is frozen and reused. The outer response triple
+      # and its headers hash MUST be built fresh per call (#unauthorized) so
+      # downstream Rack middleware can mutate them — Rack::TempfileReaper assigns
+      # response[2] and Rack::ETag adds headers. Returning a frozen array/hash
+      # raised FrozenError → 500 instead of 401 (issue #304).
+      UNAUTHORIZED_BODY = { jsonrpc: "2.0", id: nil, error: { code: -32_001, message: "Unauthorized" } }.to_json.freeze
 
       # @param data_source [Pgbus::Web::DataSource] read layer the tools query.
       #   Built once and shared: DataSource acquires Pgbus::BusRecord.connection
@@ -55,13 +56,24 @@ module Pgbus
       # the gem's transport has no auth of its own.
       def call(env)
         request = Rack::Request.new(env)
-        return UNAUTHORIZED unless authorized?(request)
+        return unauthorized unless authorized?(request)
 
         @transport.handle_request(request)
       end
 
       private
 
+      # A fresh, mutable 401 response triple each call. See UNAUTHORIZED_BODY.
+      # Header keys are lowercase per the Rack 3 SPEC (matches stream_app.rb and
+      # prometheus_exporter.rb); Rack::Headers looks them up case-insensitively.
+      def unauthorized
+        [
+          401,
+          { "content-type" => "application/json", "www-authenticate" => "Bearer" },
+          [UNAUTHORIZED_BODY]
+        ]
+      end
+
       # An explicit auth callable wins; otherwise fall back to the bearer token;
       # otherwise (neither configured) allow — the constructor already warned.
       def authorized?(request)