pgbus 0.9.9 → 0.9.11

data/lib/pgbus/web/streamer/stream_event_dispatcher.rb

@@ -35,6 +35,13 @@ module Pgbus
         WakeMessage          = Listener::WakeMessage
         ConnectMessage       = Data.define(:connection)
         DisconnectMessage    = Data.define(:connection)
+        # Posted by the OutboundPump (issue #321) after a successful off-thread
+        # durable fanout write, carrying the highest msg_id the writer actually
+        # committed for the connection. The dispatcher — the sole owner of
+        # @scanned_cursor — drains these via apply_acks and advances the scan
+        # cursor by accepted_max, so a failed/partial write never advances the
+        # cursor past a frame that didn't reach the socket.
+        WriteAckMessage      = Data.define(:connection, :accepted_max)
         # Posted by the Heartbeat once per tick with the current presence
         # connections, so the touch (a last_seen_at refresh) runs on the
         # dispatcher thread where AR connections are released each pass.
@@ -68,13 +75,21 @@ module Pgbus
         def initialize(client:, registry:, listener:, dispatch_queue:,
                        logger: Pgbus.logger, read_limit: DEFAULT_READ_LIMIT,
                        filters: nil, config: nil, stream_counter: nil,
-                       presence_provider: nil)
+                       presence_provider: nil, pump: nil, ack_queue: nil)
           @client = client
           @registry = registry
           @listener = listener
           @queue = dispatch_queue
           @logger = logger
           @read_limit = read_limit
+          # Off-thread durable fanout (issue #321). When @pump is non-nil
+          # (streams_writer_threads > 0), handle_durable_wake hands each
+          # registered-connection write to the pump instead of writing inline;
+          # the pump reports back an accepted-max on @ack_queue, which the
+          # dispatcher drains in apply_acks to advance @scanned_cursor. When
+          # nil (default), fanout writes stay inline — the pre-#321 behavior.
+          @pump = pump
+          @ack_queue = ack_queue
           # Vends a presence handle for a logical stream name. Injected so
           # tests can record join/leave/touch without a DB. Production
           # defaults to the real per-stream Presence via Pgbus.stream.
@@ -90,6 +105,14 @@ module Pgbus
           # process-wide setting. Falls back to the global config
           # for production call sites that don't specify one.
           @config = config || Pgbus.configuration
+          # Two write deadlines (issue #315 item 3). Fanout writes run serially
+          # on this single thread, so a slow client stalls the connections
+          # queued behind it — the short fanout deadline bounds that stall.
+          # Connect-replay writes run once per subscribe (not in the hot loop),
+          # so a fresh client catching up from the archive keeps the full
+          # deadline and isn't spuriously evicted.
+          @fanout_write_deadline_ms  = @config.streams_fanout_write_deadline_ms
+          @connect_write_deadline_ms = @config.streams_write_deadline_ms
           @stream_counter = stream_counter || StreamCounter.new
           # stream_name → Array<[connection, Array<Envelope>]>
           @in_flight = Hash.new { |h, k| h[k] = [] }
@@ -149,6 +172,12 @@ module Pgbus
 
         def run_loop
           while @running
+            # Apply any writer acks before doing anything else, so the scan
+            # cursor is current for the wake we're about to process — without
+            # this top-of-loop drain a wake burst would compute minimum_cursor
+            # from a stale floor and re-read/re-filter the whole un-acked
+            # window every wake (the re-read storm, issue #321).
+            apply_acks
             msg = @queue.pop
             break if msg == :__stop__
 
@@ -227,6 +256,9 @@ module Pgbus
         end
 
         def handle_durable_wake(stream, registered, in_flight_pairs, started_at)
+          # Drain writer acks before computing the read floor so a burst of
+          # wakes doesn't re-read the un-acked window (issue #321).
+          apply_acks
           min_seen = minimum_cursor(registered, in_flight_pairs)
           raw_envelopes = @client.read_after(stream, after_id: min_seen, limit: @read_limit)
           return if raw_envelopes.empty?
@@ -235,8 +267,18 @@ module Pgbus
           max_msg_id = envelopes.map(&:msg_id).max
 
           registered.each do |conn|
-            safe_enqueue(conn, visible_envelopes_for(envelopes, conn))
-            advance_scanned_cursor(conn, max_msg_id)
+            visible = visible_envelopes_for(envelopes, conn)
+            if @pump
+              # Off-thread: hand the write to the pump and DEFER the scan-cursor
+              # advance to the ack (which carries the writer's accepted max).
+              # Post even when `visible` is empty, with batch max, so an
+              # audience-filtered-away batch still advances the cursor past the
+              # hidden window (issue #321 B2). The pump raises on ephemerals.
+              @pump.post(conn, visible, max_msg_id, deadline_ms: @fanout_write_deadline_ms)
+            else
+              safe_enqueue(conn, visible)
+              advance_scanned_cursor(conn, max_msg_id)
+            end
           end
           in_flight_pairs.each do |(conn, buffer)|
             buffer.concat(visible_envelopes_for(envelopes, conn))
@@ -321,13 +363,18 @@ module Pgbus
             limit: @read_limit
           )
           initial = raw_initial.map { |e| unwrap_stream_envelope(e) }
-          safe_enqueue(connection, visible_envelopes_for(initial, connection))
+          # Connect-replay (steps 3 & 4) uses the FULL write deadline, not the
+          # short fanout one: this runs once per subscribe, outside the serial
+          # hot loop, and a new client catching up a large archive backlog must
+          # not be killed by the 250ms fanout budget (issue #315 item 3).
+          safe_enqueue(connection, visible_envelopes_for(initial, connection),
+                       deadline_ms: @connect_write_deadline_ms)
 
           # Step 4: drain the in-flight buffer (anything published between
           # step 2 and now). Connection#enqueue dedupes by cursor, so
           # overlap with step 3 is safe. The buffer entries were already
           # filtered when enqueued by handle_wake, so no re-filter here.
-          safe_enqueue(connection, buffer)
+          safe_enqueue(connection, buffer, deadline_ms: @connect_write_deadline_ms)
 
           # Step 5: promote to the main registry. From this point the
           # regular WakeMessage path handles the connection. If the
@@ -440,11 +487,46 @@ module Pgbus
           @scanned_cursor[connection] = msg_id if msg_id > current
         end
 
-        def safe_enqueue(connection, envelopes_or_buffer)
+        # Drain every pending writer ack and advance the scan cursor by the
+        # writer's accepted max (issue #321). Runs ONLY on the dispatcher
+        # thread, so @scanned_cursor keeps its single-owner invariant — the
+        # pump only reports a value via @ack_queue, it never mutates cursor
+        # state. Non-blocking: stops when the queue is empty. A no-op when
+        # offload is off (@ack_queue nil). advance_scanned_cursor is
+        # monotonic-max-guarded, so a re-applied or cross-connection-reordered
+        # ack is idempotent.
+        #
+        # Drops a stale ack for a connection that is no longer registered: the
+        # ack queue is drained independently of DisconnectMessages, so a
+        # successful-write ack can arrive AFTER handle_disconnect has already
+        # deleted @scanned_cursor[conn] (e.g. the heartbeat swept an idle client
+        # mid-write). Advancing the cursor then would resurrect the connection
+        # in @scanned_cursor and leak that entry forever. The equal? check also
+        # rejects an ack for a since-replaced connection object sharing the same
+        # id.
+        def apply_acks
+          return unless @ack_queue
+
+          loop do
+            ack = @ack_queue.pop(true)
+            next unless @registry.lookup(ack.connection.id).equal?(ack.connection)
+
+            advance_scanned_cursor(ack.connection, ack.accepted_max)
+          rescue ThreadError
+            break # queue drained
+          end
+        end
+
+        # deadline_ms defaults to the SHORT fanout deadline: every hot-loop
+        # fanout call site (handle_durable_wake, handle_ephemeral_wake) uses
+        # it implicitly. handle_connect passes @connect_write_deadline_ms
+        # explicitly so a new client's initial replay keeps the full window
+        # (issue #315 item 3).
+        def safe_enqueue(connection, envelopes_or_buffer, deadline_ms: @fanout_write_deadline_ms)
           return if connection.dead?
           return if envelopes_or_buffer.empty?
 
-          connection.enqueue(envelopes_or_buffer)
+          connection.enqueue(envelopes_or_buffer, deadline_ms: deadline_ms)
         end
 
         def prune_dead(connections)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.9.9
+  version: 0.9.11
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -158,6 +158,7 @@ files:
 - app/models/pgbus/recurring_execution.rb
 - app/models/pgbus/recurring_task.rb
 - app/models/pgbus/semaphore.rb
+- app/models/pgbus/stream_queue.rb
 - app/models/pgbus/stream_stat.rb
 - app/models/pgbus/uniqueness_key.rb
 - app/views/layouts/pgbus/application.html.erb
@@ -215,6 +216,7 @@ files:
 - lib/generators/pgbus/add_presence_generator.rb
 - lib/generators/pgbus/add_queue_states_generator.rb
 - lib/generators/pgbus/add_recurring_generator.rb
+- lib/generators/pgbus/add_stream_queues_generator.rb
 - lib/generators/pgbus/add_stream_stats_generator.rb
 - lib/generators/pgbus/add_uniqueness_keys_generator.rb
 - lib/generators/pgbus/install_generator.rb
@@ -228,11 +230,12 @@ files:
 - lib/generators/pgbus/templates/add_presence.rb.erb
 - lib/generators/pgbus/templates/add_queue_states.rb.erb
 - lib/generators/pgbus/templates/add_recurring_tables.rb.erb
+- lib/generators/pgbus/templates/add_stream_queues.rb.erb
 - lib/generators/pgbus/templates/add_stream_stats.rb.erb
 - lib/generators/pgbus/templates/add_uniqueness_keys.rb.erb
+- lib/generators/pgbus/templates/initializer.rb.erb
 - lib/generators/pgbus/templates/migrate_job_locks_to_uniqueness_keys.rb.erb
 - lib/generators/pgbus/templates/migration.rb.erb
-- lib/generators/pgbus/templates/pgbus.yml.erb
 - lib/generators/pgbus/templates/pgbus_binstub.erb
 - lib/generators/pgbus/templates/recurring.yml.erb
 - lib/generators/pgbus/templates/tune_autovacuum.rb.erb
@@ -256,10 +259,10 @@ files:
 - lib/pgbus/client/ensure_stream_queue.rb
 - lib/pgbus/client/notify_stream.rb
 - lib/pgbus/client/read_after.rb
+- lib/pgbus/client/resizable_pool.rb
 - lib/pgbus/concurrency.rb
 - lib/pgbus/concurrency/blocked_execution.rb
 - lib/pgbus/concurrency/semaphore.rb
-- lib/pgbus/config_loader.rb
 - lib/pgbus/configuration.rb
 - lib/pgbus/configuration/capsule_dsl.rb
 - lib/pgbus/dedup_cache.rb
@@ -275,7 +278,6 @@ files:
 - lib/pgbus/execution_pools/async_pool.rb
 - lib/pgbus/execution_pools/thread_pool.rb
 - lib/pgbus/failed_event_recorder.rb
-- lib/pgbus/generators/config_converter.rb
 - lib/pgbus/generators/database_target_detector.rb
 - lib/pgbus/generators/migration_detector.rb
 - lib/pgbus/instrumentation.rb
@@ -350,6 +352,8 @@ files:
 - lib/pgbus/streams/envelope.rb
 - lib/pgbus/streams/filters.rb
 - lib/pgbus/streams/key.rb
+- lib/pgbus/streams/pool_autoscaler.rb
+- lib/pgbus/streams/pool_trigger.rb
 - lib/pgbus/streams/presence.rb
 - lib/pgbus/streams/renderer.rb
 - lib/pgbus/streams/signed_name.rb
@@ -378,6 +382,7 @@ files:
 - lib/pgbus/web/streamer/instance.rb
 - lib/pgbus/web/streamer/io_writer.rb
 - lib/pgbus/web/streamer/listener.rb
+- lib/pgbus/web/streamer/outbound_pump.rb
 - lib/pgbus/web/streamer/registry.rb
 - lib/pgbus/web/streamer/stream_counter.rb
 - lib/pgbus/web/streamer/stream_event_dispatcher.rb

data/lib/generators/pgbus/templates/pgbus.yml.erb

@@ -1,76 +0,0 @@
-# Pgbus configuration
-# https://github.com/mhenrixon/pgbus
-
-default: &default
-  # Queue prefix for all PGMQ queues
-  queue_prefix: pgbus
-
-  # Default queue name (without prefix)
-  default_queue: default
-
-  # Connection pool for PGMQ client.
-  # pool_size auto-tunes from your worker thread counts:
-  #   sum(workers.threads) + sum(event_consumers.threads) + 2
-  # Set explicitly only if you need a tighter or looser pool than that.
-  # pool_size: 5
-  pool_timeout: 5
-
-  # Use PostgreSQL LISTEN/NOTIFY for instant job wake-up
-  listen_notify: true
-
-  # Visibility timeout in seconds (how long a message is invisible after being read)
-  visibility_timeout: 30
-
-  # Dead letter queue
-  max_retries: 5
-
-  # Idempotency deduplication TTL (seconds)
-  idempotency_ttl: 604800  # 7 days
-
-  # Worker definitions
-  workers:
-    - queues:
-        - critical
-        - default
-      threads: 5
-    - queues:
-        - low
-      threads: 2
-
-  # Worker recycling (prevents memory bloat)
-  max_jobs_per_worker: 10000
-  max_memory_mb: 512
-  # max_worker_lifetime: 3600  # seconds
-
-  # Dispatcher settings (maintenance tasks)
-  dispatch_interval: 1.0
-
-  # Event consumers (uncomment to enable event bus)
-  # event_consumers:
-  #   - topics:
-  #       - "orders.#"
-  #     threads: 3
-  #   - topics:
-  #       - "notifications.#"
-  #     threads: 1
-
-development:
-  <<: *default
-  workers:
-    - queues:
-        - default
-      threads: 2
-  max_jobs_per_worker: ~
-  max_memory_mb: ~
-
-test:
-  <<: *default
-  workers:
-    - queues:
-        - default
-      threads: 1
-  polling_interval: 0.01
-  visibility_timeout: 5
-
-production:
-  <<: *default

data/lib/pgbus/config_loader.rb

@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-require "yaml"
-require "erb"
-
-module Pgbus
-  module ConfigLoader
-    module_function
-
-    def load(path, env: nil)
-      env ||= (defined?(Rails) && Rails.respond_to?(:env) && Rails.env) || ENV.fetch("PGBUS_ENV", "development")
-      raw = File.read(path)
-      parsed = YAML.safe_load(ERB.new(raw).result, permitted_classes: [Symbol], aliases: true)
-      # Distinguish sectioned (top-level env keys mapping to Hashes) from
-      # flat (top-level setter keys mapping to scalars/arrays). parsed.key?(env)
-      # alone can't tell them apart, so a flat file silently lost its typo
-      # warnings whenever no env section happened to match its keys.
-      if sectioned?(parsed)
-        if parsed.key?(env)
-          apply(parsed.fetch(env))
-        else
-          warn_missing_env_section(parsed, env)
-        end
-      else
-        apply(parsed)
-      end
-    end
-
-    # A file is "sectioned" (top-level keys are env names, e.g. development:/
-    # production:) rather than "flat" (top-level keys are Configuration
-    # setters) when every top-level value is a Hash AND none of the
-    # top-level keys is itself a real Configuration setter. The setter check
-    # is what distinguishes the two: env section names (production, staging,
-    # ...) are never real setters, so a legitimate flat file whose settings
-    # all happen to be Hash-valued (connection_params:, streams_retention:,
-    # recurring_tasks:, ...) is correctly classified as flat instead of being
-    # misread as sectioned-with-no-matching-env and silently skipped.
-    def sectioned?(parsed)
-      return false unless parsed.is_a?(Hash) && parsed.any? && parsed.values.all?(Hash)
-
-      config = Pgbus.configuration
-      parsed.keys.none? { |key| config.respond_to?(:"#{key}=") }
-    end
-
-    # A sectioned file with no section for the current env has nothing to
-    # apply for this process — that's a real gap (typo'd env name, or a
-    # config file that just doesn't cover this deployment), so it must be
-    # loud instead of a silent no-op.
-    def warn_missing_env_section(parsed, env)
-      Pgbus.configuration.logger.warn do
-        "[Pgbus] No configuration section for env #{env.inspect} — " \
-          "available sections: #{parsed.keys.inspect}. Nothing was applied."
-      end
-    end
-
-    def apply(hash, warn_unknown: true)
-      config = Pgbus.configuration
-      hash.each do |key, value|
-        setter = :"#{key}="
-        if config.respond_to?(setter)
-          config.public_send(setter, value)
-        elsif warn_unknown
-          config.logger.warn { "[Pgbus] Unknown configuration key ignored: #{key.inspect} — check for typos in pgbus.yml" }
-        end
-      end
-      # Validate eagerly so a bad YAML value aborts boot with an ArgumentError
-      # naming the offending key, instead of failing later in a worker path.
-      # Honors an `eager_validation: false` key in the same hash (applied above).
-      config.validate! if config.eager_validation
-      config
-    end
-  end
-end