pgbus 0.8.4 → 0.9.2

data/lib/pgbus/configuration.rb

@@ -44,6 +44,12 @@ module Pgbus
     # Priority queues
     attr_accessor :priority_levels, :default_priority
 
+    # Grouped reads (PGMQ v1.11.0+ FIFO grouping).
+    # nil = disabled (default read_batch behavior).
+    # :fifo = use read_grouped (drains oldest group first, throughput-optimized).
+    # :round_robin = use read_grouped_rr (fair round-robin across groups).
+    attr_reader :group_mode
+
     # Archive compaction. Only the user-facing retention window is configurable;
     # the loop interval and batch size are tuned via constants on
     # Pgbus::Process::Dispatcher.
@@ -107,6 +113,7 @@ module Pgbus
                   :streams_stats_enabled, :streams_test_mode,
                   :streams_orphan_sweep_interval, :streams_orphan_threshold,
                   :streams_durable_patterns,
+                  :streams_presence_patterns, :streams_presence_member,
                   :streams_host, :streams_port, :streams_database_url
     attr_reader :streams_default_broadcast_mode # rubocop:disable Style/AccessorGrouping
 
@@ -146,6 +153,7 @@ module Pgbus
 
       @priority_levels = nil
       @default_priority = 1
+      @group_mode = nil
 
       @archive_retention = 7 * 24 * 3600 # 7 days
 
@@ -240,6 +248,15 @@ module Pgbus
       @streams_orphan_sweep_interval = 3600    # 1 hour
       @streams_orphan_threshold = 86_400       # 24 hours
       @streams_durable_patterns = []
+      # Streams matching these patterns get connection-driven presence:
+      # auto-join on SSE connect, auto-leave on disconnect, touch on the
+      # keepalive heartbeat (issue #169). Empty by default (opt-in).
+      @streams_presence_patterns = []
+      # Extracts a presence member { id:, metadata: } from a connection's
+      # authorize-hook context. Defaults to nil, which uses the built-in
+      # extractor (see #presence_member_for): a Hash with :member_id/:id,
+      # or an object responding to #id.
+      @streams_presence_member = nil
 
       # AppSignal: auto-on when the appsignal gem is loaded; probe runs in
       # the same process, so the operator can disable it independently.
@@ -300,6 +317,24 @@ module Pgbus
       @streams_default_broadcast_mode = mode
     end
 
+    VALID_GROUP_MODES = [nil, :fifo, :round_robin].freeze
+
+    def group_mode=(mode)
+      coerced = case mode
+                when nil then nil
+                when Symbol then mode
+                when String then mode.to_sym
+                else
+                  raise ArgumentError,
+                        "Invalid group_mode type: #{mode.class}. Must be nil, String, or Symbol"
+                end
+      unless VALID_GROUP_MODES.include?(coerced)
+        raise ArgumentError, "Invalid group_mode: #{coerced.inspect}. Must be nil, :fifo, or :round_robin"
+      end
+
+      @group_mode = coerced
+    end
+
     VALID_PGMQ_SCHEMA_MODES = %i[auto extension embedded].freeze
 
     def pgmq_schema_mode=(mode)
@@ -386,6 +421,12 @@ module Pgbus
 
       raise ArgumentError, "streams_durable_patterns must be an Array of strings/regex" unless streams_durable_patterns.is_a?(Array)
 
+      raise ArgumentError, "streams_presence_patterns must be an Array of strings/regex" unless streams_presence_patterns.is_a?(Array)
+
+      if !streams_presence_member.nil? && !streams_presence_member.respond_to?(:call)
+        raise ArgumentError, "streams_presence_member must respond to #call (a Proc/lambda) or be nil"
+      end
+
       return if streams_orphan_threshold.nil?
       return if streams_orphan_threshold.is_a?(Numeric) && streams_orphan_threshold.positive?
 
@@ -402,6 +443,35 @@ module Pgbus
       streams_default_broadcast_mode == :durable
     end
 
+    # Returns true if the given stream name should have connection-driven
+    # presence based on `streams_presence_patterns` (exact string or
+    # Regexp match). Presence is opt-in, so the default (no patterns) is
+    # false. See issue #169.
+    def stream_presence?(name)
+      patterns = streams_presence_patterns || []
+      patterns.any? { |p| p.is_a?(Regexp) ? p.match?(name) : p == name }
+    end
+
+    # Derives a presence member { id:, metadata: } from a connection's
+    # authorize-hook context, or nil when no member can be derived (an
+    # anonymous connection — presence is simply skipped). Uses
+    # `streams_presence_member` when configured; otherwise the built-in
+    # extractor handles the common shapes:
+    #   - Hash with :member_id (or :id) and optional :metadata
+    #   - an object responding to #id (e.g. a User model)
+    # The id is always coerced to a String and metadata defaults to {}.
+    def presence_member_for(context)
+      return nil if context.nil?
+
+      raw = streams_presence_member ? streams_presence_member.call(context) : default_presence_member(context)
+      return nil unless raw.is_a?(Hash)
+
+      id = raw[:id]
+      return nil if id.nil? || id.to_s.empty?
+
+      { id: id.to_s, metadata: raw[:metadata] || {} }
+    end
+
     # Set the worker capsule list. Accepts:
     #
     #   String — parsed via Pgbus::Configuration::CapsuleDSL into capsules
@@ -664,6 +734,20 @@ module Pgbus
 
     private
 
+    # Built-in presence-member extractor used when no custom
+    # `streams_presence_member` is configured. Returns a { id:, metadata: }
+    # Hash or nil; #presence_member_for normalizes the id/metadata.
+    def default_presence_member(context)
+      if context.is_a?(Hash)
+        id = context[:member_id] || context[:id]
+        return nil if id.nil?
+
+        { id: id, metadata: context[:metadata] || {} }
+      elsif context.respond_to?(:id)
+        { id: context.id, metadata: {} }
+      end
+    end
+
     # Coerce a duration setting value to a positive Numeric.
     #
     # Accepts an ActiveSupport::Duration (coerced to Integer seconds via .to_i)

data/lib/pgbus/execution_pools/async_pool.rb

@@ -5,8 +5,6 @@ module Pgbus
     class AsyncPool
       attr_reader :capacity
 
-      IDLE_WAIT_INTERVAL = 0.01
-
       def initialize(capacity:, on_state_change: nil)
         @capacity = capacity
         @on_state_change = on_state_change
@@ -17,6 +15,7 @@ module Pgbus
         @fatal_error = nil
         @boot_queue = Thread::Queue.new
         @pending = Thread::Queue.new
+        @wake_rd, @wake_wr = IO.pipe
 
         validate_dependencies!
         @reactor_thread = start_reactor
@@ -32,6 +31,7 @@ module Pgbus
         reserve_capacity!
         reserved = true
         @pending << block
+        wake_reactor
       rescue StandardError
         restore_capacity if reserved
         raise
@@ -52,6 +52,7 @@ module Pgbus
 
           @shutdown_flag = true
         end
+        wake_reactor
       end
 
       def shutdown?
@@ -96,7 +97,18 @@ module Pgbus
             @boot_queue << :ready
 
             wait_for_executions(semaphore)
-            wait_for_inflight
+            wait_for_inflight(task)
+          ensure
+            begin
+              @wake_rd.close
+            rescue IOError
+              nil
+            end
+            begin
+              @wake_wr.close
+            rescue IOError
+              nil
+            end
           end
         rescue Exception => e
           register_fatal_error(e)
@@ -110,10 +122,35 @@ module Pgbus
           schedule_pending(semaphore)
           break if shutdown? && @pending.empty?
 
-          sleep(IDLE_WAIT_INTERVAL) if @pending.empty?
+          wait_for_wake if @pending.empty?
         end
       end
 
+      # Fiber-aware wait: yields the reactor fiber until the main thread
+      # writes a wake byte via wake_reactor. IO#wait_readable integrates
+      # with the Async scheduler so other fibers continue running.
+      def wait_for_wake
+        return if @wake_rd.closed?
+
+        @wake_rd.wait_readable
+        drain_wake_pipe
+      rescue IOError, Errno::EBADF
+        nil
+      end
+
+      def drain_wake_pipe
+        @wake_rd.read_nonblock(256)
+      rescue IOError, SystemCallError
+        nil
+      end
+
+      # Thread-safe: called from the main thread to wake the reactor fiber.
+      def wake_reactor
+        @wake_wr.write_nonblock(".")
+      rescue IO::WaitWritable, IOError, Errno::EBADF, Errno::EPIPE
+        nil
+      end
+
       def schedule_pending(semaphore)
         while (block = next_pending)
           semaphore.async do
@@ -160,6 +197,7 @@ module Pgbus
           @available_capacity += 1
           @available_capacity.positive?
         end
+        wake_reactor
         @on_state_change&.call if should_notify
       end
 
@@ -174,8 +212,8 @@ module Pgbus
         raise error if error
       end
 
-      def wait_for_inflight
-        sleep(IDLE_WAIT_INTERVAL) while inflight?
+      def wait_for_inflight(task)
+        task.sleep(0.01) while inflight?
       end
 
       def inflight?

data/lib/pgbus/process/supervisor.rb

@@ -69,6 +69,7 @@ module Pgbus
         single_active = worker_config[:single_active_consumer] || worker_config["single_active_consumer"] || false
         priority = worker_config[:consumer_priority] || worker_config["consumer_priority"] || 0
         exec_mode = config.execution_mode_for(worker_config)
+        grp_mode = worker_config[:group_mode] || worker_config["group_mode"] || config.group_mode
 
         pid = fork do
           restore_signals
@@ -78,7 +79,7 @@ module Pgbus
           worker = Worker.new(
             queues: queues, threads: threads, config: config,
             single_active_consumer: single_active, consumer_priority: priority,
-            execution_mode: exec_mode
+            execution_mode: exec_mode, group_mode: grp_mode
           )
           worker.run
         end

data/lib/pgbus/process/worker.rb

@@ -11,12 +11,24 @@ module Pgbus
 
       def initialize(queues:, threads: 5, config: Pgbus.configuration,
                      single_active_consumer: false, consumer_priority: 0,
-                     execution_mode: :threads)
+                     execution_mode: :threads, group_mode: nil)
         @queues = Array(queues)
         @wildcard = @queues.include?("*")
         @threads = threads
         @config = config
         @execution_mode = ExecutionPools.normalize_mode(execution_mode)
+        @group_mode = case group_mode
+                      when nil then nil
+                      when Symbol then group_mode
+                      when String then group_mode.to_sym
+                      else
+                        raise ArgumentError,
+                              "Invalid group_mode type: #{group_mode.class}. Must be nil, String, or Symbol"
+                      end
+        unless Pgbus::Configuration::VALID_GROUP_MODES.include?(@group_mode)
+          raise ArgumentError,
+                "Invalid group_mode: #{@group_mode.inspect}. Must be nil, :fifo, or :round_robin"
+        end
         @single_active_consumer = single_active_consumer
         @consumer_priority = consumer_priority
         @lifecycle = Lifecycle.new
@@ -141,6 +153,8 @@ module Pgbus
 
         if priority_enabled?
           fetch_prioritized(active_queues, qty)
+        elsif @group_mode
+          fetch_grouped(active_queues, qty)
         elsif active_queues.size == 1
           queue = active_queues.first
           messages = Pgbus.client.read_batch(queue, qty: qty) || []
@@ -210,6 +224,29 @@ module Pgbus
         end
       end
 
+      # Use grouped reads for fair or throughput-optimized multi-tenant processing.
+      # Each queue is read independently with the configured group strategy.
+      def fetch_grouped(active_queues, qty)
+        remaining = qty
+        results = []
+
+        active_queues.each do |queue|
+          break if remaining <= 0
+
+          messages = case @group_mode
+                     when :round_robin
+                       Pgbus.client.read_grouped_rr(queue, qty: remaining) || []
+                     else # :fifo
+                       Pgbus.client.read_grouped(queue, qty: remaining) || []
+                     end
+
+          messages.each { |m| results << [queue, m] }
+          remaining -= messages.size
+        end
+
+        results
+      end
+
       def priority_enabled?
         config.priority_levels && config.priority_levels > 1
       end

data/lib/pgbus/serializer.rb

@@ -58,7 +58,7 @@ module Pgbus
       raise ArgumentError, "Invalid GlobalID: #{gid_string.inspect}" unless gid
 
       allowed = Pgbus.configuration.allowed_global_id_models
-      if allowed&.empty?
+      if allowed && allowed.empty?
         raise ArgumentError,
               "GlobalID deserialization is disabled (allowed_global_id_models is empty). " \
               "Set to nil to allow all models, or add permitted classes."

data/lib/pgbus/streams/coalescer.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Pgbus
+  module Streams
+    # Publish-side coalescing for high-frequency broadcasts (issue #171).
+    #
+    # A chatty reactive component — a live cursor, a typing indicator, a
+    # progress bar — can fan out many small broadcasts per second. For
+    # per-keystroke / per-frame updates that's wasteful: every frame
+    # becomes a PGMQ insert (or a NOTIFY) and a fan-out to every connection.
+    #
+    # The Coalescer batches per (stream, target) within a short window and
+    # flushes only the *latest* payload, so superseded frames never hit the
+    # bus at all. This is last-write-wins and is only safe for idempotent
+    # actions (replace / update of a stable target) — which is exactly the
+    # high-frequency case. It is strictly opt-in (`coalesce:` on broadcast).
+    #
+    # Debounce semantics: the FIRST submit for a (stream, target) schedules
+    # a flush `window_ms` later; subsequent submits within that window only
+    # overwrite the buffered payload. So latency is bounded to one window
+    # and a continuous stream of updates can't starve the flush (it is a
+    # trailing-edge-with-max-wait debounce, not a resettable one).
+    #
+    # Thread-safe: many request threads may submit concurrently. The buffer
+    # and the per-key pending-flush set are guarded by a single mutex; the
+    # flush itself runs off the mutex on the scheduler's thread.
+    class Coalescer
+      # Default coalescing window when `coalesce: true` is passed without an
+      # explicit millisecond value.
+      DEFAULT_WINDOW_MS = 50
+
+      Entry = Struct.new(:payload, :opts)
+
+      # scheduler: responds to `schedule(delay_seconds) { ... }`. Defaults
+      #   to a Concurrent::ScheduledTask-backed scheduler.
+      # flush: ->(stream_name:, target:, payload:, opts:) called once per
+      #   window per key with the latest buffered frame.
+      def initialize(flush:, scheduler: nil)
+        @flush = flush
+        @scheduler = scheduler || ScheduledTaskScheduler.new
+        @mutex = Mutex.new
+        @buffer = {}  # key => Entry (latest)
+        @pending = {} # key => true while a flush is scheduled
+      end
+
+      # Buffers a frame for (stream_name, target). Overwrites any frame
+      # already buffered for the same key within the current window. The
+      # first submit per window schedules the flush.
+      def submit(stream_name:, target:, payload:, opts:, window_ms: DEFAULT_WINDOW_MS)
+        key = [stream_name, target]
+        schedule = false
+
+        @mutex.synchronize do
+          @buffer[key] = Entry.new(payload, opts)
+          unless @pending[key]
+            @pending[key] = true
+            schedule = true
+          end
+        end
+
+        @scheduler.schedule(window_ms / 1000.0) { flush_key(key, stream_name, target) } if schedule
+      end
+
+      private
+
+      def flush_key(key, stream_name, target)
+        entry = @mutex.synchronize do
+          @pending.delete(key)
+          @buffer.delete(key)
+        end
+        return unless entry
+
+        @flush.call(stream_name: stream_name, target: target, payload: entry.payload, opts: entry.opts)
+      end
+
+      # Default scheduler backed by Concurrent::ScheduledTask. Kept as a
+      # tiny adapter so the Coalescer can be unit-tested with a synchronous
+      # fake scheduler (no real timers, no sleeps).
+      class ScheduledTaskScheduler
+        def schedule(delay_seconds, &)
+          Concurrent::ScheduledTask.execute(delay_seconds, &)
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/streams/envelope.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "json"
+
 module Pgbus
   module Streams
     # Encodes Server-Sent Events frames per https://html.spec.whatwg.org/multipage/server-sent-events.html.
@@ -29,13 +31,30 @@ module Pgbus
         raise ArgumentError, "id is required" if id.nil?
         raise ArgumentError, "event is required" if event.nil? || event.to_s.empty?
 
-        "id: #{id}\nevent: #{event}\ndata: #{strip_newlines(data.to_s)}\n\n"
+        # Strip newlines from BOTH event and data, not just data: each is
+        # interpolated into its own SSE field line, so an unescaped \r/\n in
+        # either would terminate the field early and let a crafted value
+        # inject extra SSE fields (a forged id:/data:) into the frame.
+        "id: #{id}\nevent: #{strip_newlines(event.to_s)}\ndata: #{strip_newlines(data.to_s)}\n\n"
       end
 
       def self.comment(text)
         ": #{strip_newlines(text.to_s)}\n\n"
       end
 
+      # Emits a `pgbus:connected` frame carrying the server-minted
+      # connection id as JSON. Sent once, right after the open handshake,
+      # so the page can read its own connection id and send it back as
+      # `X-Pgbus-Connection` on action requests (actor-echo suppression,
+      # issue #165). Deliberately omits an `id:` line: this is connection
+      # metadata, not a broadcast, and giving it a cursor id would corrupt
+      # the client's Last-Event-ID replay position on reconnect.
+      def self.connected(id:)
+        raise ArgumentError, "id is required" if id.nil? || id.to_s.empty?
+
+        "event: pgbus:connected\ndata: #{JSON.generate({ connectionId: id.to_s })}\n\n"
+      end
+
       def self.retry_directive(milliseconds)
         unless milliseconds.is_a?(Integer) && !milliseconds.negative?
           raise ArgumentError, "retry must be a non-negative integer (got #{milliseconds.inspect})"

data/lib/pgbus/streams/key.rb

@@ -63,9 +63,42 @@ module Pgbus
       # `to_stream_key`/`to_gid_param` implementation that forgot to
       # sanitize) raise an ArgumentError at the call site.
       def stream_key(*parts, digest_bits: DEFAULT_DIGEST_BITS)
-        fragments = Array(parts).flatten.map { |part| normalize(part, digest_bits: digest_bits) }
+        flattened = Array(parts).flatten
+
+        # Idempotency for an already-built key: a single String argument
+        # is treated as a pre-built pgbus stream key and returned
+        # unchanged (after the budget check). This lets a consumer hold
+        # one `stream_key` value and pass it to both `turbo_stream_from`
+        # and the broadcaster without the colon separator guard raising
+        # on the second call. The guard only protects against ambiguous
+        # *joins* (`stream_key('a:b', :c)` vs `stream_key('a', 'b:c')`),
+        # and there is no second fragment here to collapse against, so the
+        # hazard cannot arise. Symbols and records are NOT keys — a colon
+        # in those never came from `stream_key` and stays a mistake.
+        return stream_key!(flattened.first) if flattened.length == 1 && flattened.first.is_a?(String)
+
+        fragments = flattened.map { |part| normalize(part, digest_bits: digest_bits) }
         fragments.each { |fragment| reject_colons!(fragment) }
-        key = fragments.join(":")
+        validate_budget!(fragments.join(":"))
+      end
+
+      # Accepts an already-built stream key verbatim, skipping the
+      # per-fragment colon guard (a pre-built key legitimately contains
+      # ':' separators). Still enforces the queue-name budget so an
+      # oversized key fails at the call site rather than deep inside
+      # Client#ensure_stream_queue. Use this when you hold a key string
+      # and want to be explicit that no re-keying should happen — e.g.
+      # passing the same value to `turbo_stream_from` and a broadcaster.
+      def stream_key!(key)
+        raise ArgumentError, "stream_key! key must be a String, got #{key.class}" unless key.is_a?(String)
+
+        validate_budget!(key)
+      end
+
+      # Returns the key when it fits the pgbus queue-name budget; raises
+      # ArgumentError with an actionable message otherwise. Shared by
+      # `stream_key` and `stream_key!` so both paths fail identically.
+      def validate_budget!(key)
         budget = queue_name_budget
         return key if key.length <= budget
 

data/lib/pgbus/streams/renderer.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "cgi"
+
+module Pgbus
+  module Streams
+    # Turns a renderable into a complete `<turbo-stream>` action tag,
+    # ready to hand to Stream#broadcast. This centralises the off-request
+    # render + tag-building that every consumer would otherwise stitch
+    # together by hand (the #1 footgun in server-driven UI: rendering a
+    # component outside a request without a usable view context).
+    #
+    # pgbus deliberately has no hard dependency on turbo-rails, ActionView,
+    # or Phlex, so this builder is self-contained and matches Turbo's wire
+    # format directly. The browser's Turbo runtime consumes the tag; the
+    # exact string is the contract, not any particular Ruby library.
+    #
+    # Renderable resolution (first match wins):
+    #   - String              → used verbatim (already-rendered markup)
+    #   - responds to :call   → Phlex::HTML#call (the issue's example shape)
+    #   - responds to :render_in → ViewComponent / phlex-rails
+    #     (`render_in(view_context)`; a nil context is passed because
+    #     off-request there is no controller — components that need URL
+    #     helpers should be rendered by the app and the string passed in)
+    #   - else                → to_s
+    #
+    # Tag format mirrors Turbo::Streams::TagBuilder:
+    #   - content actions wrap the markup in a <template>
+    #   - content-less actions (remove) emit no <template>
+    module Renderer
+      # Turbo stream actions that carry no content (no <template> wrapper).
+      CONTENTLESS_ACTIONS = %w[remove].freeze
+
+      module_function
+
+      # Builds a `<turbo-stream action target><template>...</template></turbo-stream>`
+      # string. `renderable` may be nil for content-less actions.
+      def turbo_stream_tag(action:, target:, renderable: nil)
+        raise ArgumentError, "target is required" if target.nil? || target.to_s.empty?
+
+        action = action.to_s
+        attrs = %(action="#{escape(action)}" target="#{escape(target)}")
+
+        return "<turbo-stream #{attrs}></turbo-stream>" if CONTENTLESS_ACTIONS.include?(action)
+
+        content = render(renderable)
+        "<turbo-stream #{attrs}><template>#{content}</template></turbo-stream>"
+      end
+
+      # Resolves a renderable to an HTML string. See module docs for the
+      # resolution order. Returns "" for nil (a content action with no
+      # renderable still emits an empty <template>).
+      def render(renderable)
+        return "" if renderable.nil?
+        return renderable if renderable.is_a?(String)
+        return renderable.call.to_s if renderable.respond_to?(:call)
+        return renderable.render_in(nil).to_s if renderable.respond_to?(:render_in)
+
+        renderable.to_s
+      end
+
+      def escape(value)
+        CGI.escape_html(value.to_s)
+      end
+    end
+  end
+end