pgbus 0.5.0 → 0.6.0

data/lib/pgbus/configuration.rb

@@ -84,6 +84,13 @@ module Pgbus
     attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
                   :insights_default_minutes, :base_controller_class, :return_to_app_url
 
+    # Streams (turbo-rails replacement, SSE-based)
+    attr_accessor :streams_enabled, :streams_queue_prefix, :streams_signed_name_secret,
+                  :streams_default_retention, :streams_retention, :streams_heartbeat_interval,
+                  :streams_max_connections, :streams_idle_timeout, :streams_listen_health_check_ms,
+                  :streams_write_deadline_ms, :streams_falcon_streaming_body,
+                  :streams_stats_enabled
+
     def initialize
       @database_url = nil
       @connection_params = nil
@@ -150,6 +157,34 @@ module Pgbus
       @insights_default_minutes = 30 * 24 * 60 # 30 days
       @base_controller_class = "::ActionController::Base"
       @return_to_app_url = nil
+
+      @streams_enabled = true
+      @streams_queue_prefix = "pgbus_stream"
+      @streams_signed_name_secret = nil
+      @streams_default_retention = 5 * 60 # 5 minutes
+      @streams_retention = {}
+      @streams_heartbeat_interval = 15
+      @streams_max_connections = 2_000
+      @streams_idle_timeout = 3_600 # 1 hour
+      # 250ms — this value plays two roles: (1) the TCP keepalive
+      # interval for the streamer's PG LISTEN connection, and (2) the
+      # upper bound on how long Dispatcher#handle_connect waits for
+      # the Listener to acknowledge a synchronous ensure_listening
+      # call. 5s was unbounded enough to drop messages on a
+      # realistic subscribe burst; 250ms keeps the connect-path race
+      # window tight while still leaving headroom over a typical
+      # PG keepalive interval.
+      @streams_listen_health_check_ms = 250
+      @streams_write_deadline_ms = 5_000
+      @streams_falcon_streaming_body = false
+      # Opt-in: when true, the Dispatcher writes one row to
+      # pgbus_stream_stats per broadcast/connect/disconnect. Default
+      # off because stream event volume can be much higher than job
+      # volume and the Insights surface is only useful if operators
+      # actually look at it. Separate from #stats_enabled (which
+      # gates pgbus_job_stats recording) on purpose — operators
+      # usually want job stats on and stream stats off, or vice versa.
+      @streams_stats_enabled = false
     end
 
     def queue_name(name)
@@ -208,9 +243,39 @@ module Pgbus
         raise ArgumentError, "insights_default_minutes must be a positive integer"
       end
 
+      validate_streams!
+
       self
     end
 
+    def validate_streams!
+      unless streams_default_retention.is_a?(Numeric) && streams_default_retention >= 0
+        raise ArgumentError, "streams_default_retention must be a non-negative number"
+      end
+
+      unless streams_max_connections.is_a?(Integer) && streams_max_connections.positive?
+        raise ArgumentError, "streams_max_connections must be a positive integer"
+      end
+
+      unless streams_heartbeat_interval.is_a?(Numeric) && streams_heartbeat_interval.positive?
+        raise ArgumentError, "streams_heartbeat_interval must be a positive number"
+      end
+
+      unless streams_idle_timeout.is_a?(Numeric) && streams_idle_timeout.positive?
+        raise ArgumentError, "streams_idle_timeout must be a positive number"
+      end
+
+      unless streams_listen_health_check_ms.is_a?(Integer) && streams_listen_health_check_ms.positive?
+        raise ArgumentError, "streams_listen_health_check_ms must be a positive integer"
+      end
+
+      unless streams_write_deadline_ms.is_a?(Integer) && streams_write_deadline_ms.positive?
+        raise ArgumentError, "streams_write_deadline_ms must be a positive integer"
+      end
+
+      raise ArgumentError, "streams_retention must be a Hash" unless streams_retention.is_a?(Hash)
+    end
+
     # Set the worker capsule list. Accepts:
     #
     #   String — parsed via Pgbus::Configuration::CapsuleDSL into capsules
@@ -230,12 +295,34 @@ module Pgbus
     #            dispatcher-only processes).
     #
     # Raises ArgumentError for any other type.
+    #
+    # NAMING SEMANTICS for the String form:
+    #
+    # The parser produces anonymous capsules (no :name). The setter then
+    # auto-assigns a :name to capsules whose first queue would yield a
+    # *unique* name across the parsed list AND is not the bare wildcard
+    # (`*`). Anything else stays anonymous.
+    #
+    #   "critical: 5; default: 10"  -> two NAMED capsules ("critical", "default")
+    #   "*: 5"                       -> one anonymous capsule (wildcard never names)
+    #   "*: 3; *: 3; *: 3"           -> three anonymous capsules — legal,
+    #                                   represents "3 forks all reading every
+    #                                   queue", restoring the legacy YAML
+    #                                   `5 × {queues: ["*"], threads: 3}` shape
+    #   "default: 5; default: 3"     -> two anonymous capsules — same logic
+    #
+    # The point of the carve-out is the legacy "I want N forks of the same
+    # worker pool" pattern: it must keep working since PGMQ tolerates it
+    # natively (multiple processes reading the same queue with FOR UPDATE
+    # SKIP LOCKED). The CLI's --capsule selector only matches NAMED
+    # capsules, so anonymous duplicates can't be ambiguously addressed.
     def workers=(value)
       @workers = case value
                  when nil
                    nil
                  when String
-                   CapsuleDSL.parse(value).map { |entry| entry.merge(name: entry[:queues].first.to_s) }
+                   parsed = CapsuleDSL.parse(value)
+                   assign_auto_names(parsed)
                  when Array
                    value
                  else
@@ -445,33 +532,58 @@ module Pgbus
       raw&.to_s
     end
 
-    # Validates that no queue in +new_queues+ would overlap with any
-    # existing capsule. The wildcard '*' counts as overlapping with EVERY
-    # other queue (and vice versa) because at runtime '*' is expanded to
-    # all known queues. Raises ArgumentError on overlap.
+    # Auto-assign :name to parsed capsules where the first queue token would
+    # yield a unique name and is not the bare wildcard. See the long comment
+    # on +workers=+ for the why. Returns the same array with :name merged in
+    # where applicable.
+    def assign_auto_names(parsed_capsules)
+      first_queue_counts = parsed_capsules.each_with_object(Hash.new(0)) do |capsule, h|
+        h[capsule[:queues].first] += 1
+      end
+
+      parsed_capsules.map do |capsule|
+        first = capsule[:queues].first
+        nameable = first != CapsuleDSL::WILDCARD && first_queue_counts[first] == 1
+        nameable ? capsule.merge(name: first.to_s) : capsule
+      end
+    end
+
+    # Validates that the new capsule (added via +c.capsule :name, ...+) does
+    # not overlap with any existing NAMED capsule. Anonymous capsules (parsed
+    # from the string DSL with auto-naming skipped, e.g. wildcards or
+    # would-collide first-queues) are intentionally invisible here — they
+    # represent "N forks of the same pool" and are allowed to overlap with
+    # each other and with named capsules.
+    #
+    # The wildcard '*' counts as overlapping with EVERY other queue (and
+    # vice versa) because at runtime '*' is expanded to all known queues.
+    # Raises ArgumentError on overlap.
     def validate_no_queue_overlap!(new_queues)
-      existing = (@workers || []).flat_map { |c| c[:queues] || c["queues"] || [] }
-      return if existing.empty?
+      existing_named = (@workers || []).select { |c| capsule_name(c) }
+      return if existing_named.empty?
+
+      existing_queues = existing_named.flat_map { |c| c[:queues] || c["queues"] || [] }
+      return if existing_queues.empty?
 
-      if existing.include?(CapsuleDSL::WILDCARD)
+      if existing_queues.include?(CapsuleDSL::WILDCARD)
         raise ArgumentError,
-              "an existing capsule already uses '*' (matches every queue) — " \
+              "an existing named capsule already uses '*' (matches every queue) — " \
               "the new capsule's queues #{new_queues.inspect} would overlap with it"
       end
 
       if new_queues.include?(CapsuleDSL::WILDCARD)
         raise ArgumentError,
-              "the new capsule uses '*' (matches every queue) but other capsules " \
-              "are already defined with queues #{existing.inspect} — " \
+              "the new capsule uses '*' (matches every queue) but other named capsules " \
+              "are already defined with queues #{existing_queues.inspect} — " \
               "the wildcard would overlap with all of them"
       end
 
-      conflict = new_queues.find { |q| existing.include?(q) }
+      conflict = new_queues.find { |q| existing_queues.include?(q) }
       return unless conflict
 
       raise ArgumentError,
-            "queue #{conflict.inspect} is already assigned to another capsule — " \
-            "each queue can only belong to one capsule"
+            "queue #{conflict.inspect} is already assigned to another named capsule — " \
+            "named capsules cannot share queues"
     end
 
     def sum_thread_counts(entries, default_threads:, group:)

data/lib/pgbus/engine.rb

@@ -46,6 +46,7 @@ module Pgbus
 
     rake_tasks do
       load File.expand_path("../tasks/pgbus_pgmq.rake", __dir__)
+      load File.expand_path("../tasks/pgbus_streams.rake", __dir__)
     end
 
     initializer "pgbus.i18n" do
@@ -56,5 +57,35 @@ module Pgbus
       require "pgbus/web/authentication"
       require "pgbus/web/data_source"
     end
+
+    # Install the watermark cache middleware ahead of the app's own
+    # middleware so the thread-local cache is cleared between every
+    # Rack request. Without this, repeated page renders served by the
+    # same Puma thread would see stale current_msg_id values.
+    initializer "pgbus.streams.middleware" do |app|
+      app.middleware.use Pgbus::Streams::WatermarkCacheMiddleware if Pgbus.configuration.streams_enabled
+    end
+
+    # Install the Turbo::StreamsChannel patch after turbo-rails has been
+    # loaded. The patch redirects broadcast_stream_to through Pgbus.stream
+    # instead of ActionCable. When turbo-rails is not loaded, this is a
+    # no-op and pgbus_stream_from still works via the explicit
+    # Pgbus.stream(...).broadcast(...) API.
+    initializer "pgbus.streams.turbo_broadcastable", after: :load_config_initializers do
+      ActiveSupport.on_load(:after_initialize) do
+        if Pgbus.configuration.streams_enabled
+          # Touch the constant first so Zeitwerk autoloads
+          # lib/pgbus/streams/turbo_broadcastable.rb. The file defines
+          # `Pgbus::Streams::TurboBroadcastable` (the autoloaded const)
+          # AND `Pgbus::Streams.install_turbo_broadcastable_patch!`
+          # (a side-effect class method on the parent module). Without
+          # the constant reference, Zeitwerk doesn't load the file and
+          # the method call below raises NoMethodError. Assigning to
+          # `_` keeps RuboCop's Lint/Void from deleting the line.
+          _autoload_trigger = Pgbus::Streams::TurboBroadcastable
+          Pgbus::Streams.install_turbo_broadcastable_patch!
+        end
+      end
+    end
   end
 end

data/lib/pgbus/process/dispatcher.rb

@@ -33,6 +33,7 @@ module Pgbus
         @last_batch_cleanup_at = monotonic_now
         @last_recurring_cleanup_at = monotonic_now
         @last_archive_compaction_at = monotonic_now
+        @last_stream_archive_compaction_at = monotonic_now
         @last_outbox_cleanup_at = monotonic_now
         @last_job_lock_cleanup_at = monotonic_now
         @last_stats_cleanup_at = monotonic_now
@@ -79,6 +80,7 @@ module Pgbus
         run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
         run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
         run_if_due(now, :@last_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { compact_archives }
+        run_if_due(now, :@last_stream_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { prune_stream_archives }
         run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
         run_if_due(now, :@last_job_lock_cleanup_at, JOB_LOCK_CLEANUP_INTERVAL) { cleanup_job_locks }
         run_if_due(now, :@last_stats_cleanup_at, STATS_CLEANUP_INTERVAL) { cleanup_stats }
@@ -140,13 +142,20 @@ module Pgbus
       end
 
       def cleanup_stats
-        return unless config.stats_enabled
-
         retention = config.stats_retention
         return unless retention&.positive?
 
-        deleted = JobStat.cleanup!(older_than: Time.current - retention)
-        Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old job stats" } if deleted.positive?
+        cutoff = Time.current - retention
+
+        if config.stats_enabled
+          deleted = JobStat.cleanup!(older_than: cutoff)
+          Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old job stats" } if deleted.positive?
+        end
+
+        return unless config.streams_stats_enabled
+
+        deleted = StreamStat.cleanup!(older_than: cutoff)
+        Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old stream stats" } if deleted.positive?
       end
 
       def cleanup_job_locks
@@ -241,6 +250,55 @@ module Pgbus
         Pgbus.logger.warn { "[Pgbus] Archive compaction failed: #{e.message}" }
       end
 
+      # Prunes per-stream archive tables. Unlike compact_archives (which
+      # uses the global archive_retention, typically 7 days), streams need
+      # per-stream retention because a chat-history stream and a
+      # presence-ping stream have wildly different storage requirements.
+      # Lookup order for a given queue: exact string match, then regex
+      # match, then streams_default_retention (5 minutes by default).
+      def prune_stream_archives
+        prefix = config.streams_queue_prefix
+        return if prefix.nil? || prefix.empty?
+
+        batch_size = config.archive_compaction_batch_size || 1000
+        conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
+        queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
+
+        queue_names.each do |full_name|
+          next unless full_name.start_with?("#{prefix}_")
+
+          retention = retention_for_stream_queue(full_name)
+          next unless retention.positive?
+
+          cutoff = Time.current - retention
+          stripped = full_name.delete_prefix("#{config.queue_prefix}_")
+          deleted = Pgbus.client.purge_archive(stripped, older_than: cutoff, batch_size: batch_size)
+          if deleted.positive?
+            Pgbus.logger.debug do
+              "[Pgbus] Compacted #{deleted} stream archive entries from #{full_name}"
+            end
+          end
+        rescue StandardError => e
+          Pgbus.logger.warn { "[Pgbus] Stream archive compaction failed for #{full_name}: #{e.message}" }
+        end
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Stream archive compaction failed: #{e.message}" }
+      end
+
+      def retention_for_stream_queue(full_name)
+        retention_map = config.streams_retention || {}
+        # Exact-match first — cheapest and most common path
+        exact = retention_map[full_name]
+        return exact.to_f if exact
+
+        # Regex-match second for pattern-based overrides (e.g. /^chat_/)
+        retention_map.each do |key, value|
+          return value.to_f if key.is_a?(Regexp) && key.match?(full_name)
+        end
+
+        config.streams_default_retention.to_f
+      end
+
       def cleanup_recurring_executions
         retention = config.recurring_execution_retention
         return unless retention&.positive?

data/lib/pgbus/streams/cursor.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Streams
+    # Parses an SSE replay cursor from a Rack request. The cursor is the highest
+    # PGMQ msg_id the client has already seen; the streamer replays everything
+    # strictly greater on (re)connect.
+    #
+    # Two sources, by precedence:
+    #   1. Last-Event-ID header (sent automatically by EventSource on reconnect)
+    #   2. ?since= query param (sent by the custom element on first connect,
+    #      because EventSource cannot send custom headers on the initial request)
+    #
+    # Returns 0 when no cursor is present (i.e. "deliver everything from now").
+    module Cursor
+      # PGMQ msg_id is BIGINT — strictly within signed 64-bit range.
+      MAX_MSG_ID = (2**63) - 1
+
+      class InvalidCursor < ArgumentError
+      end
+
+      # Strict integer pattern: optional leading minus, then digits. No plus prefix,
+      # no decimal, no whitespace. Negatives are caught by validate! with a clearer
+      # error message than "must be numeric".
+      INTEGER_PATTERN = /\A-?\d+\z/
+
+      def self.parse(query_since:, last_event_id:)
+        raw = pick(last_event_id, query_since)
+        return 0 if raw.nil?
+
+        value = coerce(raw)
+        validate!(value)
+        value
+      end
+
+      def self.pick(last_event_id, query_since)
+        return last_event_id if present?(last_event_id)
+        return query_since   if present?(query_since)
+
+        nil
+      end
+
+      def self.present?(value)
+        return false if value.nil?
+        # Any Integer is "present" so precedence is decided here, not by
+        # the sign of the value. validate! rejects negatives separately
+        # so a caller passing a literal -1 still crashes loud rather than
+        # silently falling through to query_since.
+        return true if value.is_a?(Integer)
+
+        !value.to_s.strip.empty?
+      end
+
+      def self.coerce(raw)
+        return raw if raw.is_a?(Integer)
+
+        str = raw.to_s
+        raise InvalidCursor, "cursor must be numeric (got #{raw.inspect})" unless str.match?(INTEGER_PATTERN)
+
+        Integer(str, 10)
+      end
+
+      def self.validate!(value)
+        raise InvalidCursor, "cursor must not be negative (got #{value})" if value.negative?
+        raise InvalidCursor, "cursor #{value} is out of BIGINT range" if value > MAX_MSG_ID
+      end
+
+      private_class_method :pick, :present?, :coerce, :validate!
+    end
+  end
+end

data/lib/pgbus/streams/envelope.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Streams
+    # Encodes Server-Sent Events frames per https://html.spec.whatwg.org/multipage/server-sent-events.html.
+    #
+    # Pgbus uses three frame types:
+    #   - `message(id:, event:, data:)` — a real broadcast (carries an `id:` so the client
+    #     can resume via `Last-Event-ID` on reconnect)
+    #   - `comment(text)` — a heartbeat or sentinel that the SSE parser ignores
+    #   - `retry_directive(ms)` — tells `EventSource` how long to wait before reconnecting
+    #
+    # All frames end with `\n\n` (the SSE event terminator). `data:` lines must not
+    # contain newlines — the SSE spec uses `\n` as the field terminator, so a multi-line
+    # payload would arrive as multiple events. We strip `\r` and `\n` from data and
+    # comment text rather than splitting into multiple `data:` lines, because Turbo
+    # Stream HTML is already flat and the simpler encoding is easier to debug.
+    module Envelope
+      NEWLINES = /[\r\n]+/
+
+      RESPONSE_HEADERS = "HTTP/1.1 200 OK\r\n" \
+                         "content-type: text/event-stream\r\n" \
+                         "cache-control: no-cache, no-transform\r\n" \
+                         "x-accel-buffering: no\r\n" \
+                         "connection: keep-alive\r\n" \
+                         "\r\n"
+
+      def self.message(id:, event:, data:)
+        raise ArgumentError, "id is required" if id.nil?
+        raise ArgumentError, "event is required" if event.nil? || event.to_s.empty?
+
+        "id: #{id}\nevent: #{event}\ndata: #{strip_newlines(data.to_s)}\n\n"
+      end
+
+      def self.comment(text)
+        ": #{strip_newlines(text.to_s)}\n\n"
+      end
+
+      def self.retry_directive(milliseconds)
+        unless milliseconds.is_a?(Integer) && !milliseconds.negative?
+          raise ArgumentError, "retry must be a non-negative integer (got #{milliseconds.inspect})"
+        end
+
+        "retry: #{milliseconds}\n\n"
+      end
+
+      def self.http_response_headers
+        RESPONSE_HEADERS
+      end
+
+      def self.strip_newlines(str)
+        str.gsub(NEWLINES, "")
+      end
+
+      private_class_method :strip_newlines
+    end
+  end
+end

data/lib/pgbus/streams/filters.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Streams
+    # Process-wide registry of server-side audience filter predicates.
+    # Used by `Pgbus.stream(name).broadcast(html, visible_to: :label)`
+    # to restrict delivery to connections whose authorize-hook context
+    # matches the predicate.
+    #
+    # Typical setup at boot time:
+    #
+    #   Pgbus::Streams.filters.register(:admin_only) { |user| user.admin? }
+    #   Pgbus::Streams.filters.register(:workspace_member) do |user, stream|
+    #     user.workspaces.pluck(:id).include?(stream.split(":").last.to_i)
+    #   end
+    #
+    # Broadcasts reference filters by label:
+    #
+    #   Pgbus.stream("workspace:42").broadcast(html, visible_to: :admin_only)
+    #
+    # The Dispatcher looks up the filter in the registry, evaluates it
+    # against each connection's context (populated from the StreamApp's
+    # authorize hook return value), and only delivers to connections
+    # where the predicate returns true.
+    #
+    # Why a registry of labels instead of passing a Proc directly to
+    # broadcast: predicates can't be serialized to JSON, so they can't
+    # travel through PGMQ. The label is serialized; the predicate lives
+    # in-process on the subscriber side. This also means the predicate
+    # is evaluated on the same process that holds the SSE connection,
+    # so the user context (typically an ActiveRecord model or a session
+    # hash) is always available.
+    class Filters
+      def initialize(logger: nil)
+        @mutex = Mutex.new
+        @filters = {}
+        @logger = logger
+      end
+
+      def register(label, callable = nil, &block)
+        raise ArgumentError, "filter label must be a Symbol (got #{label.class})" unless label.is_a?(Symbol)
+
+        predicate = callable || block
+        raise ArgumentError, "filter must be given a block or callable" if predicate.nil?
+
+        @mutex.synchronize { @filters[label] = predicate }
+      end
+
+      def lookup(label)
+        @mutex.synchronize { @filters[label] }
+      end
+
+      # Evaluates the named filter against a context. The context is
+      # whatever the StreamApp's authorize hook returned when the
+      # connection was established — typically a user model.
+      #
+      # Policy decisions:
+      #   - label=nil → visible (no filter attached to the broadcast)
+      #   - unknown label → NOT visible + warning log. Fail-closed so
+      #     a typo or renamed filter doesn't turn a restricted
+      #     broadcast into a public one. The whole point of audience
+      #     filtering is data isolation; failing open on a typo
+      #     defeats the feature. The warning log is loud enough that
+      #     typos still get noticed in dev (check the log or wonder
+      #     why no subscriber sees your broadcast).
+      #   - predicate raises → NOT visible (fail-closed on runtime
+      #     error to avoid leaking data on an exception path).
+      def visible?(label, context)
+        return true if label.nil?
+
+        predicate = lookup(label)
+        if predicate.nil?
+          log_warn("unknown filter label #{label.inspect} — broadcast dropped (fail-closed)")
+          return false
+        end
+
+        begin
+          !!predicate.call(context)
+        rescue StandardError => e
+          log_error("filter #{label.inspect} raised #{e.class}: #{e.message} — dropping broadcast (fail-closed)")
+          false
+        end
+      end
+
+      private
+
+      def log_warn(message)
+        logger = @logger || (Pgbus.logger if defined?(Pgbus) && Pgbus.respond_to?(:logger))
+        logger&.warn { "[Pgbus::Streams::Filters] #{message}" }
+      end
+
+      def log_error(message)
+        logger = @logger || (Pgbus.logger if defined?(Pgbus) && Pgbus.respond_to?(:logger))
+        logger&.error { "[Pgbus::Streams::Filters] #{message}" }
+      end
+    end
+  end
+end