pgbus 0.5.1 → 0.6.0

data/lib/pgbus/streams/turbo_broadcastable.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Streams
+    # Runtime patch that redirects `Turbo::StreamsChannel.broadcast_stream_to`
+    # through pgbus instead of `ActionCable.server.broadcast`. Applied at
+    # Rails engine boot time when `defined?(::Turbo::StreamsChannel)` —
+    # see `Pgbus::Engine`'s initializer. When turbo-rails isn't loaded,
+    # this patch is a no-op and pgbus streams continue to work via the
+    # explicit `Pgbus.stream(...).broadcast(...)` API.
+    #
+    # After the patch:
+    #
+    #   class Order < ApplicationRecord
+    #     broadcasts_to :account   # existing turbo-rails API, unchanged
+    #   end
+    #
+    #   # In a controller:
+    #   @order.update!(status: "shipped")
+    #   # → Turbo::Broadcastable runs its after_update_commit callback
+    #   # → calls Turbo::StreamsChannel.broadcast_replace_to
+    #   # → which calls Turbo::StreamsChannel.broadcast_stream_to
+    #   # → which is patched to call Pgbus.stream(name).broadcast(content)
+    #   # → which inserts into PGMQ and fires NOTIFY
+    #
+    # Zero changes to user code. The entire Turbo::Broadcastable API
+    # (broadcasts_to, broadcasts_refreshes, broadcast_replace_to,
+    # broadcast_append_later_to, broadcasts_refreshes_to, etc) reuses
+    # this code path because they all funnel through broadcast_stream_to.
+    #
+    # Signed stream name reuse: we don't touch `Turbo.signed_stream_verifier`,
+    # so any existing `broadcasts_to :room` call continues to generate
+    # tokens that our `Pgbus::Streams::SignedName.verify!` accepts (as
+    # long as `Turbo.signed_stream_verifier_key` is set, which the Rails
+    # app is already responsible for).
+    module TurboBroadcastable
+      def broadcast_stream_to(*streamables, content:)
+        name = stream_name_from(streamables)
+        Pgbus.stream(name).broadcast(content)
+      end
+    end
+
+    # Apply the patch to Turbo::StreamsChannel's singleton class. Idempotent:
+    # prepending the same module twice is a no-op. Called from
+    # Pgbus::Engine's initializer when Turbo is detected.
+    def self.install_turbo_broadcastable_patch!
+      return unless defined?(::Turbo::StreamsChannel)
+      return if ::Turbo::StreamsChannel.singleton_class.include?(TurboBroadcastable)
+
+      ::Turbo::StreamsChannel.singleton_class.prepend(TurboBroadcastable)
+    end
+  end
+end

data/lib/pgbus/streams/watermark_cache_middleware.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Streams
+    # Rack middleware that clears the per-request thread-local watermark
+    # cache used by `Pgbus::StreamsHelper#pgbus_stream_from`. Without this
+    # middleware, a subsequent request served by the same thread would
+    # see stale `current_msg_id` values from the previous render — the
+    # pgbus_stream_from helper caches watermark lookups within a single
+    # request to avoid N+1 queries when a page uses multiple streams,
+    # and the cache would leak without a per-request boundary.
+    #
+    # Installed automatically by `Pgbus::Engine` at boot.
+    class WatermarkCacheMiddleware
+      CACHE_KEY = :pgbus_streams_watermark_cache
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        @app.call(env)
+      ensure
+        Thread.current[CACHE_KEY] = nil
+      end
+    end
+  end
+end

data/lib/pgbus/streams.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Pgbus::Streams is the SSE-based pub/sub subsystem that replaces
+  # `turbo_stream_from`. The user-facing entrypoint is `Pgbus.stream(name)`,
+  # which returns a `Pgbus::Streams::Stream` providing `#broadcast`,
+  # `#current_msg_id`, and `#read_after`.
+  module Streams
+    # Process-wide registry of server-side audience filter predicates.
+    # Register filters at boot time via:
+    #   Pgbus::Streams.filters.register(:admin_only) { |user| user.admin? }
+    # See lib/pgbus/streams/filters.rb for the full API.
+    def self.filters
+      @filters ||= Filters.new
+    end
+
+    # Clears the filters registry. Used by tests; not intended for runtime.
+    def self.reset_filters!
+      @filters = nil
+    end
+
+    # A handle on a single logical stream. The name can be any string, an
+    # object responding to `to_gid_param`, or an array of streamables (which
+    # are joined with colons — turbo-rails-compatible).
+    #
+    # Stream is *not* a singleton — callers create instances ad hoc, but the
+    # underlying queue is created lazily and only once per process per name.
+    class Stream
+      attr_reader :name
+
+      def initialize(streamables, client: Pgbus.client)
+        @name = self.class.name_from(streamables)
+        @client = client
+        @ensured = false
+        @ensure_mutex = Mutex.new
+      end
+
+      # Broadcasts a Turbo Stream HTML payload through the pgbus streamer.
+      # PGMQ's `message` column is JSONB, so raw HTML strings can't be passed
+      # directly. We wrap as `{"html": "..."}` on the way in and unwrap in
+      # Pgbus::Web::Streamer::Dispatcher before delivering to the SSE client.
+      # Callers pass a plain HTML string; the wrapping is an implementation
+      # detail.
+      #
+      # Transactional semantics: if this call is made inside an open
+      # ActiveRecord transaction, the PGMQ insert is deferred to an
+      # after_commit callback. If the transaction rolls back, the broadcast
+      # silently drops — clients never see the change that the database
+      # never persisted. This is the feature no other Rails real-time stack
+      # (including turbo-rails over ActionCable) can offer: the broadcast
+      # and the data mutation are atomic with respect to each other.
+      # Returns the assigned msg_id when sent synchronously, nil when
+      # deferred (the id isn't known until the after_commit callback runs).
+      #
+      # Audience filtering: pass `visible_to:` with a filter label (a
+      # Symbol previously registered via Pgbus::Streams.filters.register)
+      # to restrict delivery to connections whose authorize-hook context
+      # satisfies the predicate. The label travels with the broadcast
+      # through PGMQ; the predicate itself lives in-process on the
+      # subscriber side and is evaluated per-connection by the Dispatcher.
+      def broadcast(payload, visible_to: nil)
+        ensure_queue!
+        wrapped = { "html" => payload.to_s }
+        wrapped["visible_to"] = visible_to.to_s if visible_to
+        transaction = current_open_transaction
+        if transaction
+          transaction.after_commit { @client.send_message(@name, wrapped) }
+          nil
+        else
+          @client.send_message(@name, wrapped)
+        end
+      end
+
+      def current_msg_id
+        @client.stream_current_msg_id(@name)
+      end
+
+      def read_after(after_id:, limit: 500)
+        @client.read_after(@name, after_id: after_id, limit: limit)
+      end
+
+      def ensure!
+        ensure_queue!
+        self
+      end
+
+      # Returns a Pgbus::Streams::Presence handle for this stream.
+      # The Presence object exposes join/leave/touch/members/sweep!
+      # for tracking who is currently subscribed. See
+      # lib/pgbus/streams/presence.rb for the API.
+      def presence
+        @presence ||= Presence.new(self)
+      end
+
+      # Mirrors `Turbo::Streams::StreamName#stream_name_from`. Strings pass
+      # through; objects with `to_gid_param` or `to_param` are coerced; arrays
+      # are joined with `:`. The result is suitable both as a logical stream
+      # identifier and as the input to QueueNameValidator (after sanitisation).
+      def self.name_from(streamables)
+        if streamables.is_a?(Array)
+          streamables.map { |s| name_from(s) }.join(":")
+        elsif streamables.respond_to?(:to_gid_param)
+          streamables.to_gid_param
+        elsif streamables.respond_to?(:to_param) && !streamables.is_a?(Symbol)
+          streamables.to_param
+        else
+          streamables.to_s
+        end
+      end
+
+      private
+
+      def ensure_queue!
+        return if @ensured
+
+        @ensure_mutex.synchronize do
+          return if @ensured
+
+          @client.ensure_stream_queue(@name)
+          @ensured = true
+        end
+      end
+
+      # Returns the current AR transaction if one is open, nil otherwise.
+      # Guarded by defined? so the streams subsystem stays usable in
+      # non-Rails contexts. AR's NullTransaction yields immediately from
+      # after_commit, so we have to check #open? explicitly — otherwise
+      # every call path would hit the "deferred" branch and we'd lose the
+      # msg_id return value.
+      def current_open_transaction
+        return nil unless defined?(::ActiveRecord::Base)
+
+        connection = ::ActiveRecord::Base.connection
+        transaction = connection.current_transaction
+        transaction if transaction.open?
+      rescue StandardError => e
+        # Defensive: if AR is loaded but not yet connected (e.g. a
+        # Rake task invoked before Rails boot), don't let the transaction
+        # probe break the broadcast. Debug-logged so a misconfigured
+        # app can diagnose "why aren't my transactional broadcasts
+        # deferring?" without impacting production performance
+        # (debug is off by default).
+        Pgbus.logger.debug do
+          "[Pgbus::Streams::Stream] transaction probe failed (#{e.class}: #{e.message}); " \
+            "falling back to synchronous broadcast"
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

data/lib/pgbus/web/data_source.rb

@@ -630,6 +630,35 @@ module Pgbus
         []
       end
 
+      # Stream stats — only populated when streams_stats_enabled is
+      # true AND the migration has been run. Controllers should gate
+      # rendering on `stream_stats_available?` to avoid showing empty
+      # sections.
+      def stream_stats_available?
+        Pgbus.configuration.streams_stats_enabled && StreamStat.table_exists?
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error checking stream stats availability: #{e.message}" }
+        false
+      end
+
+      def stream_stats_summary(minutes: 60)
+        StreamStat.summary(minutes: minutes)
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching stream stats summary: #{e.message}" }
+        {
+          broadcasts: 0, connects: 0, disconnects: 0,
+          active_estimate: 0, avg_fanout: 0,
+          avg_broadcast_ms: 0, avg_connect_ms: 0
+        }
+      end
+
+      def top_streams(limit: 10, minutes: 60)
+        StreamStat.top_streams(limit: limit, minutes: minutes)
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching top streams: #{e.message}" }
+        []
+      end
+
       # Subscriber registry
       def registered_subscribers
         EventBus::Registry.instance.subscribers.map do |s|

data/lib/pgbus/web/stream_app.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "rack/request"
+
+module Pgbus
+  module Web
+    # Rack app mounted at /pgbus/streams. Not a Rails controller — this
+    # bypasses the entire Rails middleware stack for the streaming path,
+    # which avoids several classes of bug (ActionDispatch::Cookies leaking
+    # into long-lived requests, ActiveRecord::QueryCache holding the AR
+    # connection open, flash modifications after hijack, etc.). It also
+    # removes the temptation to use ActionController::Live, which has a
+    # well-documented tendency to tie up Puma threads (see puma#1009,
+    # puma#938, puma#569).
+    #
+    # Call flow (happy path, Puma 6.1+ hijack):
+    #   1. Parse the signed name from PATH_INFO
+    #   2. Verify it via Pgbus::Streams::SignedName (raises → 404)
+    #   3. Run the authorize hook (raises → 403)
+    #   4. Parse the cursor from ?since= or Last-Event-ID
+    #   5. Check streams_max_connections per worker (over → 503)
+    #   6. Check rack.hijack? (missing → 501)
+    #   7. Hijack, write the HTTP response line + SSE headers + opening
+    #      comment directly to the socket
+    #   8. Build a Connection, hand to Streamer.current.register(...)
+    #   9. Return [-1, {}, []] (Puma's async protocol)
+    #
+    # On errors, returns a normal [status, headers, body] response so
+    # Rack / the reverse proxy can log and retry. The whole thing is
+    # designed so a reviewer can read call/2 top-to-bottom and see the
+    # full request lifecycle.
+    class StreamApp
+      PATH_PREFIX = "/pgbus/streams"
+      private_constant :PATH_PREFIX
+
+      def initialize(streamer: nil, config: nil, logger: nil, authorize: nil)
+        @streamer_override = streamer
+        @config_override   = config
+        @logger_override   = logger
+        @authorize         = authorize || ->(_env, _stream_name) { true }
+      end
+
+      def call(env)
+        request = Rack::Request.new(env)
+        return not_found("only GET is supported") unless request.get?
+
+        signed_name = extract_signed_name(env)
+        return not_found("missing signed stream name") if signed_name.nil?
+
+        stream_name = verify!(signed_name)
+        return not_found("invalid signed stream name") if stream_name.nil?
+
+        authorize_result = @authorize.call(env, stream_name)
+        return forbidden unless authorize_result
+
+        # If authorize returned a non-boolean value (e.g. a User model),
+        # treat it as the connection context for audience filtering.
+        # A bare `true` means "authorized but no context" — filters that
+        # depend on a context will fail-closed on these connections.
+        context = authorize_result unless authorize_result == true
+
+        cursor = parse_cursor(env, request)
+        return bad_request("invalid cursor: #{cursor}") if cursor.is_a?(String)
+
+        return unsupported_server unless env["rack.hijack?"]
+
+        return over_capacity if streamer.registry.size >= config.streams_max_connections
+
+        hijack_and_register(env, stream_name: stream_name, since_id: cursor, context: context)
+      rescue StandardError => e
+        logger.error { "[Pgbus::StreamApp] #{e.class}: #{e.message}" }
+        server_error
+      end
+
+      private
+
+      def extract_signed_name(env)
+        path = env["PATH_INFO"].to_s
+        # PATH_INFO when mounted is relative to the mount point, so it's
+        # just "/<signed_name>". When not mounted (e.g. tests calling call
+        # directly) it's "/pgbus/streams/<signed_name>". Handle both.
+        path = path.sub(PATH_PREFIX, "")
+        path = path[1..] if path.start_with?("/") # strip leading slash
+        return nil if path.nil? || path.empty?
+
+        path
+      end
+
+      def verify!(token)
+        Pgbus::Streams::SignedName.verify!(token)
+      rescue Pgbus::Streams::SignedName::InvalidSignedName,
+             Pgbus::Streams::SignedName::MissingSecret
+        nil
+      end
+
+      def parse_cursor(env, request)
+        Pgbus::Streams::Cursor.parse(
+          query_since: request.params["since"],
+          last_event_id: env["HTTP_LAST_EVENT_ID"]
+        )
+      rescue Pgbus::Streams::Cursor::InvalidCursor => e
+        e.message
+      end
+
+      def hijack_and_register(env, stream_name:, since_id:, context: nil)
+        # Rack hijack is a one-time transition per the Rack spec. Call
+        # once, use the return value, and fall back to the side-effect
+        # `rack.hijack_io` variable only if the return was nil (some
+        # older Rack versions populate the side-effect var instead of
+        # returning the IO).
+        hijack_result = env["rack.hijack"].call
+        io = hijack_result || env["rack.hijack_io"]
+
+        write_headers(io, stream_name: stream_name, since_id: since_id)
+
+        connection = Pgbus::Web::Streamer::Connection.new(
+          id: SecureRandom.hex(8),
+          stream_name: stream_name,
+          io: io,
+          since_id: since_id,
+          writer: Pgbus::Web::Streamer::IoWriter,
+          write_deadline_ms: config.streams_write_deadline_ms,
+          context: context
+        )
+        streamer.register(connection)
+
+        [-1, {}, []]
+      end
+
+      def write_headers(io, stream_name:, since_id:)
+        io.write(Pgbus::Streams::Envelope.http_response_headers)
+        io.write(Pgbus::Streams::Envelope.retry_directive(2_000))
+        io.write(Pgbus::Streams::Envelope.comment("pgbus stream open since_id=#{since_id} stream=#{stream_name}"))
+      end
+
+      def streamer
+        @streamer_override || Pgbus::Web::Streamer.current
+      end
+
+      def config
+        @config_override || Pgbus.configuration
+      end
+
+      def logger
+        @logger_override || Pgbus.logger
+      end
+
+      # Error responses — plain text, no body parsing, no Rails. The client
+      # is EventSource which doesn't render error bodies, but we include
+      # short text for operator debugging.
+
+      def not_found(reason)
+        [404, { "content-type" => "text/plain" }, ["pgbus: #{reason}"]]
+      end
+
+      def forbidden
+        [403, { "content-type" => "text/plain" }, ["pgbus: forbidden"]]
+      end
+
+      def bad_request(reason)
+        [400, { "content-type" => "text/plain" }, ["pgbus: #{reason}"]]
+      end
+
+      def over_capacity
+        [503, { "content-type" => "text/plain", "retry-after" => "2" }, ["pgbus: streamer over capacity"]]
+      end
+
+      def unsupported_server
+        [501,
+         { "content-type" => "text/plain" },
+         ["pgbus streams require Puma 6.1+ or Falcon — current Rack server does not provide rack.hijack"]]
+      end
+
+      def server_error
+        [500, { "content-type" => "text/plain" }, ["pgbus: internal error"]]
+      end
+    end
+  end
+end

data/lib/pgbus/web/streamer/connection.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    module Streamer
+      # Wraps a single hijacked SSE client socket with its own cursor state,
+      # per-io mutex, and liveness flag. Owns no threads — the Dispatcher and
+      # Heartbeat threads call #enqueue / #write_comment on Connection instances
+      # directly, and the per-io mutex in IoWriter serialises concurrent writes.
+      #
+      # Cursor semantics: `last_msg_id_sent` is strictly monotonic. `enqueue`
+      # filters envelopes with `msg_id > last_msg_id_sent` and advances the
+      # cursor only for envelopes that actually wrote successfully. This is
+      # the client-side leg of the replay-race fix (§6.5 of the design doc).
+      class Connection
+        attr_reader :id, :stream_name, :io, :mutex, :last_msg_id_sent, :context
+
+        def initialize(id:, stream_name:, io:, since_id:, writer:, write_deadline_ms:, context: nil)
+          @id = id
+          @stream_name = stream_name
+          @io = io
+          @last_msg_id_sent = since_id.to_i
+          @writer = writer
+          @write_deadline_ms = write_deadline_ms
+          @mutex = Mutex.new
+          @dead = false
+          @closed = false
+          @created_at = monotonic
+          @last_write_at = @created_at
+          # Context is whatever the StreamApp's authorize hook returned
+          # (a truthy non-boolean value). Typically a user model or a
+          # session hash. The Dispatcher passes it to the Filters
+          # registry when evaluating visible_to predicates. Defaults to
+          # nil for tests that don't need audience filtering.
+          @context = context
+        end
+
+        def enqueue(envelopes)
+          written = []
+          envelopes.each do |envelope|
+            next if envelope.msg_id <= @last_msg_id_sent
+
+            bytes = Pgbus::Streams::Envelope.message(
+              id: envelope.msg_id,
+              event: "turbo-stream",
+              data: envelope.payload
+            )
+
+            result = @writer.write(self, bytes, deadline_ms: @write_deadline_ms)
+            if result == :ok
+              @last_msg_id_sent = envelope.msg_id
+              @last_write_at = monotonic
+              written << envelope
+            else
+              mark_dead!
+              break
+            end
+          end
+          written
+        end
+
+        def write_comment(text)
+          bytes = Pgbus::Streams::Envelope.comment(text)
+          result = @writer.write(self, bytes, deadline_ms: @write_deadline_ms)
+          if result == :ok
+            @last_write_at = monotonic
+          else
+            mark_dead!
+          end
+          result
+        end
+
+        def idle_for
+          monotonic - @last_write_at
+        end
+
+        def dead?
+          @dead
+        end
+
+        def mark_dead!
+          @dead = true
+        end
+
+        # Idempotent socket close for use by Instance#shutdown! and the
+        # heartbeat idle reaper. Wraps the respond_to? / closed? dance
+        # so callers don't need to know about StringIO-in-tests vs real
+        # Socket-in-prod or about the mark_dead! ordering.
+        #
+        # Takes the same mutex as IoWriter.write so it can't fire
+        # mid-write — otherwise the write loop could hit a half-closed
+        # socket and corrupt the `last_msg_id_sent` cursor by marking
+        # the connection dead between successful writes. The rescue
+        # narrows to IO-related exceptions; unrelated errors (bugs in
+        # the fake IO used by tests, nil-dereferences, etc.) should
+        # still propagate so the test suite catches them.
+        def close
+          @mutex.synchronize do
+            return if @closed
+
+            @closed = true
+            mark_dead!
+            return unless @io.respond_to?(:close)
+
+            @io.close unless @io.respond_to?(:closed?) && @io.closed?
+          end
+        rescue IOError, SystemCallError => e
+          Pgbus.logger&.debug { "[Pgbus::Streamer::Connection] close failed: #{e.class}: #{e.message}" }
+        end
+
+        private
+
+        def monotonic
+          # Qualify ::Process because Pgbus::Process already exists as a
+          # namespace for worker/supervisor/consumer and would otherwise
+          # shadow the top-level constant here.
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end