pgbus 0.7.9 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7661d7d684ac911e36b15267b4b6135081fa3bd9ca0a795cfceae7e9977304a
-  data.tar.gz: 7cdb802918724dafa634925a99c48c9a0ee154356ec807e69ad6ec86cb48f9be
+  metadata.gz: 60cc7178f84e5d28919085f5c5a9d824aca958180d433daf13cb04a369ed25c0
+  data.tar.gz: 5a7fdb569f90cf3e60ef5951e86d6ed0c3c31c7703b0f88be91c2df3054b8817
 SHA512:
-  metadata.gz: 917889d47343e7c8775f6be1c9bc9d72b7ca8a26de356a9434ba1470f0f4f345215e9f016f9bba74b78007c72668254b92715944574fc056b20ceb1af92ebaac
-  data.tar.gz: 011533444eec2c09e29de3938d84e6fc06efd87c82621efcccf92008d88b3a3ac76464619c959617d022e076da4bed5ca03b50f3b5997e7d81ec9b52ccce90c3
+  metadata.gz: 8cc8aa7893bdb605379f4cea9542766062b5a759a89d6149e49a39729484f56790bc131b1063b16a855357a0ab4979aeb52f4bd9f572568e793482fcabaf9939
+  data.tar.gz: '052525538d45540220e66da0d43f007727848543d8f1184f7598f481e3abe3fe6be961eb6c634f414c158cd884b856305a35ded77e39d2f0f4d7c24bc10cf1a4'

data/app/helpers/pgbus/streams_helper.rb

@@ -131,7 +131,9 @@ module Pgbus
       return nil if cache[:script_emitted]
 
       cache[:script_emitted] = true
-      script = '<script type="module">import "pgbus/stream_source_element"</script>'
+      nonce = content_security_policy_nonce if respond_to?(:content_security_policy_nonce)
+      nonce_attr = nonce ? %( nonce="#{CGI.escape_html(nonce)}") : ""
+      script = %(<script type="module"#{nonce_attr}>import "pgbus/stream_source_element"</script>)
       script.respond_to?(:html_safe) ? script.html_safe : script
     end
 

data/lib/pgbus/client/notify_stream.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Client
+    # Fire-and-forget PG NOTIFY for ephemeral stream broadcasts. No PGMQ
+    # queue is created — the payload travels via the Postgres NOTIFY channel
+    # only, matching the channel naming convention that PGMQ's trigger uses:
+    #   pgmq.q_<full_queue_name>.INSERT
+    #
+    # Subscribers already LISTEN on this channel via the Streamer's Listener.
+    # When a subscriber is connected, the StreamEventDispatcher receives the
+    # NOTIFY and fans out the payload. When no subscriber is connected,
+    # the NOTIFY is silently discarded by Postgres — no queue, no storage,
+    # no orphan tables.
+    #
+    # The payload is JSON-serialized into the NOTIFY's optional payload
+    # parameter (max 8000 bytes in Postgres). Broadcasts exceeding this
+    # limit will raise a PG::ProgramLimitExceeded error — callers needing
+    # large payloads should use durable mode (which inserts into PGMQ).
+    module NotifyStream
+      def notify_stream(stream_name, payload)
+        full_name = config.queue_name(stream_name)
+        sanitized = QueueNameValidator.sanitize!(full_name)
+        channel = "pgmq.q_#{sanitized}.INSERT"
+        json = payload.is_a?(String) ? payload : JSON.generate(payload)
+
+        Instrumentation.instrument("pgbus.stream.notify", stream: stream_name, bytes: json.bytesize) do
+          synchronized do
+            @pgmq.with_connection do |conn|
+              conn.exec_params("SELECT pg_notify($1, $2)", [channel, json])
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/client.rb

@@ -3,11 +3,13 @@
 require "json"
 require_relative "client/read_after"
 require_relative "client/ensure_stream_queue"
+require_relative "client/notify_stream"
 
 module Pgbus
   class Client
     include ReadAfter
     include EnsureStreamQueue
+    include NotifyStream
 
     attr_reader :pgmq, :config
 

data/lib/pgbus/configuration.rb

@@ -104,7 +104,9 @@ module Pgbus
                   :streams_default_retention, :streams_retention, :streams_heartbeat_interval,
                   :streams_max_connections, :streams_idle_timeout, :streams_listen_health_check_ms,
                   :streams_write_deadline_ms, :streams_falcon_streaming_body,
-                  :streams_stats_enabled, :streams_test_mode
+                  :streams_stats_enabled, :streams_test_mode,
+                  :streams_orphan_sweep_interval, :streams_orphan_threshold
+    attr_reader :streams_default_broadcast_mode # rubocop:disable Style/AccessorGrouping
 
     # AppSignal integration (auto-loaded when ::Appsignal is defined and this is true).
     # Set to false to opt out without uninstalling the appsignal gem.
@@ -216,6 +218,9 @@ module Pgbus
       # usually want job stats on and stream stats off, or vice versa.
       @streams_stats_enabled = false
       @streams_test_mode = false
+      @streams_default_broadcast_mode = :ephemeral
+      @streams_orphan_sweep_interval = 3600    # 1 hour
+      @streams_orphan_threshold = 86_400       # 24 hours
 
       # AppSignal: auto-on when the appsignal gem is loaded; probe runs in
       # the same process, so the operator can disable it independently.
@@ -264,6 +269,18 @@ module Pgbus
                           end
     end
 
+    VALID_BROADCAST_MODES = %i[ephemeral durable].freeze
+
+    def streams_default_broadcast_mode=(mode)
+      mode = mode.to_sym
+      unless VALID_BROADCAST_MODES.include?(mode)
+        raise ArgumentError,
+              "Invalid streams_default_broadcast_mode: #{mode}. Must be one of: #{VALID_BROADCAST_MODES.join(", ")}"
+      end
+
+      @streams_default_broadcast_mode = mode
+    end
+
     VALID_PGMQ_SCHEMA_MODES = %i[auto extension embedded].freeze
 
     def pgmq_schema_mode=(mode)
@@ -343,6 +360,15 @@ module Pgbus
       end
 
       raise ArgumentError, "streams_retention must be a Hash" unless streams_retention.is_a?(Hash)
+
+      if streams_orphan_sweep_interval && !(streams_orphan_sweep_interval.is_a?(Numeric) && streams_orphan_sweep_interval.positive?)
+        raise ArgumentError, "streams_orphan_sweep_interval must be a positive number or nil to disable"
+      end
+
+      return if streams_orphan_threshold.nil?
+      return if streams_orphan_threshold.is_a?(Numeric) && streams_orphan_threshold.positive?
+
+      raise ArgumentError, "streams_orphan_threshold must be a positive number or nil to disable"
     end
 
     # Set the worker capsule list. Accepts:

data/lib/pgbus/process/dispatcher.rb

@@ -15,6 +15,7 @@ module Pgbus
       OUTBOX_CLEANUP_INTERVAL = 3600 # Run outbox cleanup every hour
       JOB_LOCK_CLEANUP_INTERVAL = 300 # Run job lock cleanup every 5 minutes
       STATS_CLEANUP_INTERVAL = 3600 # Run stats cleanup every hour
+      ORPHAN_STREAM_SWEEP_INTERVAL = 3600 # Run orphan stream sweep every hour
       TABLE_MAINTENANCE_INTERVAL = Pgbus::TableMaintenance::MAINTENANCE_INTERVAL
 
       # Page size for archive compaction. Each cycle deletes up to this
@@ -38,6 +39,7 @@ module Pgbus
         @last_outbox_cleanup_at = monotonic_now
         @last_job_lock_cleanup_at = monotonic_now
         @last_stats_cleanup_at = monotonic_now
+        @last_orphan_stream_sweep_at = monotonic_now
         @last_table_maintenance_at = monotonic_now
       end
 
@@ -86,6 +88,8 @@ module Pgbus
         run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
         run_if_due(now, :@last_job_lock_cleanup_at, JOB_LOCK_CLEANUP_INTERVAL) { cleanup_job_locks }
         run_if_due(now, :@last_stats_cleanup_at, STATS_CLEANUP_INTERVAL) { cleanup_stats }
+        sweep_interval = config.streams_orphan_sweep_interval
+        run_if_due(now, :@last_orphan_stream_sweep_at, sweep_interval) { sweep_orphan_streams } if sweep_interval
         run_if_due(now, :@last_table_maintenance_at, TABLE_MAINTENANCE_INTERVAL) { run_table_maintenance }
       end
 
@@ -315,6 +319,40 @@ module Pgbus
         config.streams_default_retention.to_f
       end
 
+      def sweep_orphan_streams
+        prefix = config.streams_queue_prefix
+        return if prefix.nil? || prefix.empty?
+
+        threshold = config.streams_orphan_threshold
+        return unless threshold
+
+        conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
+        queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
+
+        dropped = 0
+        queue_names.each do |full_name|
+          next unless full_name.start_with?("#{prefix}_")
+
+          row = conn.select_one(<<~SQL, "Pgbus Orphan Check")
+            SELECT count(*) AS queue_length
+            FROM pgmq.q_#{QueueNameValidator.sanitize!(full_name)}
+          SQL
+
+          next unless row
+          next if row["queue_length"].to_i.positive?
+
+          Pgbus.client.drop_queue(full_name, prefixed: false)
+          dropped += 1
+          Pgbus.logger.info { "[Pgbus] Dropped orphan stream queue: #{full_name}" }
+        rescue StandardError => e
+          Pgbus.logger.warn { "[Pgbus] Orphan stream sweep failed for #{full_name}: #{e.message}" }
+        end
+
+        Pgbus.logger.debug { "[Pgbus] Orphan stream sweep complete: dropped #{dropped} queue(s)" } if dropped.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Orphan stream sweep failed: #{e.message}" }
+      end
+
       def cleanup_recurring_executions
         retention = config.recurring_execution_retention
         return unless retention&.positive?

data/lib/pgbus/streams/turbo_broadcastable.rb

@@ -36,7 +36,8 @@ module Pgbus
     module TurboBroadcastable
       def broadcast_stream_to(*streamables, content:)
         name = stream_name_from(streamables)
-        Pgbus.stream(name).broadcast(content)
+        mode = Pgbus.configuration.streams_default_broadcast_mode
+        Pgbus.stream(name, durable: mode == :durable).broadcast(content)
       end
     end
 

data/lib/pgbus/streams.rb

@@ -37,14 +37,19 @@ module Pgbus
     class Stream
       attr_reader :name
 
-      def initialize(streamables, client: Pgbus.client)
+      def initialize(streamables, client: Pgbus.client, durable: true)
         @name = self.class.name_from(streamables)
         self.class.validate_name_length!(@name, streamables)
         @client = client
+        @durable = durable
         @ensured = false
         @ensure_mutex = Mutex.new
       end
 
+      def durable?
+        @durable
+      end
+
       # Broadcasts a Turbo Stream HTML payload through the pgbus streamer.
       # PGMQ's `message` column is JSONB, so raw HTML strings can't be passed
       # directly. We wrap as `{"html": "..."}` on the way in and unwrap in
@@ -69,9 +74,12 @@ module Pgbus
       # through PGMQ; the predicate itself lives in-process on the
       # subscriber side and is evaluated per-connection by the Dispatcher.
       def broadcast(payload, visible_to: nil)
-        ensure_queue!
         wrapped = { "html" => payload.to_s }
         wrapped["visible_to"] = visible_to.to_s if visible_to
+
+        return broadcast_ephemeral(wrapped) unless @durable
+
+        ensure_queue!
         transaction = current_open_transaction
         instrument_payload = {
           stream: @name,
@@ -155,6 +163,11 @@ module Pgbus
 
       private
 
+      def broadcast_ephemeral(wrapped)
+        @client.notify_stream(@name, wrapped)
+        nil
+      end
+
       def ensure_queue!
         return if @ensured
 

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.7.9"
+  VERSION = "0.8.0"
 end

data/lib/pgbus/web/data_source.rb

@@ -43,11 +43,10 @@ module Pgbus
       # different connection lifecycle than the worker processes).
       def queues_with_metrics
         queue_names = connection.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
-        # paused_queue_names returns an Array; convert to Set so the
-        # per-queue membership check is O(1). With 100+ queues the
-        # Array#include? cost in the loop was O(n²) per dashboard load.
+        return [] if queue_names.empty?
+
         paused_queues = paused_queue_names.to_set
-        queue_names.map { |name| queue_metrics_via_sql(name) }.compact.map do |q|
+        batched_queue_metrics(queue_names).map do |q|
           q.merge(paused: paused_queues.include?(logical_queue_name(q[:name])))
         end
       rescue StandardError => e
@@ -1008,6 +1007,46 @@ module Pgbus
         rows.to_a.map { |r| format_message(r, r["queue_name"]) }
       end
 
+      def batched_queue_metrics(queue_names)
+        return [] if queue_names.empty?
+
+        unions = queue_names.filter_map do |name|
+          sanitized = sanitize_name(name)
+          qtable = "q_#{sanitized}"
+          seq_name = "#{qtable}_msg_id_seq"
+          <<~SQL
+            SELECT
+              #{connection.quote(name)} AS queue_name,
+              (SELECT count(*) FROM pgmq.#{qtable}) AS queue_length,
+              (SELECT count(*) FROM pgmq.#{qtable} WHERE vt <= NOW()) AS queue_visible_length,
+              (SELECT EXTRACT(epoch FROM (NOW() - max(enqueued_at)))::int FROM pgmq.#{qtable}) AS newest_msg_age_sec,
+              (SELECT EXTRACT(epoch FROM (NOW() - min(enqueued_at)))::int FROM pgmq.#{qtable}) AS oldest_msg_age_sec,
+              (SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM pgmq.#{seq_name}) AS total_messages
+          SQL
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Skipping queue metrics for #{name}: #{e.message}" }
+          nil
+        end
+
+        return [] if unions.empty?
+
+        sql = unions.join(" UNION ALL ")
+        rows = connection.select_all(sql, "Pgbus Batched Queue Metrics")
+        rows.to_a.map do |row|
+          {
+            name: row["queue_name"],
+            queue_length: row["queue_length"].to_i,
+            queue_visible_length: row["queue_visible_length"].to_i,
+            oldest_msg_age_sec: row["oldest_msg_age_sec"]&.to_i,
+            newest_msg_age_sec: row["newest_msg_age_sec"]&.to_i,
+            total_messages: row["total_messages"].to_i
+          }
+        end
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus::Web] Error fetching batched queue metrics: #{e.class}: #{e.message}" }
+        []
+      end
+
       def queue_metrics_via_sql(queue_name)
         qtable = "q_#{sanitize_name(queue_name)}"
         seq_name = "#{qtable}_msg_id_seq"

data/lib/pgbus/web/streamer/listener.rb

@@ -29,7 +29,11 @@ module Pgbus
       # For a queue named `pgbus_stream_chat` the trigger table is
       # `q_pgbus_stream_chat`, so the channel is `pgmq.q_pgbus_stream_chat.INSERT`.
       class Listener
-        WakeMessage = Data.define(:queue_name)
+        WakeMessage = Data.define(:queue_name, :payload) do
+          def initialize(queue_name:, payload: nil)
+            super
+          end
+        end
 
         CHANNEL_PREFIX = "pgmq.q_"
         CHANNEL_SUFFIX = ".INSERT"
@@ -110,8 +114,8 @@ module Pgbus
 
             timeout_s = @health_check_ms / 1000.0
             begin
-              @conn.wait_for_notify(timeout_s) do |channel, _pid, _payload|
-                handle_notify(channel)
+              @conn.wait_for_notify(timeout_s) do |channel, _pid, payload|
+                handle_notify(channel, payload)
               end || run_health_check
             rescue IOError => e
               # #stop closes the PG connection to interrupt
@@ -182,11 +186,11 @@ module Pgbus
           @listening_to.delete(channel)
         end
 
-        def handle_notify(channel)
+        def handle_notify(channel, payload = nil)
           queue_name = queue_name_from(channel)
           return unless queue_name
 
-          @dispatch_queue << WakeMessage.new(queue_name: queue_name)
+          @dispatch_queue << WakeMessage.new(queue_name: queue_name, payload: payload)
         end
 
         def run_health_check

data/lib/pgbus/web/streamer/stream_event_dispatcher.rb

@@ -92,6 +92,7 @@ module Pgbus
           # boolean assignment), the sentinel break would still fire.
           @running = false
           @thread = nil
+          @ephemeral_seq = 0
         end
 
         def start
@@ -136,7 +137,7 @@ module Pgbus
             # queue's current contents — once we hit a non-Wake or a
             # different stream, we stop and let the regular path handle
             # the rest.
-            if msg.is_a?(WakeMessage)
+            if msg.is_a?(WakeMessage) && msg.payload.nil?
               wakes, trailing = drain_wakes_for(msg)
               wakes.each { |w| handle(w) }
               handle(trailing) if trailing
@@ -165,7 +166,7 @@ module Pgbus
               return [coalesced, nil] # queue drained
             end
 
-            return [coalesced, peek] unless peek.is_a?(WakeMessage)
+            return [coalesced, peek] unless peek.is_a?(WakeMessage) && peek.payload.nil?
 
             next if seen.include?(peek.queue_name)
 
@@ -193,32 +194,26 @@ module Pgbus
 
         def handle_wake(msg)
           started_at = monotonic_ms
-          # msg.queue_name is the PGMQ full table name (pgbus_int_pbns_xxx),
-          # but connections are registered under the logical name (pbns_xxx).
-          # Translate before looking up.
           stream = @full_to_logical[msg.queue_name] || msg.queue_name
           registered = @registry.connections_for(stream)
           in_flight_pairs = @in_flight[stream]
           return if registered.empty? && in_flight_pairs.empty?
 
+          if msg.payload
+            handle_ephemeral_wake(msg, stream, registered, in_flight_pairs, started_at)
+          else
+            handle_durable_wake(stream, registered, in_flight_pairs, started_at)
+          end
+        end
+
+        def handle_durable_wake(stream, registered, in_flight_pairs, started_at)
           min_seen = minimum_cursor(registered, in_flight_pairs)
           raw_envelopes = @client.read_after(stream, after_id: min_seen, limit: @read_limit)
           return if raw_envelopes.empty?
 
           envelopes = raw_envelopes.map { |e| unwrap_stream_envelope(e) }
-          # The maximum msg_id in THIS batch. We advance every
-          # connection's scanned cursor past this value even if the
-          # filter drops everything — otherwise a 500-message run
-          # of invisible broadcasts would pin minimum_cursor and
-          # the dispatcher would re-read the same window forever,
-          # starving later public messages. Connection#enqueue still
-          # gates the client-facing cursor on actual successful
-          # writes, so this advance is invisible to clients.
           max_msg_id = envelopes.map(&:msg_id).max
 
-          # Each connection gets a per-connection filtered subset. We
-          # can't pre-filter once because different connections have
-          # different authorize contexts.
           registered.each do |conn|
             safe_enqueue(conn, visible_envelopes_for(envelopes, conn))
             advance_scanned_cursor(conn, max_msg_id)
@@ -230,11 +225,40 @@ module Pgbus
 
           prune_dead(registered)
 
-          # Record one stat row per wake. Fanout is the number of
-          # subscribers (registered + in-flight) that received the
-          # broadcast before any filter dropped it — the "intended"
-          # audience size, which is the useful operator number even
-          # when audience filtering is in play.
+          record_stat(
+            stream_name: stream,
+            event_type: "broadcast",
+            started_at: started_at,
+            fanout: registered.size + in_flight_pairs.size
+          )
+        end
+
+        def handle_ephemeral_wake(msg, stream, registered, in_flight_pairs, started_at)
+          parsed = JSON.parse(msg.payload)
+          html = parsed.is_a?(Hash) ? parsed["html"] : nil
+          return unless html.is_a?(String)
+
+          visible_to = parsed["visible_to"]
+          visible_to = visible_to.to_sym if visible_to.is_a?(String)
+
+          @ephemeral_seq += 1
+          envelope = StreamEnvelope.new(
+            msg_id: -@ephemeral_seq,
+            enqueued_at: Time.now.utc.iso8601(6),
+            payload: html,
+            source: "ephemeral",
+            visible_to: visible_to
+          )
+
+          registered.each do |conn|
+            safe_enqueue(conn, visible_envelopes_for([envelope], conn))
+          end
+          in_flight_pairs.each do |(conn, buffer)|
+            buffer.concat(visible_envelopes_for([envelope], conn))
+          end
+
+          prune_dead(registered)
+
           record_stat(
             stream_name: stream,
             event_type: "broadcast",

data/lib/pgbus.rb

@@ -104,10 +104,11 @@ module Pgbus
     # clears it. The cache key is the resolved name string, not the raw
     # streamables, so `Pgbus.stream(@order)` and `Pgbus.stream(@order)`
     # in the same process return the same instance.
-    def stream(streamables)
+    def stream(streamables, durable: true)
       name = Streams::Stream.name_from(streamables)
+      cache_key = "#{name}:#{durable ? "d" : "e"}"
       @stream_cache ||= Concurrent::Map.new
-      @stream_cache.compute_if_absent(name) { Streams::Stream.new(streamables) }
+      @stream_cache.compute_if_absent(cache_key) { Streams::Stream.new(streamables, durable: durable) }
     end
 
     # Compose a short, pgbus-safe stream identifier from any mix of

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.7.9
+  version: 0.8.0
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -250,6 +250,7 @@ files:
 - lib/pgbus/cli.rb
 - lib/pgbus/client.rb
 - lib/pgbus/client/ensure_stream_queue.rb
+- lib/pgbus/client/notify_stream.rb
 - lib/pgbus/client/read_after.rb
 - lib/pgbus/concurrency.rb
 - lib/pgbus/concurrency/blocked_execution.rb