pgbus 0.7.8 → 0.8.0

data/lib/pgbus/streams.rb

@@ -37,14 +37,19 @@ module Pgbus
     class Stream
       attr_reader :name
 
-      def initialize(streamables, client: Pgbus.client)
+      def initialize(streamables, client: Pgbus.client, durable: true)
         @name = self.class.name_from(streamables)
         self.class.validate_name_length!(@name, streamables)
         @client = client
+        @durable = durable
         @ensured = false
         @ensure_mutex = Mutex.new
       end
 
+      def durable?
+        @durable
+      end
+
       # Broadcasts a Turbo Stream HTML payload through the pgbus streamer.
       # PGMQ's `message` column is JSONB, so raw HTML strings can't be passed
       # directly. We wrap as `{"html": "..."}` on the way in and unwrap in
@@ -69,15 +74,26 @@ module Pgbus
       # through PGMQ; the predicate itself lives in-process on the
       # subscriber side and is evaluated per-connection by the Dispatcher.
       def broadcast(payload, visible_to: nil)
-        ensure_queue!
         wrapped = { "html" => payload.to_s }
         wrapped["visible_to"] = visible_to.to_s if visible_to
+
+        return broadcast_ephemeral(wrapped) unless @durable
+
+        ensure_queue!
         transaction = current_open_transaction
-        if transaction
-          transaction.after_commit { @client.send_message(@name, wrapped) }
-          nil
-        else
-          @client.send_message(@name, wrapped)
+        instrument_payload = {
+          stream: @name,
+          visible_to: visible_to,
+          deferred: !transaction.nil?,
+          bytes: wrapped["html"].bytesize
+        }
+        Instrumentation.instrument("pgbus.stream.broadcast", instrument_payload) do
+          if transaction
+            transaction.after_commit { @client.send_message(@name, wrapped) }
+            nil
+          else
+            @client.send_message(@name, wrapped)
+          end
         end
       end
 
@@ -147,6 +163,11 @@ module Pgbus
 
       private
 
+      def broadcast_ephemeral(wrapped)
+        @client.notify_stream(@name, wrapped)
+        nil
+      end
+
       def ensure_queue!
         return if @ensured
 

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.7.8"
+  VERSION = "0.8.0"
 end

data/lib/pgbus/web/data_source.rb

@@ -43,11 +43,10 @@ module Pgbus
       # different connection lifecycle than the worker processes).
       def queues_with_metrics
         queue_names = connection.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
-        # paused_queue_names returns an Array; convert to Set so the
-        # per-queue membership check is O(1). With 100+ queues the
-        # Array#include? cost in the loop was O(n²) per dashboard load.
+        return [] if queue_names.empty?
+
         paused_queues = paused_queue_names.to_set
-        queue_names.map { |name| queue_metrics_via_sql(name) }.compact.map do |q|
+        batched_queue_metrics(queue_names).map do |q|
           q.merge(paused: paused_queues.include?(logical_queue_name(q[:name])))
         end
       rescue StandardError => e
@@ -1008,6 +1007,46 @@ module Pgbus
         rows.to_a.map { |r| format_message(r, r["queue_name"]) }
       end
 
+      def batched_queue_metrics(queue_names)
+        return [] if queue_names.empty?
+
+        unions = queue_names.filter_map do |name|
+          sanitized = sanitize_name(name)
+          qtable = "q_#{sanitized}"
+          seq_name = "#{qtable}_msg_id_seq"
+          <<~SQL
+            SELECT
+              #{connection.quote(name)} AS queue_name,
+              (SELECT count(*) FROM pgmq.#{qtable}) AS queue_length,
+              (SELECT count(*) FROM pgmq.#{qtable} WHERE vt <= NOW()) AS queue_visible_length,
+              (SELECT EXTRACT(epoch FROM (NOW() - max(enqueued_at)))::int FROM pgmq.#{qtable}) AS newest_msg_age_sec,
+              (SELECT EXTRACT(epoch FROM (NOW() - min(enqueued_at)))::int FROM pgmq.#{qtable}) AS oldest_msg_age_sec,
+              (SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM pgmq.#{seq_name}) AS total_messages
+          SQL
+        rescue StandardError => e
+          Pgbus.logger.debug { "[Pgbus::Web] Skipping queue metrics for #{name}: #{e.message}" }
+          nil
+        end
+
+        return [] if unions.empty?
+
+        sql = unions.join(" UNION ALL ")
+        rows = connection.select_all(sql, "Pgbus Batched Queue Metrics")
+        rows.to_a.map do |row|
+          {
+            name: row["queue_name"],
+            queue_length: row["queue_length"].to_i,
+            queue_visible_length: row["queue_visible_length"].to_i,
+            oldest_msg_age_sec: row["oldest_msg_age_sec"]&.to_i,
+            newest_msg_age_sec: row["newest_msg_age_sec"]&.to_i,
+            total_messages: row["total_messages"].to_i
+          }
+        end
+      rescue StandardError => e
+        Pgbus.logger.error { "[Pgbus::Web] Error fetching batched queue metrics: #{e.class}: #{e.message}" }
+        []
+      end
+
       def queue_metrics_via_sql(queue_name)
         qtable = "q_#{sanitize_name(queue_name)}"
         seq_name = "#{qtable}_msg_id_seq"

data/lib/pgbus/web/streamer/listener.rb

@@ -29,7 +29,11 @@ module Pgbus
       # For a queue named `pgbus_stream_chat` the trigger table is
       # `q_pgbus_stream_chat`, so the channel is `pgmq.q_pgbus_stream_chat.INSERT`.
       class Listener
-        WakeMessage = Data.define(:queue_name)
+        WakeMessage = Data.define(:queue_name, :payload) do
+          def initialize(queue_name:, payload: nil)
+            super
+          end
+        end
 
         CHANNEL_PREFIX = "pgmq.q_"
         CHANNEL_SUFFIX = ".INSERT"
@@ -110,8 +114,8 @@ module Pgbus
 
             timeout_s = @health_check_ms / 1000.0
             begin
-              @conn.wait_for_notify(timeout_s) do |channel, _pid, _payload|
-                handle_notify(channel)
+              @conn.wait_for_notify(timeout_s) do |channel, _pid, payload|
+                handle_notify(channel, payload)
               end || run_health_check
             rescue IOError => e
               # #stop closes the PG connection to interrupt
@@ -182,11 +186,11 @@ module Pgbus
           @listening_to.delete(channel)
         end
 
-        def handle_notify(channel)
+        def handle_notify(channel, payload = nil)
           queue_name = queue_name_from(channel)
           return unless queue_name
 
-          @dispatch_queue << WakeMessage.new(queue_name: queue_name)
+          @dispatch_queue << WakeMessage.new(queue_name: queue_name, payload: payload)
         end
 
         def run_health_check

data/lib/pgbus/web/streamer/stream_event_dispatcher.rb

@@ -92,6 +92,7 @@ module Pgbus
           # boolean assignment), the sentinel break would still fire.
           @running = false
           @thread = nil
+          @ephemeral_seq = 0
         end
 
         def start
@@ -136,7 +137,7 @@ module Pgbus
             # queue's current contents — once we hit a non-Wake or a
             # different stream, we stop and let the regular path handle
             # the rest.
-            if msg.is_a?(WakeMessage)
+            if msg.is_a?(WakeMessage) && msg.payload.nil?
               wakes, trailing = drain_wakes_for(msg)
               wakes.each { |w| handle(w) }
               handle(trailing) if trailing
@@ -165,7 +166,7 @@ module Pgbus
               return [coalesced, nil] # queue drained
             end
 
-            return [coalesced, peek] unless peek.is_a?(WakeMessage)
+            return [coalesced, peek] unless peek.is_a?(WakeMessage) && peek.payload.nil?
 
             next if seen.include?(peek.queue_name)
 
@@ -193,32 +194,26 @@ module Pgbus
 
         def handle_wake(msg)
           started_at = monotonic_ms
-          # msg.queue_name is the PGMQ full table name (pgbus_int_pbns_xxx),
-          # but connections are registered under the logical name (pbns_xxx).
-          # Translate before looking up.
           stream = @full_to_logical[msg.queue_name] || msg.queue_name
           registered = @registry.connections_for(stream)
           in_flight_pairs = @in_flight[stream]
           return if registered.empty? && in_flight_pairs.empty?
 
+          if msg.payload
+            handle_ephemeral_wake(msg, stream, registered, in_flight_pairs, started_at)
+          else
+            handle_durable_wake(stream, registered, in_flight_pairs, started_at)
+          end
+        end
+
+        def handle_durable_wake(stream, registered, in_flight_pairs, started_at)
           min_seen = minimum_cursor(registered, in_flight_pairs)
           raw_envelopes = @client.read_after(stream, after_id: min_seen, limit: @read_limit)
           return if raw_envelopes.empty?
 
           envelopes = raw_envelopes.map { |e| unwrap_stream_envelope(e) }
-          # The maximum msg_id in THIS batch. We advance every
-          # connection's scanned cursor past this value even if the
-          # filter drops everything — otherwise a 500-message run
-          # of invisible broadcasts would pin minimum_cursor and
-          # the dispatcher would re-read the same window forever,
-          # starving later public messages. Connection#enqueue still
-          # gates the client-facing cursor on actual successful
-          # writes, so this advance is invisible to clients.
           max_msg_id = envelopes.map(&:msg_id).max
 
-          # Each connection gets a per-connection filtered subset. We
-          # can't pre-filter once because different connections have
-          # different authorize contexts.
           registered.each do |conn|
             safe_enqueue(conn, visible_envelopes_for(envelopes, conn))
             advance_scanned_cursor(conn, max_msg_id)
@@ -230,11 +225,40 @@ module Pgbus
 
           prune_dead(registered)
 
-          # Record one stat row per wake. Fanout is the number of
-          # subscribers (registered + in-flight) that received the
-          # broadcast before any filter dropped it — the "intended"
-          # audience size, which is the useful operator number even
-          # when audience filtering is in play.
+          record_stat(
+            stream_name: stream,
+            event_type: "broadcast",
+            started_at: started_at,
+            fanout: registered.size + in_flight_pairs.size
+          )
+        end
+
+        def handle_ephemeral_wake(msg, stream, registered, in_flight_pairs, started_at)
+          parsed = JSON.parse(msg.payload)
+          html = parsed.is_a?(Hash) ? parsed["html"] : nil
+          return unless html.is_a?(String)
+
+          visible_to = parsed["visible_to"]
+          visible_to = visible_to.to_sym if visible_to.is_a?(String)
+
+          @ephemeral_seq += 1
+          envelope = StreamEnvelope.new(
+            msg_id: -@ephemeral_seq,
+            enqueued_at: Time.now.utc.iso8601(6),
+            payload: html,
+            source: "ephemeral",
+            visible_to: visible_to
+          )
+
+          registered.each do |conn|
+            safe_enqueue(conn, visible_envelopes_for([envelope], conn))
+          end
+          in_flight_pairs.each do |(conn, buffer)|
+            buffer.concat(visible_envelopes_for([envelope], conn))
+          end
+
+          prune_dead(registered)
+
           record_stat(
             stream_name: stream,
             event_type: "broadcast",

data/lib/pgbus.rb

@@ -40,6 +40,10 @@ module Pgbus
         loader.ignore("#{__dir__}/generators")
         loader.ignore("#{__dir__}/active_job")
         loader.ignore("#{__dir__}/pgbus/testing")
+        # Vendor integrations are loaded conditionally (when the vendor gem
+        # is present) by lib/pgbus/engine.rb. Keeping them out of Zeitwerk
+        # means we don't reference vendor constants at autoload time.
+        loader.ignore("#{__dir__}/pgbus/integrations")
         # lib/puma/plugin/pgbus_streams.rb is a Puma plugin — it's required
         # explicitly by the user from config/puma.rb via `plugin :pgbus_streams`.
         # Without this ignore, Zeitwerk scans lib/puma/ under the pgbus loader
@@ -100,10 +104,11 @@ module Pgbus
     # clears it. The cache key is the resolved name string, not the raw
     # streamables, so `Pgbus.stream(@order)` and `Pgbus.stream(@order)`
     # in the same process return the same instance.
-    def stream(streamables)
+    def stream(streamables, durable: true)
       name = Streams::Stream.name_from(streamables)
+      cache_key = "#{name}:#{durable ? "d" : "e"}"
       @stream_cache ||= Concurrent::Map.new
-      @stream_cache.compute_if_absent(name) { Streams::Stream.new(streamables) }
+      @stream_cache.compute_if_absent(cache_key) { Streams::Stream.new(streamables, durable: durable) }
     end
 
     # Compose a short, pgbus-safe stream identifier from any mix of

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.8.0
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -250,6 +250,7 @@ files:
 - lib/pgbus/cli.rb
 - lib/pgbus/client.rb
 - lib/pgbus/client/ensure_stream_queue.rb
+- lib/pgbus/client/notify_stream.rb
 - lib/pgbus/client/read_after.rb
 - lib/pgbus/concurrency.rb
 - lib/pgbus/concurrency/blocked_execution.rb
@@ -273,6 +274,12 @@ files:
 - lib/pgbus/generators/database_target_detector.rb
 - lib/pgbus/generators/migration_detector.rb
 - lib/pgbus/instrumentation.rb
+- lib/pgbus/integrations/appsignal.rb
+- lib/pgbus/integrations/appsignal/dashboards/pgbus_health.json
+- lib/pgbus/integrations/appsignal/dashboards/pgbus_streams.json
+- lib/pgbus/integrations/appsignal/dashboards/pgbus_throughput.json
+- lib/pgbus/integrations/appsignal/probe.rb
+- lib/pgbus/integrations/appsignal/subscriber.rb
 - lib/pgbus/log_formatter.rb
 - lib/pgbus/outbox.rb
 - lib/pgbus/outbox/poller.rb