pgbus 0.8.4 → 0.9.2

data/lib/pgbus/web/streamer/stream_event_dispatcher.rb

@@ -32,18 +32,34 @@ module Pgbus
       # disambiguate from Pgbus::Process::Dispatcher, which is an
       # unrelated worker-side pool coordinator. See issue #98 item 8.
       class StreamEventDispatcher
-        WakeMessage       = Listener::WakeMessage
-        ConnectMessage    = Data.define(:connection)
-        DisconnectMessage = Data.define(:connection)
+        WakeMessage          = Listener::WakeMessage
+        ConnectMessage       = Data.define(:connection)
+        DisconnectMessage    = Data.define(:connection)
+        # Posted by the Heartbeat once per tick with the current presence
+        # connections, so the touch (a last_seen_at refresh) runs on the
+        # dispatcher thread where AR connections are released each pass.
+        PresenceTouchMessage = Data.define(:connections)
 
         # An unwrapped stream broadcast. Similar shape to
         # Pgbus::Client::ReadAfter::Envelope (msg_id + payload) so
         # Connection#enqueue can consume either type via duck typing,
-        # but adds the `visible_to` label carried through from
-        # Pgbus::Streams::Stream#broadcast. The Dispatcher uses
-        # visible_to to decide per-connection delivery; Connection
-        # never sees the field.
-        StreamEnvelope = Data.define(:msg_id, :enqueued_at, :payload, :source, :visible_to)
+        # but adds two delivery-control fields carried through from
+        # Pgbus::Streams::Stream#broadcast:
+        #   - `visible_to` — audience filter label (evaluated per-connection)
+        #   - `exclude`    — a connection id to skip (actor-echo suppression:
+        #                    the broadcaster's own SSE connection does not
+        #                    receive the echo of its own broadcast)
+        # The Dispatcher uses both to decide per-connection delivery;
+        # Connection never sees either field.
+        #   - `event`      — the SSE `event:` name for the delivered frame.
+        #                    nil means the default (turbo-stream); a typed
+        #                    name (e.g. "presence", "reactive") lets clients
+        #                    route without sniffing the HTML (issue #170).
+        StreamEnvelope = Data.define(:msg_id, :enqueued_at, :payload, :source, :visible_to, :exclude, :event) do
+          def initialize(msg_id:, enqueued_at:, payload:, source:, visible_to: nil, exclude: nil, event: nil)
+            super
+          end
+        end
 
         DEFAULT_READ_LIMIT = 500
 
@@ -51,13 +67,18 @@ module Pgbus
 
         def initialize(client:, registry:, listener:, dispatch_queue:,
                        logger: Pgbus.logger, read_limit: DEFAULT_READ_LIMIT,
-                       filters: nil, config: nil, stream_counter: nil)
+                       filters: nil, config: nil, stream_counter: nil,
+                       presence_provider: nil)
           @client = client
           @registry = registry
           @listener = listener
           @queue = dispatch_queue
           @logger = logger
           @read_limit = read_limit
+          # Vends a presence handle for a logical stream name. Injected so
+          # tests can record join/leave/touch without a DB. Production
+          # defaults to the real per-stream Presence via Pgbus.stream.
+          @presence_provider = presence_provider || ->(name) { Pgbus.stream(name).presence }
           # Filters default to the process-wide registry so production
           # code picks up whatever was registered at boot. Tests inject
           # a fresh Filters instance to avoid cross-test pollution.
@@ -175,9 +196,10 @@ module Pgbus
 
         def handle(msg)
           case msg
-          when WakeMessage       then handle_wake(msg)
-          when ConnectMessage    then handle_connect(msg)
-          when DisconnectMessage then handle_disconnect(msg)
+          when WakeMessage          then handle_wake(msg)
+          when ConnectMessage       then handle_connect(msg)
+          when DisconnectMessage    then handle_disconnect(msg)
+          when PresenceTouchMessage then handle_presence_touch(msg)
           else
             @logger.warn { "[Pgbus::Streamer::StreamEventDispatcher] unknown message: #{msg.class}" }
           end
@@ -239,6 +261,7 @@ module Pgbus
 
           visible_to = parsed["visible_to"]
           visible_to = visible_to.to_sym if visible_to.is_a?(String)
+          exclude = parsed["exclude"]
 
           @ephemeral_seq += 1
           envelope = StreamEnvelope.new(
@@ -246,7 +269,9 @@ module Pgbus
             enqueued_at: Time.now.utc.iso8601(6),
             payload: html,
             source: "ephemeral",
-            visible_to: visible_to
+            visible_to: visible_to,
+            exclude: exclude,
+            event: normalize_sse_event(parsed["event"])
           )
 
           registered.each do |conn|
@@ -320,6 +345,7 @@ module Pgbus
           else
             @stream_counter.increment_connections(stream)
             @registry.register(connection)
+            presence_join(connection, stream)
           end
 
           record_stat(
@@ -346,6 +372,7 @@ module Pgbus
           removed = @registry.unregister(connection)
           @scanned_cursor.delete(connection)
           @stream_counter.decrement_connections(stream) if removed
+          presence_leave(connection, stream)
           cleanup_stream_if_unused(stream)
 
           record_stat(
@@ -355,6 +382,21 @@ module Pgbus
           )
         end
 
+        # Touches (refreshes last_seen_at for) the presence members on the
+        # given connections. Posted by the Heartbeat each tick so idle but
+        # still-connected members don't get swept. Connections without a
+        # presence member (non-presence streams, anonymous) are skipped.
+        def handle_presence_touch(msg)
+          msg.connections.each do |connection|
+            member_id = presence_member_of(connection)
+            next unless member_id
+
+            @presence_provider.call(connection.stream_name).touch(member_id: member_id)
+          rescue StandardError => e
+            @logger.error { "[Pgbus::Streamer::StreamEventDispatcher] presence touch failed: #{e.class}: #{e.message}" }
+          end
+        end
+
         # If this stream has no remaining subscribers (registered or
         # in-flight), release all per-stream state so long-running
         # processes don't leak memory proportional to unique stream
@@ -426,6 +468,22 @@ module Pgbus
           @client.config.queue_name(stream_name)
         end
 
+        # Sanitizes a typed SSE event name from an untrusted broadcast
+        # payload before it reaches the SSE `event:` line. Returns nil
+        # (→ the default turbo-stream event) for non-strings, blanks, or
+        # any value containing CR/LF — a crafted event with a newline could
+        # otherwise inject extra SSE fields (a forged id:/data:) into the
+        # frame and corrupt cursor/event routing. Defense in depth with
+        # Envelope.message, which also strips newlines.
+        def normalize_sse_event(value)
+          return nil unless value.is_a?(String)
+
+          event = value.strip
+          return nil if event.empty? || event.match?(/[\r\n]/)
+
+          event
+        end
+
         # Pgbus::Streams::Stream#broadcast wraps HTML payloads as
         # {"html": "..."} so PGMQ's JSONB column accepts them. Here we
         # unwrap the html field and return a new envelope whose payload
@@ -442,13 +500,16 @@ module Pgbus
 
           visible_to = parsed["visible_to"]
           visible_to = visible_to.to_sym if visible_to.is_a?(String)
+          exclude = parsed["exclude"]
 
           StreamEnvelope.new(
             msg_id: envelope.msg_id,
             enqueued_at: envelope.enqueued_at,
             payload: html,
             source: envelope.source,
-            visible_to: visible_to
+            visible_to: visible_to,
+            exclude: exclude,
+            event: normalize_sse_event(parsed["event"])
           )
         rescue JSON::ParserError
           envelope
@@ -460,13 +521,67 @@ module Pgbus
         # Filters registry. Envelopes that predate the StreamEnvelope
         # refactor (plain ReadAfter::Envelope with no visible_to) also
         # pass through.
+        #
+        # Actor-echo suppression: an envelope carrying `exclude:` (a
+        # connection id) is dropped for the connection whose id matches.
+        # This lets the broadcaster's own SSE connection skip the echo of
+        # its own broadcast — the actor already applied the change via its
+        # action's HTTP response, so re-applying the SSE echo would
+        # double-apply (re-run animations, clobber optimistic edits). The
+        # exclude check runs *before* the audience filter so an excluded
+        # actor is skipped even when it would otherwise match visible_to.
         def visible_envelopes_for(envelopes, connection)
           envelopes.select do |envelope|
+            next false if excluded?(envelope, connection)
+
             label = envelope.respond_to?(:visible_to) ? envelope.visible_to : nil
             @filters.visible?(label, connection.context)
           end
         end
 
+        # True when the envelope names this connection in its `exclude`
+        # field. Guarded by respond_to? so plain ReadAfter::Envelopes
+        # (no exclude field) and connections without an id never match.
+        def excluded?(envelope, connection)
+          return false unless envelope.respond_to?(:exclude)
+
+          exclude = envelope.exclude
+          return false if exclude.nil? || exclude.to_s.empty?
+
+          connection.respond_to?(:id) && connection.id.to_s == exclude.to_s
+        end
+
+        # Connection-driven presence (issue #169). Auto-joins a member when
+        # the stream is configured for presence and the connection's
+        # authorize-context yields a member id. Stores the member id on the
+        # connection so handle_disconnect/handle_presence_touch can act on
+        # it. Failures are logged and swallowed: a presence DB hiccup must
+        # not knock a live SSE connection out of the registry.
+        def presence_join(connection, stream)
+          return unless @config&.stream_presence?(stream)
+
+          member = @config.presence_member_for(connection.context)
+          return unless member
+
+          @presence_provider.call(stream).join(member_id: member[:id], metadata: member[:metadata] || {})
+          connection.presence_member = member[:id] if connection.respond_to?(:presence_member=)
+        rescue StandardError => e
+          @logger.error { "[Pgbus::Streamer::StreamEventDispatcher] presence join failed: #{e.class}: #{e.message}" }
+        end
+
+        def presence_leave(connection, stream)
+          member_id = presence_member_of(connection)
+          return unless member_id
+
+          @presence_provider.call(stream).leave(member_id: member_id)
+        rescue StandardError => e
+          @logger.error { "[Pgbus::Streamer::StreamEventDispatcher] presence leave failed: #{e.class}: #{e.message}" }
+        end
+
+        def presence_member_of(connection)
+          connection.respond_to?(:presence_member) ? connection.presence_member : nil
+        end
+
         def monotonic_ms
           ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1000.0
         end

data/lib/pgbus.rb

@@ -130,6 +130,17 @@ module Pgbus
       Streams::Key.stream_key(*parts, **)
     end
 
+    # Accepts an already-built stream key verbatim (no re-keying), only
+    # enforcing the queue-name budget. Use when you hold a key string and
+    # want to pass it to both `turbo_stream_from` and a broadcaster
+    # without the colon-separator guard raising on the second call.
+    #
+    #   key = Pgbus.stream_key(chat, :messages)
+    #   Pgbus.stream(Pgbus.stream_key!(key)).broadcast(html)
+    def stream_key!(key)
+      Streams::Key.stream_key!(key)
+    end
+
     def reset!
       @client&.close
       @client = nil

data/lib/tasks/pgbus_queues.rake

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+namespace :pgbus do
+  namespace :queues do
+    desc "Create FIFO indexes on all pgbus queues (required for grouped reads)"
+    task fifo_indexes: :environment do
+      puts "Creating FIFO indexes on all pgbus queues..."
+      Pgbus.client.create_fifo_indexes_all
+      puts "Done. FIFO indexes created on all queues."
+    end
+
+    desc "Create FIFO index on a specific queue (QUEUE=name)"
+    task fifo_index: :environment do
+      queue = ENV.fetch("QUEUE") do
+        abort "Usage: rake pgbus:queues:fifo_index QUEUE=<queue_name>"
+      end
+      puts "Creating FIFO index on queue '#{queue}'..."
+      Pgbus.client.create_fifo_index(queue)
+      puts "Done."
+    end
+  end
+
+  namespace :archives do
+    desc "Convert a queue's archive table to pg_partman-managed partitions (QUEUE=name)"
+    task partition: :environment do
+      queue = ENV.fetch("QUEUE") do
+        abort "Usage: rake pgbus:archives:partition QUEUE=<queue_name> " \
+              "[INTERVAL=10000] [RETENTION=100000] [LEADING_PARTITION=10]"
+      end
+      interval = ENV.fetch("INTERVAL", "10000")
+      retention = ENV.fetch("RETENTION", "100000")
+      leading_raw = ENV.fetch("LEADING_PARTITION", "10")
+      leading = begin
+        Integer(leading_raw, 10)
+      rescue ArgumentError, TypeError
+        abort "LEADING_PARTITION must be a positive integer, got #{leading_raw.inspect}"
+      end
+      abort "LEADING_PARTITION must be a positive integer" if leading <= 0
+
+      puts "Converting archive table for queue '#{queue}' to partitioned..."
+      puts "  Partition interval: #{interval}"
+      puts "  Retention interval: #{retention}"
+      puts "  Leading partitions: #{leading}"
+
+      Pgbus.client.convert_archive_partitioned(
+        queue,
+        partition_interval: interval,
+        retention_interval: retention,
+        leading_partition: leading
+      )
+      puts "Done. Archive table is now partitioned."
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.9.2
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -63,14 +63,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.7.0
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,7 @@ files:
 - app/controllers/pgbus/api/metrics_controller.rb
 - app/controllers/pgbus/api/stats_controller.rb
 - app/controllers/pgbus/application_controller.rb
+- app/controllers/pgbus/batches_controller.rb
 - app/controllers/pgbus/dashboard_controller.rb
 - app/controllers/pgbus/dead_letter_controller.rb
 - app/controllers/pgbus/events_controller.rb
@@ -161,6 +162,9 @@ files:
 - app/models/pgbus/stream_stat.rb
 - app/models/pgbus/uniqueness_key.rb
 - app/views/layouts/pgbus/application.html.erb
+- app/views/pgbus/batches/_batches_table.html.erb
+- app/views/pgbus/batches/index.html.erb
+- app/views/pgbus/batches/show.html.erb
 - app/views/pgbus/dashboard/_processes_table.html.erb
 - app/views/pgbus/dashboard/_queue_health.html.erb
 - app/views/pgbus/dashboard/_queues_table.html.erb
@@ -309,11 +313,13 @@ files:
 - lib/pgbus/stat_buffer.rb
 - lib/pgbus/streams.rb
 - lib/pgbus/streams/broadcastable_override.rb
+- lib/pgbus/streams/coalescer.rb
 - lib/pgbus/streams/cursor.rb
 - lib/pgbus/streams/envelope.rb
 - lib/pgbus/streams/filters.rb
 - lib/pgbus/streams/key.rb
 - lib/pgbus/streams/presence.rb
+- lib/pgbus/streams/renderer.rb
 - lib/pgbus/streams/signed_name.rb
 - lib/pgbus/streams/streamable.rb
 - lib/pgbus/streams/turbo_broadcastable.rb
@@ -343,6 +349,7 @@ files:
 - lib/puma/plugin/pgbus_streams.rb
 - lib/tasks/pgbus_autovacuum.rake
 - lib/tasks/pgbus_pgmq.rake
+- lib/tasks/pgbus_queues.rake
 - lib/tasks/pgbus_streams.rake
 homepage: https://github.com/mhenrixon/pgbus
 licenses: