pgbus 0.6.8 → 0.7.0

data/lib/pgbus/streams/key.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Pgbus
+  module Streams
+    # Short, pgbus-safe stream identifiers.
+    #
+    # PGMQ queue names are bounded by two ceilings: PostgreSQL's
+    # NAMEDATALEN (63 chars for `pgmq.q_<name>`) and pgmq-ruby's own
+    # stricter runtime check (`length >= 48` in
+    # `PGMQ::Client#validate_queue_name!`). The effective budget is the
+    # lower of the two, exposed as `QueueNameValidator::MAX_QUEUE_NAME_LENGTH`
+    # (currently 47). Any stream name composed from UUID primary keys
+    # and turbo-rails-style dom ids blows past that budget almost
+    # immediately:
+    #
+    #     "gid://app/Ai::Chat/9c14e8b2-94c3-4c6f-8ca1-f50d2f5e22ca:messages"
+    #     # => 63 chars, already too long before the "pgbus_" prefix is added.
+    #
+    # `stream_key` produces a deterministic short form suitable as a
+    # pgbus stream identifier. It normalizes each part, joins with ":",
+    # and enforces the queue-name budget (derived from the configured
+    # `queue_prefix`) at the call site — raising ArgumentError rather
+    # than letting the failure surface as an opaque QueueNameValidator
+    # error deep inside `Pgbus.stream(...).broadcast(...)`.
+    #
+    # Silent truncation is intentionally NOT supported: trimming a
+    # too-long key to fit would reintroduce the collision risk that the
+    # 64-bit digest is chosen to eliminate. Callers who overflow should
+    # shorten their own identifiers or adopt `Pgbus::Streams::Streamable`
+    # on their ActiveRecord models.
+    #
+    # Usage:
+    #
+    #     Pgbus.stream_key(chat, :messages)
+    #       # => "ai_chat_3a4f9c21b7d20e18:messages"
+    #
+    #     Pgbus.stream_key([user, :notifications])
+    #       # => "user_5fa83c91d44a2701:notifications"
+    #
+    #     Pgbus.stream(Pgbus.stream_key(chat, :messages)).broadcast("<turbo-stream/>")
+    #
+    # Collision horizon: the 64-bit SHA-256 prefix gives a birthday bound
+    # of roughly 5 billion records per model class before a 50% chance of
+    # collision. For multi-tenant apps where a collision would mean two
+    # records share a stream (and receive each other's broadcasts), this
+    # is wide enough in practice. Callers with higher sensitivity can
+    # pass `digest_bits: 128`.
+    module Key
+      DEFAULT_DIGEST_BITS = 64
+
+      module_function
+
+      # Compose a short pgbus-safe stream name from any mix of records,
+      # strings, symbols, and arrays. Returns the joined key when it fits
+      # the pgbus queue-name budget; raises ArgumentError otherwise.
+      #
+      # Fragments must not contain `:` — it's the join separator, so
+      # `stream_key("a:b", :c)` and `stream_key("a", "b:c")` would both
+      # produce `"a:b:c"` and collapse two logically distinct streams
+      # onto one queue. Colons inside fragments (typically from a
+      # `to_stream_key`/`to_gid_param` implementation that forgot to
+      # sanitize) raise an ArgumentError at the call site.
+      def stream_key(*parts, digest_bits: DEFAULT_DIGEST_BITS)
+        fragments = Array(parts).flatten.map { |part| normalize(part, digest_bits: digest_bits) }
+        fragments.each { |fragment| reject_colons!(fragment) }
+        key = fragments.join(":")
+        budget = queue_name_budget
+        return key if key.length <= budget
+
+        raise ArgumentError,
+              "stream_key #{key.inspect} is #{key.length} chars, " \
+              "exceeds pgbus budget of #{budget} " \
+              "(queue_prefix=#{Pgbus.configuration.queue_prefix.inspect}, " \
+              "pgbus_max_queue_name_length=#{QueueNameValidator::MAX_QUEUE_NAME_LENGTH}). " \
+              "Shorten the streamables or use Pgbus::Streams::Streamable on the model."
+      end
+
+      # 64-bit (default) SHA-256 prefix of the record's primary key. Stdlib
+      # only, deterministic, and fixed-length. CRC32's 32-bit output is
+      # intentionally not used here: its ~77k-row birthday bound is too
+      # tight for a multi-tenant stream identifier where a collision would
+      # route two records' broadcasts to the same queue.
+      # Full output size of the backing digest, in bits. Capping
+      # digest_bits here matters because `SHA256.hexdigest` only
+      # produces 64 hex chars (256 bits) no matter what — slicing
+      # `[0, 128]` just returns all 64 chars — so a caller asking for
+      # `digest_bits: 512` would silently get the same output as
+      # `digest_bits: 256` and walk away believing they'd widened the
+      # collision horizon. Raise instead.
+      MAX_DIGEST_BITS = ::Digest::SHA256.new.digest_length * 8 # => 256
+
+      def short_id(record, digest_bits: DEFAULT_DIGEST_BITS)
+        unless digest_bits.is_a?(Integer) && digest_bits.positive? &&
+               (digest_bits % 4).zero? && digest_bits <= MAX_DIGEST_BITS
+          raise ArgumentError,
+                "digest_bits must be a positive multiple of 4 and <= #{MAX_DIGEST_BITS} " \
+                "(SHA-256 produces #{MAX_DIGEST_BITS} bits; asking for more would silently truncate)"
+        end
+
+        # Unpersisted records all share id=nil, which hashes to a single
+        # constant digest and would collapse every new instance of the
+        # same class into one stream. Fail loud at the first unsaved
+        # call site — the whole point of the 64-bit digest is to
+        # eliminate collisions, so silently producing a shared key here
+        # would reintroduce exactly what it was chosen to prevent.
+        if record.id.nil?
+          raise ArgumentError,
+                "#{record.class.name} must be persisted before generating a stream key " \
+                "(record.id is nil — all unsaved records would collide on one stream)"
+        end
+
+        hex_chars = digest_bits / 4
+        ::Digest::SHA256.hexdigest(record.id.to_s)[0, hex_chars]
+      end
+
+      # Normalize a single streamable fragment to a pgbus-safe string.
+      # Mirrors the shape accepted by Turbo::Streams::StreamName and
+      # Pgbus::Streams::Stream.name_from so the two code paths agree
+      # on the wire format.
+      #
+      # - Strings and symbols pass through verbatim.
+      # - ActiveRecord models become "<param_key>_<short_id>".
+      # - Anything else responding to `to_gid_param` / `to_param` falls
+      #   back to that; a UUID primary key would still overflow, which
+      #   is why AR models are hashed above.
+      def normalize(part, digest_bits: DEFAULT_DIGEST_BITS)
+        case part
+        when String, Symbol
+          part.to_s
+        else
+          if defined?(::ActiveRecord::Base) && part.is_a?(::ActiveRecord::Base)
+            "#{part.class.model_name.param_key}_#{short_id(part, digest_bits: digest_bits)}"
+          elsif part.respond_to?(:to_stream_key)
+            part.to_stream_key
+          elsif part.respond_to?(:to_gid_param)
+            part.to_gid_param
+          elsif part.respond_to?(:to_param)
+            part.to_param
+          else
+            part.to_s
+          end
+        end
+      end
+
+      # Budget = effective pgbus queue-name limit - "<queue_prefix>_"
+      # length. Computed at call time (not a constant) so apps that
+      # override `config.queue_prefix` get the correct budget
+      # automatically.
+      def queue_name_budget
+        QueueNameValidator::MAX_QUEUE_NAME_LENGTH -
+          Pgbus.configuration.queue_prefix.length - 1
+      end
+
+      # Raises if a normalized fragment contains the `:` separator.
+      # Kept private-ish via module_function so the guard is shared
+      # between stream_key and any future composer without becoming
+      # part of the public surface.
+      def reject_colons!(fragment)
+        return unless fragment.include?(":")
+
+        raise ArgumentError,
+              "stream_key fragment #{fragment.inspect} contains ':' which is the " \
+              "join separator — two calls with different colon placements would " \
+              "collapse to the same key (e.g. stream_key('a:b', :c) vs " \
+              "stream_key('a', 'b:c') both produce 'a:b:c'). Strip or replace " \
+              "colons in the offending streamable before calling stream_key."
+      end
+      private_class_method :reject_colons!
+    end
+  end
+end

data/lib/pgbus/streams/streamable.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Streams
+    # ActiveRecord concern that adds `short_id` and `to_stream_key`
+    # instance methods for producing pgbus-safe stream identifiers from
+    # records whose primary key is a UUID (or any other long string).
+    #
+    # Intended for inclusion in `ApplicationRecord`:
+    #
+    #     class ApplicationRecord < ActiveRecord::Base
+    #       primary_abstract_class
+    #       include Pgbus::Streams::Streamable
+    #     end
+    #
+    # Every subclass then gets:
+    #
+    #     chat.short_id
+    #     # => "3a4f9c21b7d20e18"
+    #
+    #     chat.to_stream_key
+    #     # => "ai_chat_3a4f9c21b7d20e18"
+    #
+    #     Pgbus.stream_key(chat, :messages)
+    #     # => "ai_chat_3a4f9c21b7d20e18:messages"
+    #
+    # The mixin is intentionally thin: it delegates to `Pgbus::Streams::Key`
+    # so the digest policy lives in one place.
+    #
+    # `#to_stream_key` calls `Key.short_id(self)` directly rather than
+    # dispatching through `#short_id`. Ruby does NOT warn when a class
+    # defines an instance method and a later `include` adds a module
+    # with the same name — the class method silently wins. A host app
+    # that already defines its own `#short_id` (returning, say, a
+    # display-friendly abbreviation) would therefore hijack
+    # `to_stream_key` without any indication, producing stream keys
+    # the wire format never promised. Calling `Key.short_id(self)`
+    # explicitly bypasses instance-method lookup and guarantees the
+    # advertised digest regardless of what the host class does with
+    # the unqualified name.
+    module Streamable
+      # Returns a short SHA-256 prefix (64 bits / 16 hex chars by default)
+      # of this record's primary key. See `Pgbus::Streams::Key.short_id`
+      # for the digest policy and collision horizon.
+      def short_id(digest_bits: Key::DEFAULT_DIGEST_BITS)
+        Key.short_id(self, digest_bits: digest_bits)
+      end
+
+      # Returns a stable, pgbus-safe identifier of the form
+      # `<model_key>_<short_id>` suitable for passing directly to
+      # `Pgbus.stream(...)` or composing with `Pgbus.stream_key`.
+      def to_stream_key
+        "#{self.class.model_name.param_key}_#{Key.short_id(self)}"
+      end
+    end
+  end
+end

data/lib/pgbus/streams.rb

@@ -6,6 +6,15 @@ module Pgbus
   # which returns a `Pgbus::Streams::Stream` providing `#broadcast`,
   # `#current_msg_id`, and `#read_after`.
   module Streams
+    # Raised when a composed stream name would overflow PGMQ's queue-name
+    # budget (derived from PostgreSQL's NAMEDATALEN=64, minus PGMQ's `q_`
+    # table prefix, minus the configured queue_prefix + separator).
+    #
+    # Inherits from ArgumentError so existing rescues of the underlying
+    # QueueNameValidator error keep working; callers that want to handle
+    # this specifically can rescue Pgbus::Streams::StreamNameTooLong.
+    class StreamNameTooLong < ArgumentError; end
+
     # Process-wide registry of server-side audience filter predicates.
     # Register filters at boot time via:
     #   Pgbus::Streams.filters.register(:admin_only) { |user| user.admin? }
@@ -30,6 +39,7 @@ module Pgbus
 
       def initialize(streamables, client: Pgbus.client)
         @name = self.class.name_from(streamables)
+        self.class.validate_name_length!(@name, streamables)
         @client = client
         @ensured = false
         @ensure_mutex = Mutex.new
@@ -108,6 +118,33 @@ module Pgbus
         end
       end
 
+      # Enforces the pgbus queue-name budget at the Stream-construction
+      # boundary so a forgotten call site fails with an actionable error
+      # (pointing at the offending streamables and suggesting
+      # `Pgbus.stream_key`) instead of an opaque QueueNameValidator
+      # failure three frames deep in Client#ensure_stream_queue.
+      #
+      # The budget is computed from `config.queue_prefix` at call time
+      # so apps that override the prefix get the correct limit. Does not
+      # mutate the name — silent truncation is a footgun for
+      # multi-tenant apps where collisions would mix broadcasts across
+      # records. Callers who need a short, safe identifier should use
+      # `Pgbus.stream_key(...)` or include `Pgbus::Streams::Streamable`
+      # on their ActiveRecord models.
+      def self.validate_name_length!(name, streamables)
+        budget = Key.queue_name_budget
+        return if name.length <= budget
+
+        raise StreamNameTooLong,
+              "Stream name #{name.inspect} is #{name.length} chars, " \
+              "exceeds pgbus budget of #{budget} " \
+              "(queue_prefix=#{Pgbus.configuration.queue_prefix.inspect}, " \
+              "pgbus_max_queue_name_length=#{QueueNameValidator::MAX_QUEUE_NAME_LENGTH}). " \
+              "Streamables: #{streamables.inspect}. " \
+              "Use Pgbus.stream_key(*streamables) to produce a safe short name, " \
+              "or include Pgbus::Streams::Streamable on the model."
+      end
+
       private
 
       def ensure_queue!

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.6.8"
+  VERSION = "0.7.0"
 end

data/lib/pgbus.rb

@@ -105,6 +105,20 @@ module Pgbus
       @stream_cache.compute_if_absent(name) { Streams::Stream.new(streamables) }
     end
 
+    # Compose a short, pgbus-safe stream identifier from any mix of
+    # records, strings, symbols, and arrays. Delegates to
+    # `Pgbus::Streams::Key.stream_key`; raises `ArgumentError` if the
+    # resulting key would overflow the pgbus queue-name budget. See
+    # `lib/pgbus/streams/key.rb` for the digest policy and rationale.
+    #
+    #   Pgbus.stream_key(chat, :messages)
+    #   # => "ai_chat_3a4f9c21b7d20e18:messages"
+    #
+    #   Pgbus.stream(Pgbus.stream_key(chat, :messages)).broadcast(html)
+    def stream_key(*parts, **)
+      Streams::Key.stream_key(*parts, **)
+    end
+
     def reset!
       @client&.close
       @client = nil

data/lib/tasks/pgbus_autovacuum.rake

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+namespace :pgbus do
+  desc "Apply autovacuum tuning to PGMQ queue/archive tables and high-churn pgbus tables"
+  task tune_autovacuum: :environment do
+    require "pgbus/autovacuum_tuning"
+
+    conn = Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
+
+    # Only run if pgmq schema exists (tables may not be created yet during
+    # initial setup — the install migration handles tuning itself).
+    pgmq_exists = conn.select_value(
+      "SELECT 1 FROM information_schema.schemata WHERE schema_name = 'pgmq'"
+    )
+
+    unless pgmq_exists
+      puts "[pgbus] PGMQ schema not found — skipping autovacuum tuning."
+      next
+    end
+
+    puts "[pgbus] Applying autovacuum tuning to PGMQ queue/archive tables..."
+    conn.execute(Pgbus::AutovacuumTuning.sql_for_all_queues)
+
+    puts "[pgbus] Applying autovacuum tuning to high-churn pgbus tables..."
+    conn.execute(Pgbus::AutovacuumTuning.sql_for_high_churn_tables)
+
+    puts "[pgbus] Autovacuum tuning complete."
+  end
+end
+
+# Reapply autovacuum settings after schema:load since Ruby-format schema.rb
+# does not preserve ALTER TABLE ... SET (reloptions). This is a no-op if the
+# tables don't exist yet (IF EXISTS guards in the SQL).
+%w[db:schema:load db:schema:load:pgbus].each do |task_name|
+  next unless Rake::Task.task_defined?(task_name)
+
+  Rake::Task[task_name].enhance do
+    Rake::Task["pgbus:tune_autovacuum"].invoke if Rake::Task.task_defined?("pgbus:tune_autovacuum")
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.6.8
+  version: 0.7.0
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -215,6 +215,7 @@ files:
 - lib/generators/pgbus/add_stream_stats_generator.rb
 - lib/generators/pgbus/install_generator.rb
 - lib/generators/pgbus/migrate_job_locks_generator.rb
+- lib/generators/pgbus/migration_path.rb
 - lib/generators/pgbus/templates/add_failed_events_unique_index.rb.erb
 - lib/generators/pgbus/templates/add_job_locks.rb.erb
 - lib/generators/pgbus/templates/add_job_stats.rb.erb
@@ -255,6 +256,7 @@ files:
 - lib/pgbus/configuration/capsule_dsl.rb
 - lib/pgbus/dedup_cache.rb
 - lib/pgbus/engine.rb
+- lib/pgbus/error_reporter.rb
 - lib/pgbus/event.rb
 - lib/pgbus/event_bus/handler.rb
 - lib/pgbus/event_bus/publisher.rb
@@ -268,6 +270,7 @@ files:
 - lib/pgbus/generators/database_target_detector.rb
 - lib/pgbus/generators/migration_detector.rb
 - lib/pgbus/instrumentation.rb
+- lib/pgbus/log_formatter.rb
 - lib/pgbus/outbox.rb
 - lib/pgbus/outbox/poller.rb
 - lib/pgbus/pgmq_schema.rb
@@ -298,8 +301,10 @@ files:
 - lib/pgbus/streams/cursor.rb
 - lib/pgbus/streams/envelope.rb
 - lib/pgbus/streams/filters.rb
+- lib/pgbus/streams/key.rb
 - lib/pgbus/streams/presence.rb
 - lib/pgbus/streams/signed_name.rb
+- lib/pgbus/streams/streamable.rb
 - lib/pgbus/streams/turbo_broadcastable.rb
 - lib/pgbus/streams/turbo_stream_override.rb
 - lib/pgbus/streams/watermark_cache_middleware.rb
@@ -319,6 +324,7 @@ files:
 - lib/pgbus/web/streamer/registry.rb
 - lib/pgbus/web/streamer/stream_event_dispatcher.rb
 - lib/puma/plugin/pgbus_streams.rb
+- lib/tasks/pgbus_autovacuum.rake
 - lib/tasks/pgbus_pgmq.rake
 - lib/tasks/pgbus_streams.rake
 homepage: https://github.com/mhenrixon/pgbus