pgbus 0.22.0 → 0.23.1

data/lib/pgbus/web/streamer/stream_event_dispatcher.rb

@@ -78,6 +78,7 @@ module Pgbus
         end
 
         DEFAULT_READ_LIMIT = 500
+        NO_PAIRS = [].freeze
 
         attr_reader :stream_counter
 
@@ -124,8 +125,10 @@ module Pgbus
           @fanout_write_deadline_ms  = @config.streams_fanout_write_deadline_ms
           @connect_write_deadline_ms = @config.streams_write_deadline_ms
           @stream_counter = stream_counter || StreamCounter.new
-          # stream_name → Array<[connection, Array<Envelope>]>
-          @in_flight = Hash.new { |h, k| h[k] = [] }
+          # stream_name → Array<[connection, Array<Envelope>]>. No default
+          # proc: a read for a stream nobody is connecting to (a wake after
+          # the last subscriber left) must not leave an entry behind.
+          @in_flight = {}
           # PGMQ full table name (pgbus_<prefix>_<name>) → logical stream
           # name. Populated on connect so handle_wake can translate
           # Listener::WakeMessage#queue_name (a full table name, because
@@ -265,7 +268,7 @@ module Pgbus
           started_at = monotonic_ms
           stream = @full_to_logical[msg.queue_name] || msg.queue_name
           registered = @registry.connections_for(stream)
-          in_flight_pairs = @in_flight[stream]
+          in_flight_pairs = @in_flight.fetch(stream, NO_PAIRS)
           return if registered.empty? && in_flight_pairs.empty?
 
           if msg.payload
@@ -285,6 +288,7 @@ module Pgbus
 
           envelopes = raw_envelopes.map { |e| unwrap_stream_envelope(e) }
           max_msg_id = envelopes.map(&:msg_id).max
+          frames = frames_for(envelopes) unless registered.empty?
 
           registered.each do |conn|
             visible = visible_envelopes_for(envelopes, conn)
@@ -294,9 +298,9 @@ module Pgbus
               # Post even when `visible` is empty, with batch max, so an
               # audience-filtered-away batch still advances the cursor past the
               # hidden window (issue #321 B2). The pump raises on ephemerals.
-              @pump.post(conn, visible, max_msg_id, deadline_ms: @fanout_write_deadline_ms)
+              @pump.post(conn, visible, max_msg_id, deadline_ms: @fanout_write_deadline_ms, frames: frames)
             else
-              safe_enqueue(conn, visible)
+              safe_enqueue(conn, visible, frames: frames)
               advance_scanned_cursor(conn, max_msg_id)
             end
           end
@@ -336,8 +340,9 @@ module Pgbus
             event: normalize_sse_event(parsed["event"])
           )
 
+          frames = frames_for([envelope]) unless registered.empty?
           registered.each do |conn|
-            safe_enqueue(conn, visible_envelopes_for([envelope], conn))
+            safe_enqueue(conn, visible_envelopes_for([envelope], conn), frames: frames)
           end
           in_flight_pairs.each do |(conn, buffer)|
             buffer.concat(visible_envelopes_for([envelope], conn))
@@ -367,13 +372,46 @@ module Pgbus
           # prefix: pgmq.q_<prefixed>.INSERT. Registry and the in-flight
           # buffer use the logical name. The Dispatcher is the single
           # translator between the two naming worlds.
+          #
+          # Only a stream's first subscriber waits on the listener (issue
+          # #630). remove_listening is issued in exactly one place,
+          # cleanup_stream_if_unused, and only when the registry holds no
+          # connection on the stream. So a registered connection, dead or
+          # alive, proves the stream's last ensure_listening returned after
+          # its last remove_listening was issued. The listener runs commands
+          # in order, and a LISTEN that raised is acked only once the reconnect
+          # that follows has re-LISTENed it, so the LISTEN is active. Two
+          # exceptions. A listener thread on its way out (at stop, or after an
+          # unexpected error) releases every waiting caller with nothing
+          # LISTENed. And a first connect whose ack timed out (a stuck or
+          # reconnecting listener) leaves its LISTEN queued, ahead of any
+          # later UNLISTEN, to run when the listener recovers; until then the
+          # fast-path connects share the window that first connect already
+          # had, and a durable stream's next wake re-reads from the min
+          # cursor. A listener reconnect re-LISTENs its whole set; a hub
+          # failover replays FailoverListener's subscriptions, which the
+          # first-connect / last-disconnect calls keep exact. An empty stream
+          # always takes the slow path: that covers a connect that died
+          # mid-replay and unlistened without ever registering.
+          # @full_to_logical is set here and deleted only by that same
+          # cleanup, so it is present whenever the stream is not empty.
+          #
+          # The fast path holds while the listener reconnects, on purpose.
+          # Waiting on a reconnecting listener would cost every connect the
+          # full ack budget (health_check_ms + 1 s) for as long as the
+          # database stays unreachable: the storm this rule removes. A
+          # connect in that window has the same exposure as every connection
+          # already registered: a durable stream's next wake after the
+          # re-LISTEN re-reads from the min cursor.
           full_name = notify_queue_name_for(stream)
-          @full_to_logical[full_name] = stream
-          @listener.ensure_listening(full_name)
+          if @registry.empty?(stream)
+            @full_to_logical[full_name] = stream
+            @listener.ensure_listening(full_name)
+          end
 
           # Step 2: install the in-flight buffer BEFORE any read.
           buffer = []
-          @in_flight[stream] << [connection, buffer]
+          (@in_flight[stream] ||= []) << [connection, buffer]
 
           # Step 3: read the archive for anything published before this
           # connect landed, and write to the connection.
@@ -481,7 +519,7 @@ module Pgbus
         #   2. @in_flight[stream] (cleared by remove_in_flight already)
         #   3. Listener's @listening_to set + the PG LISTEN itself
         def cleanup_stream_if_unused(stream)
-          return unless @registry.empty?(stream) && @in_flight[stream].empty?
+          return unless @registry.empty?(stream) && !@in_flight.key?(stream)
 
           full_name = @full_to_logical.key(stream)
           return unless full_name
@@ -550,11 +588,32 @@ module Pgbus
         # it implicitly. handle_connect passes @connect_write_deadline_ms
         # explicitly so a new client's initial replay keeps the full window
         # (issue #315 item 3).
-        def safe_enqueue(connection, envelopes_or_buffer, deadline_ms: @fanout_write_deadline_ms)
+        #
+        # frames: the wake's prebuilt frames (frames_for); nil on the connect
+        # replay and the in-flight drain, which frame per connection.
+        def safe_enqueue(connection, envelopes_or_buffer, deadline_ms: @fanout_write_deadline_ms, frames: nil)
           return if connection.dead?
           return if envelopes_or_buffer.empty?
 
-          connection.enqueue(envelopes_or_buffer, deadline_ms: deadline_ms)
+          connection.enqueue(envelopes_or_buffer, deadline_ms: deadline_ms, frames: frames)
+        end
+
+        # One SSE frame per envelope per wake, shared by every connection on
+        # the stream (issue #621): the bytes depend on the envelope alone, so
+        # framing per connection repeated the same work N times. Keyed by the
+        # Integer msg_id, never by the envelope (a StreamEnvelope's #hash walks
+        # its payload, which would cost what the framing saves); msg_id is
+        # unique within a wake. Frozen, values and Hash, so the frames stay
+        # Ractor-shareable across the dispatch seam (issue #527).
+        def frames_for(envelopes)
+          envelopes.to_h do |envelope|
+            frame = Pgbus::Streams::Envelope.message(
+              id: envelope.msg_id,
+              event: Pgbus::Streams::Envelope.sse_event_for(envelope),
+              data: envelope.payload
+            )
+            [envelope.msg_id, frame.freeze]
+          end.freeze
         end
 
         def prune_dead(connections)
@@ -565,6 +624,8 @@ module Pgbus
 
         def remove_in_flight(stream, connection)
           pairs = @in_flight[stream]
+          return unless pairs
+
           pairs.reject! { |(conn, _buf)| conn.equal?(connection) }
           @in_flight.delete(stream) if pairs.empty?
         end

data/lib/tasks/pgbus_queues.rake

@@ -9,6 +9,33 @@ namespace :pgbus do
       puts "Done. FIFO indexes created on all queues."
     end
 
+    # Issue #633: the pgbus:fold_queue_name_case migration runs once; a
+    # mixed-case pgmq.meta row an older process recreated after it is folded
+    # here (pgbus doctor's "Queue name case" check names them).
+    desc "Spell mixed-case queues (pgmq.meta, NOTIFY triggers, pgbus's queue-name columns) in lower case"
+    task fold_case: :environment do
+      # The fold writes pgmq.meta and pgbus's tables through one connection,
+      # the one pgbus's models (and its migrations) use; a database without
+      # pgmq.meta there (PGMQ reached through database_url elsewhere) is
+      # refused rather than half-folded.
+      pool = Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection_pool : ActiveRecord::Base.connection_pool
+      names = pool.with_connection do |connection|
+        unless connection.select_value("SELECT to_regclass('pgmq.meta')")
+          abort "pgmq.meta is not in the database pgbus's models use (#{connection.current_database}); " \
+                "run the fold where pgbus's tables and PGMQ live together"
+        end
+
+        prefix = "#{Pgbus.configuration.queue_prefix.to_s.downcase}_"
+        found = connection.select_values(
+          "SELECT queue_name FROM pgmq.meta WHERE queue_name <> lower(queue_name) " \
+          "AND left(lower(queue_name), length(#{connection.quote(prefix)})) = #{connection.quote(prefix)} ORDER BY queue_name"
+        )
+        connection.transaction { Pgbus::QueueNameFold.run!(connection) }
+        found
+      end
+      puts names.empty? ? "Every queue is already spelled in lower case." : "Folded: #{names.join(", ")}"
+    end
+
     desc "Create FIFO index on a specific queue (QUEUE=name)"
     task fifo_index: :environment do
       queue = ENV.fetch("QUEUE") do

data/sig/pgbus/active_job/adapter.rbs

@@ -19,7 +19,7 @@ module Pgbus
   module Uniqueness
     module ClassMethods
       # Any other keyword raises ArgumentError (lock_ttl: with a pointer to 1.0's removal).
-      def ensures_uniqueness: (?strategy: :until_executed | :while_executing, ?key: untyped,
+      def ensures_uniqueness: (?strategy: :until_executed | :until_executing | :while_executing, ?key: untyped,
                                ?on_conflict: :reject | :discard | :log) -> untyped
       def pgbus_uniqueness: () -> Hash[Symbol, untyped]?
     end

data/sig/pgbus/active_job/quiet_logs.rbs

@@ -0,0 +1,11 @@
+module Pgbus
+  module ActiveJob
+    # config.quiet_active_job_logs: a worker process unsubscribes
+    # ActiveJob::LogSubscriber from perform_start and perform.
+    module QuietLogs
+      EVENTS: Array[String]
+
+      def self.apply!: (?notifier: untyped) -> bool
+    end
+  end
+end

data/sig/pgbus/client.rbs

@@ -3,8 +3,8 @@
 # public method is missing here.
 module Pgbus
   class Client
-    def archive_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped) -> untyped
-    def archive_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped) -> untyped
+    def archive_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped, ?job_completion: bool) -> untyped
+    def archive_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped, ?job_completion: bool) -> untyped
     def bind_topic: (untyped pattern, untyped queue_name) -> untyped
     def close: () -> untyped
     def config: () -> untyped
@@ -14,9 +14,9 @@ module Pgbus
     def create_fifo_index: (untyped queue_name) -> untyped
     def create_fifo_indexes_all: () -> untyped
     def dead_letter_physical_name: (untyped queue_name, ?prefixed: untyped) -> untyped
-    def delete_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped) -> untyped
-    def delete_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped) -> untyped
-    def drop_queue: (untyped queue_name, ?prefixed: untyped) -> untyped
+    def delete_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped, ?job_completion: bool) -> untyped
+    def delete_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped, ?job_completion: bool) -> untyped
+    def drop_queue: (untyped queue_name, ?prefixed: untyped) ?{ () -> untyped } -> untyped
     def ensure_all_queues: () -> untyped
     def ensure_dead_letter_queue: (untyped name) -> untyped
     def ensure_fair_index: (untyped queue_name) -> untyped
@@ -32,6 +32,7 @@ module Pgbus
     def message_in_queue?: (untyped queue_name, msg_id: untyped, ?prefixed: untyped) -> bool?
     def message_with_job_id?: (untyped queue_name, job_id: untyped, ?prefixed: untyped) -> bool?
     def metrics: (?untyped queue_name) -> untyped
+    def mixed_case_queue_names: () -> Array[String]
     def move_to_dead_letter: (untyped queue_name, untyped message, ?headers: untyped) -> untyped
     def native_ready?: (String full_name) -> bool
     def notify_enabled?: (untyped queue_name) -> bool

data/sig/pgbus/configuration.rbs

@@ -36,6 +36,8 @@ module Pgbus
     def completion_batch_size=: (untyped) -> untyped
     def completion_batching: () -> bool
     def completion_batching=: (untyped) -> untyped
+    def completion_synchronous_commit: () -> bool
+    def completion_synchronous_commit=: (bool value) -> bool
     def connection_guc_mode: () -> Symbol
     def connection_guc_mode=: (untyped mode) -> untyped
     def connection_options: () -> untyped
@@ -152,6 +154,8 @@ module Pgbus
     def queue_name: (untyped name) -> untyped
     def queue_prefix: () -> String
     def queue_prefix=: (untyped) -> untyped
+    def quiet_active_job_logs: () -> bool
+    def quiet_active_job_logs=: (bool value) -> bool
     def read_ahead: () -> Integer
     def read_ahead=: (untyped value) -> untyped
     def read_ahead_for: (untyped entry) -> untyped

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.22.0
+  version: 0.23.1
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -63,14 +63,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.2
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -302,6 +302,7 @@ files:
 - lib/generators/pgbus/add_uniqueness_keys_generator.rb
 - lib/generators/pgbus/add_workflow_signals_generator.rb
 - lib/generators/pgbus/add_workflows_generator.rb
+- lib/generators/pgbus/fold_queue_name_case_generator.rb
 - lib/generators/pgbus/install_generator.rb
 - lib/generators/pgbus/migrate_job_locks_generator.rb
 - lib/generators/pgbus/migration_path.rb
@@ -325,6 +326,7 @@ files:
 - lib/generators/pgbus/templates/add_workflow_signals.rb.erb
 - lib/generators/pgbus/templates/add_workflows.rb.erb
 - lib/generators/pgbus/templates/deploy_tasks.yml.erb
+- lib/generators/pgbus/templates/fold_queue_name_case.rb.erb
 - lib/generators/pgbus/templates/initializer.rb.erb
 - lib/generators/pgbus/templates/migrate_job_locks_to_uniqueness_keys.rb.erb
 - lib/generators/pgbus/templates/migration.rb.erb
@@ -343,6 +345,7 @@ files:
 - lib/pgbus/active_job/current_attributes.rb
 - lib/pgbus/active_job/delivery.rb
 - lib/pgbus/active_job/executor.rb
+- lib/pgbus/active_job/quiet_logs.rb
 - lib/pgbus/autovacuum_tuning.rb
 - lib/pgbus/batch.rb
 - lib/pgbus/batch/builder.rb
@@ -353,6 +356,7 @@ files:
 - lib/pgbus/cli/dashboard.rb
 - lib/pgbus/cli/dlq.rb
 - lib/pgbus/client.rb
+- lib/pgbus/client/async_commit.rb
 - lib/pgbus/client/connection_health.rb
 - lib/pgbus/client/ensure_stream_queue.rb
 - lib/pgbus/client/fair_read.rb
@@ -484,6 +488,7 @@ files:
 - lib/pgbus/process/wildcard_queue_resolver.rb
 - lib/pgbus/process/worker.rb
 - lib/pgbus/queue_factory.rb
+- lib/pgbus/queue_name_fold.rb
 - lib/pgbus/queue_name_validator.rb
 - lib/pgbus/rate_counter.rb
 - lib/pgbus/recurring/already_recorded.rb
@@ -534,6 +539,8 @@ files:
 - lib/pgbus/web/data_source/list_counts.rb
 - lib/pgbus/web/data_source/process_list.rb
 - lib/pgbus/web/data_source/queue_summary.rb
+- lib/pgbus/web/data_source/resend.rb
+- lib/pgbus/web/data_source/resend_slot.rb
 - lib/pgbus/web/data_source/workflow_actions.rb
 - lib/pgbus/web/data_source/workflow_insights.rb
 - lib/pgbus/web/data_source/workflow_list.rb
@@ -615,6 +622,7 @@ files:
 - lib/tasks/pgbus_streams.rake
 - sig/pgbus.rbs
 - sig/pgbus/active_job/adapter.rbs
+- sig/pgbus/active_job/quiet_logs.rbs
 - sig/pgbus/client.rbs
 - sig/pgbus/configuration.rbs
 - sig/pgbus/configuration/snapshot.rbs