pgbus 0.21.4 → 0.23.0

data/lib/pgbus/client.rb

@@ -17,6 +17,8 @@ module Pgbus
     include NotifyLockRetry
     include EnsureStreamQueue
     include NotifyStream
+    include NativeStatements
+    include AsyncCommit
 
     attr_reader :pgmq, :config, :connection_health
 
@@ -28,6 +30,12 @@ module Pgbus
     PGMQ_META_CHECK_SQL = "SELECT 1 FROM pg_tables WHERE schemaname = 'pgmq' AND tablename = 'meta' LIMIT 1"
     private_constant :PGMQ_META_CHECK_SQL
 
+    # pgbus's own queues only ($1: the lower-case "<queue_prefix>_"): another
+    # PGMQ user of the same schema keeps its spelling.
+    MIXED_CASE_QUEUES_SQL = "SELECT queue_name FROM pgmq.meta WHERE queue_name <> lower(queue_name) " \
+                            "AND left(lower(queue_name), length($1)) = $1 ORDER BY queue_name"
+    private_constant :MIXED_CASE_QUEUES_SQL
+
     PGMQ_INSTALL_SAVEPOINT = "pgbus_pgmq_install"
     private_constant :PGMQ_INSTALL_SAVEPOINT
 
@@ -129,7 +137,11 @@ module Pgbus
         # Both raise clean PG errors — no Ruby Timeout, no Thread#raise. Only
         # safe on this dedicated-connection branch — never on the shared-AR Proc
         # path, where statement_timeout would leak into application queries.
-        conn_opts = wrap_session_gucs(apply_connection_bounds(conn_opts))
+        conn_opts = apply_connection_bounds(conn_opts)
+        # What the connect lambda SETs per session (:session GUC mode), kept so
+        # a session reset after a failed completion pipeline gets it again.
+        @session_gucs = session_gucs(conn_opts)
+        conn_opts = wrap_session_gucs(conn_opts)
         @pgmq = PGMQ::Client.new(conn_opts, pool_size: config.resolved_pool_size, pool_timeout: config.pool_timeout)
         @pgmq_mutex = nil
         # Dedicated streams pool (issue #315): isolates the durable-stream
@@ -170,6 +182,15 @@ module Pgbus
 
       @queues_created = Concurrent::Map.new
       @stream_indexes_created = Concurrent::Map.new
+      # Per physical queue: its native statements, or false (issue #605).
+      @native_statements = Concurrent::Map.new
+      @native_unprepared = config.native_statements && ar_disables_prepared_statements?
+      # Job completions commit with synchronous_commit off (issue #619).
+      # Native statements only: the pgmq functions are never wrapped. Never on
+      # ActiveRecord's own connection (the Proc path): pgbus does not pipeline
+      # on, or reset, a connection it does not own.
+      @async_completion_commit = config.native_statements && !config.completion_synchronous_commit &&
+                                 !@shared_connection
       # Guards the one-time build of the publisher autoscale trigger (issue #323).
       # NOT @pgmq_mutex — that is nil on the dedicated path (the only path the
       # trigger exists on), so it wouldn't serialize concurrent first-publishers.
@@ -299,6 +320,18 @@ module Pgbus
       end
     end
 
+    # pgmq.meta rows spelled with an upper-case letter (issue #633). PGMQ
+    # lower-cases table names, so each shares its table with the lower-case
+    # spelling pgbus now uses; `rails generate pgbus:fold_queue_name_case`
+    # renames them. Pgbus::Doctor reports them.
+    def mixed_case_queue_names
+      synchronized do
+        @pgmq.with_connection do |conn|
+          conn.exec_params(MIXED_CASE_QUEUES_SQL, ["#{config.queue_prefix.to_s.downcase}_"]).column_values(0)
+        end
+      end
+    end
+
     # The most recently recorded installed PGMQ schema version string (e.g.
     # "1.5.0"), read from the pgbus_pgmq_schema_versions tracking table. Returns
     # nil when nothing is tracked yet or the table does not exist — the same
@@ -356,7 +389,7 @@ module Pgbus
       Instrumentation.instrument("pgbus.client.send_message", queue: target) do
         with_stale_connection_retry do
           ensure_queue(queue_name)
-          synchronized { @pgmq.produce(target, serialize(payload), headers: headers && serialize(headers), delay: delay) }
+          synchronized { exec_produce(target, serialize(payload), headers: headers && serialize(headers), delay: delay) }
         end
       end
     end
@@ -403,7 +436,7 @@ module Pgbus
       Instrumentation.instrument("pgbus.client.send_batch", queue: target, size: payloads.size) do
         with_stale_connection_retry do
           ensure_queue(queue_name)
-          synchronized { @pgmq.produce_batch(target, serialized, headers: serialized_headers, delay: delay) }
+          synchronized { exec_produce_batch(target, serialized, headers: serialized_headers, delay: delay) }
         end
       end
     end
@@ -424,7 +457,7 @@ module Pgbus
       guarded_read do
         Instrumentation.instrument("pgbus.client.read_batch", queue: full_name, qty: qty) do
           with_stale_connection_retry do
-            synchronized { with_read_timeout { @pgmq.read_batch(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
+            synchronized { with_read_timeout { exec_read_batch(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
           end
         end
       end
@@ -547,35 +580,45 @@ module Pgbus
 
     # Delete a message. Pass prefixed: false when queue_name is already
     # the full PGMQ queue name (e.g. from priority sub-queues or dashboard).
-    def delete_message(queue_name, msg_id, prefixed: true)
+    #
+    # job_completion: true marks the call as a job's completion (the executor
+    # and the completer pass it). Under config.completion_synchronous_commit =
+    # false only those commit with synchronous_commit off (issue #619); every
+    # other caller (the event consumer, whose replay reads the archive, and the
+    # dashboard) keeps the durable commit. The same holds for the three below.
+    def delete_message(queue_name, msg_id, prefixed: true, job_completion: false)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      async = job_completion && @async_completion_commit
       with_stale_connection_retry do
-        synchronized { @pgmq.delete(name, msg_id) }
+        synchronized { exec_delete(name, msg_id, async: async) }
       end
     end
 
     # Archive a message. Pass prefixed: false when queue_name is already
     # the full PGMQ queue name.
-    def archive_message(queue_name, msg_id, prefixed: true)
+    def archive_message(queue_name, msg_id, prefixed: true, job_completion: false)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      async = job_completion && @async_completion_commit
       with_stale_connection_retry do
-        synchronized { @pgmq.archive(name, msg_id) }
+        synchronized { exec_archive(name, msg_id, async: async) }
       end
     end
 
     # Batch archive — moves multiple messages to the archive table in one call.
-    def archive_batch(queue_name, msg_ids, prefixed: true)
+    def archive_batch(queue_name, msg_ids, prefixed: true, job_completion: false)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      async = job_completion && @async_completion_commit
       with_stale_connection_retry do
-        synchronized { @pgmq.archive_batch(name, msg_ids) }
+        synchronized { exec_archive_batch(name, msg_ids, async: async) }
       end
     end
 
     # Batch delete — permanently removes multiple messages in one call.
-    def delete_batch(queue_name, msg_ids, prefixed: true)
+    def delete_batch(queue_name, msg_ids, prefixed: true, job_completion: false)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      async = job_completion && @async_completion_commit
       with_stale_connection_retry do
-        synchronized { @pgmq.delete_batch(name, msg_ids) }
+        synchronized { exec_delete_batch(name, msg_ids, async: async) }
       end
     end
 
@@ -700,12 +743,23 @@ module Pgbus
       end
     end
 
-    def drop_queue(queue_name, prefixed: true)
+    # +before_drop+ runs only when the table itself is dropped (not for a
+    # case duplicate, whose table stays), decided once: the dashboard
+    # releases the dropped messages' uniqueness locks there.
+    def drop_queue(queue_name, prefixed: true, &before_drop)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      if case_duplicate?(name)
+        forget_case_duplicate(name)
+        return true
+      end
+
+      before_drop&.call
+
       result = with_stale_connection_retry do
         synchronized { @pgmq.drop_queue(name) }
       end
       @queues_created.delete(name)
+      forget_other_spellings(name)
       # Failed-event rows are keyed by the name the executor saw (physical or
       # logical); a recreated queue reuses msg_ids, so neither may survive.
       FailedEventRecorder.clear_queue!([name, name.delete_prefix("#{config.queue_prefix}_")].uniq)
@@ -1602,6 +1656,67 @@ module Pgbus
       duplicate_install_error?(error) || duplicate_install_error?(error.cause)
     end
 
+    # A mixed-case pgmq.meta row from before issue #633 whose lower-case
+    # spelling is registered too: both name one table, the lower-case row is
+    # the queue every current process uses. Dropping the duplicate (the
+    # dashboard lists both until pgbus:fold_queue_name_case runs, and the
+    # orphan stream sweep sees both) must not drop that table.
+    def case_duplicate?(name)
+      return false if name == name.downcase
+
+      synchronized { queue_registered?(name) && queue_registered?(name.downcase) }
+    end
+
+    # Removes only the duplicate's pgmq.meta row (its throttle and topic
+    # bindings cascade); the table and the lower-case queue stay. The shared
+    # table's insert trigger may carry the duplicate's name (an older process
+    # armed it last), so it is re-armed under the lower-case name with the
+    # throttle the lower-case row had (else the duplicate's): with the
+    # duplicate's throttle row gone, that trigger would NOTIFY no more.
+    def forget_case_duplicate(name)
+      canonical = name.downcase
+      with_stale_connection_retry do
+        synchronized do
+          throttle = notify_throttle_for(canonical, name)
+          @pgmq.with_connection do |conn|
+            conn.exec_params("DELETE FROM pgmq.meta WHERE queue_name = $1", [name])
+          end
+          @pgmq.enable_notify_insert(canonical, throttle_interval_ms: throttle) if throttle && !notify_trigger_current?(canonical, throttle)
+        end
+      end
+      @queues_created.delete(name)
+    end
+
+    # The NOTIFY throttle of the first of +names+ that has one, or nil (no
+    # throttle row, or a PGMQ without notify_insert_throttle).
+    def notify_throttle_for(*names)
+      @pgmq.with_connection do |conn|
+        next nil unless conn.exec("SELECT to_regclass('pgmq.notify_insert_throttle')").getvalue(0, 0)
+
+        rows = conn.exec_params(
+          "SELECT queue_name, throttle_interval_ms FROM pgmq.notify_insert_throttle WHERE queue_name = ANY($1::text[])",
+          ["{#{names.join(",")}}"]
+        ).to_a
+        row = names.lazy.filter_map { |n| rows.find { |r| r["queue_name"] == n } }.first
+        row && Integer(row["throttle_interval_ms"])
+      end
+    end
+
+    # pgmq.drop_queue removes the table and only its own pgmq.meta row; a
+    # mixed-case spelling from before issue #633 would be left naming a table
+    # that is gone. Idempotent, so a stale pooled socket retries it.
+    def forget_other_spellings(name)
+      with_stale_connection_retry do
+        synchronized do
+          @pgmq.with_connection do |conn|
+            conn.exec_params(
+              "DELETE FROM pgmq.meta WHERE lower(queue_name) = lower($1) AND queue_name <> $1", [name]
+            )
+          end
+        end
+      end
+    end
+
     # Whether pgmq.meta records the queue — the authoritative "create
     # committed" signal (pgmq.create writes it atomically with the
     # tables). The pooled checkout is a sequential sibling of the failed
@@ -1618,7 +1733,13 @@ module Pgbus
     end
 
     # Check whether the NOTIFY trigger already exists on this queue with the
-    # expected throttle interval. When it does, we can skip the destructive
+    # expected throttle interval, armed under this spelling of its name. PGMQ
+    # 1.13.1's trigger NOTIFYs on the channel of the name it was armed with
+    # (its argument), and every spelling of a name shares the queue's table
+    # (issue #633): a process from before #633 that re-arms it under a
+    # mixed-case name moves its NOTIFYs to a channel no listener uses. A
+    # trigger without an argument (older PGMQ) NOTIFYs on the table's name,
+    # which is the lower-case spelling. When it does, we can skip the destructive
     # DROP TRIGGER + CREATE TRIGGER cycle that causes deadlocks when multiple
     # forked processes race during bootstrap.
     #
@@ -1640,6 +1761,7 @@ module Pgbus
           WHERE n.nspname = 'pgmq'
             AND c.relname = pgmq.format_table_name($1, 'q')
             AND t.tgname = '#{NOTIFY_TRIGGER_NAME}'
+            AND (t.tgnargs = 0 OR t.tgargs = convert_to($1, 'UTF8') || '\\000'::bytea)
             AND EXISTS (
               SELECT 1 FROM pgmq.notify_insert_throttle
               WHERE queue_name = $1
@@ -2027,20 +2149,30 @@ module Pgbus
     # unchanged, as does :options mode (where forward_connection_variables
     # already baked the GUCs into `options`). See issue #332.
     def wrap_session_gucs(conn_opts)
-      return conn_opts unless config.connection_guc_mode == :session
-      return conn_opts unless conn_opts.is_a?(Hash)
-
-      variables = conn_opts[:variables]
-      return conn_opts if variables.nil? || variables.empty?
+      variables = session_gucs(conn_opts)
+      return conn_opts unless variables
 
       pg_opts = conn_opts.except(:variables)
       lambda do
         conn = PG.connect(pg_opts)
-        variables.each { |name, value| conn.exec("SET #{name} = '#{value}'") }
+        apply_session_gucs(conn, variables)
         conn
       end
     end
 
+    # The database.yml variables :session GUC mode applies with SET after
+    # connect, or nil when nothing is applied that way.
+    def session_gucs(conn_opts)
+      return unless config.connection_guc_mode == :session && conn_opts.is_a?(Hash)
+
+      variables = conn_opts[:variables]
+      variables unless variables.nil? || variables.empty?
+    end
+
+    def apply_session_gucs(conn, variables)
+      variables.each { |name, value| conn.exec("SET #{name} = '#{value}'") }
+    end
+
     def apply_connection_bounds(conn_opts)
       timeout = config.read_timeout
       return conn_opts unless timeout&.positive?
@@ -2182,6 +2314,10 @@ module Pgbus
     end
 
     def stale_connection_error?(error)
+      # A completion whose re-run came back short may have committed: never
+      # re-run it again (NativeStatements#run_native).
+      return false if NativeStatements.completion_lost?(error)
+
       # Substring match against any stale-connection pattern. Deliberately a
       # single Regexp rather than `STALE_CONNECTION_PATTERNS.any? { |p| msg.include?(p) }`:
       # Style/ArrayIntersect misfires on that shape and "corrects" it to

data/lib/pgbus/concurrency/blocked_execution.rb

@@ -1,10 +1,17 @@
 # frozen_string_literal: true
 
 require "json"
+require "time"
 
 module Pgbus
   module Concurrency
     module BlockedExecution
+      # Headers a parked job is to be sent with. pgbus_blocked_executions has
+      # no headers column, so a dashboard retry parked behind its limit
+      # (Web::DataSource::ResendSlot) carries them in its payload under this
+      # key; promote_next takes them out and sends them as headers.
+      HEADERS_KEY = "pgbus_park_headers"
+
       class << self
         # Insert a blocked execution for a job that hit the concurrency limit.
         #
@@ -63,8 +70,10 @@ module Pgbus
 
             # Carry the enqueuer's priority through: under priority routing it
             # picks the _pN sub-queue, not just the release order (issue #423).
+            headers = released[:payload].is_a?(Hash) ? released[:payload].delete(HEADERS_KEY) : nil
             msg_id = client.send_message(released[:queue_name], released[:payload],
-                                         delay: actual_delay, priority: released[:priority])
+                                         delay: actual_delay, priority: released[:priority],
+                                         **(headers ? { headers: headers } : {}))
           end
 
           return false unless released && msg_id

data/lib/pgbus/configuration.rb

@@ -24,6 +24,65 @@ module Pgbus
     # one. 0 (default) is off. Overridable per capsule / event_consumers entry
     # (`read_ahead:`); prefetch_limit still caps everything claimed.
     attr_reader :read_ahead
+    # completion_batching (issue #606, opt-in, default false): a worker archives
+    # the jobs that finish close together with one archive_batch per flush
+    # instead of one pgmq.archive per job (Process::Completer). No timer: a lone
+    # job is archived at once, and whatever finishes during a flush goes in the
+    # next. completion_batch_size (default 100) caps the ids per statement.
+    # Threads only: validate! rejects it next to execution_mode :async, global
+    # or per capsule. The crash window widens from one job to one flush: on a
+    # hard crash (SIGKILL, OOM) the jobs waiting in the current flush re-run
+    # after visibility_timeout, and a flush that committed just before the
+    # crash leaves its jobs completed but unsignalled (uniqueness lock, batch,
+    # concurrency slot) until the reapers and sweeps heal them. Worker shutdown
+    # waits for a flush in flight at most immediate_shutdown_grace (QUIT) or
+    # the rest of the 5 s pool residual (TERM); a flush wedged past that is
+    # abandoned, and its jobs fail (Completer::Stopped) and re-run after
+    # visibility_timeout, or stay unsignalled if it had committed.
+    attr_accessor :completion_batching, :completion_batch_size
+    # completed_jobs (issue #607): how a successful job's message leaves its
+    # queue. :archive (default) is pgmq.archive, DELETE + INSERT into the
+    # queue's a_ table, which archive_retention compacts. :delete is
+    # pgmq.delete, the DELETE alone: no a_ rows, no a_ index upkeep, nothing for
+    # the compaction to trim, and no completed-job history in the archive.
+    # Equally reliable: either way the message is gone once the statement
+    # commits. Through the completer (completion_batching) it is delete_batch.
+    # ActiveJob only: EventBus handler queues keep archiving (event replay
+    # reads their archive), and failures and dead letters are unchanged.
+    # Batch::Sweep then cannot tell a deleted batch job from one whose message
+    # vanished another way (a purged queue): both count as completed, silently.
+    attr_reader :completed_jobs
+    # completion_synchronous_commit (issue #619, opt-in): false commits a job's
+    # completion (the executor's archive or delete, the completer's
+    # archive_batch or delete_batch) with `SET LOCAL synchronous_commit TO off`,
+    # in one libpq pipeline: BEGIN, the SET LOCAL, the prepared statement and
+    # COMMIT go out with one sync, so the job stops waiting for its WAL flush.
+    # A completion is then acknowledged before its WAL record reaches disk: a
+    # Postgres server crash (not a client crash, not a planned switchover), or a
+    # failover to a standby that has not yet received that WAL (it also stops
+    # waiting for a synchronous standby), within about three times
+    # wal_writer_delay can lose completions committed in that window, and each
+    # lost completion redelivers its job after visibility_timeout. Sends, retries, dead-letter moves, the EventBus
+    # consumer's archive and dashboard actions keep full durability, and a
+    # completion inside a caller's transaction is left to that transaction.
+    # Native statements on pgbus's own connections only: with
+    # native_statements = false, or on ActiveRecord's shared connection (a
+    # connection_params Proc), it has no effect (pgbus doctor warns). Needs
+    # libpq 14+ and pg 1.5+ (pipeline mode), checked by validate!. true
+    # (default) commits as before.
+    attr_reader :completion_synchronous_commit
+    # quiet_active_job_logs (issue #618): true makes each worker process
+    # unsubscribe ActiveJob's LogSubscriber from perform_start.active_job and
+    # perform.active_job (Pgbus::ActiveJob::QuietLogs), so the per-job
+    # "Performing …" / "Performed …" info lines stop, and with them ActiveJob's
+    # "Error performing …" line (the same perform listener writes it): an
+    # exception that escapes ActiveJob's handlers is still logged by
+    # ErrorReporter and recorded in pgbus_failed_events, one retry_on catches
+    # still logs ActiveJob's "Retrying …" line, and a job a before_perform
+    # callback halted logs nothing.
+    # Enqueued, Retrying, Stopped retrying and Discarded still log, and the web
+    # process is untouched. Without ActiveJob it does nothing.
+    attr_reader :quiet_active_job_logs
     # visibility_heartbeat / visibility_heartbeat_interval: while a job runs,
     # its message's visibility timeout is re-armed every interval seconds
     # (default: a third of visibility_timeout), so a job that outlives the
@@ -214,6 +273,22 @@ module Pgbus
     # See issue #332.
     attr_reader :connection_guc_mode
 
+    # Run Client's read_batch, archive, delete, send and their batch forms as
+    # pgbus-owned static SQL in named prepared statements on the pooled
+    # connection, instead of pgmq's plpgsql functions, whose dynamic EXECUTE
+    # Postgres plans again on every call (issue #605). Default true. A queue
+    # whose table lacks a column the statements use (a pgmq older than the
+    # `headers` column) stays on the pgmq functions by itself.
+    #
+    # Behind a transaction-mode PgBouncer, prepared statements need PgBouncer
+    # >= 1.21 with `max_prepared_statements` set. Without it, a process that
+    # sees its statements disappear switches to sending the same SQL
+    # unprepared and logs a warning once, and connections built from a
+    # database.yml with `prepared_statements: false` send it unprepared from
+    # the start. false uses the pgmq functions everywhere: the full escape
+    # hatch.
+    attr_reader :native_statements
+
     # Zombie message detection — logs a warning when a message is redelivered
     # (read_ct > 1) without any prior failure recorded in pgbus_failed_events.
     attr_accessor :zombie_detection
@@ -320,6 +395,11 @@ module Pgbus
 
       @prefetch_limit = nil
       @read_ahead = 0
+      @completion_batching = false
+      @completion_batch_size = 100
+      @completed_jobs = :archive
+      @completion_synchronous_commit = true
+      @quiet_active_job_logs = false
       @execution_mode = :threads
 
       @max_jobs_per_worker = nil
@@ -404,6 +484,7 @@ module Pgbus
       @connects_to = nil
       @require_primary = false
       @connection_guc_mode = :options
+      @native_statements = true
 
       @mcp_write_tools = false
 
@@ -516,12 +597,13 @@ module Pgbus
       @streams_idle_timeout = 3_600 # 1 hour
       # 250ms — this value plays two roles: (1) the TCP keepalive
       # interval for the streamer's PG LISTEN connection, and (2) the
-      # upper bound on how long Dispatcher#handle_connect waits for
-      # the Listener to acknowledge a synchronous ensure_listening
-      # call. 5s was unbounded enough to drop messages on a
-      # realistic subscribe burst; 250ms keeps the connect-path race
-      # window tight while still leaving headroom over a typical
-      # PG keepalive interval.
+      # budget (plus 1s) a stream's first connect waits for a stuck local
+      # Listener to acknowledge its ensure_listening (with
+      # streams_listen_scope :master, HubClient waits this cycle plus 2s
+      # before failing over). A live listener
+      # is woken through a pipe and acks within a thread hop, and a
+      # connect on a stream that already has a subscriber does not
+      # wait at all (issue #630), so connects no longer pay this cycle.
       @streams_listen_health_check_ms = 250
       @streams_write_deadline_ms = 5_000
       # Deadline (ms) for a socket write made INSIDE the dispatcher's
@@ -797,6 +879,39 @@ module Pgbus
       @connection_guc_mode = mode
     end
 
+    VALID_COMPLETED_JOBS = %i[archive delete].freeze
+
+    def completed_jobs=(value)
+      mode = value.to_sym if value.is_a?(String) || value.is_a?(Symbol)
+      unless VALID_COMPLETED_JOBS.include?(mode)
+        raise Pgbus::ConfigurationError, "completed_jobs must be :archive or :delete (got #{value.inspect})"
+      end
+
+      @completed_jobs = mode
+    end
+
+    def completion_synchronous_commit=(value)
+      unless [true, false].include?(value)
+        raise Pgbus::ConfigurationError, "completion_synchronous_commit must be true or false (got #{value.inspect})"
+      end
+
+      @completion_synchronous_commit = value
+    end
+
+    def quiet_active_job_logs=(value)
+      unless [true, false].include?(value)
+        raise Pgbus::ConfigurationError, "quiet_active_job_logs must be true or false (got #{value.inspect})"
+      end
+
+      @quiet_active_job_logs = value
+    end
+
+    def native_statements=(value)
+      raise Pgbus::ConfigurationError, "native_statements must be true or false" unless [true, false].include?(value)
+
+      @native_statements = value
+    end
+
     VALID_PGMQ_SCHEMA_MODES = %i[auto extension embedded].freeze
 
     def pgmq_schema_mode=(mode)
@@ -986,6 +1101,8 @@ module Pgbus
       end
 
       validate_read_ahead!
+      validate_completion_batching!
+      validate_completion_synchronous_commit!
       validate_capsule_processes!
 
       if priority_levels && !(priority_levels.is_a?(Integer) && priority_levels >= 1 && priority_levels <= 10)
@@ -1756,6 +1873,41 @@ module Pgbus
 
     private
 
+    # The completer's job threads block on their flush, which would stall an
+    # Async reactor, so batching is threads-only (issue #606). Every capsule
+    # resolves its mode through execution_mode_for, as the supervisor does.
+    def validate_completion_batching!
+      unless [true, false].include?(completion_batching)
+        raise Pgbus::ConfigurationError, "completion_batching must be true or false (got #{completion_batching.inspect})"
+      end
+      unless completion_batch_size.is_a?(Integer) && completion_batch_size.positive?
+        raise Pgbus::ConfigurationError,
+              "completion_batch_size must be a positive Integer (got #{completion_batch_size.inspect})"
+      end
+      return unless completion_batching
+
+      modes = [ExecutionPools.normalize_mode(execution_mode)] + Array(workers).map { |w| execution_mode_for(w) }
+      return unless modes.include?(:async)
+
+      raise Pgbus::ConfigurationError,
+            "completion_batching supports execution_mode :threads only; turn it off for a global or capsule " \
+            "execution_mode :async"
+    end
+
+    # The asynchronous completion commit runs in libpq pipeline mode (issue
+    # #619): libpq 14+ and a pg gem that wraps it (1.5+). Checked here, once,
+    # rather than on every completion.
+    def validate_completion_synchronous_commit!
+      return if completion_synchronous_commit
+
+      require "pg" unless defined?(::PG::Connection)
+      return if ::PG::Connection.method_defined?(:enter_pipeline_mode) && ::PG.library_version >= 140_000
+
+      raise Pgbus::ConfigurationError,
+            "completion_synchronous_commit = false needs libpq 14+ and pg 1.5+ (pipeline mode); " \
+            "this process has libpq #{::PG.library_version} and pg #{::PG::VERSION}"
+    end
+
     # Shared resolver for a dedicated connection (streamer LISTEN / worker
     # NotifyListener) that can override the base `connection_options` to bypass a
     # pooler. Precedence: a full `url` wins outright; otherwise `host`/`port`

data/lib/pgbus/dead_letter_header.rb

@@ -20,6 +20,9 @@ module Pgbus
     ORIGINAL_KEY = "pgbus_original_headers"
     VERSION = 1
     REASON_MAX_RETRIES = "max_retries_exceeded"
+    # A :while_executing duplicate skipped on every read while another
+    # message held its key, until read_ct passed max_retries (issue #629).
+    REASON_UNIQUENESS_CONFLICT = "uniqueness_conflict"
     # The release that started writing the block: rows without it predate it.
     SINCE = "0.18.0"
     MESSAGE_LIMIT = 1_000

data/lib/pgbus/doctor.rb

@@ -5,7 +5,7 @@ require "pgbus/mcp/health_analyzer"
 
 module Pgbus
   # Preflight diagnostics for a pgbus deployment — the single command that
-  # answers "is this environment healthy enough to run?". Runs twelve checks and
+  # answers "is this environment healthy enough to run?". Runs fourteen checks and
   # returns a machine-readable result plus a human report, so `pgbus doctor`
   # and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
   # 1 on any failure).
@@ -36,13 +36,15 @@ module Pgbus
       "Database" => :check_database,
       "PGMQ schema" => :check_pgmq_schema,
       "Queues" => :check_queues,
+      "Queue name case" => :check_queue_name_case,
       "LISTEN/NOTIFY" => :check_notify,
       "Process liveness" => :check_processes,
       "GlobalID allowlist" => :check_allowed_global_id_models,
       "Broadcast queue" => :check_broadcast_queue,
       "Primary affinity" => :check_primary,
       "Dedicated connections" => :check_dedicated_connections,
-      "Connection budget" => :check_connection_budget
+      "Connection budget" => :check_connection_budget,
+      "Completion commit" => :check_completion_commit
     }.freeze
 
     # Process liveness reads the pgbus_processes table (via HealthAnalyzer), so
@@ -128,6 +130,7 @@ module Pgbus
         resolved_pool_size: safe { @config.resolved_pool_size },
         listen_notify: @config.listen_notify,
         roles: @config.roles || "all",
+        completion_commit: @config.completion_synchronous_commit ? "sync" : "async",
         capsules: capsule_summary,
         database_url: redacted_database_url,
         connection_params: redacted_connection_params
@@ -252,6 +255,30 @@ module Pgbus
       Check.new(name: "Queues", status: :fail, detail: "#{e.class}: #{e.message}")
     end
 
+    # How many mixed-case queue names the queue-name case check lists.
+    CASE_NAMES_SHOWN = 5
+    private_constant :CASE_NAMES_SHOWN
+
+    # 4b. Queue-name case (issue #633). PGMQ spells every queue table in lower
+    # case, and pgbus now spells every queue name that way; a pgmq.meta row an
+    # older pgbus created as "pgbus_Orders" shares pgmq.q_pgbus_orders with the
+    # current spelling. Its messages are reachable, so this warns rather than
+    # fails: the dashboard lists the queue twice until the fold migration runs.
+    def check_queue_name_case
+      names = @client.mixed_case_queue_names
+      return Check.new(name: "Queue name case", status: :ok, detail: "every queue is spelled in lower case") if names.empty?
+
+      shown = names.first(CASE_NAMES_SHOWN).join(", ")
+      shown += ", …" if names.size > CASE_NAMES_SHOWN
+      Check.new(name: "Queue name case", status: :warn,
+                detail: "#{names.size} queue(s) in pgmq.meta spelled in mixed case (#{shown}); their tables are " \
+                        "lower case and pgbus now uses that spelling — once every process runs this pgbus, run " \
+                        "`rails generate pgbus:fold_queue_name_case` and migrate " \
+                        "(`rake pgbus:queues:fold_case` once that migration has run)")
+    rescue StandardError => e
+      Check.new(name: "Queue name case", status: :warn, detail: "could not read pgmq.meta (#{e.class}: #{e.message})")
+    end
+
     # 5. LISTEN/NOTIFY — when configured on, every configured queue should carry
     # the insert-NOTIFY trigger. A missing trigger is a warning, not a failure:
     # pgbus still works (it falls back to polling), it just loses instant wakeup.
@@ -395,6 +422,32 @@ module Pgbus
       Check.new(name: "Dedicated connections", status: :fail, detail: "#{e.class}: #{e.message}")
     end
 
+    # config.completion_synchronous_commit (issue #619): informational, except
+    # that false does nothing with native_statements off or on ActiveRecord's
+    # shared connection, which warns.
+    def check_completion_commit
+      return Check.new(name: "Completion commit", status: :ok, detail: "synchronous (default)") if
+        @config.completion_synchronous_commit
+
+      unless @config.native_statements
+        return Check.new(name: "Completion commit", status: :warn,
+                         detail: "completion_synchronous_commit = false has no effect: completions run through " \
+                                 "pgmq's functions (native_statements = false)")
+      end
+
+      if @client.respond_to?(:shared_connection?) && @client.shared_connection?
+        return Check.new(name: "Completion commit", status: :warn,
+                         detail: "completion_synchronous_commit = false has no effect: pgbus shares ActiveRecord's " \
+                                 "connection (connection_params Proc); give pgbus its own connections to use it")
+      end
+
+      Check.new(name: "Completion commit", status: :ok,
+                detail: "asynchronous: completions commit with synchronous_commit off; a Postgres server crash can " \
+                        "lose the last completions, whose jobs run again after visibility_timeout")
+    rescue StandardError => e
+      Check.new(name: "Completion commit", status: :warn, detail: "#{e.class}: #{e.message}")
+    end
+
     # How many direct LISTEN connections this config pins at steady state
     # (issue #381) — informational (always :ok) so operators can do capacity
     # math on the pooler's direct-connection budget from the doctor output.