pgbus 0.22.0 → 0.23.1

data/lib/pgbus/client/async_commit.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Client
+    # Asynchronous completion commit (issue #619,
+    # `config.completion_synchronous_commit = false`). A completion statement
+    # (archive, delete and their batches) runs in its own transaction with
+    # synchronous_commit off, so the job is acknowledged without waiting for
+    # its WAL flush. The four queries go out in one libpq pipeline with one
+    # sync, so it is still one round trip (pg writes each query to the socket
+    # as it is sent, but waits for a reply only once, after the sync):
+    #
+    #   BEGIN
+    #   SET LOCAL synchronous_commit TO off
+    #   <the prepared completion statement>   <- the result the caller gets
+    #   COMMIT
+    #
+    # SET LOCAL ends with the transaction, so the pooled connection never
+    # keeps the setting, and the statement stays prepared (no planning cost
+    # back, issue #605). Only a job's completion asks for it (the executor and
+    # the completer pass job_completion: true; the event consumer and the
+    # dashboard keep the durable commit), and NativeStatements#exec_native
+    # calls it only outside a caller's transaction: a completion inside one is
+    # the caller's to commit.
+    #
+    # A failed query aborts the rest of the pipeline and leaves the explicit
+    # BEGIN open in a failed state: the connection leaves pipeline mode, the
+    # transaction is rolled back, and the error is raised for run_native's
+    # re-run and short-row checks. A statement not yet prepared on this session
+    # runs through prepare_and_exec outside the pipeline, committed
+    # synchronously, once; the next call pipelines again.
+    module AsyncCommit
+      ASYNC_COMMIT_BEGIN = "BEGIN"
+      ASYNC_COMMIT_OFF = "SET LOCAL synchronous_commit TO off"
+      ASYNC_COMMIT_COMMIT = "COMMIT"
+      NO_PARAMS = [].freeze
+
+      private
+
+      def exec_async_commit(conn, statement, params, prepared)
+        results = pipeline_async_commit(conn, statement, params, prepared)
+        failed = results.find { |result| result.result_status == PG::PGRES_FATAL_ERROR }
+        unless failed
+          NativeStatements.note_executed if prepared
+          return results[2]
+        end
+
+        rollback_async_commit(conn)
+        begin
+          failed.check
+        rescue PG::InvalidSqlStatementName
+          raise unless prepared
+
+          prepare_and_exec(conn, statement, params)
+        end
+      end
+
+      # Sends the four queries and one sync, and reads every result up to the
+      # sync: each query answers its result and then nil.
+      def pipeline_async_commit(conn, statement, params, prepared)
+        conn.enter_pipeline_mode
+        begin
+          conn.send_query_params(ASYNC_COMMIT_BEGIN, NO_PARAMS)
+          conn.send_query_params(ASYNC_COMMIT_OFF, NO_PARAMS)
+          if prepared
+            conn.send_query_prepared(statement.name, params)
+          else
+            conn.send_query_params(statement.sql, params)
+          end
+          conn.send_query_params(ASYNC_COMMIT_COMMIT, NO_PARAMS)
+          conn.pipeline_sync
+          read_pipeline(conn)
+        ensure
+          leave_pipeline(conn)
+        end
+      end
+
+      def read_pipeline(conn)
+        results = []
+        ended = false
+        loop do
+          result = conn.get_result
+          if result.nil?
+            # One nil closes each query's results; two in a row means libpq
+            # has nothing more, so the sync will not come. That session's
+            # state cannot be trusted: pgmq-ruby re-runs on a fresh one.
+            if ended
+              discard_session(conn, "A completion pipeline ended before its sync")
+              raise PG::ConnectionBad, "pipeline ended before its sync"
+            end
+
+            ended = true
+            next
+          end
+          ended = false
+          break if result.result_status == PG::PGRES_PIPELINE_SYNC
+
+          results << result
+        end
+        results
+      end
+
+      # A connection back in the pool in pipeline mode would break the next
+      # caller. A lost one is left as it is: pgmq-ruby resets a bad connection
+      # at its next checkout, and a reset leaves pipeline mode. One that cannot
+      # leave (results left unread by an interrupted read) is reset here.
+      def leave_pipeline(conn)
+        return if conn.pipeline_status == PG::PQ_PIPELINE_OFF || conn.status == PG::CONNECTION_BAD
+
+        conn.exit_pipeline_mode
+      rescue PG::Error => e
+        discard_session(conn, "Could not leave pipeline mode after a completion (#{e.message})")
+      end
+
+      # Resets a session whose pipeline state cannot be trusted. A reset that
+      # fails (the server is gone) is logged and left there, so the error the
+      # caller is already raising is the one that surfaces; pgmq-ruby resets a
+      # bad connection at its next checkout.
+      # A reset reconnects with the same conninfo (the read bounds ride in it)
+      # but does not re-run the connect lambda, so the :session-mode SETs are
+      # applied again here.
+      def discard_session(conn, reason)
+        Pgbus.logger.warn { "[Pgbus::Client] #{reason}, resetting the connection" }
+        begin
+          conn.reset
+        rescue PG::Error => e
+          Pgbus.logger.warn { "[Pgbus::Client] Resetting the connection failed: #{e.message}" }
+          return
+        end
+        reapply_session_gucs(conn)
+      end
+
+      def reapply_session_gucs(conn)
+        apply_session_gucs(conn, @session_gucs) if @session_gucs
+      rescue PG::Error => e
+        Pgbus.logger.warn do
+          "[Pgbus::Client] Re-applying the session's database.yml variables after the reset failed: #{e.message}"
+        end
+      end
+
+      # The failed BEGIN block (INERROR) is still open after the sync. A failed
+      # COMMIT has already ended it (IDLE); a lost connection reports UNKNOWN
+      # and has nothing to roll back.
+      def rollback_async_commit(conn)
+        conn.exec("ROLLBACK") if [PG::PQTRANS_INERROR, PG::PQTRANS_INTRANS].include?(conn.transaction_status)
+      end
+    end
+  end
+end

data/lib/pgbus/client/native_statements.rb

@@ -299,8 +299,11 @@ module Pgbus
       # call otherwise. Return values match pgmq-ruby's: Messages with String
       # fields, String ids, true/false for one archive/delete.
 
+      # The send statements bind the delay as $3::integer. is_a?(Integer) also
+      # takes a whole-second ActiveSupport::Duration, which binds as its seconds;
+      # a fractional delay or a Time goes through pgmq-ruby.
       def exec_produce(name, message, headers:, delay:)
-        statements = delay.is_a?(Numeric) && native_statements_for(name)
+        statements = delay.is_a?(Integer) && native_statements_for(name)
         return @pgmq.produce(name, message, headers: headers, delay: delay) unless statements
 
         run_native(statements[:send], [message, headers, delay])[0]["msg_id"]
@@ -308,7 +311,7 @@ module Pgbus
 
       def exec_produce_batch(name, messages, headers:, delay:)
         statements = messages.any? && (headers.nil? || headers.length == messages.length) &&
-                     delay.is_a?(Numeric) && native_statements_for(name)
+                     delay.is_a?(Integer) && native_statements_for(name)
         return @pgmq.produce_batch(name, messages, headers: headers, delay: delay) unless statements
 
         encoder = PG::TextEncoder::Array.new
@@ -325,32 +328,37 @@ module Pgbus
         result.map { |row| PGMQ::Message.new(row) }
       end
 
-      def exec_archive(name, msg_id)
+      # async: commit with synchronous_commit off (AsyncCommit, issue #619);
+      # true only for a job's completion under completion_synchronous_commit
+      # = false.
+      def exec_archive(name, msg_id, async: false)
         statements = native_statements_for(name)
         return @pgmq.archive(name, msg_id) unless statements
 
-        run_native(statements[:archive], [msg_id], rows: 1).ntuples.positive?
+        run_native(statements[:archive], [msg_id], rows: 1, async: async).ntuples.positive?
       end
 
-      def exec_archive_batch(name, msg_ids)
+      def exec_archive_batch(name, msg_ids, async: false)
         statements = msg_ids.any? && native_statements_for(name)
         return @pgmq.archive_batch(name, msg_ids) unless statements
 
-        run_native(statements[:archive_batch], [PG::TextEncoder::Array.new.encode(msg_ids)], rows: msg_ids).map { |row| row["msg_id"] }
+        params = [PG::TextEncoder::Array.new.encode(msg_ids)]
+        run_native(statements[:archive_batch], params, rows: msg_ids, async: async).map { |row| row["msg_id"] }
       end
 
-      def exec_delete(name, msg_id)
+      def exec_delete(name, msg_id, async: false)
         statements = native_statements_for(name)
         return @pgmq.delete(name, msg_id) unless statements
 
-        run_native(statements[:delete], [msg_id], rows: 1).ntuples.positive?
+        run_native(statements[:delete], [msg_id], rows: 1, async: async).ntuples.positive?
       end
 
-      def exec_delete_batch(name, msg_ids)
+      def exec_delete_batch(name, msg_ids, async: false)
         statements = msg_ids.any? && native_statements_for(name)
         return @pgmq.delete_batch(name, msg_ids) unless statements
 
-        run_native(statements[:delete_batch], [PG::TextEncoder::Array.new.encode(msg_ids)], rows: msg_ids).map { |row| row["msg_id"] }
+        params = [PG::TextEncoder::Array.new.encode(msg_ids)]
+        run_native(statements[:delete_batch], params, rows: msg_ids, async: async).map { |row| row["msg_id"] }
       end
 
       # rows: the rows an archive or delete returns when every id was still
@@ -366,12 +374,12 @@ module Pgbus
       # id another worker had already taken also makes a re-run come back
       # short; every waiter of that batch then claims :ambiguous, the side the
       # executor already errs on.
-      def run_native(statement, params, rows: nil)
+      def run_native(statement, params, rows: nil, async: false)
         attempt = 0
         lost = nil
         @pgmq.with_connection do |conn|
           attempt += 1
-          result = exec_native(conn, statement, params)
+          result = exec_native(conn, statement, params, async)
           if rows && attempt > 1 && result.ntuples < (rows.is_a?(Array) ? rows.uniq.size : rows)
             raise PGMQ::Errors::ConnectionError, "#{COMPLETION_NOT_RESENT}: #{lost}"
           end
@@ -383,10 +391,14 @@ module Pgbus
         end
       end
 
-      def exec_native(conn, statement, params)
-        if @native_unprepared || NativeStatements.unprepared? || inside_caller_transaction?(conn)
-          return conn.exec_params(statement.sql, params)
-        end
+      # async (a job's completion under completion_synchronous_commit = false)
+      # goes through AsyncCommit, unless it runs inside a caller's transaction.
+      def exec_native(conn, statement, params, async)
+        return conn.exec_params(statement.sql, params) if inside_caller_transaction?(conn)
+
+        unprepared = @native_unprepared || NativeStatements.unprepared?
+        return exec_async_commit(conn, statement, params, !unprepared) if async
+        return conn.exec_params(statement.sql, params) if unprepared
 
         begin
           result = conn.exec_prepared(statement.name, params)

data/lib/pgbus/client.rb

@@ -18,6 +18,7 @@ module Pgbus
     include EnsureStreamQueue
     include NotifyStream
     include NativeStatements
+    include AsyncCommit
 
     attr_reader :pgmq, :config, :connection_health
 
@@ -29,6 +30,12 @@ module Pgbus
     PGMQ_META_CHECK_SQL = "SELECT 1 FROM pg_tables WHERE schemaname = 'pgmq' AND tablename = 'meta' LIMIT 1"
     private_constant :PGMQ_META_CHECK_SQL
 
+    # pgbus's own queues only ($1: the lower-case "<queue_prefix>_"): another
+    # PGMQ user of the same schema keeps its spelling.
+    MIXED_CASE_QUEUES_SQL = "SELECT queue_name FROM pgmq.meta WHERE queue_name <> lower(queue_name) " \
+                            "AND left(lower(queue_name), length($1)) = $1 ORDER BY queue_name"
+    private_constant :MIXED_CASE_QUEUES_SQL
+
     PGMQ_INSTALL_SAVEPOINT = "pgbus_pgmq_install"
     private_constant :PGMQ_INSTALL_SAVEPOINT
 
@@ -130,7 +137,11 @@ module Pgbus
         # Both raise clean PG errors — no Ruby Timeout, no Thread#raise. Only
         # safe on this dedicated-connection branch — never on the shared-AR Proc
         # path, where statement_timeout would leak into application queries.
-        conn_opts = wrap_session_gucs(apply_connection_bounds(conn_opts))
+        conn_opts = apply_connection_bounds(conn_opts)
+        # What the connect lambda SETs per session (:session GUC mode), kept so
+        # a session reset after a failed completion pipeline gets it again.
+        @session_gucs = session_gucs(conn_opts)
+        conn_opts = wrap_session_gucs(conn_opts)
         @pgmq = PGMQ::Client.new(conn_opts, pool_size: config.resolved_pool_size, pool_timeout: config.pool_timeout)
         @pgmq_mutex = nil
         # Dedicated streams pool (issue #315): isolates the durable-stream
@@ -174,6 +185,12 @@ module Pgbus
       # Per physical queue: its native statements, or false (issue #605).
       @native_statements = Concurrent::Map.new
       @native_unprepared = config.native_statements && ar_disables_prepared_statements?
+      # Job completions commit with synchronous_commit off (issue #619).
+      # Native statements only: the pgmq functions are never wrapped. Never on
+      # ActiveRecord's own connection (the Proc path): pgbus does not pipeline
+      # on, or reset, a connection it does not own.
+      @async_completion_commit = config.native_statements && !config.completion_synchronous_commit &&
+                                 !@shared_connection
       # Guards the one-time build of the publisher autoscale trigger (issue #323).
       # NOT @pgmq_mutex — that is nil on the dedicated path (the only path the
       # trigger exists on), so it wouldn't serialize concurrent first-publishers.
@@ -303,6 +320,18 @@ module Pgbus
       end
     end
 
+    # pgmq.meta rows spelled with an upper-case letter (issue #633). PGMQ
+    # lower-cases table names, so each shares its table with the lower-case
+    # spelling pgbus now uses; `rails generate pgbus:fold_queue_name_case`
+    # renames them. Pgbus::Doctor reports them.
+    def mixed_case_queue_names
+      synchronized do
+        @pgmq.with_connection do |conn|
+          conn.exec_params(MIXED_CASE_QUEUES_SQL, ["#{config.queue_prefix.to_s.downcase}_"]).column_values(0)
+        end
+      end
+    end
+
     # The most recently recorded installed PGMQ schema version string (e.g.
     # "1.5.0"), read from the pgbus_pgmq_schema_versions tracking table. Returns
     # nil when nothing is tracked yet or the table does not exist — the same
@@ -551,35 +580,45 @@ module Pgbus
 
     # Delete a message. Pass prefixed: false when queue_name is already
     # the full PGMQ queue name (e.g. from priority sub-queues or dashboard).
-    def delete_message(queue_name, msg_id, prefixed: true)
+    #
+    # job_completion: true marks the call as a job's completion (the executor
+    # and the completer pass it). Under config.completion_synchronous_commit =
+    # false only those commit with synchronous_commit off (issue #619); every
+    # other caller (the event consumer, whose replay reads the archive, and the
+    # dashboard) keeps the durable commit. The same holds for the three below.
+    def delete_message(queue_name, msg_id, prefixed: true, job_completion: false)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      async = job_completion && @async_completion_commit
       with_stale_connection_retry do
-        synchronized { exec_delete(name, msg_id) }
+        synchronized { exec_delete(name, msg_id, async: async) }
       end
     end
 
     # Archive a message. Pass prefixed: false when queue_name is already
     # the full PGMQ queue name.
-    def archive_message(queue_name, msg_id, prefixed: true)
+    def archive_message(queue_name, msg_id, prefixed: true, job_completion: false)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      async = job_completion && @async_completion_commit
       with_stale_connection_retry do
-        synchronized { exec_archive(name, msg_id) }
+        synchronized { exec_archive(name, msg_id, async: async) }
       end
     end
 
     # Batch archive — moves multiple messages to the archive table in one call.
-    def archive_batch(queue_name, msg_ids, prefixed: true)
+    def archive_batch(queue_name, msg_ids, prefixed: true, job_completion: false)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      async = job_completion && @async_completion_commit
       with_stale_connection_retry do
-        synchronized { exec_archive_batch(name, msg_ids) }
+        synchronized { exec_archive_batch(name, msg_ids, async: async) }
       end
     end
 
     # Batch delete — permanently removes multiple messages in one call.
-    def delete_batch(queue_name, msg_ids, prefixed: true)
+    def delete_batch(queue_name, msg_ids, prefixed: true, job_completion: false)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      async = job_completion && @async_completion_commit
       with_stale_connection_retry do
-        synchronized { exec_delete_batch(name, msg_ids) }
+        synchronized { exec_delete_batch(name, msg_ids, async: async) }
       end
     end
 
@@ -704,12 +743,23 @@ module Pgbus
       end
     end
 
-    def drop_queue(queue_name, prefixed: true)
+    # +before_drop+ runs only when the table itself is dropped (not for a
+    # case duplicate, whose table stays), decided once: the dashboard
+    # releases the dropped messages' uniqueness locks there.
+    def drop_queue(queue_name, prefixed: true, &before_drop)
       name = prefixed ? config.queue_name(queue_name) : queue_name
+      if case_duplicate?(name)
+        forget_case_duplicate(name)
+        return true
+      end
+
+      before_drop&.call
+
       result = with_stale_connection_retry do
         synchronized { @pgmq.drop_queue(name) }
       end
       @queues_created.delete(name)
+      forget_other_spellings(name)
       # Failed-event rows are keyed by the name the executor saw (physical or
       # logical); a recreated queue reuses msg_ids, so neither may survive.
       FailedEventRecorder.clear_queue!([name, name.delete_prefix("#{config.queue_prefix}_")].uniq)
@@ -1606,6 +1656,67 @@ module Pgbus
       duplicate_install_error?(error) || duplicate_install_error?(error.cause)
     end
 
+    # A mixed-case pgmq.meta row from before issue #633 whose lower-case
+    # spelling is registered too: both name one table, the lower-case row is
+    # the queue every current process uses. Dropping the duplicate (the
+    # dashboard lists both until pgbus:fold_queue_name_case runs, and the
+    # orphan stream sweep sees both) must not drop that table.
+    def case_duplicate?(name)
+      return false if name == name.downcase
+
+      synchronized { queue_registered?(name) && queue_registered?(name.downcase) }
+    end
+
+    # Removes only the duplicate's pgmq.meta row (its throttle and topic
+    # bindings cascade); the table and the lower-case queue stay. The shared
+    # table's insert trigger may carry the duplicate's name (an older process
+    # armed it last), so it is re-armed under the lower-case name with the
+    # throttle the lower-case row had (else the duplicate's): with the
+    # duplicate's throttle row gone, that trigger would NOTIFY no more.
+    def forget_case_duplicate(name)
+      canonical = name.downcase
+      with_stale_connection_retry do
+        synchronized do
+          throttle = notify_throttle_for(canonical, name)
+          @pgmq.with_connection do |conn|
+            conn.exec_params("DELETE FROM pgmq.meta WHERE queue_name = $1", [name])
+          end
+          @pgmq.enable_notify_insert(canonical, throttle_interval_ms: throttle) if throttle && !notify_trigger_current?(canonical, throttle)
+        end
+      end
+      @queues_created.delete(name)
+    end
+
+    # The NOTIFY throttle of the first of +names+ that has one, or nil (no
+    # throttle row, or a PGMQ without notify_insert_throttle).
+    def notify_throttle_for(*names)
+      @pgmq.with_connection do |conn|
+        next nil unless conn.exec("SELECT to_regclass('pgmq.notify_insert_throttle')").getvalue(0, 0)
+
+        rows = conn.exec_params(
+          "SELECT queue_name, throttle_interval_ms FROM pgmq.notify_insert_throttle WHERE queue_name = ANY($1::text[])",
+          ["{#{names.join(",")}}"]
+        ).to_a
+        row = names.lazy.filter_map { |n| rows.find { |r| r["queue_name"] == n } }.first
+        row && Integer(row["throttle_interval_ms"])
+      end
+    end
+
+    # pgmq.drop_queue removes the table and only its own pgmq.meta row; a
+    # mixed-case spelling from before issue #633 would be left naming a table
+    # that is gone. Idempotent, so a stale pooled socket retries it.
+    def forget_other_spellings(name)
+      with_stale_connection_retry do
+        synchronized do
+          @pgmq.with_connection do |conn|
+            conn.exec_params(
+              "DELETE FROM pgmq.meta WHERE lower(queue_name) = lower($1) AND queue_name <> $1", [name]
+            )
+          end
+        end
+      end
+    end
+
     # Whether pgmq.meta records the queue — the authoritative "create
     # committed" signal (pgmq.create writes it atomically with the
     # tables). The pooled checkout is a sequential sibling of the failed
@@ -1622,7 +1733,13 @@ module Pgbus
     end
 
     # Check whether the NOTIFY trigger already exists on this queue with the
-    # expected throttle interval. When it does, we can skip the destructive
+    # expected throttle interval, armed under this spelling of its name. PGMQ
+    # 1.13.1's trigger NOTIFYs on the channel of the name it was armed with
+    # (its argument), and every spelling of a name shares the queue's table
+    # (issue #633): a process from before #633 that re-arms it under a
+    # mixed-case name moves its NOTIFYs to a channel no listener uses. A
+    # trigger without an argument (older PGMQ) NOTIFYs on the table's name,
+    # which is the lower-case spelling. When it does, we can skip the destructive
     # DROP TRIGGER + CREATE TRIGGER cycle that causes deadlocks when multiple
     # forked processes race during bootstrap.
     #
@@ -1644,6 +1761,7 @@ module Pgbus
           WHERE n.nspname = 'pgmq'
             AND c.relname = pgmq.format_table_name($1, 'q')
             AND t.tgname = '#{NOTIFY_TRIGGER_NAME}'
+            AND (t.tgnargs = 0 OR t.tgargs = convert_to($1, 'UTF8') || '\\000'::bytea)
             AND EXISTS (
               SELECT 1 FROM pgmq.notify_insert_throttle
               WHERE queue_name = $1
@@ -1758,6 +1876,16 @@ module Pgbus
     # restart or a brief failover — that outlasts the first immediate retry
     # still gets a second, backed-off chance rather than failing an enqueue
     # the caller may never retry.
+    #
+    # The last retry runs on freshly reloaded pools (issue #645). A reset that
+    # Thread#raise interrupted mid-poll leaves the pooled PG::Connection in
+    # CONNECTION_STARTED: neither finished? nor CONNECTION_BAD, so pgmq-ruby's
+    # checkout health check keeps handing it out and libpq refuses every send
+    # ("no connection to the server"). The first retry still goes through that
+    # health check alone, which replaces a socket the server closed without
+    # touching the rest of the pool; only when it fails too are both pools'
+    # idle connections dropped (reload_pools_before_last_stale_retry), as the
+    # #354 wedged-read path does.
     STALE_RETRY_ATTEMPTS = 2
     private_constant :STALE_RETRY_ATTEMPTS
 
@@ -1889,6 +2017,31 @@ module Pgbus
       end
     end
 
+    # Drop both pools' idle connections before the last stale retry (issue
+    # #645; see STALE_RETRY_ATTEMPTS), so it checks out a fresh connection
+    # instead of the one pgmq-ruby's health check kept handing back. Skipped
+    # silently on the shared-AR Proc path: those slots wrap ActiveRecord's own
+    # socket (#reload refuses there with a warning that would be noise on every
+    # retry). Each pool reloads on its own, so one failing does not keep the
+    # other's connections. A reload failure is logged, never raised: the stale
+    # ConnectionError the retry may raise next is the actionable error.
+    def reload_pools_before_last_stale_retry(attempt, error)
+      return if @shared_connection
+
+      Pgbus.logger.warn do
+        "[Pgbus::Client] Dropping pooled connections before the last stale-connection retry " \
+          "(attempt #{attempt}/#{STALE_RETRY_ATTEMPTS}): #{error.message}"
+      end
+      { "job" => @pgmq, "streams" => @streams_pool }.each do |name, pool|
+        pool.reload
+      rescue StandardError => e
+        Pgbus.logger.warn do
+          "[Pgbus::Client] #{name} pool reload before the last stale-connection retry failed: " \
+            "#{e.class}: #{e.message}"
+        end
+      end
+    end
+
     # True when libpq's connection-baked read bounds (statement_timeout +
     # tcp_user_timeout + keepalives) fully cover both a slow query AND a
     # dead/hung socket, so with_read_timeout can skip the Ruby Timeout entirely.
@@ -2031,20 +2184,30 @@ module Pgbus
     # unchanged, as does :options mode (where forward_connection_variables
     # already baked the GUCs into `options`). See issue #332.
     def wrap_session_gucs(conn_opts)
-      return conn_opts unless config.connection_guc_mode == :session
-      return conn_opts unless conn_opts.is_a?(Hash)
-
-      variables = conn_opts[:variables]
-      return conn_opts if variables.nil? || variables.empty?
+      variables = session_gucs(conn_opts)
+      return conn_opts unless variables
 
       pg_opts = conn_opts.except(:variables)
       lambda do
         conn = PG.connect(pg_opts)
-        variables.each { |name, value| conn.exec("SET #{name} = '#{value}'") }
+        apply_session_gucs(conn, variables)
         conn
       end
     end
 
+    # The database.yml variables :session GUC mode applies with SET after
+    # connect, or nil when nothing is applied that way.
+    def session_gucs(conn_opts)
+      return unless config.connection_guc_mode == :session && conn_opts.is_a?(Hash)
+
+      variables = conn_opts[:variables]
+      variables unless variables.nil? || variables.empty?
+    end
+
+    def apply_session_gucs(conn, variables)
+      variables.each { |name, value| conn.exec("SET #{name} = '#{value}'") }
+    end
+
     def apply_connection_bounds(conn_opts)
       timeout = config.read_timeout
       return conn_opts unless timeout&.positive?
@@ -2165,6 +2328,7 @@ module Pgbus
           "[Pgbus::Client] Retrying after stale pgmq connection " \
             "(attempt #{attempts}/#{STALE_RETRY_ATTEMPTS}): #{e.message}"
         end
+        reload_pools_before_last_stale_retry(attempts, e) if attempts == STALE_RETRY_ATTEMPTS
         retry
       end
     end

data/lib/pgbus/concurrency/blocked_execution.rb

@@ -6,6 +6,12 @@ require "time"
 module Pgbus
   module Concurrency
     module BlockedExecution
+      # Headers a parked job is to be sent with. pgbus_blocked_executions has
+      # no headers column, so a dashboard retry parked behind its limit
+      # (Web::DataSource::ResendSlot) carries them in its payload under this
+      # key; promote_next takes them out and sends them as headers.
+      HEADERS_KEY = "pgbus_park_headers"
+
       class << self
         # Insert a blocked execution for a job that hit the concurrency limit.
         #
@@ -64,8 +70,10 @@ module Pgbus
 
             # Carry the enqueuer's priority through: under priority routing it
             # picks the _pN sub-queue, not just the release order (issue #423).
+            headers = released[:payload].is_a?(Hash) ? released[:payload].delete(HEADERS_KEY) : nil
             msg_id = client.send_message(released[:queue_name], released[:payload],
-                                         delay: actual_delay, priority: released[:priority])
+                                         delay: actual_delay, priority: released[:priority],
+                                         **(headers ? { headers: headers } : {}))
           end
 
           return false unless released && msg_id