pgbus 0.9.7 → 0.9.8

data/lib/pgbus/client.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
 require "json"
+require "socket"
 require "timeout"
 require_relative "client/read_after"
 require_relative "client/ensure_stream_queue"
 require_relative "client/notify_stream"
+require_relative "client/connection_health"
 
 module Pgbus
   class Client
@@ -12,7 +14,7 @@ module Pgbus
     include EnsureStreamQueue
     include NotifyStream
 
-    attr_reader :pgmq, :config
+    attr_reader :pgmq, :config, :connection_health
 
     PGMQ_REQUIRE_MUTEX = Mutex.new
     private_constant :PGMQ_REQUIRE_MUTEX
@@ -25,15 +27,26 @@ module Pgbus
     # re-running the trigger DDL on every queue.
     NOTIFY_THROTTLE_MS = 250
 
-    def initialize(config = Pgbus.configuration)
-      # Define the PGMQ module before requiring the gem so that Zeitwerk's
-      # eager_load (called inside pgmq.rb) can resolve the constant.
-      # Without this, Ruby 4.0 + Zeitwerk 2.7.5 raises NameError because
-      # eager_load runs const_get(:Client) on PGMQ before the module is defined.
+    # Load the pgmq-ruby gem, defining the PGMQ module before requiring it so
+    # Zeitwerk's eager_load (called inside pgmq.rb) can resolve the constant.
+    # Without the pre-definition, Ruby 4.0 + Zeitwerk 2.7.5 raises NameError
+    # because eager_load runs const_get(:Client) on PGMQ before the module is
+    # defined. Extracted as a class method so unit specs that fake PGMQ::Client
+    # can stub *this* (a per-example class-method stub, torn down cleanly)
+    # instead of the global Kernel#require, which — if stubbed before pgmq is
+    # genuinely loaded — permanently prevents the real gem from ever loading.
+    def self.load_pgmq_gem!
       PGMQ_REQUIRE_MUTEX.synchronize do
         Object.const_set(:PGMQ, Module.new) unless defined?(::PGMQ)
         require "pgmq"
       end
+    end
+
+    # `schema_ensured:` lets a caller (in practice, tests) skip the one-time
+    # PGMQ schema install probe by asserting the schema already exists. Defaults
+    # to false so production always runs the check on first queue access.
+    def initialize(config = Pgbus.configuration, schema_ensured: false)
+      self.class.load_pgmq_gem!
       @config = config
       conn_opts = config.connection_options
       @shared_connection = conn_opts.is_a?(Proc)
@@ -52,6 +65,14 @@ module Pgbus
         # Use the resolved pool size (auto-tuned from worker thread counts
         # unless explicitly set) and let pgmq-ruby's connection_pool handle
         # concurrency internally (no mutex needed).
+        #
+        # Bound reads with libpq-native mechanisms baked into the connection
+        # options (issue #198): a server-side statement_timeout for a slow query,
+        # plus client-side tcp_user_timeout + keepalives for a dead/hung socket.
+        # Both raise clean PG errors — no Ruby Timeout, no Thread#raise. Only
+        # safe on this dedicated-connection branch — never on the shared-AR Proc
+        # path, where statement_timeout would leak into application queries.
+        conn_opts = apply_connection_bounds(conn_opts)
         @pgmq = PGMQ::Client.new(conn_opts, pool_size: config.resolved_pool_size, pool_timeout: config.pool_timeout)
         @pgmq_mutex = nil
       end
@@ -59,7 +80,130 @@ module Pgbus
       @queues_created = Concurrent::Map.new
       @stream_indexes_created = Concurrent::Map.new
       @queue_strategy = QueueFactory.for(config)
-      @schema_ensured = false
+      @schema_ensured = schema_ensured
+      @connection_health = ConnectionHealth.new(
+        on_open: method(:log_circuit_open),
+        on_close: method(:log_circuit_close)
+      )
+      # Snapshot whether libpq's baked-in read bounds fully cover a hung socket
+      # on this host/connection, so the read path can skip the Ruby Timeout
+      # last resort. Computed once: @shared_connection, config.read_timeout
+      # (which apply_connection_bounds also snapshots), the platform, and the
+      # linked libpq version are all fixed for a Client's lifetime.
+      @libpq_read_bounds_effective = libpq_read_bounds_effective?
+      warn_shared_connection_read_bounds
+    end
+
+    # True when this client shares ActiveRecord's connection (the Proc
+    # connection_options path): pool_size is forced to 1 and every operation is
+    # serialized through @pgmq_mutex. False on the dedicated-connection path,
+    # where pgmq-ruby owns its own pool and no mutex is needed.
+    def shared_connection?
+      @shared_connection
+    end
+
+    # Whether the shared-connection serialization mutex is currently held. False
+    # on the dedicated-connection path (no mutex). Lets callers assert that a
+    # code path (e.g. a retry backoff sleep) runs OUTSIDE the mutex without
+    # reaching into the mutex object itself.
+    def synchronizing?
+      @pgmq_mutex ? @pgmq_mutex.locked? : false
+    end
+
+    # Actively open a database connection and run `SELECT 1` so a bad
+    # database_url / connection_params surfaces at boot instead of on the
+    # first operation. PGMQ::Client's pool is lazy — nothing touches the
+    # database at init — so without this the supervisor forks children that
+    # crash-loop against an unreachable DB. Called from Supervisor#run before
+    # any queue bootstrap or forking.
+    #
+    # Raises Pgbus::ConfigurationError (not a transient PGMQ error) because a
+    # failure here means the operator's connection config is wrong: the message
+    # carries the underlying error plus which config source was in use.
+    def verify_connection!
+      synchronized do
+        @pgmq.with_connection { |conn| conn.exec("SELECT 1") }
+      end
+      true
+    rescue PGMQ::Errors::ConnectionError, PG::Error => e
+      raise ConfigurationError, "Database connection failed via #{connection_source}: #{e.message}"
+    end
+
+    # Lightweight liveness probe used by the doctor: open a raw connection and
+    # run `SELECT 1`. Unlike verify_connection! (which wraps failures as
+    # ConfigurationError for the supervisor boot path), ping lets the raw
+    # PG/PGMQ error propagate so the caller can render the underlying reason.
+    # Returns true on success; a bad connection raises rather than returning
+    # false — the caller renders the underlying reason — so this is a probe,
+    # not a boolean predicate, hence no `?` suffix.
+    def ping # rubocop:disable Naming/PredicateMethod
+      with_raw_connection { |conn| conn.exec("SELECT 1") }
+      true
+    end
+
+    # The logical queue names pgbus expects to exist based on the configuration
+    # (default queue + worker capsules + recurring tasks). Public wrapper around
+    # collect_configured_queues so the doctor can diff configured-vs-existing
+    # queues without reaching into PGMQ or config internals directly.
+    def configured_queues
+      collect_configured_queues
+    end
+
+    # Whether the given logical queue currently has a live PGMQ insert-NOTIFY
+    # trigger with pgbus's throttle interval on every physical table it maps to.
+    # Uses the same physical-name resolution as bootstrap (@queue_strategy), so
+    # a priority queue's _p0.._pN sub-tables — where the trigger actually lives —
+    # are all checked, not the bare prefixed name that priority mode never
+    # creates. Returns false when any physical table lacks the trigger or the
+    # check can't run.
+    def notify_enabled?(queue_name)
+      names = @queue_strategy.physical_queue_names(queue_name)
+      names.all? { |physical| notify_trigger_current?(physical, NOTIFY_THROTTLE_MS) }
+    end
+
+    # The physical PGMQ queue table names a logical queue maps to — one for a
+    # standard queue, or the _p0.._pN sub-queues when priority is enabled. This
+    # is the SAME resolution the bootstrap path uses (@queue_strategy), so a
+    # caller diffing configured-vs-existing queues compares the exact names PGMQ
+    # actually holds rather than the bare prefixed name.
+    def physical_queue_names(logical_name)
+      @queue_strategy.physical_queue_names(logical_name)
+    end
+
+    # Whether the PGMQ schema itself is present (the pgmq.meta table exists),
+    # independent of pgbus's own version-tracking table. Lets a caller tell
+    # "PGMQ installed via the extension / before version tracking" (schema
+    # present, no tracking row) apart from "PGMQ not installed at all".
+    def pgmq_installed?
+      with_raw_connection do |conn|
+        result = conn.exec(
+          "SELECT 1 FROM pg_tables WHERE schemaname = 'pgmq' AND tablename = 'meta' LIMIT 1"
+        )
+        result.ntuples.positive?
+      end
+    end
+
+    # The most recently recorded installed PGMQ schema version string (e.g.
+    # "1.5.0"), read from the pgbus_pgmq_schema_versions tracking table. Returns
+    # nil when nothing is tracked yet or the table does not exist — the same
+    # logic the `pgbus:pgmq:status` rake task uses, kept here so the doctor and
+    # the rake task share one raw-SQL path (never SQL outside the Client).
+    def pgmq_schema_version
+      with_raw_connection do |conn|
+        result = conn.exec(
+          "SELECT version FROM pgbus_pgmq_schema_versions ORDER BY installed_at DESC LIMIT 1"
+        )
+        row = result.first
+        row && row["version"]
+      end
+    rescue ActiveRecord::StatementInvalid => e
+      raise unless undefined_table_error?(e)
+
+      nil
+    rescue StandardError => e
+      raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
+
+      nil
     end
 
     def ensure_queue(name)
@@ -109,18 +253,22 @@ module Pgbus
 
     def read_message(queue_name, vt: nil)
       full_name = config.queue_name(queue_name)
-      Instrumentation.instrument("pgbus.client.read_message", queue: full_name) do
-        with_stale_connection_retry do
-          synchronized { with_read_timeout { @pgmq.read(full_name, vt: vt || config.visibility_timeout) } }
+      guarded_read do
+        Instrumentation.instrument("pgbus.client.read_message", queue: full_name) do
+          with_stale_connection_retry do
+            synchronized { with_read_timeout { @pgmq.read(full_name, vt: vt || config.visibility_timeout) } }
+          end
         end
       end
     end
 
     def read_batch(queue_name, qty:, vt: nil)
       full_name = config.queue_name(queue_name)
-      Instrumentation.instrument("pgbus.client.read_batch", queue: full_name, qty: qty) do
-        with_stale_connection_retry do
-          synchronized { with_read_timeout { @pgmq.read_batch(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
+      guarded_read do
+        Instrumentation.instrument("pgbus.client.read_batch", queue: full_name, qty: qty) do
+          with_stale_connection_retry do
+            synchronized { with_read_timeout { @pgmq.read_batch(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
+          end
         end
       end
     end
@@ -128,42 +276,51 @@ module Pgbus
     # Read from priority sub-queues, highest priority (p0) first.
     # Returns [priority_queue_name, messages] pairs.
     def read_batch_prioritized(queue_name, qty:, vt: nil)
+      # Non-priority fast path delegates to read_batch, which is already gated
+      # by the connection-health breaker — no extra guard needed here.
       unless @queue_strategy.priority?
         return (read_batch(queue_name, qty: qty, vt: vt) || []).map do |m|
           [config.queue_name(queue_name), m]
         end
       end
 
-      remaining = qty
-      results = []
+      # The priority loop issues its own reads, so gate the whole loop: an open
+      # breaker fails fast before any sub-queue is touched, and the loop as a
+      # unit records one success/failure with the latch.
+      guarded_read do
+        remaining = qty
+        results = []
 
-      config.priority_queue_names(queue_name).each do |pq_name|
-        break if remaining <= 0
+        config.priority_queue_names(queue_name).each do |pq_name|
+          break if remaining <= 0
 
-        msgs = Instrumentation.instrument("pgbus.client.read_batch", queue: pq_name, qty: remaining) do
-          with_stale_connection_retry do
-            synchronized { with_read_timeout { @pgmq.read_batch(pq_name, vt: vt || config.visibility_timeout, qty: remaining) } }
-          end
-        end || []
+          msgs = Instrumentation.instrument("pgbus.client.read_batch", queue: pq_name, qty: remaining) do
+            with_stale_connection_retry do
+              synchronized { with_read_timeout { @pgmq.read_batch(pq_name, vt: vt || config.visibility_timeout, qty: remaining) } }
+            end
+          end || []
 
-        msgs.each { |m| results << [pq_name, m] }
-        remaining -= msgs.size
-      end
+          msgs.each { |m| results << [pq_name, m] }
+          remaining -= msgs.size
+        end
 
-      results
+        results
+      end
     end
 
     def read_with_poll(queue_name, qty:, vt: nil, max_poll_seconds: 5, poll_interval_ms: 100)
       full_name = config.queue_name(queue_name)
-      with_stale_connection_retry do
-        synchronized do
-          @pgmq.read_with_poll(
-            full_name,
-            vt: vt || config.visibility_timeout,
-            qty: qty,
-            max_poll_seconds: max_poll_seconds,
-            poll_interval_ms: poll_interval_ms
-          )
+      guarded_read do
+        with_stale_connection_retry do
+          synchronized do
+            @pgmq.read_with_poll(
+              full_name,
+              vt: vt || config.visibility_timeout,
+              qty: qty,
+              max_poll_seconds: max_poll_seconds,
+              poll_interval_ms: poll_interval_ms
+            )
+          end
         end
       end
     end
@@ -178,11 +335,13 @@ module Pgbus
     # otherwise the pool can overflow on multi-queue reads (issue #123).
     def read_multi(queue_names, qty:, vt: nil, limit: nil)
       full_names = queue_names.map { |q| config.queue_name(q) }
-      Instrumentation.instrument("pgbus.client.read_multi", queues: full_names, qty: qty, limit: limit) do
-        with_stale_connection_retry do
-          synchronized do
-            with_read_timeout do
-              @pgmq.read_multi(full_names, vt: vt || config.visibility_timeout, qty: qty, limit: limit)
+      guarded_read do
+        Instrumentation.instrument("pgbus.client.read_multi", queues: full_names, qty: qty, limit: limit) do
+          with_stale_connection_retry do
+            synchronized do
+              with_read_timeout do
+                @pgmq.read_multi(full_names, vt: vt || config.visibility_timeout, qty: qty, limit: limit)
+              end
             end
           end
         end
@@ -268,6 +427,25 @@ module Pgbus
       end
     end
 
+    # Snapshot of the PGMQ connection pool: {size:, available:, pool_timeout:}.
+    #
+    # Reads pgmq-ruby's own pool counters (@pgmq.stats -> {size:, available:})
+    # and adds the configured pool_timeout so alerting has the full picture:
+    # how many connections exist, how many are free right now, and how long a
+    # checkout waits before raising a pool-timeout error. Works on both the
+    # dedicated-pool path and the shared-Proc path (where size is 1).
+    #
+    # Purely observational — wrapped in a rescue that returns {} so a probe or
+    # heartbeat reading the pool can never break job processing. Not routed
+    # through with_stale_connection_retry: reading in-memory counters touches no
+    # socket, and a failing read must degrade to {} rather than retry.
+    def pool_stats
+      @pgmq.stats.merge(pool_timeout: config.pool_timeout)
+    rescue StandardError => e
+      Pgbus.logger.debug { "[Pgbus::Client] pool_stats unavailable: #{e.class}: #{e.message}" }
+      {}
+    end
+
     def list_queues
       with_stale_connection_retry do
         synchronized { @pgmq.list_queues }
@@ -358,26 +536,32 @@ module Pgbus
 
     def read_grouped(queue_name, qty:, vt: nil)
       full_name = config.queue_name(queue_name)
-      Instrumentation.instrument("pgbus.client.read_grouped", queue: full_name, qty: qty) do
-        with_stale_connection_retry do
-          synchronized { with_read_timeout { @pgmq.read_grouped(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
+      guarded_read do
+        Instrumentation.instrument("pgbus.client.read_grouped", queue: full_name, qty: qty) do
+          with_stale_connection_retry do
+            synchronized { with_read_timeout { @pgmq.read_grouped(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
+          end
         end
       end
     end
 
     def read_grouped_rr(queue_name, qty:, vt: nil)
       full_name = config.queue_name(queue_name)
-      Instrumentation.instrument("pgbus.client.read_grouped_rr", queue: full_name, qty: qty) do
-        with_stale_connection_retry do
-          synchronized { with_read_timeout { @pgmq.read_grouped_rr(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
+      guarded_read do
+        Instrumentation.instrument("pgbus.client.read_grouped_rr", queue: full_name, qty: qty) do
+          with_stale_connection_retry do
+            synchronized { with_read_timeout { @pgmq.read_grouped_rr(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
+          end
         end
       end
     end
 
     def read_grouped_head(queue_name, qty:, vt: nil)
       full_name = config.queue_name(queue_name)
-      with_stale_connection_retry do
-        synchronized { with_read_timeout { @pgmq.read_grouped_head(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
+      guarded_read do
+        with_stale_connection_retry do
+          synchronized { with_read_timeout { @pgmq.read_grouped_head(full_name, vt: vt || config.visibility_timeout, qty: qty) } }
+        end
       end
     end
 
@@ -463,6 +647,19 @@ module Pgbus
 
     private
 
+    # Human-readable label for which config knob supplied the connection
+    # options, mirroring Configuration#connection_options' precedence. Used in
+    # verify_connection!'s error so the operator knows which setting to fix.
+    def connection_source
+      if config.database_url
+        "database_url"
+      elsif config.connection_params
+        "connection_params"
+      else
+        "ActiveRecord-derived connection"
+      end
+    end
+
     # Accept either a logical name ("default") or an already-prefixed
     # physical name ("pgbus_default") and return the physical name.
     # Coerces symbols to strings so callers can pass either form.
@@ -505,7 +702,7 @@ module Pgbus
 
       # Queues from worker configs
       (config.workers || []).each do |w|
-        worker_queues = w[:queues] || w["queues"] || [config.default_queue]
+        worker_queues = w[:queues] || [config.default_queue]
         worker_queues.each { |q| queues << q unless q == "*" }
       end
 
@@ -610,8 +807,17 @@ module Pgbus
     # expected throttle interval. When it does, we can skip the destructive
     # DROP TRIGGER + CREATE TRIGGER cycle that causes deadlocks when multiple
     # forked processes race during bootstrap.
+    #
+    # Routes through the pooled @pgmq.with_connection (health-checked, reused)
+    # rather than opening a fresh PG.connect per queue: on the String/Hash path
+    # with_raw_connection did a full TCP/TLS/auth setup for every queue at every
+    # supervisor boot — and again in each forked child — churning short-lived
+    # connections through the pooler. The checkout here is a sequential sibling
+    # of the @pgmq.create call above it (create's own checkout has already been
+    # returned), so there is no nested checkout: safe even on the shared-Proc
+    # pool_size=1 path.
     def notify_trigger_current?(full_name, throttle_ms)
-      with_raw_connection do |conn|
+      @pgmq.with_connection do |conn|
         result = conn.exec_params(<<~SQL, [full_name, throttle_ms])
           SELECT 1
           FROM pg_trigger t
@@ -687,11 +893,29 @@ module Pgbus
     ].freeze
     private_constant :STALE_CONNECTION_PATTERNS
 
-    # Rescue PGMQ::Errors::ConnectionError once if its message matches a
-    # known stale-socket pattern. pgmq-ruby's auto_reconnect + verify_connection!
-    # already recovers on the *next* checkout, so a single retry is sufficient.
-    # Other connection errors (pool timeout, misconfiguration, truly unreachable
-    # DB) propagate.
+    # How many times a matched stale-connection error is retried before it
+    # propagates. Two attempts (not one) so a transient window — a PgBouncer
+    # restart or a brief failover — that outlasts the first immediate retry
+    # still gets a second, backed-off chance rather than failing an enqueue
+    # the caller may never retry.
+    STALE_RETRY_ATTEMPTS = 2
+    private_constant :STALE_RETRY_ATTEMPTS
+
+    # Backoff before each retry, indexed by (attempt - 1): ~0.1s before the
+    # first retry, ~0.5s before the second. Short enough to stay invisible on
+    # a healthy path (error-path only — never slept on success) and to not
+    # stall a worker loop, long enough to let a pooler/failover window clear.
+    STALE_RETRY_DELAYS = [0.1, 0.5].freeze
+    private_constant :STALE_RETRY_DELAYS
+
+    # Rescue PGMQ::Errors::ConnectionError if its message matches a known
+    # stale-socket pattern, retrying up to STALE_RETRY_ATTEMPTS times with a
+    # short backoff (STALE_RETRY_DELAYS) between attempts. pgmq-ruby's
+    # auto_reconnect + verify_connection! recovers a single dead pooled socket
+    # on the *next* checkout, but a transient window — a PgBouncer restart or a
+    # brief failover — can outlast an immediate retry; the backed-off second
+    # attempt gives that window time to clear. Other connection errors (pool
+    # timeout, misconfiguration, truly unreachable DB) propagate immediately.
     #
     # Wraps every @pgmq.* call site. Pattern matching is intentionally narrow
     # (pre-flight / idle-socket signals only), so retry is safe even for
@@ -699,28 +923,281 @@ module Pgbus
     # connection was dead *before* pgmq-ruby tried to use it, so no SQL was
     # ever sent. Mid-flight errors like "server closed the connection" are
     # excluded from the pattern list for this reason.
-    # Bound a read at config.read_timeout so a dead socket raises
-    # Pgbus::ReadTimeoutError instead of blocking the worker loop forever.
+
+    # Seconds by which the outer bounds (client-side tcp_user_timeout and the
+    # Ruby Timeout last resort) exceed the server-side statement_timeout. Sizing
+    # the outer bounds a little higher lets a live-but-slow server's clean
+    # statement_timeout cancel win the race, so the outer bounds fire only when
+    # the peer is genuinely gone. See apply_connection_bounds and with_read_timeout.
+    READ_TIMEOUT_SLACK = 5
+    private_constant :READ_TIMEOUT_SLACK
+
+    # Bound a read and surface a timeout as Pgbus::ReadTimeoutError. Prefer
+    # libpq-native bounds baked into the connection; the Ruby Timeout is a
+    # narrow, last-resort fallback used only where libpq cannot bound a hung
+    # socket. In order, cleanest to last-resort:
+    #
+    #   1. statement_timeout (server GUC, baked into the connection) — a slow
+    #      query is cancelled by Postgres → PG::QueryCanceled, which pgmq-ruby
+    #      wraps as PGMQ::Errors::ConnectionError ("canceling statement due to
+    #      statement timeout"); mapping_statement_timeout re-raises it as
+    #      Pgbus::ReadTimeoutError. The clean path for a live-but-slow server.
+    #   2. tcp_user_timeout / keepalives (client-side libpq, baked into the
+    #      connection) — a dead/hung socket makes libpq raise PG::ConnectionBad
+    #      synchronously, which pgmq-ruby recognises and reconnects. NO
+    #      Thread#raise, no buffer corruption. Linux + libpq >= 12 only.
+    #
+    # When @libpq_read_bounds_effective (the common production case: Linux,
+    # dedicated connection, read_timeout set, libpq >= 12) BOTH bounds are in
+    # force and Ruby Timeout is never wired in — pure libpq.
+    #
+    #   3. Ruby Timeout.timeout — the LAST resort, reached ONLY on a *dedicated*
+    #      connection where libpq's socket bound is a no-op: non-Linux hosts
+    #      (macOS/BSD/Windows) or a libpq < 12. It interrupts via Thread#raise —
+    #      the mechanism issue #198 flags as unsafe — so it is slack-delayed and
+    #      used only when there is no libpq alternative on that host.
     #
-    # MUST wrap only the bare `@pgmq.read*` call, sitting *inside* both
-    # `synchronized` and `with_stale_connection_retry`:
+    #      The shared-AR Proc path deliberately gets NEITHER a baked-in bound NOR
+    #      this Ruby Timeout: we don't own that socket, and Thread#raise on a
+    #      connection ActiveRecord also queries is the most dangerous place to use
+    #      it. Instead the operator configures libpq timeouts in database.yml
+    #      (statement_timeout via `variables:`, plus tcp_user_timeout/keepalives),
+    #      which AR passes straight through to the connection. #initialize logs a
+    #      one-time hint when read_timeout is set on a Proc connection.
+    #
+    #      KNOWN LIMITATION: when (3) fires on a genuinely hung socket, libpq may
+    #      leave the pooled PG::Connection reporting CONNECTION_OK while it will
+    #      in fact re-hang on reuse, and pgmq-ruby's health check won't discard
+    #      it (it isn't CONNECTION_BAD). The proper fix is a public pool-reload on
+    #      pgmq-ruby (follow-up, cf. mensfeld/pgmq-ruby#94); until then it's
+    #      documented and confined to the non-Linux dedicated path.
+    #
+    # MUST wrap only the bare `@pgmq.read*` call, inside both `synchronized` and
+    # `with_stale_connection_retry`, so the Timeout clock starts only after the
+    # mutex is acquired (a thread queued behind another read is not charged for
+    # the wait) and each stale-retry attempt gets its own full budget:
     #
     #   with_stale_connection_retry { synchronized { with_read_timeout { @pgmq.read* } } }
+    def with_read_timeout(&block)
+      # libpq covers everything (Linux, dedicated conn, read_timeout, libpq>=12),
+      # OR this is the Proc path where we defer to AR/database.yml — either way,
+      # no Ruby Timeout. Only the dedicated-but-libpq-can't-bound-the-socket case
+      # (non-Linux / libpq<12) falls through to the Timeout fallback below.
+      return mapping_statement_timeout(&block) if @libpq_read_bounds_effective || @shared_connection
+
+      timeout = config.read_timeout
+      return mapping_statement_timeout(&block) unless timeout&.positive?
+
+      # rubocop:disable Pgbus/NoRubyTimeout -- deliberate last-resort bound; see above
+      Timeout.timeout(timeout + READ_TIMEOUT_SLACK, Pgbus::ReadTimeoutError) do
+        mapping_statement_timeout(&block)
+      end
+      # rubocop:enable Pgbus/NoRubyTimeout
+    end
+
+    # True when libpq's connection-baked read bounds (statement_timeout +
+    # tcp_user_timeout + keepalives) fully cover both a slow query AND a
+    # dead/hung socket, so with_read_timeout can skip the Ruby Timeout entirely.
+    # Requires ALL of:
+    #   * a dedicated connection — the shared-AR Proc path has no baked-in bounds
+    #     (we don't own that socket; statement_timeout would leak into app queries)
+    #   * read_timeout set — apply_connection_bounds no-ops on nil, so skipping
+    #     Timeout with no bound installed would leave a read unbounded forever
+    #   * TCP_USER_TIMEOUT available — macOS/BSD/Windows no-op it, and keepalives
+    #     alone can't bound a stall mid-reply (data sent, never ACKed)
+    #   * libpq >= 12 — older libpq rejects the tcp_user_timeout conninfo keyword
+    #     outright (it fails the whole connection), so we must not have baked it in
+    def libpq_read_bounds_effective?
+      return false if @shared_connection
+      return false unless config.read_timeout&.positive?
+      return false unless Socket.const_defined?(:TCP_USER_TIMEOUT)
+
+      PG.library_version >= 120_000
+    end
+
+    # On the shared-AR (Proc) path pgbus doesn't own the connection, so it bakes
+    # in no read bounds and deliberately does NOT wrap reads in Ruby Timeout
+    # (Thread#raise on a socket ActiveRecord also uses is the most dangerous
+    # place for it). When read_timeout is set the operator likely expects reads
+    # to be bounded, so point them at the libpq timeouts AR passes through from
+    # database.yml — the same bounds pgbus's dedicated path installs itself.
+    def warn_shared_connection_read_bounds
+      return unless @shared_connection && config.read_timeout&.positive?
+
+      Pgbus.logger.warn do
+        "[Pgbus::Client] read_timeout is set but pgbus is sharing ActiveRecord's " \
+          "connection, so it can't bound reads itself. Configure libpq timeouts on " \
+          "the pgbus connection in database.yml instead: `variables: { statement_timeout: <ms> }` " \
+          "plus `tcp_user_timeout: <ms>` and `keepalives: 1` (Linux). Use a dedicated " \
+          "database_url/connection_params for pgbus to have it apply these automatically."
+      end
+    end
+
+    # Substring pgmq-ruby surfaces (wrapped as PGMQ::Errors::ConnectionError)
+    # when Postgres cancels a query that overran statement_timeout. Detected in
+    # the read paths and re-raised as Pgbus::ReadTimeoutError so the server-side
+    # bound preserves the same public contract the Ruby Timeout gave callers.
+    STATEMENT_TIMEOUT_PATTERN = "canceling statement due to statement timeout"
+    private_constant :STATEMENT_TIMEOUT_PATTERN
+
+    def statement_timeout_error?(error)
+      error.message.to_s.downcase.include?(STATEMENT_TIMEOUT_PATTERN)
+    end
+
+    # Run a read block, re-raising a server-side statement_timeout cancellation
+    # as Pgbus::ReadTimeoutError. Wraps the read call sites so the public
+    # contract (ReadTimeoutError on a timed-out read) holds whether the bound
+    # fired server-side (the normal case) or via the Ruby Timeout fallback.
+    def mapping_statement_timeout
+      yield
+    rescue PGMQ::Errors::ConnectionError => e
+      raise Pgbus::ReadTimeoutError, e.message if statement_timeout_error?(e)
+
+      raise
+    end
+
+    # Idle seconds before libpq starts probing a quiet connection with TCP
+    # keepalives, and the interval/count of those probes. Sized to detect a
+    # dead peer (or a NAT/LB that silently dropped an idle flow) well inside a
+    # typical cloud idle-drop window (~350–600s). Pool and LISTEN connections
+    # sit idle between reads, so keepalives are what catch a peer that vanished
+    # while nothing was in flight. Client-side libpq keywords — never sent in
+    # the startup packet, so a pooler (PgBouncer) can't reject them.
+    KEEPALIVE_IDLE_SECONDS = 30
+    KEEPALIVE_INTERVAL_SECONDS = 10
+    KEEPALIVE_COUNT = 3
+    private_constant :KEEPALIVE_IDLE_SECONDS, :KEEPALIVE_INTERVAL_SECONDS, :KEEPALIVE_COUNT
+
+    # Bake libpq-native read/connection bounds into the connection options of a
+    # dedicated pgmq-ruby connection (issue #198). Two independent libpq
+    # mechanisms, deliberately NOT Ruby's Timeout — Timeout interrupts via
+    # Thread#raise, which can fire mid-libpq call and corrupt the pooled
+    # PG::Connection's result buffer:
+    #
+    #   1. statement_timeout (server GUC, via `options=-c`) — bounds a query the
+    #      server is actively running. Postgres cancels it and sends back
+    #      PG::QueryCanceled, which the read paths map to Pgbus::ReadTimeoutError.
+    #      This is the bound for a live-but-slow server.
+    #   2. tcp_user_timeout + keepalives (client-side libpq conninfo keywords) —
+    #      bound a dead/hung socket the server never answers on, where
+    #      statement_timeout structurally cannot fire (no live server to cancel).
+    #      libpq forces the socket closed and raises PG::ConnectionBad /
+    #      PG::UnableToSend synchronously on the calling thread — a clean error
+    #      through the normal pgmq path, no Thread#raise, no buffer corruption.
+    #      tcp_user_timeout catches death mid-read (data sent, never ACKed);
+    #      keepalives catch death on an idle connection.
+    #
+    # tcp_user_timeout is sized at read_timeout + a small slack so statement_timeout
+    # (the clean server-side cancel) wins whenever the server is still answering;
+    # the socket bound only fires when the peer is genuinely gone.
     #
-    # Two reasons for this nesting:
-    #   1. On the shared-connection path @pgmq_mutex serializes all reads.
-    #      The timeout clock must start only after the mutex is acquired,
-    #      otherwise a thread queued behind another read is charged for the
-    #      wait and raises a false ReadTimeoutError for a socket it never
-    #      touched.
-    #   2. with_stale_connection_retry can retry once; with the timeout
-    #      inside, each socket attempt gets its own full timeout budget
-    #      rather than sharing one across both attempts.
-    def with_read_timeout(&)
+    # Called only on the dedicated-connection branch (String URL / Hash params).
+    # Never on the shared-AR Proc path — statement_timeout is connection-wide and
+    # would leak into application queries, and the socket there is AR's to own.
+    #
+    # NOTE: statement_timeout is connection-wide, so writes on these connections
+    # gain the same bound. Acceptable — an enqueue that can't complete within
+    # read_timeout is already failing — and keeps a single server-side mechanism.
+    #
+    # Returns conn_opts unchanged when read_timeout is nil (bounding disabled).
+    def apply_connection_bounds(conn_opts)
       timeout = config.read_timeout
-      return yield unless timeout&.positive?
+      return conn_opts unless timeout&.positive?
+
+      statement_ms = (timeout * 1000).to_i
+      # Socket-death bound sits just above the server-side query bound so a live
+      # server's clean statement_timeout cancel always wins the race.
+      socket_ms = ((timeout + READ_TIMEOUT_SLACK) * 1000).to_i
+      # tcp_user_timeout is a libpq 12+ conninfo keyword; libpq < 12 rejects it
+      # outright and fails the whole connection. So only bake in the socket-level
+      # keywords when the linked libpq understands them — statement_timeout (a
+      # server GUC via `options`) is always safe. Older libpq keeps just the
+      # query bound; the Ruby Timeout fallback covers the socket there.
+      with_socket = libpq_supports_socket_bounds?
+
+      case conn_opts
+      when Hash
+        merge_connection_bounds(conn_opts, statement_ms, socket_ms, with_socket: with_socket)
+      when String
+        append_connection_bounds(conn_opts, statement_ms, socket_ms, with_socket: with_socket)
+      else
+        conn_opts
+      end
+    end
+
+    # Whether the linked libpq accepts the tcp_user_timeout / keepalives conninfo
+    # keywords. Added in libpq 12; an older libpq raises "invalid connection
+    # option" and fails the connection, so we must not emit them there.
+    def libpq_supports_socket_bounds?
+      defined?(PG) && PG.respond_to?(:library_version) && PG.library_version >= 120_000
+    end
 
-      Timeout.timeout(timeout, Pgbus::ReadTimeoutError, &)
+    # Hash form maps 1:1 to libpq keywords, so no escaping/encoding is needed.
+    # The GUC stays nested in `options`; the socket keywords are top-level.
+    # Preserve any caller-supplied `:options` (e.g. `-c search_path=…`) by
+    # appending our `-c statement_timeout=…` rather than overwriting it.
+    def merge_connection_bounds(conn_opts, statement_ms, socket_ms, with_socket:)
+      options = [conn_opts[:options], "-c statement_timeout=#{statement_ms}"].compact.join(" ")
+      merged = conn_opts.merge(options: options)
+      return merged unless with_socket
+
+      merged.merge(
+        keepalives: 1,
+        keepalives_idle: KEEPALIVE_IDLE_SECONDS,
+        keepalives_interval: KEEPALIVE_INTERVAL_SECONDS,
+        keepalives_count: KEEPALIVE_COUNT,
+        tcp_user_timeout: socket_ms
+      )
+    end
+
+    # libpq accepts two connection-string forms. URI form (postgres:// or
+    # postgresql://) carries keywords as URL-encoded query params — the GUC in
+    # `options` must percent-encode its space (%20) and `=` (%3D). key=value
+    # conninfo form carries them space-separated, with the GUC single-quoted so
+    # the outer parser keeps `-c statement_timeout=…` as one value.
+    def append_connection_bounds(conn_opts, statement_ms, socket_ms, with_socket:)
+      if conn_opts.start_with?("postgres://", "postgresql://")
+        separator = conn_opts.include?("?") ? "&" : "?"
+        socket = if with_socket
+                   "keepalives=1&keepalives_idle=#{KEEPALIVE_IDLE_SECONDS}" \
+                     "&keepalives_interval=#{KEEPALIVE_INTERVAL_SECONDS}" \
+                     "&keepalives_count=#{KEEPALIVE_COUNT}&tcp_user_timeout=#{socket_ms}&"
+                 else
+                   ""
+                 end
+        "#{conn_opts}#{separator}#{socket}options=-c%20statement_timeout%3D#{statement_ms}"
+      else
+        socket = if with_socket
+                   "keepalives=1 keepalives_idle=#{KEEPALIVE_IDLE_SECONDS} " \
+                     "keepalives_interval=#{KEEPALIVE_INTERVAL_SECONDS} " \
+                     "keepalives_count=#{KEEPALIVE_COUNT} tcp_user_timeout=#{socket_ms} "
+                 else
+                   ""
+                 end
+        "#{conn_opts} #{socket}options='-c statement_timeout=#{statement_ms}'"
+      end
+    end
+
+    # Gate a read through the in-memory connection-health circuit breaker.
+    # When the breaker is open the block never runs — Pgbus::ConnectionCircuitOpenError
+    # is raised before any pool checkout, sparing a dead database from the whole
+    # fleet re-polling and the error tracker from per-poll noise. A completed
+    # read records success (closing/resetting the breaker); a
+    # PGMQ::Errors::ConnectionError records a failure (and still propagates).
+    # Writes are intentionally NOT gated — callers must see enqueue failures.
+    def guarded_read(&)
+      @connection_health.run_guarded(&)
+    end
+
+    def log_circuit_open(backoff)
+      Pgbus.logger.warn do
+        "[Pgbus::Client] Connection circuit opened after #{ConnectionHealth::OPEN_THRESHOLD}+ " \
+          "consecutive connection failures — reads fail fast for ~#{backoff}s"
+      end
+    end
+
+    def log_circuit_close
+      Pgbus.logger.info { "[Pgbus::Client] Connection circuit closed — database reachable again" }
     end
 
     def with_stale_connection_retry
@@ -729,10 +1206,19 @@ module Pgbus
         yield
       rescue PGMQ::Errors::ConnectionError => e
         attempts += 1
-        raise unless attempts == 1 && stale_connection_error?(e)
+        raise enrich_pool_timeout_error(e) unless attempts <= STALE_RETRY_ATTEMPTS && stale_connection_error?(e)
+
+        # Sleep here — in the rescue, *outside* the yielded block — so the
+        # backoff never runs while @pgmq_mutex is held: on the shared-connection
+        # path the mutex lives inside `synchronized` within the yielded block,
+        # and the raise unwinds out of it (releasing the mutex) before we get
+        # here. See STALE_RETRY_DELAYS. Clamp the index to the last delay so a
+        # future STALE_RETRY_ATTEMPTS > STALE_RETRY_DELAYS.size never sleeps nil.
+        sleep STALE_RETRY_DELAYS[[attempts - 1, STALE_RETRY_DELAYS.size - 1].min]
 
         Pgbus.logger.warn do
-          "[Pgbus::Client] Retrying after stale pgmq connection: #{e.message}"
+          "[Pgbus::Client] Retrying after stale pgmq connection " \
+            "(attempt #{attempts}/#{STALE_RETRY_ATTEMPTS}): #{e.message}"
         end
         retry
       end
@@ -743,6 +1229,39 @@ module Pgbus
       STALE_CONNECTION_PATTERNS.any? { |pattern| message.include?(pattern) }
     end
 
+    # Substring pgmq-ruby uses when a pool checkout times out — a
+    # ConnectionPool::TimeoutError re-raised as PGMQ::Errors::ConnectionError
+    # "Connection pool timeout: ..." (see PGMQ::Connection#with_connection).
+    # Deliberately NOT in STALE_CONNECTION_PATTERNS: a saturated pool must not
+    # be retried (that just piles more waiters onto an already-exhausted pool).
+    POOL_TIMEOUT_MARKER = "connection pool timeout"
+    private_constant :POOL_TIMEOUT_MARKER
+
+    def pool_timeout_error?(error)
+      error.message.to_s.downcase.include?(POOL_TIMEOUT_MARKER)
+    end
+
+    # A bare "Connection pool timeout" tells an operator nothing actionable.
+    # For that (and only that) error, return a same-class replacement whose
+    # message carries the live pool state and a concrete next step, so the
+    # first signal of saturation is diagnosable. The class is preserved so
+    # callers rescuing PGMQ::Errors::ConnectionError behave identically. Any
+    # other ConnectionError is returned untouched. Enrichment never raises:
+    # pool_stats already rescues to {}, and a formatting failure falls back to
+    # the original error.
+    def enrich_pool_timeout_error(error)
+      return error unless pool_timeout_error?(error)
+
+      stats = pool_stats
+      detail = stats.empty? ? "" : " (pool #{stats})"
+      error.class.new(
+        "#{error.message}#{detail} — " \
+        "raise Pgbus.configuration.pool_size or reduce worker threads"
+      )
+    rescue StandardError
+      error
+    end
+
     def serialize(data)
       case data
       when String