pgbus 0.9.6 → 0.9.8

data/lib/pgbus/process/notify_probe.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    # One-shot self-probe that verifies a freshly built LISTEN connection can
+    # actually receive a NOTIFY it sends to itself. Two real-world failure modes
+    # break LISTEN/NOTIFY silently, and both are caught here:
+    #
+    #   1. Transaction-mode PgBouncer — LISTEN registrations do not survive the
+    #      transaction boundaries the pooler inserts, so a self-NOTIFY is never
+    #      delivered (the wait times out).
+    #   2. Read-only replica — `pg_notify()` runs inside a read-only transaction
+    #      and raises, so the probe never even reaches the wait.
+    #
+    # In both cases the caller "connected successfully" but would never wake on a
+    # real INSERT, degrading to slow fallback polling with no explanation. The
+    # probe surfaces the condition with an actionable error naming the direct-
+    # connection overrides, then returns false so the caller can degrade
+    # gracefully (workers keep polling; the streamer starts anyway).
+    #
+    # These listeners own raw PG::Connections by design — the documented
+    # exception to the "all PGMQ access through Pgbus::Client" rule — so the
+    # probe operates directly on a PG::Connection.
+    module NotifyProbe
+      PROBE_TIMEOUT_SECONDS = 2
+
+      class << self
+        # Runs the probe on an already-built LISTEN connection. Returns true when
+        # the self-NOTIFY is delivered within PROBE_TIMEOUT_SECONDS, false on
+        # timeout or when pg_notify raises. Never raises: a probe is diagnostic,
+        # so a failure must not take down listener startup.
+        def probe_notify_delivery!(conn, logger: Pgbus.logger)
+          channel = probe_channel
+          begin
+            conn.exec(%(LISTEN "#{channel}"))
+            conn.exec_params("SELECT pg_notify($1, '')", [channel])
+            delivered = wait_for_probe(conn, channel)
+            log_failure(logger) unless delivered
+            delivered
+          rescue PG::Error => e
+            # Read-only replica: pg_notify raised inside a read-only transaction.
+            log_failure(logger, error: e)
+            false
+          ensure
+            safe_unlisten(conn, channel, logger: logger)
+          end
+        end
+
+        private
+
+        # Unique per process AND per invocation so two probes (worker + streamer
+        # in the same process, or a retry) never collide on the same channel.
+        def probe_channel
+          "pgbus_probe_#{::Process.pid}_#{probe_nonce}"
+        end
+
+        def probe_nonce
+          require "securerandom"
+          SecureRandom.hex(4)
+        end
+
+        # wait_for_notify yields the NEXT notification on the connection, not one
+        # filtered to a channel, so confirm the delivered channel is our probe's.
+        # The probe runs on a fresh connection with only this channel LISTENed,
+        # but scoping the check here keeps the method correct regardless of the
+        # caller's LISTEN ordering — an incidental NOTIFY must not read as a
+        # successful probe (that would mask the exact failure this catches).
+        def wait_for_probe(conn, channel)
+          delivered = false
+          conn.wait_for_notify(PROBE_TIMEOUT_SECONDS) do |notified_channel, _pid, _payload|
+            delivered = (notified_channel == channel)
+          end
+          delivered
+        end
+
+        def safe_unlisten(conn, channel, logger: Pgbus.logger)
+          conn.exec(%(UNLISTEN "#{channel}"))
+        rescue PG::Error => e
+          logger.debug { "[Pgbus::NotifyProbe] best-effort UNLISTEN \"#{channel}\" failed: #{e.class}: #{e.message}" }
+        end
+
+        def log_failure(logger, error: nil)
+          detail = error ? " (#{error.class}: #{error.message})" : ""
+          logger.error do
+            "[Pgbus::NotifyProbe] LISTEN/NOTIFY self-probe failed#{detail}: a self-NOTIFY was not delivered " \
+              "within #{PROBE_TIMEOUT_SECONDS}s. This connection cannot receive wake-ups — it is likely a " \
+              "transaction-mode pooler (e.g. PgBouncer) that drops LISTEN, or a read-only replica. The " \
+              "listener will fall back to slow polling. Point the LISTEN connection at a DIRECT database " \
+              "port: for workers set worker_notify_database_url (or worker_notify_host / worker_notify_port); " \
+              "for the streamer set streams_database_url (or streams_host / streams_port)."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/process/primary_validator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    # Rejects a freshly built LISTEN connection that landed on a read-only
+    # replica. After a PostgreSQL failover, stale DNS can point a fresh
+    # `PG.connect` at the demoted master — now a replica. NOTIFY fires only on
+    # the primary, so the listener would connect "successfully" and then sit
+    # deaf forever, waking on nothing and degrading to slow fallback polling
+    # with no explanation.
+    #
+    # `SELECT pg_is_in_recovery()` returns `t` on a replica (a standby in
+    # recovery) and `f` on the primary. On a replica we raise
+    # ReplicaConnectionError so the caller's existing reconnect loop closes the
+    # connection, backs off, and retries. Each retry calls `PG.connect` fresh,
+    # which re-resolves DNS, so the loop converges on the promoted master once
+    # DNS catches up.
+    #
+    # These listeners own raw PG::Connections by design — the documented
+    # exception to the "all PGMQ access through Pgbus::Client" rule — so the
+    # validator operates directly on a PG::Connection. It is a sibling of
+    # NotifyProbe (delivery self-probe); PrimaryValidator is the cheap
+    # every-connection recovery check, NotifyProbe the one-shot start-only
+    # delivery check.
+    module PrimaryValidator
+      RECOVERY_QUERY = "SELECT pg_is_in_recovery()"
+
+      class << self
+        # Runs the recovery check. Returns the connection unchanged on a primary
+        # so callers can write `validate_primary!(build_connection)`. Raises
+        # ReplicaConnectionError on a replica. A PG::Error from exec (dead
+        # connection, etc.) propagates unchanged — callers already rescue
+        # PG::Error and treat it as a reconnect trigger.
+        def validate_primary!(conn)
+          return conn unless in_recovery?(conn)
+
+          raise ReplicaConnectionError,
+                "LISTEN connection landed on a read-only replica " \
+                "(pg_is_in_recovery() => t). After a failover, stale DNS can " \
+                "point a fresh connection at the demoted master. NOTIFY fires " \
+                "only on the primary, so this connection would never wake. " \
+                "Rejecting it; the reconnect loop will re-resolve DNS and retry."
+        end
+
+        private
+
+        def in_recovery?(conn)
+          conn.exec(RECOVERY_QUERY).getvalue(0, 0) == "t"
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/process/signal_handler.rb

@@ -7,6 +7,12 @@ module Pgbus
         base.attr_reader :signal_queue
       end
 
+      # The signals setup_signals has installed handlers for (and will restore on
+      # restore_signals). Empty until setup_signals runs.
+      def handled_signals
+        (@previous_handlers || {}).keys
+      end
+
       def setup_signals
         @signal_queue = Queue.new
         @previous_handlers = {}