pgbus 0.9.7 → 0.9.8

data/lib/pgbus/process/dispatcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "concurrent"
+
 module Pgbus
   module Process
     class Dispatcher
@@ -24,23 +26,106 @@ module Pgbus
       # too-small value just delays cleanup, never breaks anything.
       ARCHIVE_COMPACTION_BATCH_SIZE = 1000
 
-      attr_reader :config
+      # Maintenance backoff during systemic outages. When every attempted
+      # maintenance task fails for two consecutive cycles (e.g. the database
+      # is down), skip maintenance entirely for an exponentially growing
+      # window instead of retrying ~12 tasks every dispatch interval and
+      # flooding logs and error trackers. The window doubles per consecutive
+      # failed cycle and is capped. Constants (not config) per the precedent
+      # above: the values rarely need tuning and only affect noise, never
+      # correctness. Backoff exits the moment any task succeeds again.
+      MAINTENANCE_BACKOFF_BASE = 30    # seconds; first backoff window
+      MAINTENANCE_BACKOFF_MAX = 600    # seconds; window ceiling (10 minutes)
+
+      # The per-task monotonic timestamps run_maintenance consults via run_if_due.
+      # Enumerated so set_maintenance_timestamp can validate its argument.
+      MAINTENANCE_TIMESTAMPS = %i[
+        @last_cleanup_at @last_reap_at @last_concurrency_at @last_batch_cleanup_at
+        @last_recurring_cleanup_at @last_archive_compaction_at @last_stream_archive_compaction_at
+        @last_outbox_cleanup_at @last_job_lock_cleanup_at @last_stats_cleanup_at
+        @last_orphan_stream_sweep_at @last_table_maintenance_at
+      ].freeze
+
+      # Outcome of a single maintenance task in a cycle. Lets run_maintenance
+      # distinguish success/failed/skipped and, on failure, carry the error
+      # and task name for reporting or summarizing.
+      MaintenanceResult = ::Struct.new(:status, :task, :error) do
+        def success? = status == :success
+        def skipped? = status == :skipped
+      end
+
+      # Accumulates the per-item outcomes of a task that loops over queues.
+      # A task should only be considered failed when it attempted work and
+      # *every* attempt failed — a partial failure (some items succeed) must
+      # not trip maintenance backoff. #result returns the first error in the
+      # total-failure case and nil otherwise, matching the "return e on
+      # failure" contract run_if_due expects.
+      class LoopOutcome
+        def initialize
+          @successes = 0
+          @failures = 0
+          @first_error = nil
+        end
+
+        def record_success
+          @successes += 1
+        end
+
+        def record_failure(error)
+          @failures += 1
+          @first_error = error if @first_error.nil?
+        end
+
+        def result
+          @first_error if @failures.positive? && @successes.zero?
+        end
+      end
+
+      attr_reader :config, :maintenance_failure_streak, :maintenance_backoff_until
+
+      def shutting_down?
+        @shutting_down
+      end
+
+      # The last wall-clock timestamp (Time.now.to_f) stamped by the run loop
+      # via the heartbeat's loop_tick_supplier. Wall-clock — NOT monotonic — so
+      # it stays comparable across the parent/child process boundary the
+      # supervisor watchdog reads it over. nil until the first stamp_loop_tick.
+      def last_loop_tick
+        @loop_tick_at.get
+      end
+
+      # Test seam: set one of the @last_*_at maintenance timestamps so a task
+      # becomes (or stops being) due on the next run_maintenance. The timestamps
+      # are per-task monotonic clocks with no constructor injection point (all 12
+      # default to monotonic_now at construction), so a post-construction setter
+      # is the minimal way to drive the due/not-due logic deterministically.
+      # ivar must be one of the recognized @last_*_at names.
+      def set_maintenance_timestamp(ivar, monotonic_value)
+        raise ArgumentError, "unknown maintenance timestamp #{ivar}" unless MAINTENANCE_TIMESTAMPS.include?(ivar.to_sym)
+
+        instance_variable_set(ivar, monotonic_value)
+      end
+
+      # Read one of the @last_*_at maintenance timestamps (companion to
+      # set_maintenance_timestamp) so a test can assert a timestamp did or did
+      # not advance without reaching into the ivar directly.
+      def maintenance_timestamp(ivar)
+        raise ArgumentError, "unknown maintenance timestamp #{ivar}" unless MAINTENANCE_TIMESTAMPS.include?(ivar.to_sym)
+
+        instance_variable_get(ivar)
+      end
 
       def initialize(config: Pgbus.configuration)
         @config = config
         @shutting_down = false
-        @last_cleanup_at = monotonic_now
-        @last_reap_at = monotonic_now
-        @last_concurrency_at = monotonic_now
-        @last_batch_cleanup_at = monotonic_now
-        @last_recurring_cleanup_at = monotonic_now
-        @last_archive_compaction_at = monotonic_now
-        @last_stream_archive_compaction_at = monotonic_now
-        @last_outbox_cleanup_at = monotonic_now
-        @last_job_lock_cleanup_at = monotonic_now
-        @last_stats_cleanup_at = monotonic_now
-        @last_orphan_stream_sweep_at = monotonic_now
-        @last_table_maintenance_at = monotonic_now
+        # Seed every per-task timestamp from the single source of truth so the
+        # list can never drift from set_maintenance_timestamp's allow-list.
+        now = monotonic_now
+        MAINTENANCE_TIMESTAMPS.each { |ivar| instance_variable_set(ivar, now) }
+        @maintenance_failure_streak = 0
+        @maintenance_backoff_until = nil
+        @loop_tick_at = Concurrent::AtomicReference.new(nil)
       end
 
       def run
@@ -51,6 +136,7 @@ module Pgbus
         end
 
         loop do
+          stamp_loop_tick
           break if @shutting_down
 
           process_signals
@@ -75,33 +161,120 @@ module Pgbus
 
       private
 
-      def run_maintenance
-        now = monotonic_now
+      def run_maintenance(now = monotonic_now)
+        return if in_backoff?(now)
+
+        results = run_maintenance_tasks(now)
+        update_maintenance_backoff(now, results)
+      end
 
-        run_if_due(now, :@last_cleanup_at, CLEANUP_INTERVAL) { cleanup_processed_events }
-        run_if_due(now, :@last_reap_at, REAP_INTERVAL) { reap_stale_processes }
-        run_if_due(now, :@last_concurrency_at, CONCURRENCY_INTERVAL) { cleanup_concurrency }
-        run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
-        run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
-        run_if_due(now, :@last_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { compact_archives }
-        run_if_due(now, :@last_stream_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { prune_stream_archives }
-        run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
-        run_if_due(now, :@last_job_lock_cleanup_at, JOB_LOCK_CLEANUP_INTERVAL) { cleanup_job_locks }
-        run_if_due(now, :@last_stats_cleanup_at, STATS_CLEANUP_INTERVAL) { cleanup_stats }
+      # Runs every due maintenance task and returns their results. Each entry
+      # is a MaintenanceResult carrying the outcome (:success/:failed/:skipped)
+      # and, on failure, the exception and task name so run_maintenance can
+      # decide whether to report per-task or summarize.
+      def run_maintenance_tasks(now)
+        results = []
+        results << run_if_due(now, :@last_cleanup_at, CLEANUP_INTERVAL) { cleanup_processed_events }
+        results << run_if_due(now, :@last_reap_at, REAP_INTERVAL) { reap_stale_processes }
+        results << run_if_due(now, :@last_concurrency_at, CONCURRENCY_INTERVAL) { cleanup_concurrency }
+        results << run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
+        results << run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
+        results << run_if_due(now, :@last_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { compact_archives }
+        results << run_if_due(now, :@last_stream_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { prune_stream_archives }
+        results << run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
+        results << run_if_due(now, :@last_job_lock_cleanup_at, JOB_LOCK_CLEANUP_INTERVAL) { cleanup_job_locks }
+        results << run_if_due(now, :@last_stats_cleanup_at, STATS_CLEANUP_INTERVAL) { cleanup_stats }
         sweep_interval = config.streams_orphan_sweep_interval
-        run_if_due(now, :@last_orphan_stream_sweep_at, sweep_interval) { sweep_orphan_streams } if sweep_interval
-        run_if_due(now, :@last_table_maintenance_at, TABLE_MAINTENANCE_INTERVAL) { run_table_maintenance }
+        results << run_if_due(now, :@last_orphan_stream_sweep_at, sweep_interval) { sweep_orphan_streams } if sweep_interval
+        results << run_if_due(now, :@last_table_maintenance_at, TABLE_MAINTENANCE_INTERVAL) { run_table_maintenance }
+        results
       end
 
-      # Only update the timestamp when the block succeeds.
-      # On failure, the next tick retries instead of waiting the full interval.
+      # Runs the block when the interval has elapsed and reports the outcome.
+      # Only updates the timestamp when the block succeeds — on failure the
+      # next tick retries instead of waiting the full interval. Errors are NOT
+      # reported here; run_maintenance decides whether to report per-task or
+      # summarize into a single backoff warning.
+      #
+      # A task fails one of two ways: it raises (a few tasks let errors
+      # propagate), or its own rescue returns the exception (most tasks catch
+      # StandardError to log a descriptive warning, then hand the error back
+      # here via `return e` so the cycle can still see the failure). Both are
+      # normalized to a :failed result carrying the exception.
       def run_if_due(now, ivar, interval)
-        return unless now - instance_variable_get(ivar) >= interval
+        return MaintenanceResult.new(:skipped, task_name(ivar), nil) unless now - instance_variable_get(ivar) >= interval
+
+        outcome = yield
+        return MaintenanceResult.new(:failed, task_name(ivar), outcome) if outcome.is_a?(StandardError)
 
-        yield
         instance_variable_set(ivar, now)
+        MaintenanceResult.new(:success, task_name(ivar), nil)
       rescue StandardError => e
-        ErrorReporter.report(e, { action: "dispatcher_maintenance", task: ivar.to_s.delete_prefix("@last_").delete_suffix("_at") })
+        MaintenanceResult.new(:failed, task_name(ivar), e)
+      end
+
+      def task_name(ivar)
+        ivar.to_s.delete_prefix("@last_").delete_suffix("_at")
+      end
+
+      def in_backoff?(now)
+        @maintenance_backoff_until && now < @maintenance_backoff_until
+      end
+
+      # Given this cycle's results, advance or reset the failure streak and
+      # the backoff window. Any success restores normal cadence. An all-failed
+      # cycle (at least one task attempted, none succeeded) advances the
+      # streak; the first such cycle reports every error individually, and the
+      # cycle that reaches the streak threshold enters backoff with a single
+      # summary warning instead.
+      def update_maintenance_backoff(now, results)
+        attempted = results.reject(&:skipped?)
+        return if attempted.empty?
+
+        if attempted.any?(&:success?)
+          reset_maintenance_backoff
+          return
+        end
+
+        enter_or_advance_backoff(now, attempted)
+      end
+
+      def reset_maintenance_backoff
+        @maintenance_failure_streak = 0
+        @maintenance_backoff_until = nil
+      end
+
+      def enter_or_advance_backoff(now, failures)
+        first_failing_cycle = @maintenance_failure_streak.zero?
+        @maintenance_failure_streak += 1
+
+        if first_failing_cycle
+          report_maintenance_failures(failures)
+        elsif @maintenance_failure_streak >= 2
+          begin_backoff(now, failures)
+        end
+      end
+
+      def report_maintenance_failures(failures)
+        failures.each do |result|
+          ErrorReporter.report(result.error, { action: "dispatcher_maintenance", task: result.task })
+        end
+      end
+
+      def begin_backoff(now, failures)
+        delay = backoff_delay(@maintenance_failure_streak)
+        @maintenance_backoff_until = now + delay
+        first = failures.first
+        Pgbus.logger.warn do
+          "[Pgbus] Dispatcher maintenance backoff: #{failures.size} task(s) failing " \
+            "(#{first.error.class}: #{first.error.message}); skipping maintenance for #{delay}s"
+        end
+      end
+
+      # Exponential window: BASE * 2**(streak - 2), capped at MAX. Streak 2 is
+      # the first backoff (2**0 == 1 == BASE), streak 3 doubles it, and so on.
+      def backoff_delay(streak)
+        [MAINTENANCE_BACKOFF_BASE * (2**(streak - 2)), MAINTENANCE_BACKOFF_MAX].min
       end
 
       def cleanup_processed_events
@@ -112,6 +285,7 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} expired processed events" } if deleted.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Idempotency cleanup failed: #{e.message}" }
+        e
       end
 
       def reap_stale_processes
@@ -120,6 +294,7 @@ module Pgbus
         Pgbus.logger.info { "[Pgbus] Reaped #{deleted} stale processes" } if deleted.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Stale process reaping failed: #{e.message}" }
+        e
       end
 
       def cleanup_concurrency
@@ -132,6 +307,7 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus] Expired #{orphaned} orphaned blocked executions" } if orphaned.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Concurrency cleanup failed: #{e.message}" }
+        e
       end
 
       def release_blocked_for_key(key)
@@ -146,6 +322,7 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} finished batches" } if deleted.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Batch cleanup failed: #{e.message}" }
+        e
       end
 
       def cleanup_stats
@@ -171,11 +348,25 @@ module Pgbus
         maintained = TableMaintenance.run_maintenance(
           raw_conn,
           threshold: TableMaintenance::BLOAT_THRESHOLD,
-          reindex: true
+          reindex: true,
+          stop_check: -> { @shutting_down }
         )
         Pgbus.logger.info { "[Pgbus] Table maintenance completed: #{maintained} table(s) vacuumed" } if maintained.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Table maintenance failed: #{e.message}" }
+        e
+      end
+
+      # Polled at the top of each per-queue maintenance loop. Returns true when
+      # a shutdown has begun, logging one summary line so the interruption is
+      # visible. `index` is how many queues were already processed, so the
+      # message reads "after N of M". An interrupted (early-returned) task
+      # counts as success in run_if_due and simply runs again next interval.
+      def interrupted?(label, index, total)
+        return false unless @shutting_down
+
+        Pgbus.logger.info { "[Pgbus] #{label} interrupted by shutdown after #{index} of #{total}" }
+        true
       end
 
       def cleanup_job_locks
@@ -186,6 +377,9 @@ module Pgbus
         # that fail and retry can hold locks for hours.
         reaped = reap_orphaned_uniqueness_keys
         Pgbus.logger.info { "[Pgbus] Reaped #{reaped} orphaned uniqueness keys" } if reaped.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Job lock cleanup failed: #{e.message}" }
+        e
       end
 
       def reap_orphaned_uniqueness_keys
@@ -208,9 +402,6 @@ module Pgbus
         return 0 if orphaned.empty?
 
         UniquenessKey.where(lock_key: orphaned.map(&:lock_key)).delete_all
-      rescue StandardError => e
-        Pgbus.logger.warn { "[Pgbus] Uniqueness key cleanup failed: #{e.message}" }
-        0
       end
 
       # Returns true if the message referenced by this lock is definitely gone
@@ -244,6 +435,7 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} published outbox entries" } if deleted.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Outbox cleanup failed: #{e.message}" }
+        e
       end
 
       def compact_archives
@@ -257,17 +449,23 @@ module Pgbus
         conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
         queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
 
-        queue_names.each do |full_name|
+        outcome = LoopOutcome.new
+        queue_names.each_with_index do |full_name, index|
+          break if interrupted?("Archive compaction", index, queue_names.size)
           next unless full_name.start_with?("#{prefix}_")
 
           stripped = full_name.delete_prefix("#{prefix}_")
           deleted = Pgbus.client.purge_archive(stripped, older_than: cutoff, batch_size: batch_size)
           Pgbus.logger.debug { "[Pgbus] Compacted #{deleted} archive entries from #{full_name}" } if deleted.positive?
+          outcome.record_success
         rescue StandardError => e
           Pgbus.logger.warn { "[Pgbus] Archive compaction failed for #{full_name}: #{e.message}" }
+          outcome.record_failure(e)
         end
+        outcome.result
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Archive compaction failed: #{e.message}" }
+        e
       end
 
       # Prunes per-stream archive tables. Unlike compact_archives (which
@@ -280,11 +478,13 @@ module Pgbus
         prefix = config.streams_queue_prefix
         return if prefix.nil? || prefix.empty?
 
-        batch_size = config.archive_compaction_batch_size || 1000
+        batch_size = ARCHIVE_COMPACTION_BATCH_SIZE
         conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
         queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
 
-        queue_names.each do |full_name|
+        outcome = LoopOutcome.new
+        queue_names.each_with_index do |full_name, index|
+          break if interrupted?("Stream archive compaction", index, queue_names.size)
           next unless full_name.start_with?("#{prefix}_")
 
           retention = retention_for_stream_queue(full_name)
@@ -298,11 +498,15 @@ module Pgbus
               "[Pgbus] Compacted #{deleted} stream archive entries from #{full_name}"
             end
           end
+          outcome.record_success
         rescue StandardError => e
           Pgbus.logger.warn { "[Pgbus] Stream archive compaction failed for #{full_name}: #{e.message}" }
+          outcome.record_failure(e)
         end
+        outcome.result
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Stream archive compaction failed: #{e.message}" }
+        e
       end
 
       def retention_for_stream_queue(full_name)
@@ -330,7 +534,9 @@ module Pgbus
         queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
 
         dropped = 0
-        queue_names.each do |full_name|
+        outcome = LoopOutcome.new
+        queue_names.each_with_index do |full_name, index|
+          break if interrupted?("Orphan stream sweep", index, queue_names.size)
           next unless full_name.start_with?("#{prefix}_")
 
           row = conn.select_one(<<~SQL, "Pgbus Orphan Check")
@@ -344,13 +550,17 @@ module Pgbus
           Pgbus.client.drop_queue(full_name, prefixed: false)
           dropped += 1
           Pgbus.logger.info { "[Pgbus] Dropped orphan stream queue: #{full_name}" }
+          outcome.record_success
         rescue StandardError => e
           Pgbus.logger.warn { "[Pgbus] Orphan stream sweep failed for #{full_name}: #{e.message}" }
+          outcome.record_failure(e)
         end
 
         Pgbus.logger.debug { "[Pgbus] Orphan stream sweep complete: dropped #{dropped} queue(s)" } if dropped.positive?
+        outcome.result
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Orphan stream sweep failed: #{e.message}" }
+        e
       end
 
       def cleanup_recurring_executions
@@ -361,14 +571,28 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old recurring executions" } if deleted.positive?
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Recurring execution cleanup failed: #{e.message}" }
+        e
       end
 
       def monotonic_now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
 
+      # Wall clock (not monotonic) so the dashboard can compare the beacon
+      # against Time.now across processes; the heartbeat timer thread reads it
+      # via loop_tick_supplier. A dispatcher wedged inside run_maintenance stops
+      # advancing this while heartbeats keep firing, so the beacon ages and the
+      # processes page can surface the stall.
+      def stamp_loop_tick
+        @loop_tick_at.set(Time.now.to_f)
+      end
+
       def start_heartbeat
-        @heartbeat = Heartbeat.new(kind: "dispatcher", metadata: { pid: ::Process.pid })
+        @heartbeat = Heartbeat.new(
+          kind: "dispatcher",
+          metadata: { pid: ::Process.pid },
+          loop_tick_supplier: -> { @loop_tick_at.get }
+        )
         @heartbeat.start
       end
 

data/lib/pgbus/process/heartbeat.rb

@@ -11,11 +11,12 @@ module Pgbus
 
       attr_reader :process_entry
 
-      def initialize(kind:, metadata: {}, on_beat: nil, loop_tick_supplier: nil)
+      def initialize(kind:, metadata: {}, on_beat: nil, loop_tick_supplier: nil, metadata_supplier: nil)
         @kind = kind
         @metadata = metadata
         @on_beat = on_beat
         @loop_tick_supplier = loop_tick_supplier
+        @metadata_supplier = metadata_supplier
         @timer = nil
       end
 
@@ -35,10 +36,8 @@ module Pgbus
 
         @on_beat&.call
         updates = { last_heartbeat_at: Time.current }
-        if @loop_tick_supplier
-          tick = @loop_tick_supplier.call
-          updates[:metadata] = @metadata.merge("loop_tick_at" => tick&.to_f)
-        end
+        metadata = beat_metadata
+        updates[:metadata] = metadata unless metadata.nil?
         ProcessEntry.where(id: @process_id).update_all(updates)
       rescue StandardError => e
         Pgbus.logger.warn { "[Pgbus] Heartbeat failed: #{e.message}" }
@@ -46,6 +45,20 @@ module Pgbus
 
       private
 
+      # Build the metadata hash to persist on this beat. Returns nil when no
+      # supplier is configured so heartbeats without dynamic metadata
+      # (supervisor/dispatcher/scheduler) leave the column untouched, exactly
+      # as before. The loop_tick and metadata suppliers are called here — after
+      # @on_beat has run — so any snapshot!/refresh in on_beat is reflected.
+      def beat_metadata
+        return nil unless @loop_tick_supplier || @metadata_supplier
+
+        metadata = @metadata.dup
+        metadata["loop_tick_at"] = @loop_tick_supplier.call&.to_f if @loop_tick_supplier
+        metadata.merge!(@metadata_supplier.call) if @metadata_supplier
+        metadata
+      end
+
       def register_process
         record = ProcessEntry.create!(
           kind: @kind,

data/lib/pgbus/process/memory_usage.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    module MemoryUsage
+      MEMORY_CHECK_TTL = 5
+
+      @mutex = Mutex.new
+      @cached_value = nil
+      @cached_at = nil
+
+      class << self
+        def current_mb
+          @mutex.synchronize do
+            now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+            return @cached_value if @cached_value && @cached_at && (now - @cached_at) < MEMORY_CHECK_TTL
+
+            @cached_value = read_memory
+            @cached_at = now
+            @cached_value
+          end
+        end
+
+        def reset!
+          @mutex.synchronize do
+            @cached_value = nil
+            @cached_at = nil
+          end
+        end
+
+        private
+
+        def read_memory
+          if RUBY_PLATFORM.include?("darwin")
+            `ps -o rss= -p #{::Process.pid}`.to_i / 1024
+          else
+            begin
+              File.read("/proc/#{::Process.pid}/statm").split[1].to_i * 4096 / (1024 * 1024)
+            rescue Errno::ENOENT
+              0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/process/notify_listener.rb

@@ -95,14 +95,29 @@ module Pgbus
         @commands << [:unlisten, physical_queue]
       end
 
-      private
-
+      # Public so the owning worker can detect a listener whose thread died
+      # (run_loop hit a fatal error and cleared @running in its ensure) and
+      # restart it. Guarded by @state_mutex like every other @running access.
       def running?
         @state_mutex.synchronize { @running }
       end
 
+      private
+
       def run_loop
         conn = build_connection
+        # Reject a connection that landed on a read-only replica before doing
+        # anything else. After a failover, stale DNS can point this fresh
+        # connection at the demoted master; NOTIFY fires only on the primary,
+        # so we'd sit deaf forever. A replica here raises ReplicaConnectionError
+        # and the rescue below runs the fatal/ensure path (worker-level startup
+        # retry is a separate concern); a reconnect converges on the primary.
+        PrimaryValidator.validate_primary!(conn)
+        # One-shot delivery self-probe on the initial connection only. A pooler
+        # or replica that silently breaks LISTEN/NOTIFY is surfaced here with an
+        # actionable error; the listener still runs and degrades to polling.
+        # Reconnects skip the probe to stay cheap.
+        NotifyProbe.probe_notify_delivery!(conn, logger: @logger)
         @state_mutex.synchronize { @conn = conn }
         drain_commands
 
@@ -200,12 +215,16 @@ module Pgbus
           new_conn = nil
           begin
             new_conn = build_connection
+            # Reject a replica before re-LISTENing. A fresh PG.connect re-resolves
+            # DNS, so backing off and retrying converges on the promoted master
+            # once DNS catches up after a failover.
+            PrimaryValidator.validate_primary!(new_conn)
             channels.each { |channel| new_conn.exec(%(LISTEN "#{channel}")) }
-          rescue PG::Error => e
-            # build_connection may have succeeded before a later LISTEN raised.
-            # Without this close, the partially-built conn is orphaned and the
-            # next retry just allocates another one — leaking PG connections on
-            # repeated failures.
+          rescue PG::Error, ReplicaConnectionError => e
+            # build_connection may have succeeded before a later LISTEN raised,
+            # or validate_primary! rejected a replica. Without this close, the
+            # partially-built conn is orphaned and the next retry just allocates
+            # another one — leaking PG connections on repeated failures.
             close_quietly(new_conn)
             @logger.error { "[Pgbus::NotifyListener] reconnect failed: #{e.class}: #{e.message}" }
             sleep RECONNECT_BACKOFF_SECONDS