pgbus 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5027c420c26dac65291e11e07ca0291da540b7d2aef9d963e9fdebc2217e7943
-  data.tar.gz: 9e66b85229c55528b3b5b2093883935c1333733018508bb6d9f7f7836666363d
+  metadata.gz: 562d32e977685559437bdb1d3817fd02feb4fd9485b9101241a64d25d37162b1
+  data.tar.gz: eeacf0de199bfdc49c5b68523c5bbae7b1eff7970637742f3ec3e4391e4a3b06
 SHA512:
-  metadata.gz: 9c476c7de9c0db40e0d2183a11f7d502704a468e68f49b28934619cfab07f1130f3988d6f0fbfa6e77a57d6450aa183746b301367e0d4af7dcd2779319f96ff7
-  data.tar.gz: ed62152e6c2c361c5b8f38c61e31ea4e3af5d031cf68f71d0ea95177ab04f538bfbb4fefc84f5d08a2acd8efb23b0024d6433d4a6db2272e7c05521a59e83a3b
+  metadata.gz: a06cdb8eeecef32d6fa024f1fadac4f8c22758a11c91b1e9f5b00266869ea92b068c58a239a543c9409375d7155a59a5ec05f17cad59d7a23a49c90ecf74d7cd
+  data.tar.gz: e2a941feb09d3462bb9a83cda9b48821cf93649b53e9e7305af4a3b43b95bccca7690f40ebcb42857c4b9118ebeb592bcfacc317302e3a71b39be1c4f78493aa

data/lib/pgbus/configuration.rb

@@ -117,6 +117,14 @@ module Pgbus
                   :streams_host, :streams_port, :streams_database_url
     attr_reader :streams_default_broadcast_mode # rubocop:disable Style/AccessorGrouping
 
+    # NOTIFY-gated worker wakeups. When true, each Worker fork owns a
+    # dedicated NotifyListener PG connection that LISTENs on its queues'
+    # INSERT channels and wakes the loop on a real insert. Defaults to the
+    # value of listen_notify. The worker_notify_* overrides mirror
+    # streams_* so the LISTEN connection can bypass PgBouncer.
+    attr_accessor :worker_notify_wakeup,
+                  :worker_notify_host, :worker_notify_port, :worker_notify_database_url
+
     # AppSignal integration (auto-loaded when ::Appsignal is defined and this is true).
     # Set to false to opt out without uninstalling the appsignal gem.
     attr_accessor :appsignal_enabled, :appsignal_probe_enabled
@@ -171,6 +179,11 @@ module Pgbus
 
       @listen_notify = true
 
+      @worker_notify_wakeup = nil
+      @worker_notify_host = nil
+      @worker_notify_port = nil
+      @worker_notify_database_url = nil
+
       @pgmq_schema_mode = :auto
 
       @event_consumers = nil
@@ -732,6 +745,41 @@ module Pgbus
       end
     end
 
+    # Connection options for the Worker's dedicated NotifyListener connection.
+    # Mirrors streams_connection_options: defaults to the base connection_options,
+    # overridable via worker_notify_database_url / worker_notify_host /
+    # worker_notify_port so the LISTEN connection can bypass PgBouncer.
+    def worker_notify_connection_options
+      return worker_notify_database_url if worker_notify_database_url
+
+      base = connection_options
+      return base unless worker_notify_host || worker_notify_port
+
+      case base
+      when Hash
+        result = base.dup
+        result[:host] = worker_notify_host if worker_notify_host
+        result[:port] = worker_notify_port if worker_notify_port
+        result
+      when String
+        parts = [base]
+        parts << "host=#{worker_notify_host}" if worker_notify_host
+        parts << "port=#{worker_notify_port}" if worker_notify_port
+        parts.join(" ")
+      else
+        base
+      end
+    end
+
+    # Resolved notify wakeup flag: defaults to listen_notify when nil.
+    def worker_notify_wakeup?
+      if @worker_notify_wakeup.nil?
+        listen_notify
+      else
+        @worker_notify_wakeup
+      end
+    end
+
     private
 
     # Built-in presence-member extractor used when no custom

data/lib/pgbus/process/notify_listener.rb

@@ -0,0 +1,268 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Process
+    # Owns a single dedicated PG::Connection that LISTENs on the INSERT NOTIFY
+    # channel of every queue a Worker/Consumer reads, and fires a WakeSignal the
+    # moment any of them receives a row. This converts the worker/consumer loop
+    # from "blind-read every polling_interval" into "sleep until a real insert,
+    # poll only as a fallback" — eliminating the empty-read storm that dominates
+    # DB load on idle queues.
+    #
+    # pgmq-ruby's `wait_for_notify(queue, timeout:)` is single-queue and wraps
+    # the wait in `with_connection`, which only watches one channel and holds the
+    # pooled connection for the whole wait. Neither fits a worker that reads N
+    # queues on a small shared pool. So we own ONE raw PG::Connection and
+    # hand-roll per-channel LISTEN on it.
+    #
+    # A persistent LISTEN connection silently dies under a transaction-pool
+    # PgBouncer (LISTEN does not survive COMMIT boundaries). Point this
+    # connection at a DIRECT port via `config.worker_notify_*` overrides.
+    # The health-check-on-timeout catches a connection killed out from under us
+    # and re-LISTENs everything.
+    #
+    # NOTIFY channel naming (pgmq trigger): PG_NOTIFY('pgmq.' || table || '.' ||
+    # TG_OP). For queue `pgbus_default` the table is `q_pgbus_default`, so the
+    # channel is `pgmq.q_pgbus_default.INSERT`.
+    #
+    # Thread safety: @running, @conn, and @listening_to are guarded by
+    # @state_mutex. The listener thread owns @conn during wait_for_notify (a
+    # blocking IO call where the mutex MUST NOT be held), so wait_once reads
+    # the connection out of the mutex first and operates on a local. Reconnect
+    # publishes the new connection + channel set under the mutex.
+    class NotifyListener
+      CHANNEL_PREFIX = "pgmq.q_"
+      CHANNEL_SUFFIX = ".INSERT"
+
+      RECONNECT_BACKOFF_SECONDS = 0.5
+
+      def initialize(physical_queues:, on_wake:, connection_options:,
+                     health_check_ms: 1000, logger: Pgbus.logger)
+        @physical_queues = Array(physical_queues)
+        @on_wake = on_wake
+        @connection_options = connection_options
+        @health_check_ms = health_check_ms
+        @logger = logger
+        @state_mutex = Mutex.new
+        @listening_to = Set.new
+        @commands = Queue.new
+        @running = false
+        @thread = nil
+        @conn = nil
+      end
+
+      def listening_to
+        @state_mutex.synchronize { @listening_to.dup }
+      end
+
+      def start
+        @state_mutex.synchronize do
+          return self if @running
+
+          @running = true
+        end
+        @physical_queues.each { |q| @commands << [:listen, q] }
+        @thread = Thread.new { run_loop }
+        self
+      end
+
+      def stop
+        conn_to_close = nil
+        @state_mutex.synchronize do
+          return self unless @running
+
+          @running = false
+          conn_to_close = @conn
+        end
+        @commands << [:stop]
+        # Interrupt the blocking wait by closing the socket; the rescue in
+        # wait_once sees @running == false and exits cleanly.
+        begin
+          conn_to_close&.close if conn_to_close.respond_to?(:close)
+        rescue StandardError
+          nil
+        end
+        @thread&.join(5)
+        @thread = nil
+        self
+      end
+
+      def add_queue(physical_queue)
+        @commands << [:listen, physical_queue]
+      end
+
+      def remove_queue(physical_queue)
+        @commands << [:unlisten, physical_queue]
+      end
+
+      private
+
+      def running?
+        @state_mutex.synchronize { @running }
+      end
+
+      def run_loop
+        conn = build_connection
+        @state_mutex.synchronize { @conn = conn }
+        drain_commands
+
+        loop do
+          break unless running?
+
+          drain_commands
+          break unless running?
+
+          wait_once
+        end
+      rescue StandardError => e
+        @logger.error { "[Pgbus::NotifyListener] fatal: #{e.class}: #{e.message}" } if running?
+      ensure
+        # Clear @running so #start can spawn a fresh thread after a fatal exit
+        # (e.g. build_connection raising at boot). Without this, the dead
+        # thread's @running stays true and #start returns early forever.
+        @state_mutex.synchronize { @running = false }
+        safe_unlisten_all
+        safe_close
+      end
+
+      def wait_once
+        conn = @state_mutex.synchronize { @conn }
+        return reconnect! unless conn
+
+        timeout_s = @health_check_ms / 1000.0
+        got_notify = conn.wait_for_notify(timeout_s) do |_channel, _pid, _payload|
+          @on_wake.call
+        end
+        run_health_check(conn) unless got_notify
+      rescue IOError, PG::Error => e
+        return unless running?
+
+        @logger.warn { "[Pgbus::NotifyListener] connection error (#{e.class}: #{e.message}) — reconnecting" }
+        reconnect!
+      end
+
+      def drain_commands
+        loop do
+          cmd = @commands.pop(true)
+          case cmd[0]
+          when :listen   then do_listen(cmd[1])
+          when :unlisten then do_unlisten(cmd[1])
+          when :stop
+            @state_mutex.synchronize { @running = false }
+            return
+          end
+        rescue ThreadError
+          return
+        end
+      end
+
+      def do_listen(physical_queue)
+        channel = channel_for(physical_queue)
+        conn = @state_mutex.synchronize do
+          return if @listening_to.include?(channel)
+
+          @conn
+        end
+        return unless conn
+
+        conn.exec(%(LISTEN "#{channel}"))
+        @state_mutex.synchronize { @listening_to.add(channel) }
+      end
+
+      def do_unlisten(physical_queue)
+        channel = channel_for(physical_queue)
+        conn = @state_mutex.synchronize do
+          return unless @listening_to.include?(channel)
+
+          @conn
+        end
+        return unless conn
+
+        conn.exec(%(UNLISTEN "#{channel}"))
+        @state_mutex.synchronize { @listening_to.delete(channel) }
+      end
+
+      def run_health_check(conn)
+        conn.exec("SELECT 1")
+      end
+
+      # Retry reconnect until either we succeed (new conn + every channel
+      # re-LISTENed) or @running flips to false. Without the loop, a single
+      # PG::Error during build/LISTEN left @conn nil and the listener degraded
+      # silently — wait_once would re-enter and fail forever or run with an
+      # incomplete subscription set.
+      def reconnect!
+        channels = @state_mutex.synchronize { @listening_to.to_a }
+        loop do
+          return unless running?
+
+          safe_close
+          new_conn = nil
+          begin
+            new_conn = build_connection
+            channels.each { |channel| new_conn.exec(%(LISTEN "#{channel}")) }
+          rescue PG::Error => e
+            # build_connection may have succeeded before a later LISTEN raised.
+            # Without this close, the partially-built conn is orphaned and the
+            # next retry just allocates another one — leaking PG connections on
+            # repeated failures.
+            close_quietly(new_conn)
+            @logger.error { "[Pgbus::NotifyListener] reconnect failed: #{e.class}: #{e.message}" }
+            sleep RECONNECT_BACKOFF_SECONDS
+            next
+          end
+
+          @state_mutex.synchronize do
+            @conn = new_conn
+            @listening_to = Set.new(channels)
+          end
+          return
+        end
+      end
+
+      def build_connection
+        require "pg" unless defined?(::PG::Connection)
+        case @connection_options
+        when String then ::PG.connect(@connection_options)
+        when Hash   then ::PG.connect(**@connection_options)
+        else
+          raise Pgbus::ConfigurationError,
+                "NotifyListener cannot build a PG connection from #{@connection_options.class}. " \
+                "Set worker_notify_database_url / worker_notify_host / worker_notify_port, " \
+                "or a base database_url, so the listener owns a dedicated connection."
+        end
+      end
+
+      def safe_unlisten_all
+        channels, conn = @state_mutex.synchronize { [@listening_to.to_a, @conn] }
+        channels.each do |channel|
+          conn&.exec(%(UNLISTEN "#{channel}"))
+        rescue PG::Error
+          nil
+        end
+        @state_mutex.synchronize { @listening_to.clear }
+      end
+
+      def safe_close
+        conn = @state_mutex.synchronize do
+          c = @conn
+          @conn = nil
+          c
+        end
+        close_quietly(conn)
+      end
+
+      # Close an unpublished PG::Connection (one that never made it into @conn,
+      # e.g. a half-built reconnect attempt where LISTEN raised). Best-effort.
+      def close_quietly(conn)
+        conn&.close if conn.respond_to?(:close)
+      rescue StandardError
+        nil
+      end
+
+      def channel_for(physical_queue)
+        "#{CHANNEL_PREFIX}#{physical_queue}#{CHANNEL_SUFFIX}"
+      end
+    end
+  end
+end

data/lib/pgbus/process/worker.rb

@@ -13,6 +13,7 @@ module Pgbus
                      single_active_consumer: false, consumer_priority: 0,
                      execution_mode: :threads, group_mode: nil)
         @queues = Array(queues)
+        @initial_queues = @queues.dup.freeze
         @wildcard = @queues.include?("*")
         @threads = threads
         @config = config
@@ -49,6 +50,7 @@ module Pgbus
         )
         @circuit_breaker = Pgbus::CircuitBreaker.new(config: config)
         @queue_lock = QueueLock.new if @single_active_consumer
+        @notify_listener = nil
       end
 
       def stats
@@ -66,14 +68,17 @@ module Pgbus
         }.merge(@pool.metadata)
       end
 
+      NOTIFY_FALLBACK_POLL_SECONDS = 15
+
       def run
         setup_signals
         start_heartbeat
         resolve_wildcard_queues
+        start_notify_listener
         @lifecycle.transition_to!(:running)
         Pgbus.logger.info do
           "[Pgbus] Worker started: queues=#{queues.join(",")} threads=#{threads} " \
-            "mode=#{@execution_mode} pid=#{::Process.pid}"
+            "mode=#{@execution_mode} notify_wakeup=#{notify_wakeup?} pid=#{::Process.pid}"
         end
 
         loop do
@@ -117,7 +122,7 @@ module Pgbus
       private
 
       def claim_and_execute
-        poll_interval = effective_polling_interval
+        poll_interval = wake_timeout
 
         idle = @pool.available_capacity
         return @wake_signal.wait(timeout: poll_interval) if idle <= 0
@@ -147,9 +152,19 @@ module Pgbus
       # Returns an array of [queue_name, message] pairs so we always know
       # which queue each message came from.
       def fetch_messages(qty)
+        restore_evicted_queues if queues.empty? && !@wildcard
+
         active_queues = queues.reject { |q| @circuit_breaker.paused?(q) }
         active_queues = active_queues.select { |q| @queue_lock.try_lock(q) } if @single_active_consumer
-        return [] if active_queues.empty?
+
+        if active_queues.empty?
+          Pgbus.logger.debug do
+            paused = queues.select { |q| @circuit_breaker.paused?(q) }
+            "[Pgbus] Worker fetch: all queues filtered — queues=#{queues.join(",")} " \
+              "paused=#{paused.join(",")}"
+          end
+          return []
+        end
 
         if priority_enabled?
           fetch_prioritized(active_queues, qty)
@@ -295,6 +310,7 @@ module Pgbus
           Pgbus.logger.info { "[Pgbus] Wildcard queue '*' resolved to: #{@queues.join(", ")}" } unless @last_wildcard_resolve
         end
         @last_wildcard_resolve = monotonic_now
+        sync_notify_listener_queues
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus] Failed to resolve wildcard queues: #{e.message} — falling back to default" }
         @queues = [config.default_queue] unless @last_wildcard_resolve
@@ -321,6 +337,15 @@ module Pgbus
           end
         end
         Pgbus.logger.error { "[Pgbus] Queue table missing: #{error.message}" }
+        restore_evicted_queues if @queues.empty? && !@wildcard
+      end
+
+      def restore_evicted_queues
+        @queues = @initial_queues.dup
+        Pgbus.logger.warn do
+          "[Pgbus] Worker queue list was empty after eviction — " \
+            "restoring initial queues: #{@queues.join(", ")}"
+        end
       end
 
       def detect_zombie(queue_name, message)
@@ -401,6 +426,16 @@ module Pgbus
         end
       end
 
+      def notify_wakeup?
+        config.respond_to?(:worker_notify_wakeup?) && config.worker_notify_wakeup?
+      end
+
+      def wake_timeout
+        return effective_polling_interval unless notify_wakeup? && @notify_listener
+
+        [effective_polling_interval, config.polling_interval, NOTIFY_FALLBACK_POLL_SECONDS].max
+      end
+
       def effective_polling_interval
         return config.polling_interval if @consumer_priority.zero?
 
@@ -413,6 +448,43 @@ module Pgbus
         config.polling_interval
       end
 
+      def start_notify_listener
+        return unless notify_wakeup?
+
+        @notify_listener = NotifyListener.new(
+          physical_queues: physical_queue_names,
+          on_wake: -> { @wake_signal.notify! },
+          connection_options: config.worker_notify_connection_options,
+          health_check_ms: (config.polling_interval * 1000).to_i.clamp(250, 5_000),
+          logger: Pgbus.logger
+        ).start
+      rescue StandardError => e
+        @notify_listener = nil
+        Pgbus.logger.error do
+          "[Pgbus] NotifyListener failed to start, falling back to polling: #{e.class}: #{e.message}"
+        end
+      end
+
+      def sync_notify_listener_queues
+        return unless @notify_listener
+
+        desired = physical_queue_names.to_set
+        current = @notify_listener.listening_to.to_set { |c| channel_to_physical(c) }
+        (desired - current).each { |q| @notify_listener.add_queue(q) }
+        (current - desired).each { |q| @notify_listener.remove_queue(q) }
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] NotifyListener queue sync failed: #{e.class}: #{e.message}" }
+      end
+
+      def physical_queue_names
+        prefix = "#{config.queue_prefix}_"
+        queues.map { |q| "#{prefix}#{q}" }
+      end
+
+      def channel_to_physical(channel)
+        channel.delete_prefix(NotifyListener::CHANNEL_PREFIX).delete_suffix(NotifyListener::CHANNEL_SUFFIX)
+      end
+
       def start_heartbeat
         @heartbeat = Heartbeat.new(
           kind: "worker",
@@ -427,6 +499,7 @@ module Pgbus
 
       def shutdown
         Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
+        @notify_listener&.stop
         @pool.shutdown
         @pool.wait_for_termination(30)
         @stat_buffer&.stop

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.9.2"
+  VERSION = "0.9.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.3
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -294,6 +294,7 @@ files:
 - lib/pgbus/process/dispatcher.rb
 - lib/pgbus/process/heartbeat.rb
 - lib/pgbus/process/lifecycle.rb
+- lib/pgbus/process/notify_listener.rb
 - lib/pgbus/process/queue_lock.rb
 - lib/pgbus/process/signal_handler.rb
 - lib/pgbus/process/supervisor.rb