pgbus 0.7.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dca12989cdb8d34cd026facd0ecb71cc9df51fb966135806a858e1c4bdb0f7cc
-  data.tar.gz: 4f94274ec1bda644087f5711f2a75f8b1c70f5840f5e0d1d9a1b7ea137f6e3ea
+  metadata.gz: b1a647a2e485cace147ca201861ff4613e9c14f72f8fd8be9060f64d6eccfff5
+  data.tar.gz: bf3ca37850d549c6e9ed86bedbad1d88f9a50462b86a2d8a09113de7c1227d0f
 SHA512:
-  metadata.gz: decefd5c55ee2f4ca0c1aa6d52e1e1732f89ba3e789a0ae64c6b0bbd1cf60fe0d0c138dc87398f0a80b5e28d7e5cd7cd09eb44e2265265265adcfd9c0ee829f5
-  data.tar.gz: ad140f317e5de14558842881f2dd428281a8fb141986e7d8099976969a2d85b87dd08e7d01f64ebf5128f5b3174dd485b7af5c1f8d00b07bb4037ec095256c8c
+  metadata.gz: b32dac214071005fa5974478f2cb8bd4d8a270efc3e6bc94141b74c28cebfea0dfa84e41ad9bf1ec2a10c8ab0d09ae08d0df989f0894b8d5cbc6a461698ecf5b
+  data.tar.gz: ea55e1a29e53e491063c34aaea3a643498d1567968e64833d9a38deef2bcd7a48dfb35e377601d347fa8e05e55a5647a77dbe958d9584fcdc8669a362ff253a8

data/lib/pgbus/active_job/executor.rb

@@ -13,9 +13,17 @@ module Pgbus
         @stat_buffer = stat_buffer
       end
 
+      # Exceptions we never want to swallow — let the process die/signal propagate.
+      FATAL_EXCEPTIONS = [SystemExit, Interrupt, SignalException, NoMemoryError, SystemStackError].freeze
+      private_constant :FATAL_EXCEPTIONS
+
       def execute(message, queue_name, source_queue: nil)
         execution_start = monotonic_now
+        tag = "msg_id=#{message.msg_id} queue=#{queue_name} read_ct=#{message.read_ct}"
+        Pgbus.logger.debug { "[Pgbus::Executor] start #{tag}" }
+
         payload = JSON.parse(message.message)
+        job_class = payload["job_class"]
         read_count = message.read_ct.to_i
 
         if read_count > config.max_retries
@@ -25,10 +33,9 @@ module Pgbus
           signal_batch_discarded(payload)
           Uniqueness.release_lock(Uniqueness.extract_key(payload))
           record_stat(payload, queue_name, "dead_lettered", execution_start, message: message)
+          Pgbus.logger.debug { "[Pgbus::Executor] dead_lettered #{tag} job_class=#{job_class}" }
           return :dead_lettered
         end
-
-        job_class = payload["job_class"]
         uniqueness_key = Uniqueness.extract_key(payload)
         uniqueness_strategy = Uniqueness.extract_strategy(payload)
 
@@ -49,23 +56,38 @@ module Pgbus
           end
         end
 
+        Pgbus.logger.debug { "[Pgbus::Executor] deserialized #{tag} job_class=#{job_class}" }
         job_succeeded = false
 
+        msg_id = message.msg_id.to_i
         Instrumentation.instrument("pgbus.executor.execute", queue: queue_name, job_class: job_class) do
           job = ::ActiveJob::Base.deserialize(payload)
+          Pgbus.logger.debug { "[Pgbus::Executor] running #{tag} job_class=#{job_class}" }
           execute_job(job)
-          archive_from(queue_name, message.msg_id.to_i, source_queue: source_queue)
-          FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i)
+          Pgbus.logger.debug { "[Pgbus::Executor] perform_returned #{tag} job_class=#{job_class}" }
+          archive_from(queue_name, msg_id, source_queue: source_queue)
+          Pgbus.logger.debug { "[Pgbus::Executor] archived #{tag} job_class=#{job_class}" }
+          FailedEventRecorder.clear!(queue_name: queue_name, msg_id: msg_id)
           job_succeeded = true
         end
 
         instrument("pgbus.job_completed", queue: queue_name, job_class: job_class)
         record_stat(payload, queue_name, "success", execution_start, message: message)
+        Pgbus.logger.debug { "[Pgbus::Executor] done #{tag} job_class=#{job_class}" }
         :success
-      rescue StandardError => e
+      rescue *FATAL_EXCEPTIONS
+        # Process-fatal: propagate so the supervisor/OS can react.
+        raise
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        # Widened from StandardError to catch Async::Stop / Async::Cancel
+        # (both inherit from Exception, not StandardError) under execution_mode: :async.
+        # Before this, a fiber interruption between perform_now and archive_from
+        # silently lost control flow — no failed event row, no job_failed
+        # notification, uniqueness lock held until VT expired. See issue #126.
         handle_failure(message, queue_name, e, payload: payload)
         instrument("pgbus.job_failed", queue: queue_name, job_class: payload&.dig("job_class"), error: e.class.name)
         record_stat(payload, queue_name, "failed", execution_start, message: message)
+        Pgbus.logger.debug { "[Pgbus::Executor] failed #{tag} job_class=#{payload&.dig("job_class")} error=#{e.class}" }
         # Don't signal concurrency on transient failure — the job will be retried.
         # Semaphore is released only on success or dead-lettering.
         :failed

data/lib/pgbus/client/ensure_stream_queue.rb

@@ -27,7 +27,9 @@ module Pgbus
         # sensitive and need every broadcast to fire a NOTIFY, even
         # when several are batched within a single millisecond.
         # Override the throttle to 0 specifically for stream queues.
-        synchronized { @pgmq.enable_notify_insert(full_name, throttle_interval_ms: 0) } if config.listen_notify
+        # Use the idempotent path to avoid deadlocks when multiple
+        # processes race to set up the same stream queue.
+        synchronized { enable_notify_if_needed(full_name, 0) }
 
         # CREATE INDEX IF NOT EXISTS is idempotent in Postgres but still
         # requires a roundtrip and a brief ACCESS SHARE lock on the archive

data/lib/pgbus/client.rb

@@ -457,12 +457,48 @@ module Pgbus
         synchronized do
           @pgmq.create(full_name)
           tune_autovacuum(full_name)
-          @pgmq.enable_notify_insert(full_name, throttle_interval_ms: NOTIFY_THROTTLE_MS) if config.listen_notify
+          enable_notify_if_needed(full_name, NOTIFY_THROTTLE_MS)
         end
         true
       end
     end
 
+    def enable_notify_if_needed(full_name, throttle_ms)
+      return unless config.listen_notify
+      return if notify_trigger_current?(full_name, throttle_ms)
+
+      @pgmq.enable_notify_insert(full_name, throttle_interval_ms: throttle_ms)
+    end
+
+    # Check whether the NOTIFY trigger already exists on this queue with the
+    # expected throttle interval. When it does, we can skip the destructive
+    # DROP TRIGGER + CREATE TRIGGER cycle that causes deadlocks when multiple
+    # forked processes race during bootstrap.
+    def notify_trigger_current?(full_name, throttle_ms)
+      with_raw_connection do |conn|
+        result = conn.exec_params(<<~SQL, [full_name, throttle_ms])
+          SELECT 1
+          FROM pg_trigger t
+          JOIN pg_class c ON t.tgrelid = c.oid
+          JOIN pg_namespace n ON c.relnamespace = n.oid
+          WHERE n.nspname = 'pgmq'
+            AND c.relname = pgmq.format_table_name($1, 'q')
+            AND t.tgname = 'trigger_notify_queue_insert_listeners'
+            AND EXISTS (
+              SELECT 1 FROM pgmq.notify_insert_throttle
+              WHERE queue_name = $1
+                AND throttle_interval_ms = $2
+            )
+          LIMIT 1
+        SQL
+        result.ntuples.positive?
+      end
+    rescue StandardError
+      # If we can't check (e.g. pgmq schema not fully ready), fall back to
+      # the unconditional path — same behavior as before this fix.
+      false
+    end
+
     def tune_autovacuum(queue_name)
       with_raw_connection do |conn|
         conn.exec(AutovacuumTuning.sql_for_queue(queue_name))

data/lib/pgbus/configuration.rb

@@ -85,6 +85,10 @@ module Pgbus
     # Requires a matching entry in config/database.yml under the "pgbus" key.
     attr_accessor :connects_to
 
+    # Zombie message detection — logs a warning when a message is redelivered
+    # (read_ct > 1) without any prior failure recorded in pgbus_failed_events.
+    attr_accessor :zombie_detection
+
     # Job stats
     attr_accessor :stats_enabled
     attr_reader :stats_retention # rubocop:disable Style/AccessorGrouping
@@ -160,6 +164,8 @@ module Pgbus
       @skip_recurring = false
       @recurring_execution_retention = 7 * 24 * 3600 # 7 days
 
+      @zombie_detection = true
+
       @stats_enabled = true
       @stats_retention = 30 * 24 * 3600 # 30 days
 

data/lib/pgbus/execution_pools/async_pool.rb

@@ -128,9 +128,20 @@ module Pgbus
         nil
       end
 
+      # Supervisor-level rescue: catch any Exception raised from the user
+      # block so capacity is always restored and the failure is logged.
+      # The `async` gem uses Async::Stop / Async::Cancel (Exception subclasses,
+      # NOT StandardError) to cancel tasks, and prior to issue #126 those
+      # would leak past `rescue StandardError` and silently vanish.
+      # Process-fatal signals still propagate so the supervisor can react.
+      FATAL_EXCEPTIONS = [SystemExit, Interrupt, SignalException, NoMemoryError, SystemStackError].freeze
+      private_constant :FATAL_EXCEPTIONS
+
       def perform(block)
         block.call
-      rescue StandardError => e
+      rescue *FATAL_EXCEPTIONS
+        raise
+      rescue Exception => e # rubocop:disable Lint/RescueException
         Pgbus.logger.error { "[Pgbus] Async pool fiber error: #{e.class}: #{e.message}" }
       ensure
         restore_capacity

data/lib/pgbus/failed_event_recorder.rb

@@ -35,6 +35,18 @@ module Pgbus
         ErrorReporter.report(e, { action: "record_failed_event", queue: queue_name, msg_id: msg_id })
       end
 
+      def exists?(queue_name:, msg_id:)
+        result = connection.select_value(
+          "SELECT 1 FROM pgbus_failed_events WHERE queue_name = $1 AND msg_id = $2 LIMIT 1",
+          "FailedEvent Exists",
+          [queue_name, msg_id.to_i]
+        )
+        !result.nil?
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] FailedEvent exists? check failed: #{e.class}: #{e.message}" }
+        false
+      end
+
       def clear!(queue_name:, msg_id:)
         connection.exec_delete(
           "DELETE FROM pgbus_failed_events WHERE queue_name = $1 AND msg_id = $2",

data/lib/pgbus/process/supervisor.rb

@@ -21,6 +21,14 @@ module Pgbus
 
         Pgbus.logger.info { "[Pgbus] Supervisor starting pid=#{::Process.pid}" }
 
+        # Bootstrap queues once in the parent process before forking children.
+        # This avoids the deadlock that occurs when multiple forked children
+        # race to call enable_notify_insert (DROP TRIGGER + CREATE TRIGGER)
+        # concurrently on the same queue tables. Children still call
+        # bootstrap_queues post-fork but the idempotent check in
+        # notify_trigger_current? makes those calls cheap no-ops.
+        bootstrap_queues
+
         boot_processes
         monitor_loop
       ensure

data/lib/pgbus/process/worker.rb

@@ -126,6 +126,7 @@ module Pgbus
 
         @rate_counter.increment(:dequeued, tagged_messages.size)
         tagged_messages.each do |queue_name, message, source_queue|
+          detect_zombie(queue_name, message)
           @in_flight.increment
           @pool.post { process_message(message, queue_name, source_queue: source_queue) }
         end
@@ -285,6 +286,21 @@ module Pgbus
         Pgbus.logger.error { "[Pgbus] Queue table missing: #{error.message}" }
       end
 
+      def detect_zombie(queue_name, message)
+        return unless config.zombie_detection
+        return unless message.read_ct.to_i > 1
+
+        return if FailedEventRecorder.exists?(queue_name: queue_name, msg_id: message.msg_id.to_i)
+
+        Pgbus.logger.warn do
+          "[Pgbus] Zombie message redelivered: queue=#{queue_name} msg_id=#{message.msg_id} " \
+            "read_ct=#{message.read_ct} — previous read did not record a failure. " \
+            "The worker may have crashed mid-execute or the executor silently dropped the job."
+        end
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] Zombie detection failed: #{e.class}: #{e.message}" }
+      end
+
       def check_recycle
         return unless @lifecycle.running? && recycle_needed?
 

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.7.0"
+  VERSION = "0.7.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Mikael Henriksson