pgbus 0.21.0 → 0.21.2

data/lib/pgbus/failed_event_recorder.rb

@@ -4,45 +4,49 @@ require "time"
 
 module Pgbus
   # Records job failures to pgbus_failed_events for dashboard visibility.
-  # Uses upsert (INSERT ON CONFLICT UPDATE) keyed on (queue_name, msg_id)
-  # so each message has at most one failed_event row tracking its latest error.
+  # Uses upsert (INSERT ON CONFLICT UPDATE) keyed on the message, so each
+  # message has at most one failed_event row tracking its latest error.
+  #
+  # A message is (queue_name, msg_id, source_queue): queue_name is the logical
+  # queue, source_queue the physical table the message was read from. Under
+  # priority routing each `_pN` sub-queue numbers its messages from its own
+  # sequence, so the logical name and msg_id alone can name two jobs (issue
+  # #583). A row written before the source_queue column existed has NULL there
+  # and is read as belonging to any table of its queue. Until the column is
+  # migrated, every query keeps its (queue_name, msg_id) shape.
   class FailedEventRecorder
+    # How long a "column missing" answer stands before the schema is probed
+    # again, so a process started before the migration picks the column up.
+    SOURCE_QUEUE_RECHECK_SECONDS = 60
+
+    @source_queue_column_mutex = Mutex.new
+
     class << self
-      def record!(queue_name:, msg_id:, payload:, headers:, error:, retry_count:)
-        connection.exec_query(
-          <<~SQL.squish,
-            INSERT INTO pgbus_failed_events
-              (queue_name, msg_id, payload, headers, error_class, error_message, backtrace, retry_count, failed_at)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, CURRENT_TIMESTAMP)
-            ON CONFLICT (queue_name, msg_id) DO UPDATE SET
-              error_class = EXCLUDED.error_class,
-              error_message = EXCLUDED.error_message,
-              backtrace = EXCLUDED.backtrace,
-              retry_count = EXCLUDED.retry_count,
-              failed_at = EXCLUDED.failed_at
-          SQL
-          "FailedEvent Record",
-          [
-            queue_name,
-            msg_id.to_i,
-            payload.is_a?(String) ? payload : JSON.generate(payload),
-            headers.is_a?(String) ? headers : headers&.then { |h| JSON.generate(h) },
-            error.class.name,
-            error.message.to_s.truncate(10_000),
-            error.backtrace&.first(30)&.join("\n"),
-            retry_count
-          ]
-        )
+      def record!(queue_name:, msg_id:, source_queue:, payload:, headers:, error:, retry_count:)
+        details = [
+          payload.is_a?(String) ? payload : JSON.generate(payload),
+          headers.is_a?(String) ? headers : headers&.then { |h| JSON.generate(h) },
+          error.class.name,
+          error.message.to_s.truncate(10_000),
+          error.backtrace&.first(30)&.join("\n"),
+          retry_count
+        ]
+        legacy = !source_queue_column?
+        if legacy
+          connection.exec_query(LEGACY_RECORD_SQL, "FailedEvent Record", [queue_name, msg_id.to_i, *details])
+        else
+          connection.exec_query(RECORD_SQL, "FailedEvent Record", [queue_name, source_queue, msg_id.to_i, *details])
+        end
       rescue StandardError => e
+        # A process started before the migration would keep sending the old
+        # ON CONFLICT target, whose index the migration drops: probe again.
+        reset_source_queue_column_check! if legacy
         ErrorReporter.report(e, { action: "record_failed_event", queue: queue_name, msg_id: msg_id })
       end
 
-      def exists?(queue_name:, msg_id:)
-        result = connection.select_value(
-          "SELECT 1 FROM pgbus_failed_events WHERE queue_name = $1 AND msg_id = $2 LIMIT 1",
-          "FailedEvent Exists",
-          [queue_name, msg_id.to_i]
-        )
+      def exists?(queue_name:, msg_id:, source_queue:)
+        sql, binds = message_scope(queue_name, msg_id, source_queue)
+        result = connection.select_value("SELECT 1 FROM pgbus_failed_events WHERE #{sql} LIMIT 1", "FailedEvent Exists", binds)
         !result.nil?
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus] FailedEvent exists? check failed: #{e.class}: #{e.message}" }
@@ -53,12 +57,15 @@ module Pgbus
       # dead-letters the message and clears the row (issue #495). failed_at
       # comes back as an ISO 8601 UTC string whatever the adapter decodes it
       # to. A failed read returns nil: it must never stop a dead-letter.
-      def last_error(queue_name:, msg_id:)
+      # The message's own row wins over one written before source_queue.
+      def last_error(queue_name:, msg_id:, source_queue:)
+        sql, binds = message_scope(queue_name, msg_id, source_queue)
+        order = binds.size > 2 ? " ORDER BY source_queue IS NULL, failed_at DESC" : ""
         row = connection.select_one(
           "SELECT error_class, error_message, backtrace, retry_count, failed_at " \
-          "FROM pgbus_failed_events WHERE queue_name = $1 AND msg_id = $2 LIMIT 1",
+          "FROM pgbus_failed_events WHERE #{sql}#{order} LIMIT 1",
           "FailedEvent Last Error",
-          [queue_name, msg_id.to_i]
+          binds
         )
         return unless row
 
@@ -91,12 +98,11 @@ module Pgbus
         end
       end
 
-      def clear!(queue_name:, msg_id:)
-        connection.exec_delete(
-          "DELETE FROM pgbus_failed_events WHERE queue_name = $1 AND msg_id = $2",
-          "FailedEvent Clear",
-          [queue_name, msg_id.to_i]
-        )
+      # Deletes the message's own row, and a row written before source_queue
+      # existed: a sibling sub-queue's row with the same msg_id stays.
+      def clear!(queue_name:, msg_id:, source_queue:)
+        sql, binds = message_scope(queue_name, msg_id, source_queue)
+        connection.exec_delete("DELETE FROM pgbus_failed_events WHERE #{sql}", "FailedEvent Clear", binds)
       rescue StandardError => e
         # ERROR-level: a failed clear leaves a stale row in the dashboard
         # AFTER the job actually succeeded — confusing and load-bearing
@@ -107,8 +113,69 @@ module Pgbus
         end
       end
 
+      # Whether pgbus_failed_events has the source_queue column (issue #583).
+      # Probed like ProcessedEvent.completion_column?: a gem upgraded before
+      # its migration gets false, a one-time warning, and the (queue_name,
+      # msg_id) queries. A present column is remembered for good; a missing
+      # one for SOURCE_QUEUE_RECHECK_SECONDS. A probe that raises is not
+      # memoized.
+      def source_queue_column?
+        # Double-checked: the unlocked read only skips the lock while the
+        # answer stands; it is written under @source_queue_column_mutex. The
+        # state is true, nil (never probed) or the monotonic time a missing
+        # column's answer runs out.
+        known = source_queue_column_answer(@source_queue_column) # rubocop:disable ThreadSafety/ClassInstanceVariable
+        return known unless known.nil?
+
+        @source_queue_column_mutex.synchronize do
+          known = source_queue_column_answer(@source_queue_column)
+          next known unless known.nil?
+
+          supported = detect_source_queue_column(warn: @source_queue_column.nil?)
+          @source_queue_column = supported || (monotonic_now + SOURCE_QUEUE_RECHECK_SECONDS)
+          supported
+        end
+      end
+
+      # Clears the memoized probe: specs exercise both schema shapes, and a
+      # failed legacy write re-probes after the migration lands.
+      def reset_source_queue_column_check!
+        @source_queue_column_mutex.synchronize { @source_queue_column = nil }
+      end
+
       private
 
+      # WHERE fragment and binds naming one message's rows.
+      def message_scope(queue_name, msg_id, source_queue)
+        return ["queue_name = $1 AND msg_id = $2", [queue_name, msg_id.to_i]] unless source_queue_column?
+
+        ["queue_name = $1 AND msg_id = $2 AND (source_queue = $3 OR source_queue IS NULL)",
+         [queue_name, msg_id.to_i, source_queue]]
+      end
+
+      # true, false while a missing column's answer stands, nil to probe.
+      def source_queue_column_answer(state)
+        return true if state == true
+
+        false if state && monotonic_now < state
+      end
+
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
+      def detect_source_queue_column(warn:)
+        supported = connection.column_exists?(:pgbus_failed_events, :source_queue)
+        if warn && !supported
+          Pgbus.logger.warn do
+            "[Pgbus] pgbus_failed_events is missing the source_queue column; under priority routing two " \
+              "sub-queues' failures with the same msg_id share one row. " \
+              "Run `rails generate pgbus:add_failed_events_source_queue` (or pgbus:update) and migrate."
+          end
+        end
+        supported
+      end
+
       def iso_utc(value)
         time = value.is_a?(String) ? Time.parse(value) : value
         time&.utc&.iso8601(6)
@@ -124,5 +191,31 @@ module Pgbus
         end
       end
     end
+
+    UPSERT_ERROR_COLUMNS = <<~SQL.gsub(/\s+/, " ").strip.freeze
+      DO UPDATE SET
+        error_class = EXCLUDED.error_class,
+        error_message = EXCLUDED.error_message,
+        backtrace = EXCLUDED.backtrace,
+        retry_count = EXCLUDED.retry_count,
+        failed_at = EXCLUDED.failed_at
+    SQL
+    private_constant :UPSERT_ERROR_COLUMNS
+
+    RECORD_SQL = <<~SQL.gsub(/\s+/, " ").strip.freeze
+      INSERT INTO pgbus_failed_events
+        (queue_name, source_queue, msg_id, payload, headers, error_class, error_message, backtrace, retry_count, failed_at)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, CURRENT_TIMESTAMP)
+      ON CONFLICT (queue_name, msg_id, source_queue) #{UPSERT_ERROR_COLUMNS}
+    SQL
+    private_constant :RECORD_SQL
+
+    LEGACY_RECORD_SQL = <<~SQL.gsub(/\s+/, " ").strip.freeze
+      INSERT INTO pgbus_failed_events
+        (queue_name, msg_id, payload, headers, error_class, error_message, backtrace, retry_count, failed_at)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, CURRENT_TIMESTAMP)
+      ON CONFLICT (queue_name, msg_id) #{UPSERT_ERROR_COLUMNS}
+    SQL
+    private_constant :LEGACY_RECORD_SQL
   end
 end

data/lib/pgbus/generators/migration_detector.rb

@@ -97,6 +97,7 @@ module Pgbus
         add_workflows: "pgbus:add_workflows",
         add_workflow_signals: "pgbus:add_workflow_signals",
         add_failed_events_index: "pgbus:add_failed_events_index",
+        add_failed_events_source_queue: "pgbus:add_failed_events_source_queue",
         add_processed_event_completion: "pgbus:add_processed_event_completion",
         add_batch_executions: "pgbus:add_batch_executions",
         add_batch_callback_jobs: "pgbus:add_batch_callback_jobs",
@@ -122,6 +123,7 @@ module Pgbus
         add_workflows: "workflow tables (durable multi-step workflows: runs, step attempts, audit trail)",
         add_workflow_signals: "waiting workflow steps (fan_out, wait_for: signals, wait timeouts)",
         add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
+        add_failed_events_source_queue: "source_queue on pgbus_failed_events (one row per priority sub-queue's failure)",
         add_processed_event_completion: "completed_at on pgbus_processed_events (two-phase idempotency claim)",
         add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
         add_batch_callback_jobs: "batch callback-job columns (configured ActiveJob instances as batch callbacks)",
@@ -151,7 +153,7 @@ module Pgbus
           *recurring_migrations,
           *deploy_tasks_migrations,
           *workflows_migrations,
-          *failed_events_index_migrations,
+          *failed_events_migrations,
           *processed_event_completion_migrations,
           *batch_executions_migrations,
           *batch_callback_jobs_migrations,
@@ -266,16 +268,21 @@ module Pgbus
         migrations
       end
 
-      def failed_events_index_migrations
-        # pgbus_failed_events is created by pgbus:install, but the
-        # unique (queue_name, msg_id) index was added later via its own
-        # generator. If the table exists without the unique index,
-        # FailedEventRecorder's upsert silently swallows ON CONFLICT
-        # errors — so this is a real bug waiting to bite.
+      # FailedEventRecorder's upsert needs the unique index that matches its
+      # ON CONFLICT target, or it fails and the failure goes unrecorded. The
+      # source_queue migration (issue #583) adds the column, creates the
+      # (queue_name, msg_id, source_queue) index and drops the old
+      # (queue_name, msg_id) one, so it is the only migration queued here:
+      # add_failed_events_index would put back the index that collapses two
+      # priority sub-queues' failures into one row. That index still present
+      # (or put back by running the old generator by name) queues it again.
+      def failed_events_migrations
         return [] unless table_exists?("pgbus_failed_events")
-        return [] if index_exists?("pgbus_failed_events", "idx_pgbus_failed_events_queue_msg")
+        return [] if column_names("pgbus_failed_events").include?("source_queue") &&
+                     index_exists?("pgbus_failed_events", "idx_pgbus_failed_events_queue_msg_source") &&
+                     !index_exists?("pgbus_failed_events", "idx_pgbus_failed_events_queue_msg")
 
-        [:add_failed_events_index]
+        [:add_failed_events_source_queue]
       end
 
       # completed_at backs the two-phase idempotency claim (issue #385).

data/lib/pgbus/process/consumer.rb

@@ -267,7 +267,10 @@ module Pgbus
         Pgbus.client.archive_message(queue_name, message.msg_id.to_i)
         # A redelivery that succeeds leaves no stale failure behind. First
         # deliveries never had one: no query on the hot path (#484's gate).
-        FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i) if message.read_ct.to_i > 1
+        if message.read_ct.to_i > 1
+          FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i,
+                                     source_queue: config.queue_name(queue_name))
+        end
         @circuit_breaker.record_success(queue_name)
         record_stat(message, queue_name, "success", execution_start)
       rescue StandardError => e
@@ -293,7 +296,8 @@ module Pgbus
       # failed-event row, then announce it. A move that raises keeps the row,
       # so the next attempt still carries the reason, and announces nothing.
       def dead_letter(message, queue_name)
-        last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i)
+        last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i,
+                                                    source_queue: config.queue_name(queue_name))
         headers = DeadLetterHeader.build(
           existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "consumer",
           source_queue: config.queue_name(queue_name), source_msg_id: message.msg_id.to_i,
@@ -301,7 +305,8 @@ module Pgbus
           max_retries: config.max_retries, error: last_error
         )
         Pgbus.client.move_to_dead_letter(queue_name, message, headers: headers)
-        FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i)
+        FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i,
+                                   source_queue: config.queue_name(queue_name))
         instrument_dead_lettered(message, queue_name, last_error)
       end
 
@@ -361,10 +366,12 @@ module Pgbus
       # failure, under the LOGICAL queue name, so the dashboard reads the event
       # as retrying with its error. No ErrorReporter call: Handler#process!
       # already instruments pgbus.event_failed, which feeds the reporters.
+      # A handler queue has no priority sub-queues: its one table is the source.
       def record_failed_event(message, queue_name, error)
         FailedEventRecorder.record!(
-          queue_name: queue_name, msg_id: message.msg_id.to_i, payload: message.message,
-          headers: message.headers, error: error, retry_count: [message.read_ct.to_i - 1, 0].max
+          queue_name: queue_name, msg_id: message.msg_id.to_i, source_queue: config.queue_name(queue_name),
+          payload: message.message, headers: message.headers, error: error,
+          retry_count: [message.read_ct.to_i - 1, 0].max
         )
       end
 

data/lib/pgbus/process/heartbeat.rb

@@ -32,6 +32,7 @@ module Pgbus
       end
 
       def start
+        @started = true
         register_process
         @timer = Concurrent::TimerTask.new(execution_interval: INTERVAL) { beat }
         @timer.execute
@@ -44,7 +45,8 @@ module Pgbus
       end
 
       def beat
-        return unless @process_id && !@stopped
+        return if @stopped
+        return retry_registration unless @process_id
 
         @on_beat&.call
         updates = { last_heartbeat_at: Time.current }
@@ -57,6 +59,15 @@ module Pgbus
 
       private
 
+      # The registration at start failed (a DB blip at boot). Retry it here so
+      # the process does not stay invisible: a pipeless child checks in with
+      # the supervisor's boot deadline only through this row (issue #587).
+      def retry_registration
+        return unless @started
+
+        @mutex.synchronize { register_process unless @stopped }
+      end
+
       # Build the metadata hash to persist on this beat. Returns nil when no
       # supplier is configured so heartbeats without dynamic metadata
       # (supervisor/dispatcher/scheduler) leave the column untouched, exactly

data/lib/pgbus/process/readiness_snapshot.rb

@@ -7,12 +7,20 @@ module Pgbus
     # server's accept thread — the immutability is what makes the cross-thread
     # handoff safe without a lock (issue #386).
     #
-    # `expected` is the child count forked by boot_processes; `live` is the
-    # current fork-table size. A child sitting in crash-restart backoff keeps
-    # `live < expected`, which is exactly the signal a rolling deploy's health
-    # gate needs to fail on: the replacement container never goes ready, and
-    # the orchestrator keeps the old container running.
-    ReadinessSnapshot = Data.define(:booted, :shutting_down, :expected, :live) do
+    # `expected` is the child count forked by boot_processes; `forked` is the
+    # current fork-table size; `live` counts the forks that have checked in
+    # (a process row on this host, or a liveness byte). A child sitting in
+    # crash-restart backoff, or one that forked and hangs before checking in
+    # (issue #587), keeps `live < expected`, which is exactly the signal a
+    # rolling deploy's health gate needs to fail on: the replacement
+    # container never goes ready, and the orchestrator keeps the old
+    # container running. `forked` defaults to `live` for callers that do not
+    # track check-ins.
+    ReadinessSnapshot = Data.define(:booted, :shutting_down, :expected, :live, :forked) do
+      def initialize(booted:, shutting_down:, expected:, live:, forked: live)
+        super
+      end
+
       def ready?
         booted && !shutting_down && live >= expected
       end