pgbus 0.21.1 → 0.21.3

data/lib/pgbus/process/worker.rb

@@ -296,7 +296,7 @@ module Pgbus
         @rate_counter.increment(:dequeued, tagged_messages.size)
         client = Pgbus.client
         tagged_messages.each do |queue_name, message, source_queue|
-          detect_zombie(queue_name, message)
+          detect_zombie(queue_name, message, source_queue)
           @in_flight.increment
           @claim_buffer.push(queue_name, message, source_queue, client: client)
         end
@@ -603,11 +603,12 @@ module Pgbus
         [RESTORE_COOLDOWN_BASE * (2**@restore_streak), RESTORE_COOLDOWN_MAX].min
       end
 
-      def detect_zombie(queue_name, message)
+      def detect_zombie(queue_name, message, source_queue = nil)
         return unless config.zombie_detection
         return unless message.read_ct.to_i > 1
 
-        return if FailedEventRecorder.exists?(queue_name: queue_name, msg_id: message.msg_id.to_i)
+        return if FailedEventRecorder.exists?(queue_name: queue_name, msg_id: message.msg_id.to_i,
+                                              source_queue: source_queue || config.queue_name(queue_name))
 
         Pgbus.logger.warn do
           "[Pgbus] Zombie message redelivered: queue=#{queue_name} msg_id=#{message.msg_id} " \

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.21.1"
+  VERSION = "0.21.3"
 end

data/lib/pgbus/web/data_source/job_list.rb

@@ -10,7 +10,8 @@ module Pgbus
       # Sources, one UNION ALL fragment each:
       # - every non-DLQ queue table, LEFT JOINed to its pgbus_failed_events row
       #   (failed_events.queue_name is the LOGICAL name; the physical one is
-      #   matched too for rows older code wrote)
+      #   matched too for rows older code wrote; source_queue, when set, names
+      #   the one table the row's message was read from — issue #583)
       # - failed rows whose message is no longer in its queue ("orphans")
       # - pgbus_blocked_executions (jobs parked by limits_concurrency)
       #
@@ -187,7 +188,7 @@ module Pgbus
               SELECT f.id, f.error_class::text AS error_class, f.error_message,
                      f.failed_at::timestamptz AS failed_at
               FROM pgbus_failed_events f
-              WHERE f.queue_name IN ('#{logical}', '#{qtable}') AND f.msg_id = m.msg_id
+              WHERE f.queue_name IN ('#{logical}', '#{qtable}') AND f.msg_id = m.msg_id#{failed_in(qtable)}
               ORDER BY f.failed_at DESC LIMIT 1
             ) f ON true
           SQL
@@ -197,7 +198,9 @@ module Pgbus
         # queue, plus (unfiltered) one for rows whose queue no longer exists.
         def orphan_fragments(scope, fetch)
           fragments = scope[:by_logical].map do |logical, tables|
-            missing = tables.map { |(qtable, _)| "NOT EXISTS (SELECT 1 FROM pgmq.q_#{qtable} m WHERE m.msg_id = f.msg_id)" }
+            missing = tables.map do |(qtable, _)|
+              "NOT EXISTS (SELECT 1 FROM pgmq.q_#{qtable} m WHERE m.msg_id = f.msg_id#{failed_in(qtable)})"
+            end
             names = quoted_list([logical, *tables.map(&:first)])
             orphan_fragment("'#{logical}'::text", "f.queue_name IN (#{names}) AND #{missing.join(" AND ")}", fetch)
           end
@@ -209,6 +212,16 @@ module Pgbus
           fragments << orphan_fragment("f.queue_name::text", where, fetch)
         end
 
+        # Under priority routing each _pN sub-queue numbers its messages from
+        # its own sequence: a row that names its table belongs to that table's
+        # message only. A row written before source_queue (NULL), or any row
+        # before the column is migrated, matches every table of its queue.
+        def failed_in(qtable)
+          return "" unless FailedEventRecorder.source_queue_column?
+
+          " AND (f.source_queue IS NULL OR f.source_queue = '#{qtable}')"
+        end
+
         def orphan_fragment(logical_sql, where, fetch)
           <<~SQL.strip
             (SELECT 'failed'::text AS source, f.id, f.queue_name::text AS queue_name, #{logical_sql} AS logical_queue,

data/lib/pgbus/web/data_source.rb

@@ -1340,7 +1340,8 @@ module Pgbus
               (SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM pgmq.#{seq_name}) AS total_messages,
               (SELECT max(read_ct) FROM pgmq.#{qtable}) AS max_read_ct,
               (SELECT count(*) FROM pgmq.#{qtable} WHERE vt <= NOW() AND read_ct = 0) AS visible_unread_length,
-              (SELECT EXTRACT(epoch FROM (NOW() - min(vt)))::int FROM pgmq.#{qtable} WHERE vt <= NOW()) AS oldest_claimable_age_sec
+              (SELECT EXTRACT(epoch FROM (NOW() - min(vt)))::int FROM pgmq.#{qtable} WHERE vt <= NOW()) AS oldest_claimable_age_sec,
+              (SELECT EXTRACT(epoch FROM (NOW() - min(vt)))::int FROM pgmq.#{qtable} WHERE vt <= NOW() AND read_ct = 0) AS oldest_unread_claimable_age_sec
           SQL
         rescue StandardError => e
           Pgbus.logger.debug { "[Pgbus::Web] Skipping queue metrics for #{name}: #{e.message}" }
@@ -1362,7 +1363,8 @@ module Pgbus
             newest_msg_age_sec: row["newest_msg_age_sec"]&.to_i,
             total_messages: row["total_messages"].to_i,
             max_read_ct: row["max_read_ct"]&.to_i,
-            visible_unread_length: row["visible_unread_length"].to_i
+            visible_unread_length: row["visible_unread_length"].to_i,
+            oldest_unread_claimable_age_sec: row["oldest_unread_claimable_age_sec"]&.to_i
           }
         end
       rescue StandardError => e
@@ -1382,6 +1384,7 @@ module Pgbus
               EXTRACT(epoch FROM (NOW() - max(enqueued_at)))::int AS newest_msg_age_sec,
               EXTRACT(epoch FROM (NOW() - min(enqueued_at)))::int AS oldest_msg_age_sec,
               EXTRACT(epoch FROM (NOW() - min(vt) FILTER (WHERE vt <= NOW())))::int AS oldest_claimable_age_sec,
+              EXTRACT(epoch FROM (NOW() - min(vt) FILTER (WHERE vt <= NOW() AND read_ct = 0)))::int AS oldest_unread_claimable_age_sec,
               max(read_ct) AS max_read_ct,
               count(CASE WHEN vt <= NOW() AND read_ct = 0 THEN 1 END) AS visible_unread_length
             FROM pgmq.#{qtable}
@@ -1398,6 +1401,7 @@ module Pgbus
             q_summary.oldest_claimable_age_sec,
             q_summary.max_read_ct,
             q_summary.visible_unread_length,
+            q_summary.oldest_unread_claimable_age_sec,
             all_metrics.total_messages
           FROM q_summary, all_metrics
         SQL
@@ -1414,7 +1418,8 @@ module Pgbus
           newest_msg_age_sec: row["newest_msg_age_sec"]&.to_i,
           total_messages: row["total_messages"].to_i,
           max_read_ct: row["max_read_ct"]&.to_i,
-          visible_unread_length: row["visible_unread_length"].to_i
+          visible_unread_length: row["visible_unread_length"].to_i,
+          oldest_unread_claimable_age_sec: row["oldest_unread_claimable_age_sec"]&.to_i
         }
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus::Web] Error fetching metrics for #{queue_name}: #{e.class}: #{e.message}" }
@@ -1627,20 +1632,22 @@ module Pgbus
       # lock handed to a retry before the failure would count as the job's.
       # A message that is gone has no send time, so only a bound lock goes.
       def release_locks_held_by_failed_events(condition, binds)
+        source = FailedEventRecorder.source_queue_column? ? ", source_queue" : ""
         rows = connection.select_all(
-          "SELECT queue_name, msg_id, payload FROM pgbus_failed_events WHERE #{condition}",
+          "SELECT queue_name, msg_id, payload#{source} FROM pgbus_failed_events WHERE #{condition}",
           "Pgbus Collect Failed Keys", binds
         )
 
         # queue_name is the logical name the executor saw: the message is looked
-        # up under it, and the lock is released under its table spelling.
+        # up under it (or in the table source_queue names), and the lock is
+        # released under its table spelling.
         # A key proc may return a number; the queue lookup reads it as text.
         holders = rows.to_a.filter_map do |row|
           key = extract_uniqueness_key_from_payload_str(row["payload"])&.to_s
-          [key, row["queue_name"], row["msg_id"].to_i] if key && row["msg_id"]
+          [key, row["queue_name"], row["msg_id"].to_i, row["source_queue"]] if key && row["msg_id"]
         end
-        holders = holders.group_by { |_key, queue, _msg_id| queue }.flat_map do |queue, entries|
-          failed_queue_lock_holders(queue, entries)
+        holders = holders.group_by { |_key, queue, _msg_id, table| [queue, table] }.flat_map do |(queue, table), entries|
+          failed_queue_lock_holders(queue, table, entries)
         end
         UniquenessKey.release_all_held_by!(holders) if holders.any?
       rescue StandardError => e
@@ -1650,8 +1657,8 @@ module Pgbus
       # The release tuples for one queue's failed jobs. A queue that cannot
       # be resolved (a stored name no longer valid as a table name) is skipped
       # with a warning, so it does not stop the other queues' release.
-      def failed_queue_lock_holders(queue, entries)
-        sent = queued_send_times(queue, entries)
+      def failed_queue_lock_holders(queue, table, entries)
+        sent = queued_send_times(queue, table, entries)
         lock_queue = Uniqueness.lock_queue_name(queue)
         entries.map { |key, _queue, msg_id| [key, lock_queue, msg_id, sent[[msg_id, key]]] }
       rescue ActiveRecord::StatementInvalid
@@ -1662,13 +1669,15 @@ module Pgbus
       end
 
       # { [msg_id, uniqueness key] => enqueued_at } for the messages of
-      # +entries+ ([key, queue, msg_id]) still in the logical queue. Every
-      # priority sub-queue is searched: a failed event records the logical
-      # queue only, and msg_ids repeat across sub-queues, so the key must
+      # +entries+ ([key, queue, msg_id, table]) still queued. A row that names
+      # its table (issue #583) is that table's message. A row written before
+      # source_queue records the logical queue only, so every priority
+      # sub-queue is searched: msg_ids repeat across them, so the key must
       # match too, and the earliest send wins.
-      def queued_send_times(queue, entries)
+      def queued_send_times(queue, table, entries)
         msg_ids = "{#{entries.map { |_key, _queue, msg_id| msg_id }.uniq.join(",")}}"
-        @client.physical_queue_names(queue).each_with_object({}) do |physical, found|
+        tables = table ? [table] : @client.physical_queue_names(queue)
+        tables.each_with_object({}) do |physical, found|
           queued_rows(physical, msg_ids).each do |row|
             pair = [row["msg_id"].to_i, row["lock_key"]]
             found[pair] = [found[pair], row["enqueued_at"]].compact.min
@@ -1722,15 +1731,22 @@ module Pgbus
       end
 
       # The physical queue holding a failed job's message, or nil when it is
-      # gone. The row records the logical queue only, and under priority
-      # routing a msg_id repeats across the _pN sub-queues, so the job's
-      # job_id must match too (issue #577). Two sub-queues' failures with the
-      # same msg_id share one row, which keeps the first one's payload: that
-      # is the message found (issue #583).
+      # gone. The job's job_id must match too: under priority routing a msg_id
+      # repeats across the _pN sub-queues (issue #577). A row that names the
+      # table its message was read from is looked up there alone (issue
+      # #583); a row written before source_queue, under every table of its
+      # logical queue.
       def failed_message_queue(event)
         return unless event["queue_name"] && event["msg_id"]
 
-        @client.locate_message(event["queue_name"], msg_id: event["msg_id"].to_i, job_id: failed_job_id(event))
+        name, prefixed = failed_message_scope(event)
+        @client.locate_message(name, msg_id: event["msg_id"].to_i, job_id: failed_job_id(event), prefixed: prefixed)
+      end
+
+      # [queue, prefixed] to look a failed row's message up under: the physical
+      # table the row names, else its logical queue.
+      def failed_message_scope(event)
+        event["source_queue"] ? [event["source_queue"], false] : [event["queue_name"], true]
       end
 
       def failed_job_id(event)
@@ -1756,16 +1772,19 @@ module Pgbus
       # per-row roundtrips. Falls back to per-row archive inside a rescue if
       # a batch fails, so one bad queue can't block progress on the others.
       def archive_all_failed_messages
+        source = FailedEventRecorder.source_queue_column? ? ", source_queue" : ""
         rows = connection.select_all(
-          "SELECT id, queue_name, msg_id, payload FROM pgbus_failed_events WHERE msg_id IS NOT NULL AND #{NOT_HANDLER_QUEUE}",
+          "SELECT id, queue_name, msg_id, payload#{source} FROM pgbus_failed_events " \
+          "WHERE msg_id IS NOT NULL AND #{NOT_HANDLER_QUEUE}",
           "Pgbus Collect Failed Messages", [handler_queue_names_literal]
         )
 
-        grouped = rows.to_a.group_by { |row| row["queue_name"] }
-        grouped.each do |queue_name, events|
+        grouped = rows.to_a.group_by { |row| failed_message_scope(row) }
+        grouped.each do |(queue_name, prefixed), events|
           # A priority queue's msg_ids repeat across its _pN sub-queues, so
           # each message is located first and archived from its own table.
-          located = @client.locate_messages(queue_name, events.map { |row| [row["msg_id"].to_i, failed_job_id(row)] })
+          located = @client.locate_messages(queue_name, events.map { |row| [row["msg_id"].to_i, failed_job_id(row)] },
+                                            prefixed: prefixed)
           located.group_by { |_msg_id, physical| physical }.each do |physical, pairs|
             @client.archive_batch(physical, pairs.map(&:first), prefixed: false)
           end

data/lib/pgbus/web/health_app.rb

@@ -98,7 +98,7 @@ module Pgbus
       def local_readyz
         snapshot = @local_readiness.call
         status = snapshot.ready? ? 200 : 503
-        body = { status: snapshot.status, expected: snapshot.expected, live: snapshot.live }
+        body = { status: snapshot.status, expected: snapshot.expected, live: snapshot.live, forked: snapshot.forked }
         [status, JSON_HEADERS.dup, [body.to_json]]
       rescue StandardError => e
         Pgbus.logger.error { "[Pgbus::Web::HealthApp] local readiness check failed: #{e.class}: #{e.message}" }

data/sig/pgbus/configuration.rbs

@@ -24,6 +24,8 @@ module Pgbus
     def batch_stall_threshold=: (untyped value) -> untyped
     def batch_sweep_interval: () -> Numeric?
     def batch_sweep_interval=: (untyped value) -> untyped
+    def boot_timeout: () -> Numeric?
+    def boot_timeout=: (untyped) -> untyped
     def capsule: (untyped name, queues: untyped, threads: untyped, **untyped options) -> untyped
     def capsule_named: (untyped name) -> untyped
     def circuit_breaker_enabled: () -> bool
@@ -83,6 +85,8 @@ module Pgbus
     def health_bind: () -> String
     def health_bind=: (untyped) -> untyped
     def health_port: () -> untyped
+    def health_unclaimed_grace: () -> untyped
+    def health_unclaimed_grace=: (untyped value) -> untyped
     def health_port=: (untyped) -> untyped
     def idempotency_ttl: () -> Numeric?
     def idempotency_ttl=: (untyped value) -> untyped

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.21.1
+  version: 0.21.3
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -287,6 +287,7 @@ files:
 - lib/generators/pgbus/add_batch_executions_generator.rb
 - lib/generators/pgbus/add_deploy_tasks_generator.rb
 - lib/generators/pgbus/add_failed_events_index_generator.rb
+- lib/generators/pgbus/add_failed_events_source_queue_generator.rb
 - lib/generators/pgbus/add_job_stats_generator.rb
 - lib/generators/pgbus/add_job_stats_latency_generator.rb
 - lib/generators/pgbus/add_job_stats_queue_index_generator.rb
@@ -307,6 +308,7 @@ files:
 - lib/generators/pgbus/templates/add_batch_callback_jobs.rb.erb
 - lib/generators/pgbus/templates/add_batch_executions.rb.erb
 - lib/generators/pgbus/templates/add_deploy_tasks_tables.rb.erb
+- lib/generators/pgbus/templates/add_failed_events_source_queue.rb.erb
 - lib/generators/pgbus/templates/add_failed_events_unique_index.rb.erb
 - lib/generators/pgbus/templates/add_job_stats.rb.erb
 - lib/generators/pgbus/templates/add_job_stats_latency.rb.erb