pgbus 0.20.0 → 0.20.1

data/lib/pgbus/process/dispatcher.rb

@@ -505,12 +505,22 @@ module Pgbus
       # Unbound rows (synthetic "pending" queue and/or msg_id=0) must not probe
       # pgmq.q_<prefix>_pending — that table does not exist, message_exists?
       # returns nil, and the reaper would keep the lock forever (issue #418).
-      # Scan every live queue for the payload key instead.
+      # Scan every live queue for the payload key instead. A job parked behind
+      # its concurrency slot is in no queue: its key counts as live while it
+      # is parked (issue #564). The parked jobs are asked first: a promotion
+      # sends the job's message before its parked row's delete commits (or
+      # both in one transaction), so a job promoted between the two reads is
+      # still parked for the first or already queued for the second. Read the
+      # other way round it could be in neither.
       def gone_unbound_locks(unbound)
         return [] if unbound.empty?
 
-        present = Pgbus.client.uniqueness_keys_present(unbound.map(&:lock_key))
-        unbound.reject { |key| present.include?(key.lock_key) }
+        parked = BlockedExecution.parked_uniqueness_keys(unbound.map(&:lock_key))
+        unparked = unbound.reject { |key| parked.include?(key.lock_key) }
+        return [] if unparked.empty?
+
+        present = Pgbus.client.uniqueness_keys_present(unparked.map(&:lock_key))
+        unparked.reject { |key| present.include?(key.lock_key) }
       end
 
       # Returns true if the message referenced by this bound lock is definitely

data/lib/pgbus/process/worker.rb

@@ -407,12 +407,21 @@ module Pgbus
       # so we also pass `limit: qty` to cap the total across all queues —
       # otherwise we get `queue_count * qty` messages and overflow the
       # execution pool, crashing the worker fork (issue #123).
+      #
+      # The logical name is the configured one the table was made from, not
+      # the table name with its prefix cut: a name normalized for its table
+      # ("orders-handler" → "orders_handler") would otherwise reach the
+      # executor spelled unlike the name its jobs' uniqueness locks are bound
+      # under, and unlike every other fetch path (issue #564).
       def fetch_multi(active_queues, qty)
         messages = Pgbus.client.read_multi(active_queues, qty: qty, limit: qty) || []
+        return messages if messages.empty?
+
         prefix = "#{config.queue_prefix}_"
+        configured = active_queues.to_h { |q| [config.queue_name(q), q] }
 
         messages.map do |m|
-          logical = m.queue_name&.delete_prefix(prefix) || active_queues.first
+          logical = configured[m.queue_name] || m.queue_name&.delete_prefix(prefix) || active_queues.first
           [logical, m]
         end
       end

data/lib/pgbus/recurring/schedule.rb

@@ -143,7 +143,7 @@ module Pgbus
         key = resolve_uniqueness_key(job_class, config, task)
         return nil unless key
 
-        acquired = UniquenessKey.acquire!(key, queue_name: resolve_queue(task), msg_id: 0)
+        acquired = UniquenessKey.acquire!(key, queue_name: Uniqueness.lock_queue_name(resolve_queue(task)), msg_id: 0)
 
         if acquired
           key

data/lib/pgbus/streams/presence.rb

@@ -72,6 +72,7 @@ module Pgbus
       @schema_mutex = Mutex.new
       @seats_supported = nil
       @probed_at = nil
+      @schema_generation = 0
 
       class << self
         # True when pgbus_presence_members has the seat columns (the
@@ -81,17 +82,29 @@ module Pgbus
         # The answer is read and written under @schema_mutex only (issue
         # #527). A successful probe is kept as above; a probe that raises is
         # not, so the next call probes again.
+        #
+        # The probe itself runs outside @schema_mutex (issue #566): it waits
+        # for the adapter's lock, and a request inside a transaction holds
+        # that lock while it asks seats_supported?. Two threads may probe at
+        # once; a true answer one of them stored is never taken back, and a
+        # probe that a reset_schema_cache! overtook stores nothing.
         def seats_supported?
+          cached, generation = @schema_mutex.synchronize do
+            [(@seats_supported if schema_answer_fresh?(@seats_supported, @probed_at)), @schema_generation]
+          end
+          return cached unless cached.nil?
+
+          rows = exec_params(<<~SQL, [])
+            SELECT 1 AS present FROM information_schema.columns
+            WHERE table_name = 'pgbus_presence_members' AND column_name = 'connection_id'
+              AND table_schema = ANY (current_schemas(false))
+          SQL
+          supported = !rows.to_a.empty?
           @schema_mutex.synchronize do
-            next @seats_supported if schema_answer_fresh?(@seats_supported, @probed_at)
+            next supported unless generation == @schema_generation
 
-            rows = connection.exec_params(<<~SQL, [])
-              SELECT 1 AS present FROM information_schema.columns
-              WHERE table_name = 'pgbus_presence_members' AND column_name = 'connection_id'
-                AND table_schema = ANY (current_schemas(false))
-            SQL
             @probed_at = monotonic_now
-            @seats_supported = !rows.to_a.empty?
+            @seats_supported ||= supported
           end
         end
 
@@ -108,6 +121,7 @@ module Pgbus
 
         def reset_schema_cache!
           @schema_mutex.synchronize do
+            @schema_generation += 1
             @probed_at = nil
             @seats_supported = nil
           end
@@ -124,7 +138,7 @@ module Pgbus
         def expire_seats!(ttl_for:, min_ttl:)
           return [] unless seats_supported?
 
-          candidates = connection.exec_params(<<~SQL, [min_ttl])
+          candidates = exec_params(<<~SQL, [min_ttl])
             SELECT stream_name, connection_id FROM pgbus_presence_members
             WHERE connection_id IS NOT NULL AND last_seen_at < NOW() - make_interval(secs => $1)
           SQL
@@ -133,23 +147,23 @@ module Pgbus
           end
         end
 
+        # The PG::Connection under the adapter. Run SQL through exec_params,
+        # not on this: the adapter's lock is released once it is returned.
         def connection
-          # Respect multi-database setups (connects_to). When
-          # Pgbus.configuration.connects_to is set, the pgbus tables
-          # live in a separate database accessed via Pgbus::BusRecord;
-          # the default path uses ActiveRecord::Base. Matches the
-          # canonical pattern in Pgbus::Process::QueueLock#connection
-          # and Pgbus::Configuration's connection probe.
-          unless defined?(::ActiveRecord::Base)
-            raise Pgbus::ConfigurationError,
-                  "Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
-          end
+          adapter.raw_connection
+        end
 
-          if Pgbus.configuration.connects_to
-            Pgbus::BusRecord.connection.raw_connection
-          else
-            ::ActiveRecord::Base.connection.raw_connection
-          end
+        # Runs one statement on the adapter's PG::Connection while holding
+        # the adapter's lock (issue #566). raw_connection takes that lock
+        # only to hand the connection out; on a pinned pool (transactional
+        # tests) every thread shares the one adapter, and its lock is all
+        # that keeps a second thread's statement off the libpq connection
+        # while this one is in flight. Unpinned, the lock is a NullLock. The
+        # lock is reentrant, so a thread already holding it (a request
+        # inside a transaction) re-enters it.
+        def exec_params(sql, binds)
+          adapter = self.adapter
+          adapter.lock.synchronize { adapter.raw_connection.exec_params(sql, binds) }
         end
 
         # Reports a seat join/leave to streams_presence_on_change (issue #515).
@@ -171,6 +185,21 @@ module Pgbus
 
         private
 
+        # Respect multi-database setups (connects_to). When
+        # Pgbus.configuration.connects_to is set, the pgbus tables
+        # live in a separate database accessed via Pgbus::BusRecord;
+        # the default path uses ActiveRecord::Base. Matches the
+        # canonical pattern in Pgbus::Process::QueueLock#connection
+        # and Pgbus::Configuration's connection probe.
+        def adapter
+          unless defined?(::ActiveRecord::Base)
+            raise Pgbus::ConfigurationError,
+                  "Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
+          end
+
+          Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection : ::ActiveRecord::Base.connection
+        end
+
         def schema_answer_fresh?(supported, probed_at)
           return false if supported.nil? || probed_at.nil?
 
@@ -178,7 +207,7 @@ module Pgbus
         end
 
         def expire_stream_seats(stream_name, connection_ids, ttl)
-          rows = connection.exec_params(<<~SQL, [stream_name, text_array(connection_ids), ttl])
+          rows = exec_params(<<~SQL, [stream_name, text_array(connection_ids), ttl])
             DELETE FROM pgbus_presence_members
             WHERE stream_name = $1 AND connection_id = ANY($2::text[])
               AND last_seen_at < NOW() - make_interval(secs => $3)
@@ -306,7 +335,7 @@ module Pgbus
         return touch_seats(Array(connection_id)) if connection_id
 
         member_id = member_id.to_s
-        connection.exec_params(<<~SQL, [@stream.name, member_id])
+        exec_params(<<~SQL, [@stream.name, member_id])
           UPDATE pgbus_presence_members
           SET last_seen_at = NOW()
           WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
@@ -320,7 +349,7 @@ module Pgbus
       def set_state(connection_id:, state:)
         raise ArgumentError, "set_state takes a Hash (got #{state.class})" unless state.is_a?(Hash)
 
-        rows = connection.exec_params(<<~SQL, [@stream.name, connection_id.to_s, JSON.generate(state)])
+        rows = exec_params(<<~SQL, [@stream.name, connection_id.to_s, JSON.generate(state)])
           UPDATE pgbus_presence_members
           SET state = state || $3::jsonb, last_seen_at = NOW()
           WHERE stream_name = $1 AND connection_id = $2
@@ -335,7 +364,7 @@ module Pgbus
       # table each row is one seat and also carries connection_id, seat
       # and state; group by "id" for one entry per member.
       def members
-        rows = connection.exec_params(<<~SQL, [@stream.name])
+        rows = exec_params(<<~SQL, [@stream.name])
           SELECT #{seats? ? SEAT_RECORD_COLUMNS : RECORD_COLUMNS}
           FROM pgbus_presence_members
           WHERE stream_name = $1
@@ -347,7 +376,7 @@ module Pgbus
       # Returns the count of current members. Faster than members.size
       # because it doesn't deserialize metadata.
       def count
-        rows = connection.exec_params(<<~SQL, [@stream.name])
+        rows = exec_params(<<~SQL, [@stream.name])
           SELECT COUNT(*) AS n FROM pgbus_presence_members WHERE stream_name = $1
         SQL
         rows.first["n"].to_i
@@ -362,7 +391,7 @@ module Pgbus
       # workers running the sweep concurrently won't double-emit leave
       # events. Yields each expired record when a block is given.
       def sweep!(older_than:)
-        rows = connection.exec_params(<<~SQL, [@stream.name, older_than])
+        rows = exec_params(<<~SQL, [@stream.name, older_than])
           DELETE FROM pgbus_presence_members
           WHERE stream_name = $1 AND last_seen_at < $2#{" AND connection_id IS NULL" if seats_now?}
           RETURNING #{RECORD_COLUMNS}
@@ -388,7 +417,7 @@ module Pgbus
 
       def upsert_member_once(member_id, metadata)
         conflict = seats? ? "(stream_name, member_id) WHERE connection_id IS NULL" : "(stream_name, member_id)"
-        rows = connection.exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata)])
+        rows = exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata)])
           INSERT INTO pgbus_presence_members (stream_name, member_id, metadata, joined_at, last_seen_at)
           VALUES ($1, $2, $3::jsonb, NOW(), NOW())
           ON CONFLICT #{conflict}
@@ -399,7 +428,7 @@ module Pgbus
       end
 
       def upsert_seat(member_id, metadata, connection_id, seat)
-        rows = connection.exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata), connection_id, seat])
+        rows = exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata), connection_id, seat])
           INSERT INTO pgbus_presence_members
             (stream_name, member_id, metadata, connection_id, seat, joined_at, last_seen_at)
           VALUES ($1, $2, $3::jsonb, $4, $5, NOW(), NOW())
@@ -412,7 +441,7 @@ module Pgbus
       end
 
       def delete_member(member_id)
-        rows = connection.exec_params(<<~SQL, [@stream.name, member_id])
+        rows = exec_params(<<~SQL, [@stream.name, member_id])
           DELETE FROM pgbus_presence_members
           WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
           RETURNING #{RECORD_COLUMNS}
@@ -422,7 +451,7 @@ module Pgbus
       end
 
       def delete_seat(connection_id)
-        rows = connection.exec_params(<<~SQL, [@stream.name, connection_id])
+        rows = exec_params(<<~SQL, [@stream.name, connection_id])
           DELETE FROM pgbus_presence_members
           WHERE stream_name = $1 AND connection_id = $2
           RETURNING #{SEAT_RECORD_COLUMNS}
@@ -434,7 +463,7 @@ module Pgbus
       def touch_seats(connection_ids)
         return [] if connection_ids.empty?
 
-        rows = connection.exec_params(<<~SQL, [@stream.name, self.class.text_array(connection_ids)])
+        rows = exec_params(<<~SQL, [@stream.name, self.class.text_array(connection_ids)])
           UPDATE pgbus_presence_members
           SET last_seen_at = NOW()
           WHERE stream_name = $1 AND connection_id = ANY($2::text[])
@@ -443,8 +472,8 @@ module Pgbus
         rows.to_a.map { |row| row["connection_id"] }
       end
 
-      def connection
-        self.class.connection
+      def exec_params(sql, binds)
+        self.class.exec_params(sql, binds)
       end
     end
   end

data/lib/pgbus/uniqueness.rb

@@ -16,8 +16,14 @@ module Pgbus
   #      or executing.
   #   2. Execution: PGMQ's visibility timeout is the execution lock —
   #      no separate claim_for_execution step needed.
-  #   3. Completion/DLQ: DELETE FROM pgbus_uniqueness_keys WHERE lock_key = ?.
-  #   4. Crash recovery: if a worker dies, VT expires, the message becomes
+  #   3. Retry: a retry_job / retry_on re-enqueue passes through the key the
+  #      running attempt holds, and the row is handed to the retry: unbound
+  #      and re-stamped before the retry is sent or parked (unbind!), bound
+  #      to it once sent (bind!).
+  #   4. Completion/DLQ: the finishing message deletes the row only while it
+  #      holds it (release_held_by!), so a row handed to a retry, or a
+  #      successor's, stays. The dashboard's discards do the same.
+  #   5. Crash recovery: if a worker dies, VT expires, the message becomes
   #      readable again. The uniqueness key row stays (correctly — the job
   #      hasn't finished). The next worker picks it up and executes.
   #
@@ -179,14 +185,25 @@ module Pgbus
         return :no_lock unless config[:strategy] == :until_executed
 
         acquired = if msg_id && queue_name
-                     UniquenessKey.acquire!(key, queue_name: queue_name, msg_id: msg_id)
+                     UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id)
                    else
                      # Pre-produce check: use advisory lock + ON CONFLICT
-                     UniquenessKey.acquire!(key, queue_name: queue_name || PLACEHOLDER_QUEUE, msg_id: msg_id || 0)
+                     UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name || PLACEHOLDER_QUEUE),
+                                                 msg_id: msg_id || 0)
                    end
         acquired ? :acquired : :locked
       end
 
+      # The queue a job's uniqueness lock records: the logical name as its
+      # table spells it ("orders-handler" → "orders_handler"), the inverse of
+      # stripping the prefix off the table name. Every writer and releaser of
+      # a job's lock goes through it, so a release that knows only the table
+      # (the dashboard) or a worker that spells the queue the table's way
+      # still finds the row it was bound under (issue #564).
+      def lock_queue_name(queue_name, config: Pgbus.configuration)
+        config.queue_name(queue_name).delete_prefix("#{config.queue_prefix}_")
+      end
+
       # Acquire the uniqueness lock at execution time (:while_executing only).
       # Returns true if acquired, false if another instance is running.
       #
@@ -199,7 +216,7 @@ module Pgbus
         return true unless strategy == :while_executing
 
         queue_name ||= payload["queue_name"] || "unknown"
-        UniquenessKey.acquire!(key, queue_name: queue_name, msg_id: msg_id.to_i,
+        UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id.to_i,
                                     reacquire_same_message: msg_id.to_i.positive?)
       end
 
@@ -217,7 +234,7 @@ module Pgbus
         return unless key
         return unless msg_id.to_i.positive?
 
-        UniquenessKey.bind!(key, queue_name: queue_name, msg_id: msg_id)
+        UniquenessKey.bind!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id)
       end
 
       # True when the uniqueness row is not yet bound to a real PGMQ message.

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.20.0"
+  VERSION = "0.20.1"
 end

data/lib/pgbus/web/data_source/workflow_list.rb

@@ -17,6 +17,7 @@ module Pgbus
         # The pause events the timeline pairs, read on their own so a long
         # audit trail never drops a run's earliest pauses from the chart.
         PAUSE_HISTORY = 1_000
+        WAIT_TIMEOUT_HISTORY = 200
         SIGNAL_HISTORY = 100
         RERUN_LINKS = 10
         ATTENTION_GROUPS = 200
@@ -46,7 +47,8 @@ module Pgbus
 
           steps = WorkflowStep.where(workflow_id: workflow_id).order(id: :desc).limit(STEP_HISTORY).to_a
           progress = batch_progress([step_attribute(steps.first, :batch_id)].compact)
-          workflow_row(record, steps.first, progress).merge(workflow_extras(record, steps))
+          late = late_since(steps.first&.status == "waiting" ? [steps.first] : [])
+          workflow_row(record, steps.first, progress, late).merge(workflow_extras(record, steps))
         rescue StandardError => e
           Pgbus.logger.error { "[Pgbus::Web] Error fetching workflow #{workflow_id}: #{e.class}: #{e.message}" }
           nil
@@ -96,7 +98,8 @@ module Pgbus
           latest = latest_steps(records.map(&:workflow_id))
           waiting = latest.values.select { |step| step.status == "waiting" }
           progress = batch_progress(waiting.filter_map { |step| step_attribute(step, :batch_id) })
-          records.map { |record| workflow_row(record, latest[record.workflow_id], progress) }
+          late = late_since(waiting)
+          records.map { |record| workflow_row(record, latest[record.workflow_id], progress, late) }
         rescue StandardError => e
           Pgbus.logger.error { "[Pgbus::Web] Error fetching workflows: #{e.class}: #{e.message}" }
           []
@@ -155,7 +158,37 @@ module Pgbus
           end
         end
 
-        def workflow_row(record, step, progress)
+        # { step_id => the first deadline its wait missed } for waiting steps
+        # with a deadline, one query: the first wait_timed_out event of each.
+        # Only on_timeout: :notify leaves a step waiting after that event
+        # (issue #565): every other on_timeout ends the attempt.
+        def late_since(waiting)
+          steps = waiting.select { |step| step_attribute(step, :timeout_at) }
+          return {} if steps.empty?
+
+          WorkflowEvent.where(workflow_id: steps.map(&:workflow_id), step_id: steps.map(&:id),
+                              event: WorkflowTimeline::WAIT_TIMED_OUT)
+                       .select("DISTINCT ON (step_id) step_id, details, created_at").order(Arel.sql("step_id, id"))
+                       .to_h { |event| [event.step_id, missed_deadline(event)] }
+        end
+
+        # The deadline the event reports; when it went out, for an event
+        # written without one.
+        def missed_deadline(event)
+          deadline = event.details.to_h["timeout_at"]
+          deadline ? Time.iso8601(deadline) : event.created_at
+        end
+
+        # The waiting step's on_timeout as its class declares it: nil without
+        # a wait_timeout:, or when the class is not loaded in this process.
+        def wait_on_timeout(record, step)
+          return nil unless step_attribute(step, :timeout_at)
+
+          definition = record.workflow_klass&.workflow_steps&.fetch(step.step_name)
+          definition.on_timeout.to_s if definition.is_a?(Workflow::StepDefinition)
+        end
+
+        def workflow_row(record, step, progress, late = {})
           waiting = step&.status == "waiting"
           { workflow_id: record.workflow_id, workflow_class: record.workflow_class, subject_type: record.subject_type,
             subject_id: record.subject_id, status: record.status, current_step: record.current_step,
@@ -163,6 +196,7 @@ module Pgbus
             waiting_for: (step_attribute(step, :wait_for) if waiting),
             waiting_since: (step_attribute(step, :waiting_since) if waiting),
             timeout_at: (step_attribute(step, :timeout_at) if waiting),
+            on_timeout: (wait_on_timeout(record, step) if waiting), late_since: (late[step.id] if waiting),
             batch_id: step_attribute(step, :batch_id),
             batch_progress: (progress[step_attribute(step, :batch_id)] if waiting),
             scheduled_at: (step.scheduled_at if step&.status == "scheduled"),
@@ -192,7 +226,7 @@ module Pgbus
             definition_outdated: record.definition_outdated?, class_loaded: !klass.nil?,
             step_names: klass ? klass.workflow_steps.names : [], resume_action: resume_action, resume_step: resume_step,
             signal_names: WorkflowActionPlan.signal_names(steps.first, klass), reruns: workflow_reruns(record.workflow_id),
-            timeline: WorkflowTimeline.chart(step_hashes, pause_events(record.workflow_id), status: record.status, now: now) }
+            timeline: WorkflowTimeline.chart(step_hashes, timeline_events(record.workflow_id), status: record.status, now: now) }
         end
 
         def workflow_step_hash(step)
@@ -207,9 +241,18 @@ module Pgbus
             signal_id: step_attribute(step, :signal_id) }
         end
 
-        def pause_events(workflow_id)
-          WorkflowEvent.where(workflow_id: workflow_id, event: WorkflowTimeline::PAUSE_EVENTS).order(:id).limit(PAUSE_HISTORY)
-                       .map { |event| { event: event.event, created_at: event.created_at } }
+        # The pauses (oldest first) and the newest wait_timed_out events the
+        # timeline marks, read apart so a wait late for weeks cannot crowd out
+        # the pauses.
+        def timeline_events(workflow_id)
+          pauses = WorkflowEvent.where(workflow_id: workflow_id, event: WorkflowTimeline::PAUSE_EVENTS).order(:id)
+                                .limit(PAUSE_HISTORY).map { |event| { event: event.event, created_at: event.created_at } }
+          timeouts = WorkflowEvent.where(workflow_id: workflow_id, event: WorkflowTimeline::WAIT_TIMED_OUT)
+                                  .order(id: :desc).limit(WAIT_TIMEOUT_HISTORY).map do |event|
+            { event: event.event, created_at: event.created_at, step_name: event.step_name, attempt: event.attempt,
+              details: event.details }
+          end
+          pauses + timeouts
         end
 
         def workflow_signals(workflow_id)

data/lib/pgbus/web/data_source.rb

@@ -321,7 +321,7 @@ module Pgbus
       def discard_failed_event(id)
         event = failed_event(id)
         if event
-          release_lock_for_payload(event["payload"])
+          release_locks_held_by_failed_events("id = $1", [id.to_i])
           archive_failed_message(event)
         end
 
@@ -920,7 +920,8 @@ module Pgbus
         # memory — otherwise the message is archived but the lock row stays
         # behind, blocking later publishes with the same key. Mirrors
         # discard_event.
-        release_lock_for_payload(detail[:message])
+        release_lock_held_by(detail[:message], queue_name: lock_queue_for_table(queue_name), msg_id: detail[:msg_id],
+                                               enqueued_at: detail[:enqueued_at])
         @client.archive_message(queue_name, msg_id.to_i, prefixed: false)
         clear_event_failure(queue_name, msg_id)
         true
@@ -1554,6 +1555,13 @@ module Pgbus
       end
 
       # --- Lock cleanup helpers ---
+      #
+      # Discarding a message releases only the uniqueness lock that message
+      # holds (UniquenessKey.release_held_by!): its job may have handed the
+      # lock to a retry that is still waiting (issue #564), or a successor may
+      # have taken the key, and neither is the discarded message's to free.
+      # The Locks page's own deletes stay key-only: there the key is the
+      # operator's target.
 
       # Extract uniqueness key from a queue message and release its lock.
       def release_lock_for_message(queue_name, msg_id)
@@ -1564,39 +1572,139 @@ module Pgbus
         )
         return unless row
 
-        release_lock_for_payload(row["message"])
+        holder_queue, holder_msg_id, sent_at = lock_holder(queue_name, row["msg_id"], row["headers"], row["enqueued_at"])
+        release_lock_held_by(row["message"], queue_name: holder_queue, msg_id: holder_msg_id, enqueued_at: sent_at)
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error releasing lock for message #{msg_id}: #{e.message}" }
       end
 
-      # Extract uniqueness key from a JSON payload string and release its lock.
-      def release_lock_for_payload(payload_str)
+      # [queue, msg_id, sent_at] of the message whose lock a discard may
+      # release. A live message is its own holder. A dead-lettered copy's is
+      # the message it was moved from, which its dead-letter header names:
+      # the copy's own enqueued_at is the move to the DLQ, later than a lock
+      # its job may have handed to a retry. A copy whose header does not name
+      # the original (dead-lettered by an earlier release, which freed its
+      # lock then) has no send time to go by, so only a bound lock could
+      # match, and none is bound to a DLQ copy.
+      def lock_holder(queue_name, msg_id, headers, enqueued_at)
+        own = [lock_queue_for_table(queue_name), msg_id.to_i]
+        return [*own, enqueued_at] unless queue_name.to_s.end_with?(Pgbus::DEAD_LETTER_SUFFIX)
+
+        block = DeadLetterHeader.parse(headers)
+        source_msg_id = block && Integer(block["source_msg_id"], exception: false)
+        return [*own, nil] unless source_msg_id && block["source_queue"]
+
+        [lock_queue_for_table(block["source_queue"]), source_msg_id, block["source_enqueued_at"]]
+      end
+
+      # Release the lock held by the message whose JSON payload this is.
+      def release_lock_held_by(payload_str, queue_name:, msg_id:, enqueued_at:)
         key = extract_uniqueness_key_from_payload_str(payload_str)
-        UniquenessKey.release!(key) if key
+        return unless key
+
+        UniquenessKey.release_held_by!(key, queue_name: queue_name, msg_id: msg_id.to_i, enqueued_at: enqueued_at)
       end
 
-      # Extract uniqueness keys from a collection of formatted messages and
-      # release all associated locks in a single query.
+      # Release the locks a collection of formatted messages hold, in a single
+      # query.
       def release_locks_for_messages(messages)
-        keys = messages.filter_map { |m| extract_uniqueness_key_from_payload_str(m[:message]) }
-        UniquenessKey.where(lock_key: keys).delete_all if keys.any?
+        holders = messages.filter_map do |m|
+          key = extract_uniqueness_key_from_payload_str(m[:message])
+          [key, *lock_holder(m[:queue_name], m[:msg_id], m[:headers], m[:enqueued_at])] if key
+        end
+        UniquenessKey.release_all_held_by!(holders) if holders.any?
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for messages: #{e.message}" }
       end
 
-      # Collect uniqueness keys from all failed events and release their locks.
       def release_locks_for_failed_events
+        release_locks_held_by_failed_events(NOT_HANDLER_QUEUE, [handler_queue_names_literal])
+      end
+
+      # Release the locks the messages of the failed jobs matching +condition+
+      # hold. A failed job's message is still in its queue, waiting for its
+      # next read, and its enqueued_at is the send time: failed_at is not, a
+      # lock handed to a retry before the failure would count as the job's.
+      # A message that is gone has no send time, so only a bound lock goes.
+      def release_locks_held_by_failed_events(condition, binds)
         rows = connection.select_all(
-          "SELECT payload FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE}", "Pgbus Collect Failed Keys",
-          [handler_queue_names_literal]
+          "SELECT queue_name, msg_id, payload FROM pgbus_failed_events WHERE #{condition}",
+          "Pgbus Collect Failed Keys", binds
         )
 
-        keys = rows.to_a.filter_map { |row| extract_uniqueness_key_from_payload_str(row["payload"]) }
-        UniquenessKey.where(lock_key: keys).delete_all if keys.any?
+        # queue_name is the logical name the executor saw: the message is looked
+        # up under it, and the lock is released under its table spelling.
+        # A key proc may return a number; the queue lookup reads it as text.
+        holders = rows.to_a.filter_map do |row|
+          key = extract_uniqueness_key_from_payload_str(row["payload"])&.to_s
+          [key, row["queue_name"], row["msg_id"].to_i] if key && row["msg_id"]
+        end
+        holders = holders.group_by { |_key, queue, _msg_id| queue }.flat_map do |queue, entries|
+          failed_queue_lock_holders(queue, entries)
+        end
+        UniquenessKey.release_all_held_by!(holders) if holders.any?
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for failed events: #{e.message}" }
       end
 
+      # The release tuples for one queue's failed jobs. A queue that cannot
+      # be resolved (a stored name no longer valid as a table name) is skipped
+      # with a warning, so it does not stop the other queues' release.
+      def failed_queue_lock_holders(queue, entries)
+        sent = queued_send_times(queue, entries)
+        lock_queue = Uniqueness.lock_queue_name(queue)
+        entries.map { |key, _queue, msg_id| [key, lock_queue, msg_id, sent[[msg_id, key]]] }
+      rescue ActiveRecord::StatementInvalid
+        raise
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus::Web] Skipped releasing locks for failed jobs on #{queue.inspect}: #{e.message}" }
+        []
+      end
+
+      # { [msg_id, uniqueness key] => enqueued_at } for the messages of
+      # +entries+ ([key, queue, msg_id]) still in the logical queue. Every
+      # priority sub-queue is searched: a failed event records the logical
+      # queue only, and msg_ids repeat across sub-queues, so the key must
+      # match too, and the earliest send wins.
+      def queued_send_times(queue, entries)
+        msg_ids = "{#{entries.map { |_key, _queue, msg_id| msg_id }.uniq.join(",")}}"
+        @client.physical_queue_names(queue).each_with_object({}) do |physical, found|
+          queued_rows(physical, msg_ids).each do |row|
+            pair = [row["msg_id"].to_i, row["lock_key"]]
+            found[pair] = [found[pair], row["enqueued_at"]].compact.min
+          end
+        end
+      end
+
+      # Each read runs in its own savepoint: a sub-queue the strategy names
+      # but that does not exist raises, and inside a caller's transaction
+      # that would abort the discard's own statements after it.
+      def queued_rows(physical, msg_ids)
+        connection.transaction(requires_new: true) do
+          connection.select_all(
+            "SELECT msg_id, enqueued_at, message->>'#{Uniqueness::METADATA_KEY}' AS lock_key " \
+            "FROM pgmq.q_#{sanitize_name(physical)} WHERE msg_id = ANY($1::bigint[])",
+            "Pgbus Failed Message Sent At", [msg_ids]
+          ).to_a
+        end
+      rescue ActiveRecord::StatementInvalid => e
+        raise unless defined?(PG::UndefinedTable) && e.cause.is_a?(PG::UndefinedTable)
+
+        []
+      end
+
+      # The queue a uniqueness lock records for a message in this physical
+      # queue: the table name without its prefix and a priority sub-queue's
+      # _pN, which is how Uniqueness.lock_queue_name spells the queue the lock
+      # was bound for. A dead-letter queue keeps its suffix, so its copy's
+      # msg_id (another sequence) never matches a lock bound to a live message.
+      def lock_queue_for_table(physical_name)
+        name = physical_name.to_s.delete_prefix("#{Pgbus.configuration.queue_prefix}_")
+        return name if name.end_with?(Pgbus::DEAD_LETTER_SUFFIX) || Pgbus.configuration.priority_levels.to_i <= 1
+
+        name.sub(/_p\d+\z/, "")
+      end
+
       # Single unwrap point for PGMQ message / failed_event payload strings.
       # Accepts a raw JSON string or an already-parsed Hash and returns the
       # uniqueness metadata key, or nil when the payload is blank, unparseable,