pgbus 0.20.0 → 0.21.0

data/lib/pgbus/uniqueness.rb

@@ -16,8 +16,14 @@ module Pgbus
   #      or executing.
   #   2. Execution: PGMQ's visibility timeout is the execution lock —
   #      no separate claim_for_execution step needed.
-  #   3. Completion/DLQ: DELETE FROM pgbus_uniqueness_keys WHERE lock_key = ?.
-  #   4. Crash recovery: if a worker dies, VT expires, the message becomes
+  #   3. Retry: a retry_job / retry_on re-enqueue passes through the key the
+  #      running attempt holds, and the row is handed to the retry: unbound
+  #      and re-stamped before the retry is sent or parked (unbind!), bound
+  #      to it once sent (bind!).
+  #   4. Completion/DLQ: the finishing message deletes the row only while it
+  #      holds it (release_held_by!), so a row handed to a retry, or a
+  #      successor's, stays. The dashboard's discards do the same.
+  #   5. Crash recovery: if a worker dies, VT expires, the message becomes
   #      readable again. The uniqueness key row stays (correctly — the job
   #      hasn't finished). The next worker picks it up and executes.
   #
@@ -179,14 +185,25 @@ module Pgbus
         return :no_lock unless config[:strategy] == :until_executed
 
         acquired = if msg_id && queue_name
-                     UniquenessKey.acquire!(key, queue_name: queue_name, msg_id: msg_id)
+                     UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id)
                    else
                      # Pre-produce check: use advisory lock + ON CONFLICT
-                     UniquenessKey.acquire!(key, queue_name: queue_name || PLACEHOLDER_QUEUE, msg_id: msg_id || 0)
+                     UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name || PLACEHOLDER_QUEUE),
+                                                 msg_id: msg_id || 0)
                    end
         acquired ? :acquired : :locked
       end
 
+      # The queue a job's uniqueness lock records: the logical name as its
+      # table spells it ("orders-handler" → "orders_handler"), the inverse of
+      # stripping the prefix off the table name. Every writer and releaser of
+      # a job's lock goes through it, so a release that knows only the table
+      # (the dashboard) or a worker that spells the queue the table's way
+      # still finds the row it was bound under (issue #564).
+      def lock_queue_name(queue_name, config: Pgbus.configuration)
+        config.queue_name(queue_name).delete_prefix("#{config.queue_prefix}_")
+      end
+
       # Acquire the uniqueness lock at execution time (:while_executing only).
       # Returns true if acquired, false if another instance is running.
       #
@@ -199,7 +216,7 @@ module Pgbus
         return true unless strategy == :while_executing
 
         queue_name ||= payload["queue_name"] || "unknown"
-        UniquenessKey.acquire!(key, queue_name: queue_name, msg_id: msg_id.to_i,
+        UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id.to_i,
                                     reacquire_same_message: msg_id.to_i.positive?)
       end
 
@@ -217,7 +234,7 @@ module Pgbus
         return unless key
         return unless msg_id.to_i.positive?
 
-        UniquenessKey.bind!(key, queue_name: queue_name, msg_id: msg_id)
+        UniquenessKey.bind!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id)
       end
 
       # True when the uniqueness row is not yet bound to a real PGMQ message.

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.20.0"
+  VERSION = "0.21.0"
 end

data/lib/pgbus/web/data_source/workflow_list.rb

@@ -17,6 +17,7 @@ module Pgbus
         # The pause events the timeline pairs, read on their own so a long
         # audit trail never drops a run's earliest pauses from the chart.
         PAUSE_HISTORY = 1_000
+        WAIT_TIMEOUT_HISTORY = 200
         SIGNAL_HISTORY = 100
         RERUN_LINKS = 10
         ATTENTION_GROUPS = 200
@@ -46,7 +47,8 @@ module Pgbus
 
           steps = WorkflowStep.where(workflow_id: workflow_id).order(id: :desc).limit(STEP_HISTORY).to_a
           progress = batch_progress([step_attribute(steps.first, :batch_id)].compact)
-          workflow_row(record, steps.first, progress).merge(workflow_extras(record, steps))
+          late = late_since(steps.first&.status == "waiting" ? [steps.first] : [])
+          workflow_row(record, steps.first, progress, late).merge(workflow_extras(record, steps))
         rescue StandardError => e
           Pgbus.logger.error { "[Pgbus::Web] Error fetching workflow #{workflow_id}: #{e.class}: #{e.message}" }
           nil
@@ -96,7 +98,8 @@ module Pgbus
           latest = latest_steps(records.map(&:workflow_id))
           waiting = latest.values.select { |step| step.status == "waiting" }
           progress = batch_progress(waiting.filter_map { |step| step_attribute(step, :batch_id) })
-          records.map { |record| workflow_row(record, latest[record.workflow_id], progress) }
+          late = late_since(waiting)
+          records.map { |record| workflow_row(record, latest[record.workflow_id], progress, late) }
         rescue StandardError => e
           Pgbus.logger.error { "[Pgbus::Web] Error fetching workflows: #{e.class}: #{e.message}" }
           []
@@ -155,7 +158,39 @@ module Pgbus
           end
         end
 
-        def workflow_row(record, step, progress)
+        # { step_id => the first deadline its wait missed } for steps waiting
+        # on a signal, one query: the first wait_timed_out event of each.
+        # Only on_timeout: :notify leaves a step waiting after that event
+        # (issue #565): every other on_timeout ends the attempt. A per-run
+        # wait_timeout: may have cleared the deadline at that notice (issue
+        # #574), so a step without one is looked up too.
+        def late_since(waiting)
+          steps = waiting.select { |step| step_attribute(step, :wait_for) }
+          return {} if steps.empty?
+
+          WorkflowEvent.where(workflow_id: steps.map(&:workflow_id), step_id: steps.map(&:id),
+                              event: WorkflowTimeline::WAIT_TIMED_OUT)
+                       .select("DISTINCT ON (step_id) step_id, details, created_at").order(Arel.sql("step_id, id"))
+                       .to_h { |event| [event.step_id, missed_deadline(event)] }
+        end
+
+        # The deadline the event reports; when it went out, for an event
+        # written without one.
+        def missed_deadline(event)
+          deadline = event.details.to_h["timeout_at"]
+          deadline ? Time.iso8601(deadline) : event.created_at
+        end
+
+        # The waiting step's on_timeout as its class declares it: nil without
+        # a wait_timeout:, or when the class is not loaded in this process.
+        def wait_on_timeout(record, step)
+          return nil unless step_attribute(step, :timeout_at)
+
+          definition = record.workflow_klass&.workflow_steps&.fetch(step.step_name)
+          definition.on_timeout.to_s if definition.is_a?(Workflow::StepDefinition)
+        end
+
+        def workflow_row(record, step, progress, late = {})
           waiting = step&.status == "waiting"
           { workflow_id: record.workflow_id, workflow_class: record.workflow_class, subject_type: record.subject_type,
             subject_id: record.subject_id, status: record.status, current_step: record.current_step,
@@ -163,6 +198,7 @@ module Pgbus
             waiting_for: (step_attribute(step, :wait_for) if waiting),
             waiting_since: (step_attribute(step, :waiting_since) if waiting),
             timeout_at: (step_attribute(step, :timeout_at) if waiting),
+            on_timeout: (wait_on_timeout(record, step) if waiting), late_since: (late[step.id] if waiting),
             batch_id: step_attribute(step, :batch_id),
             batch_progress: (progress[step_attribute(step, :batch_id)] if waiting),
             scheduled_at: (step.scheduled_at if step&.status == "scheduled"),
@@ -192,7 +228,7 @@ module Pgbus
             definition_outdated: record.definition_outdated?, class_loaded: !klass.nil?,
             step_names: klass ? klass.workflow_steps.names : [], resume_action: resume_action, resume_step: resume_step,
             signal_names: WorkflowActionPlan.signal_names(steps.first, klass), reruns: workflow_reruns(record.workflow_id),
-            timeline: WorkflowTimeline.chart(step_hashes, pause_events(record.workflow_id), status: record.status, now: now) }
+            timeline: WorkflowTimeline.chart(step_hashes, timeline_events(record.workflow_id), status: record.status, now: now) }
         end
 
         def workflow_step_hash(step)
@@ -207,9 +243,18 @@ module Pgbus
             signal_id: step_attribute(step, :signal_id) }
         end
 
-        def pause_events(workflow_id)
-          WorkflowEvent.where(workflow_id: workflow_id, event: WorkflowTimeline::PAUSE_EVENTS).order(:id).limit(PAUSE_HISTORY)
-                       .map { |event| { event: event.event, created_at: event.created_at } }
+        # The pauses (oldest first) and the newest wait_timed_out events the
+        # timeline marks, read apart so a wait late for weeks cannot crowd out
+        # the pauses.
+        def timeline_events(workflow_id)
+          pauses = WorkflowEvent.where(workflow_id: workflow_id, event: WorkflowTimeline::PAUSE_EVENTS).order(:id)
+                                .limit(PAUSE_HISTORY).map { |event| { event: event.event, created_at: event.created_at } }
+          timeouts = WorkflowEvent.where(workflow_id: workflow_id, event: WorkflowTimeline::WAIT_TIMED_OUT)
+                                  .order(id: :desc).limit(WAIT_TIMEOUT_HISTORY).map do |event|
+            { event: event.event, created_at: event.created_at, step_name: event.step_name, attempt: event.attempt,
+              details: event.details }
+          end
+          pauses + timeouts
         end
 
         def workflow_signals(workflow_id)

data/lib/pgbus/web/data_source.rb

@@ -321,7 +321,7 @@ module Pgbus
       def discard_failed_event(id)
         event = failed_event(id)
         if event
-          release_lock_for_payload(event["payload"])
+          release_locks_held_by_failed_events("id = $1", [id.to_i])
           archive_failed_message(event)
         end
 
@@ -920,7 +920,8 @@ module Pgbus
         # memory — otherwise the message is archived but the lock row stays
         # behind, blocking later publishes with the same key. Mirrors
         # discard_event.
-        release_lock_for_payload(detail[:message])
+        release_lock_held_by(detail[:message], queue_name: lock_queue_for_table(queue_name), msg_id: detail[:msg_id],
+                                               enqueued_at: detail[:enqueued_at])
         @client.archive_message(queue_name, msg_id.to_i, prefixed: false)
         clear_event_failure(queue_name, msg_id)
         true
@@ -1554,6 +1555,13 @@ module Pgbus
       end
 
       # --- Lock cleanup helpers ---
+      #
+      # Discarding a message releases only the uniqueness lock that message
+      # holds (UniquenessKey.release_held_by!): its job may have handed the
+      # lock to a retry that is still waiting (issue #564), or a successor may
+      # have taken the key, and neither is the discarded message's to free.
+      # The Locks page's own deletes stay key-only: there the key is the
+      # operator's target.
 
       # Extract uniqueness key from a queue message and release its lock.
       def release_lock_for_message(queue_name, msg_id)
@@ -1564,39 +1572,139 @@ module Pgbus
         )
         return unless row
 
-        release_lock_for_payload(row["message"])
+        holder_queue, holder_msg_id, sent_at = lock_holder(queue_name, row["msg_id"], row["headers"], row["enqueued_at"])
+        release_lock_held_by(row["message"], queue_name: holder_queue, msg_id: holder_msg_id, enqueued_at: sent_at)
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error releasing lock for message #{msg_id}: #{e.message}" }
       end
 
-      # Extract uniqueness key from a JSON payload string and release its lock.
-      def release_lock_for_payload(payload_str)
+      # [queue, msg_id, sent_at] of the message whose lock a discard may
+      # release. A live message is its own holder. A dead-lettered copy's is
+      # the message it was moved from, which its dead-letter header names:
+      # the copy's own enqueued_at is the move to the DLQ, later than a lock
+      # its job may have handed to a retry. A copy whose header does not name
+      # the original (dead-lettered by an earlier release, which freed its
+      # lock then) has no send time to go by, so only a bound lock could
+      # match, and none is bound to a DLQ copy.
+      def lock_holder(queue_name, msg_id, headers, enqueued_at)
+        own = [lock_queue_for_table(queue_name), msg_id.to_i]
+        return [*own, enqueued_at] unless queue_name.to_s.end_with?(Pgbus::DEAD_LETTER_SUFFIX)
+
+        block = DeadLetterHeader.parse(headers)
+        source_msg_id = block && Integer(block["source_msg_id"], exception: false)
+        return [*own, nil] unless source_msg_id && block["source_queue"]
+
+        [lock_queue_for_table(block["source_queue"]), source_msg_id, block["source_enqueued_at"]]
+      end
+
+      # Release the lock held by the message whose JSON payload this is.
+      def release_lock_held_by(payload_str, queue_name:, msg_id:, enqueued_at:)
         key = extract_uniqueness_key_from_payload_str(payload_str)
-        UniquenessKey.release!(key) if key
+        return unless key
+
+        UniquenessKey.release_held_by!(key, queue_name: queue_name, msg_id: msg_id.to_i, enqueued_at: enqueued_at)
       end
 
-      # Extract uniqueness keys from a collection of formatted messages and
-      # release all associated locks in a single query.
+      # Release the locks a collection of formatted messages hold, in a single
+      # query.
       def release_locks_for_messages(messages)
-        keys = messages.filter_map { |m| extract_uniqueness_key_from_payload_str(m[:message]) }
-        UniquenessKey.where(lock_key: keys).delete_all if keys.any?
+        holders = messages.filter_map do |m|
+          key = extract_uniqueness_key_from_payload_str(m[:message])
+          [key, *lock_holder(m[:queue_name], m[:msg_id], m[:headers], m[:enqueued_at])] if key
+        end
+        UniquenessKey.release_all_held_by!(holders) if holders.any?
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for messages: #{e.message}" }
       end
 
-      # Collect uniqueness keys from all failed events and release their locks.
       def release_locks_for_failed_events
+        release_locks_held_by_failed_events(NOT_HANDLER_QUEUE, [handler_queue_names_literal])
+      end
+
+      # Release the locks the messages of the failed jobs matching +condition+
+      # hold. A failed job's message is still in its queue, waiting for its
+      # next read, and its enqueued_at is the send time: failed_at is not, a
+      # lock handed to a retry before the failure would count as the job's.
+      # A message that is gone has no send time, so only a bound lock goes.
+      def release_locks_held_by_failed_events(condition, binds)
         rows = connection.select_all(
-          "SELECT payload FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE}", "Pgbus Collect Failed Keys",
-          [handler_queue_names_literal]
+          "SELECT queue_name, msg_id, payload FROM pgbus_failed_events WHERE #{condition}",
+          "Pgbus Collect Failed Keys", binds
         )
 
-        keys = rows.to_a.filter_map { |row| extract_uniqueness_key_from_payload_str(row["payload"]) }
-        UniquenessKey.where(lock_key: keys).delete_all if keys.any?
+        # queue_name is the logical name the executor saw: the message is looked
+        # up under it, and the lock is released under its table spelling.
+        # A key proc may return a number; the queue lookup reads it as text.
+        holders = rows.to_a.filter_map do |row|
+          key = extract_uniqueness_key_from_payload_str(row["payload"])&.to_s
+          [key, row["queue_name"], row["msg_id"].to_i] if key && row["msg_id"]
+        end
+        holders = holders.group_by { |_key, queue, _msg_id| queue }.flat_map do |queue, entries|
+          failed_queue_lock_holders(queue, entries)
+        end
+        UniquenessKey.release_all_held_by!(holders) if holders.any?
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for failed events: #{e.message}" }
       end
 
+      # The release tuples for one queue's failed jobs. A queue that cannot
+      # be resolved (a stored name no longer valid as a table name) is skipped
+      # with a warning, so it does not stop the other queues' release.
+      def failed_queue_lock_holders(queue, entries)
+        sent = queued_send_times(queue, entries)
+        lock_queue = Uniqueness.lock_queue_name(queue)
+        entries.map { |key, _queue, msg_id| [key, lock_queue, msg_id, sent[[msg_id, key]]] }
+      rescue ActiveRecord::StatementInvalid
+        raise
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus::Web] Skipped releasing locks for failed jobs on #{queue.inspect}: #{e.message}" }
+        []
+      end
+
+      # { [msg_id, uniqueness key] => enqueued_at } for the messages of
+      # +entries+ ([key, queue, msg_id]) still in the logical queue. Every
+      # priority sub-queue is searched: a failed event records the logical
+      # queue only, and msg_ids repeat across sub-queues, so the key must
+      # match too, and the earliest send wins.
+      def queued_send_times(queue, entries)
+        msg_ids = "{#{entries.map { |_key, _queue, msg_id| msg_id }.uniq.join(",")}}"
+        @client.physical_queue_names(queue).each_with_object({}) do |physical, found|
+          queued_rows(physical, msg_ids).each do |row|
+            pair = [row["msg_id"].to_i, row["lock_key"]]
+            found[pair] = [found[pair], row["enqueued_at"]].compact.min
+          end
+        end
+      end
+
+      # Each read runs in its own savepoint: a sub-queue the strategy names
+      # but that does not exist raises, and inside a caller's transaction
+      # that would abort the discard's own statements after it.
+      def queued_rows(physical, msg_ids)
+        connection.transaction(requires_new: true) do
+          connection.select_all(
+            "SELECT msg_id, enqueued_at, message->>'#{Uniqueness::METADATA_KEY}' AS lock_key " \
+            "FROM pgmq.q_#{sanitize_name(physical)} WHERE msg_id = ANY($1::bigint[])",
+            "Pgbus Failed Message Sent At", [msg_ids]
+          ).to_a
+        end
+      rescue ActiveRecord::StatementInvalid => e
+        raise unless defined?(PG::UndefinedTable) && e.cause.is_a?(PG::UndefinedTable)
+
+        []
+      end
+
+      # The queue a uniqueness lock records for a message in this physical
+      # queue: the table name without its prefix and a priority sub-queue's
+      # _pN, which is how Uniqueness.lock_queue_name spells the queue the lock
+      # was bound for. A dead-letter queue keeps its suffix, so its copy's
+      # msg_id (another sequence) never matches a lock bound to a live message.
+      def lock_queue_for_table(physical_name)
+        name = physical_name.to_s.delete_prefix("#{Pgbus.configuration.queue_prefix}_")
+        return name if name.end_with?(Pgbus::DEAD_LETTER_SUFFIX) || Pgbus.configuration.priority_levels.to_i <= 1
+
+        name.sub(/_p\d+\z/, "")
+      end
+
       # Single unwrap point for PGMQ message / failed_event payload strings.
       # Accepts a raw JSON string or an already-parsed Hash and returns the
       # uniqueness metadata key, or nil when the payload is blank, unparseable,

data/lib/pgbus/web/workflow_insights_report.rb

@@ -47,14 +47,14 @@ module Pgbus
         )
       end
 
-      # [{ name:, kind:, removed:, declared: }] in pipeline order.
+      # [{ name:, kind:, removed:, declared:, wait_timeout: }] in pipeline order.
       def step_list(klass, stats, ongoing, reached, stopped)
         seen = (stats.keys + ongoing.map { |row| row[:step] } + reached.keys + stopped.keys.map(&:first)).compact.uniq
         declared = klass ? klass.workflow_steps.entries.map { |entry| declared_step(entry) } : []
         extra = (seen - declared.map { |entry| entry[:name] }).sort_by do |name|
           [reached.dig(name, :position) || Float::INFINITY, stats.dig(name, :first_seen_at)&.to_f || Float::INFINITY, name]
         end
-        declared + extra.map { |name| { name: name, kind: nil, removed: false, declared: false } }
+        declared + extra.map { |name| { name: name, kind: nil, removed: false, declared: false, wait_timeout: nil } }
       end
 
       def declared_step(entry)
@@ -63,7 +63,15 @@ module Pgbus
                elsif entry.waits_for_signal? then "signal"
                else "step"
                end
-        { name: entry.name, kind: kind, removed: entry.removed?, declared: true }
+        { name: entry.name, kind: kind, removed: entry.removed?, declared: true, wait_timeout: wait_timeout(entry) }
+      end
+
+      # A signal step's wait_timeout: in seconds, "dynamic" for a per-run one
+      # (issue #574), nil without one.
+      def wait_timeout(entry)
+        return nil if entry.removed?
+
+        entry.dynamic_wait_timeout? ? "dynamic" : entry.wait_timeout
       end
 
       def step(entry, stats, ongoing)

data/lib/pgbus/web/workflow_state.rb

@@ -6,8 +6,8 @@ module Pgbus
     # (its status, or "scheduled" for a ready run whose step waits for a
     # time still ahead) and what it is doing and what happens next, as a
     # pgbus.workflows.next.<key>_html key and its arguments. Arguments named
-    # :step, :signal and :batch are names, :ago a past moment and :time a
-    # future one; the helper renders each. Pure.
+    # :step, :signal and :batch are names, :ago and :late past moments and
+    # :time a future one; the helper renders each. Pure.
     module WorkflowState
       BADGE_TONES = { "ready" => :gray, "scheduled" => :gray, "performing" => :blue, "waiting" => :yellow,
                       "paused" => :yellow, "finished" => :green, "cancelled" => :red }.freeze
@@ -51,7 +51,23 @@ module Pgbus
         return ["waiting", "waiting_step", { step: step }] unless row[:waiting_for]
 
         args = { signal: row[:waiting_for], ago: row[:waiting_since] }
-        row[:timeout_at] ? ["waiting", "waiting_signal_timeout", args.merge(time: row[:timeout_at])] : ["waiting", "waiting_signal", args]
+        return signal_deadline(row, args.merge(time: row[:timeout_at])) if row[:timeout_at]
+        # A per-run wait_timeout: reported once under on_timeout: :notify,
+        # then left without a deadline (issue #574).
+        return ["waiting", "waiting_signal_late_no_deadline", args.merge(late: row[:late_since])] if row[:late_since]
+
+        ["waiting", "waiting_signal", args]
+      end
+
+      # An on_timeout: :notify deadline sends a late notice and the wait goes
+      # on (issue #565); late_since is the first deadline the wait missed.
+      def signal_deadline(row, args)
+        # A deploy that changed on_timeout since the last notice: the next
+        # deadline does something else (nil: the class is not loaded here).
+        late = row[:late_since] && [nil, "notify"].include?(row[:on_timeout]&.to_s)
+        return ["waiting", "waiting_signal_late", args.merge(late: row[:late_since])] if late
+
+        ["waiting", row[:on_timeout].to_s == "notify" ? "waiting_signal_notify" : "waiting_signal_timeout", args]
       end
 
       def paused(row, step)

data/lib/pgbus/web/workflow_timeline.rb

@@ -11,13 +11,15 @@ module Pgbus
     #   waiting  waiting_since → …           parked on a signal or its fan-out batch
     #
     # and a paused run's pauses (paused → resumed, from the events) are their
-    # own segments, so a long gap explains itself. The show page's step table
-    # prints durations; the range-bar chart (charts.js renderTimeline) draws
-    # chart. Pure: rows in, hashes out.
+    # own segments, so a long gap explains itself; each wait_timed_out event
+    # is a marker. The show page's step table prints durations; the
+    # range-bar chart (charts.js renderTimeline) draws chart. Pure: rows in,
+    # hashes out.
     module WorkflowTimeline
       OPEN = %w[scheduled waiting in_progress].freeze
       PAUSE_ENDS = %w[resumed cancelled finished].freeze
       PAUSE_EVENTS = (%w[paused] + PAUSE_ENDS).freeze
+      WAIT_TIMED_OUT = "wait_timed_out"
       RAN_SERIES = { "completed" => "ran_ok", "skipped" => "ran_ok", "failed" => "ran_failed",
                      "in_progress" => "ran_open", "waiting" => "ran_open" }.freeze
 
@@ -69,8 +71,8 @@ module Pgbus
         periods
       end
 
-      # The chart payload: one row per attempt (oldest first) and the paused
-      # periods. Times are epoch milliseconds; each segment carries its
+      # The chart payload: one row per attempt (oldest first), the paused
+      # periods and the wait_timed_out marks (wait_timeouts). Times are epoch milliseconds; each segment carries its
       # series (its colour: ran is coloured by how the attempt ended) and its
       # duration in words for the tooltip.
       def chart(steps, events, status:, now:)
@@ -82,7 +84,20 @@ module Pgbus
         paused = paused_periods(events, status: status, now: now).map do |period|
           chart_segment(period.merge(kind: :paused), "paused")
         end
-        { rows: rows, paused: paused }
+        { rows: rows, paused: paused, wait_timeouts: wait_timeouts(events) }
+      end
+
+      # [{ at:, step:, action: }] per wait_timed_out event, oldest first, at
+      # the deadline it reports (when it went out, without one): the chart
+      # draws each as a line across the rows, the page lists them. An
+      # on_timeout: :notify wait has one per late notice (issue #565).
+      def wait_timeouts(events)
+        timed_out = events.select { |event| event[:event] == WAIT_TIMED_OUT }.sort_by { |event| event[:created_at] }
+        timed_out.map do |event|
+          details = event[:details].to_h
+          { at: millis(deadline(details["timeout_at"]) || event[:created_at]),
+            step: "#{event[:step_name]} ##{event[:attempt]}", action: details["action"] }
+        end
       end
 
       def series_for(kind, step)
@@ -98,6 +113,8 @@ module Pgbus
 
       def millis(time) = (time.to_f * 1000).round
 
+      def deadline(iso) = iso && Time.iso8601(iso)
+
       # Drops a piece without both ends or of no length, and ends every piece
       # at the attempt's finished_at (a step cancelled before its scheduled_at).
       def clip(pieces, finished)

data/lib/pgbus/workflow/instance.rb

@@ -94,6 +94,33 @@ module Pgbus
         @timed_out == true
       end
 
+      # Inside a step (its guards and its body): true when this attempt holds
+      # a signal before its gate runs: signal! woke the step, or an earlier
+      # attempt of it took one and reattempted or failed into a pause. False
+      # on an attempt that has not waited yet (even one whose gate then takes
+      # a buffered signal), after an on_timeout: :run wake, and in a step
+      # without wait_for:. The same answer for the whole attempt.
+      def woken?
+        step_context!(:woken?)
+        @woken = !@signal_peek&.held.nil? if @woken.nil?
+        @woken
+      end
+
+      # Inside a step (its guards and its body): true when the gate runs this
+      # attempt with a signal: the one it holds (woken?), or a pending one its
+      # wait_for: matcher takes. The guard for "skip a wait the record already
+      # passed", which neither skips a signalled step nor leaves its signal
+      # pending. False after an on_timeout: :run wake (the gate takes no
+      # signal then) and in a step without wait_for:. Reads only: the gate
+      # claims the signal after the guards.
+      #
+      #   step :await_shipped, skip_if: -> { subject.shipped? && !signal_ready? }, wait_for: :shipped do … end
+      def signal_ready?
+        step_context!(:signal_ready?)
+        @signal_ready = !@signal_peek&.ready.nil? if @signal_ready.nil?
+        @signal_ready
+      end
+
       # Deliver a signal to this run. Stored under the run's lock; a step
       # waiting for it (wait_for:) is woken at once, otherwise it waits,
       # pending, for the step that claims it. With an idempotency_key a
@@ -164,19 +191,25 @@ module Pgbus
         @guarding = false
       end
 
-      # Used by the StepJob only.
-      def enter_step(context)
+      # Used by the StepJob only. +signals+ is the attempt's SignalPeek, which
+      # woken? and signal_ready? read until the gate answers them.
+      def enter_step(context, signals: nil)
         @step_context = context
+        @signal_peek = signals
         @output_values = nil
         @pending_output = nil
         @received_signal = nil
         @timed_out = false
+        @woken = nil
+        @signal_ready = nil
       end
 
       # Used by the StepJob's signal gate only.
-      def enter_wait(signal:, timed_out:)
+      def enter_wait(signal:, timed_out:, woken: false)
         @received_signal = signal
         @timed_out = timed_out
+        @woken = woken
+        @signal_ready = !signal.nil?
       end
     end
   end

data/lib/pgbus/workflow/runner.rb

@@ -68,11 +68,11 @@ module Pgbus
 
       # --- start! and rerun! --------------------------------------------
 
-      def start(klass, subject:, arguments:, idempotency_key: nil)
+      def start(klass, subject:, arguments:, idempotency_key: nil, actor: DEFAULT_ACTOR, reason: nil)
         attrs = run_attributes(klass, idempotency_key: idempotency_key)
                 .merge(Payload.subject_attributes(subject))
                 .merge(arguments: Payload.dump(arguments, what: "arguments of #{klass.name}"))
-        insert_run(klass, attrs, actor: DEFAULT_ACTOR)
+        insert_run(klass, attrs, actor: actor, reason: reason)
       end
 
       # A finished or cancelled run started again: same subject and

data/lib/pgbus/workflow/signal_peek.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # What a wait_for: step's attempt holds and what its gate would run it
+    # with (issue #572), read without claiming anything. The StepJob builds
+    # one per attempt: the step readers woken? and signal_ready? read it in
+    # the guards, and the gate decides with the same one, so the gate takes
+    # exactly the signal signal_ready? promised. All of it runs under the
+    # run's row lock, so each answer is looked up (and a matcher run) once.
+    class SignalPeek
+      def initialize(record, step, definition, now: Time.current)
+        @record = record
+        @step = step
+        @definition = definition
+        @now = now
+      end
+
+      # The claimed signal the step holds (signal! woke it, or an earlier
+      # attempt took one and reattempted or paused), or nil. Always nil for a
+      # step without wait_for:.
+      def held
+        return @held if defined?(@held)
+
+        @held = waits? ? WorkflowSignal.held_by_step(@record.workflow_id, @step.step_name) : nil
+      end
+
+      # The sweep woke this attempt with on_timeout: :run: its body runs
+      # without a signal (Signals.timed_out?).
+      def timed_out? = Signals.timed_out?(@step, @now)
+
+      # The signal the gate runs this attempt with: the one the step holds,
+      # else the oldest pending one its matcher takes. nil when the wait timed
+      # out (the body runs without one), when nothing matches (the step
+      # parks), and for a step without wait_for:.
+      def ready
+        return @ready if defined?(@ready)
+
+        @ready = waits? && !timed_out? ? held || Signals.match(@record, @definition.wait_for) : nil
+      end
+
+      private
+
+      def waits? = @definition.waits_for_signal?
+    end
+  end
+end