pgbus 0.19.4 → 0.20.1

data/app/helpers/pgbus/workflows_helper.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+
+module Pgbus
+  # The Workflows pages' words (issue #542): status badges, the "what happens
+  # next" sentence, the subject link, the Resume label, the audit-trail
+  # sentence and the attention reasons. Moments go through pgbus_time; the
+  # _html keys let the <time>, <code> and links through, translate escapes
+  # every other argument.
+  module WorkflowsHelper
+    BADGE_CSS = {
+      blue: "bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-400",
+      gray: "bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-300",
+      yellow: "bg-yellow-100 text-yellow-800 dark:bg-yellow-900/30 dark:text-yellow-400",
+      green: "bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400",
+      red: "bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-400"
+    }.freeze
+
+    # The outside verbs the show page has a button for (Send signal is its
+    # own form; pause and reschedule stay in the console for now).
+    DASHBOARD_ACTIONS = %i[resume resume_at run_now skip rerun cancel].freeze
+
+    # A link inside a sentence: inline (the button helper's inline-flex would
+    # break the sentence) and underlined; WCAG 2.5.8 exempts inline targets.
+    INLINE_LINK_CSS = "font-medium underline text-indigo-700 hover:text-indigo-900 dark:text-indigo-300 dark:hover:text-indigo-200"
+
+    # Text inputs, selects and textareas on the Workflows pages.
+    INPUT_CSS = "mt-1 block w-full rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-900 " \
+                "px-2 py-1.5 text-sm text-gray-900 dark:text-white focus:border-indigo-500 focus:ring-indigo-500"
+
+    STEP_TONES = { "scheduled" => :gray, "waiting" => :yellow, "in_progress" => :blue, "completed" => :green,
+                   "failed" => :red, "cancelled" => :gray, "skipped" => :gray }.freeze
+
+    SIGNAL_TONES = { "pending" => :yellow, "claimed" => :blue, "consumed" => :green }.freeze
+
+    # The verbs that need the run's class (its step list): resume!, resume_at!
+    # and rerun! raise StepNotDefined without it, and skip! would find no next
+    # step and finish the run.
+    CLASS_ACTIONS = %i[resume resume_at rerun skip].freeze
+
+    # The show page's buttons: available_actions it has a button for, without
+    # CLASS_ACTIONS when the run's class is not loaded here.
+    def pgbus_workflow_dashboard_actions(workflow)
+      actions = Array(workflow[:available_actions]) & DASHBOARD_ACTIONS
+      workflow[:class_loaded] == false ? actions - CLASS_ACTIONS : actions
+    end
+
+    def pgbus_workflow_state(row)
+      Web::WorkflowState.present(row, threshold: Pgbus.configuration.workflow_stall_threshold)
+    end
+
+    def pgbus_workflow_badge(result)
+      pgbus_workflow_tone_badge(t("pgbus.workflows.statuses.#{result.badge}"), result.badge_tone)
+    end
+
+    def pgbus_workflow_step_badge(status)
+      pgbus_workflow_tone_badge(t("pgbus.workflows.step_statuses.#{status}"), STEP_TONES.fetch(status.to_s, :gray))
+    end
+
+    def pgbus_workflow_signal_badge(status)
+      pgbus_workflow_tone_badge(t("pgbus.workflows.signal_statuses.#{status}"), SIGNAL_TONES.fetch(status.to_s, :gray))
+    end
+
+    def pgbus_workflow_tone_badge(label, tone)
+      tag.span(label, class: "#{ApplicationHelper::BATCH_BADGE_BASE} #{BADGE_CSS.fetch(tone)}")
+    end
+
+    # "Waiting for batch OrderWorkflow#notify_vendors (7 of 12 jobs done)."
+    def pgbus_workflow_next(row, result = pgbus_workflow_state(row))
+      args = result.next_args.to_h do |key, value|
+        case key
+        when :step, :signal then [key, tag.code(value, class: "font-mono")]
+        when :batch then [key, pgbus_workflow_batch_link(value, row[:batch_id])]
+        when :ago, :late then [key, pgbus_time(value)]
+        when :time then [key, pgbus_time(value, clock: true)]
+        else [key, value]
+        end
+      end
+      translate("pgbus.workflows.next.#{result.next_key}_html", **args)
+    end
+
+    def pgbus_workflow_batch_link(name, batch_id)
+      label = tag.code(name, class: "font-mono")
+      batch_id ? link_to(label, pgbus.batch_path(batch_id), class: INLINE_LINK_CSS, data: { turbo_frame: "_top" }) : label
+    end
+
+    # "Order 1042", linked through config.workflow_subject_url when it gives
+    # an http(s) URL or a same-site path; plain text otherwise, and "—"
+    # for a subject-less run.
+    def pgbus_workflow_subject(row)
+      return Web::TimeFormat::NONE unless row[:subject_type]
+
+      label = "#{row[:subject_type]} #{row[:subject_id]}"
+      url = pgbus_workflow_subject_url(row[:subject_type], row[:subject_id])
+      url ? pgbus_link_to(label, url, data: { turbo_frame: "_top" }) : label
+    end
+
+    def pgbus_workflow_subject_url(type, id)
+      builder = Pgbus.configuration.workflow_subject_url
+      return nil unless builder
+
+      pgbus_workflow_safe_url(builder.call(type, id))
+    rescue StandardError => e
+      Pgbus.logger.error { "[Pgbus::Web] workflow_subject_url raised #{e.class}: #{e.message}" }
+      nil
+    end
+
+    # An absolute http(s) URL with a host, or a path on this site ("/…",
+    # never "//host" or "/\host"). Anything else (javascript:, data:, a
+    # relative path) is not linked.
+    def pgbus_workflow_safe_url(value)
+      url = value.to_s.strip
+      return nil if url.empty?
+      return url if url.start_with?("/") && !url.start_with?("//", "/\\")
+
+      uri = URI.parse(url)
+      %w[http https].include?(uri.scheme&.downcase) && uri.host.present? ? url : nil
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    # The Resume button's label: what resume! will do (WorkflowActionPlan).
+    def pgbus_workflow_resume_label(workflow)
+      action = workflow[:resume_action]
+      return t("pgbus.workflows.actions.resume") unless action
+
+      t("pgbus.workflows.resume.#{action}", step: workflow[:resume_step])
+    end
+
+    # "Resumed by dashboard:alice: SMTP fixed", "Step failed by system (capture #2)".
+    def pgbus_workflow_event_sentence(event)
+      what = t("pgbus.workflows.events.#{event[:event]}", default: event[:event].to_s.humanize)
+      sentence = t("pgbus.workflows.audit.by", event: what, actor: event[:actor])
+      if event[:step_name]
+        sentence = t("pgbus.workflows.audit.with_step", sentence: sentence, step: event[:step_name], attempt: event[:attempt])
+      end
+      event[:reason].present? ? t("pgbus.workflows.audit.with_reason", sentence: sentence, reason: event[:reason]) : sentence
+    end
+
+    def pgbus_workflow_value(value)
+      value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : value.to_s
+    end
+
+    # The bulk confirm: "Resume all 3 runs paused at capture on Timeout::Error?"
+    def pgbus_workflow_bulk_confirm(action, group)
+      key = group[:error_class] ? "#{action}_error" : action.to_s
+      t("pgbus.workflows.attention.confirm.#{key}", count: group[:count], step: group[:step_name], error: group[:error_class])
+    end
+
+    # The index URL with the current query, merged with +extra+ (nil drops a key).
+    def pgbus_workflows_path_with(query, **extra)
+      pgbus.workflows_path(query.to_param(**extra))
+    end
+
+    # Series names and words the timeline chart prints (charts.js renderTimeline).
+    def pgbus_workflow_chart_i18n
+      series = %w[waited queued waiting paused ran_ok ran_failed ran_open ran_cancelled]
+      { series: series.index_with { |name| t("pgbus.workflows.timeline.series.#{name}") },
+        paused_row: t("pgbus.workflows.timeline.paused_row"), wait_timed_out: t("pgbus.workflows.events.wait_timed_out"),
+        statuses: Workflow::States::STEP.index_with { |status| t("pgbus.workflows.step_statuses.#{status}") } }
+    end
+  end
+end

data/app/models/pgbus/blocked_execution.rb

@@ -62,5 +62,17 @@ module Pgbus
         .limit(limit)
         .pluck(:concurrency_key)
     end
+
+    # The uniqueness keys, out of +lock_keys+, that a parked job carries. Its
+    # :until_executed row stays unbound until it runs and it sits in no queue,
+    # so the uniqueness reaper asks here before calling the row an orphan. A
+    # row parked before the double-encoding fix is read through its string,
+    # since the reaper can run before the promote sweep repairs it.
+    def self.parked_uniqueness_keys(lock_keys)
+      key = Uniqueness::METADATA_KEY
+      key_path = "CASE WHEN jsonb_typeof(payload) = 'string' " \
+                 "THEN (payload #>> '{}')::jsonb ->> '#{key}' ELSE payload ->> '#{key}' END"
+      where("#{key_path} IN (?)", lock_keys).distinct.pluck(Arel.sql(key_path)).to_set
+    end
   end
 end

data/app/models/pgbus/uniqueness_key.rb

@@ -40,10 +40,11 @@ module Pgbus
     # Does not touch created_at — the reaper's age floor is from acquire time.
     # Restricted to this enqueue's unbound row (msg_id=0, matching created_at
     # when acquire! stamped one) so a completed job's bind cannot retarget a
-    # successor that re-acquired the key.
-    def self.bind!(lock_key, queue_name:, msg_id:)
+    # successor that re-acquired the key. A retry passes the stamp unbind!
+    # returned as +acquired_at+.
+    def self.bind!(lock_key, queue_name:, msg_id:, acquired_at: nil)
       stamps = Thread.current[:pgbus_uniqueness_created_at]
-      created_at = stamps&.delete(lock_key)
+      created_at = acquired_at || stamps&.delete(lock_key)
       sql = "UPDATE #{table_name} SET queue_name = $2, msg_id = $3 " \
             "WHERE lock_key = $1 AND msg_id = 0"
       binds = [lock_key, queue_name, msg_id]
@@ -86,6 +87,90 @@ module Pgbus
       )
     end
 
+    # The row a message holds: bound to that message (queue + msg_id, see
+    # release_if_bound!), or still unbound — a parked job's row, or one whose
+    # bind failed — and acquired no later than the message was sent ($4). A
+    # lock is always taken before its message exists, so an unbound row
+    # acquired after the send is a successor's. created_at is a timestamp
+    # without time zone written in the session's zone, and the cast compares
+    # it in that same zone; PGMQ's enqueued_at string carries its offset.
+    # Unbound means msg_id <= 0 only: "pending" (the placeholder queue) is a
+    # name a real queue can have too, and a lock bound to one of its messages
+    # is bound.
+    HELD_BY_CONDITION = "lock_key = $1 AND ((queue_name = $2 AND msg_id = $3 AND msg_id > 0) OR " \
+                        "(msg_id <= 0 AND created_at <= $4::timestamptz))"
+    private_constant :HELD_BY_CONDITION
+
+    # Release the lock a finishing message holds (success or DLQ). Unlike the
+    # key-only release!, it leaves a row that has moved on: handed to the
+    # message's retry (unbind!), or taken by a successor after this
+    # message's own row went. +enqueued_at+ is when that message was sent;
+    # without it only the row bound to the message goes (an unbound row could
+    # be a retry's it handed over), and the reaper takes an orphan.
+    def self.release_held_by!(lock_key, queue_name:, msg_id:, enqueued_at:)
+      Thread.current[:pgbus_uniqueness_created_at]&.delete(lock_key)
+      connection.exec_delete(
+        "DELETE FROM #{table_name} WHERE #{HELD_BY_CONDITION}",
+        "UniquenessKey Release Held By", held_by_binds(lock_key, queue_name, msg_id, sent_at_literal(enqueued_at))
+      )
+    end
+
+    # Hand the running attempt's lock to its retry, which goes out through the
+    # key the attempt still holds (issue #333) and never acquires one. Called
+    # before the retry exists, as a first enqueue acquires before its send:
+    # the row is left unbound on the retry's queue and stamped now, after the
+    # attempt was sent, so the attempt's release no longer counts it as its
+    # own while the retry's (sent later) does, even if the retry finishes
+    # before bind! points the row at it. Returns the new stamp for that
+    # bind!, or nil when the attempt no longer holds the key. Without the
+    # attempt's enqueued_at only a row bound to it is taken.
+    def self.unbind!(lock_key, from_queue_name:, from_msg_id:, from_enqueued_at:, queue_name:)
+      binds = held_by_binds(lock_key, from_queue_name, from_msg_id, sent_at_literal(from_enqueued_at))
+      result = connection.exec_query(
+        "UPDATE #{table_name} SET queue_name = $5, msg_id = 0, created_at = CURRENT_TIMESTAMP " \
+        "WHERE #{HELD_BY_CONDITION} RETURNING created_at",
+        "UniquenessKey Unbind", binds + [queue_name.to_s]
+      )
+      result.rows.first&.first
+    end
+
+    # release_held_by! for many messages at once — the dashboard's bulk
+    # discards and queue purges — as [lock_key, queue_name, msg_id,
+    # enqueued_at] tuples, one statement per thousand.
+    def self.release_all_held_by!(holders)
+      holders.each_slice(1000).sum do |slice|
+        values = slice.map do |lock_key, queue_name, msg_id, enqueued_at|
+          "(#{connection.quote(lock_key.to_s)}, #{connection.quote(queue_name.to_s)}, #{msg_id.to_i}, " \
+            "#{connection.quote(sent_at_literal(enqueued_at))}::timestamptz)"
+        end
+        connection.exec_delete(
+          "DELETE FROM #{table_name} AS u USING (VALUES #{values.join(", ")}) AS m(lock_key, queue_name, msg_id, sent_at) " \
+          "WHERE u.lock_key = m.lock_key AND ((u.queue_name = m.queue_name AND u.msg_id = m.msg_id AND u.msg_id > 0) OR " \
+          "(u.msg_id <= 0 AND u.created_at <= m.sent_at))",
+          "UniquenessKey Release All Held By"
+        )
+      end
+    end
+
+    def self.held_by_binds(lock_key, queue_name, msg_id, sent_at)
+      [lock_key, queue_name.to_s, msg_id.to_i, sent_at]
+    end
+    private_class_method :held_by_binds
+
+    # A message's enqueued_at as a timestamptz literal that carries its own
+    # offset. PGMQ hands over a String that already does; a Time (the
+    # dashboard reads it through ActiveRecord) or epoch seconds would
+    # otherwise be bound as zone-less digits and read in the session's zone.
+    # An unknown send time is -infinity: no unbound row counts as the
+    # message's own.
+    def self.sent_at_literal(enqueued_at)
+      sent_at = enqueued_at.is_a?(Numeric) ? Time.at(enqueued_at) : enqueued_at
+      return "-infinity" if sent_at.nil?
+
+      sent_at.respond_to?(:iso8601) ? sent_at.iso8601(6) : sent_at.to_s
+    end
+    private_class_method :sent_at_literal
+
     # Release a lock only while it is still UNBOUND (msg_id = 0) AND was
     # acquired no later than +acquired_before+. Used when discarding a parked
     # job: a parked job never got a msg_id, so its lock was never bound, and

data/app/models/pgbus/workflow_entry.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  # One workflow run (pgbus_workflows). Pgbus::Workflow is the DSL and the
+  # handle; this is its row. Every status change goes through
+  # Pgbus::Workflow::Runner, as a guarded UPDATE under the row lock.
+  class WorkflowEntry < BusRecord
+    self.table_name = "pgbus_workflows"
+
+    scope :ongoing, -> { where(status: Workflow::States::ONGOING) }
+    scope :terminal, -> { where(status: Workflow::States::TERMINAL) }
+    scope :stale, ->(before:) { terminal.where(updated_at: ...before) }
+
+    # The run's row, locked FOR UPDATE for the rest of the transaction.
+    def self.lock_for(workflow_id)
+      lock.find_by(workflow_id: workflow_id)
+    end
+
+    # Single-winner status change: 1 when this call moved the row, 0 when
+    # its status was not in +from+.
+    def self.transition!(workflow_id, from:, to:, **attrs) # rubocop:disable Naming/MethodParameterName
+      where(workflow_id: workflow_id, status: from).update_all(attrs.merge(status: to, updated_at: Time.current))
+    end
+
+    # Inserts a run. With an idempotency_key a conflict on
+    # (workflow_class, idempotency_key) inserts nothing and returns false; a
+    # conflict on the one-ongoing-per-subject index raises
+    # ActiveRecord::RecordNotUnique. Raw SQL rather than
+    # insert_all(unique_by:), which resolves the index through the schema
+    # cache (issue #401).
+    def self.insert_run!(attrs) # rubocop:disable Naming/PredicateMethod
+      columns = attrs.keys
+      values = attrs.values.map { |value| value.is_a?(Hash) ? JSON.generate(value) : value }
+      placeholders = (1..columns.size).map { |i| "$#{i}" }.join(", ")
+      conflict = (" ON CONFLICT (workflow_class, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING" if attrs[:idempotency_key])
+      result = connection.exec_query(
+        "INSERT INTO #{table_name} (#{columns.join(", ")}) VALUES (#{placeholders})#{conflict} RETURNING id",
+        "WorkflowEntry Insert", values
+      )
+      result.rows.any?
+    end
+
+    # { [workflow_class, status] => count } over every run.
+    def self.status_counts
+      group(:workflow_class, :status).count
+    end
+
+    # Why ongoing runs need a person, as { [workflow_class, reason] => count }
+    # (Workflow::States::STUCK_REASONS). One query: each paused or ready run
+    # (and, with +waiting_threshold+, each waiting run) with its latest step row.
+    #   stalled       paused on a step that failed with StepStalled
+    #   paused_error  paused on a step that failed with any other error
+    #   paused_manual paused otherwise (the step called pause!, or an outside pause!)
+    #   overdue       ready, its scheduled step due more than +threshold+ ago
+    #   waiting_overdue waiting longer than +waiting_threshold+ on a step
+    #                 without a wait_timeout: (only with +waiting_threshold+,
+    #                 which needs the waiting-step columns)
+    # With +workflow_class+, that class's runs only (the workflow insights,
+    # issue #549).
+    def self.stuck_reason_counts(threshold:, now: Time.current, waiting_threshold: nil, workflow_class: nil)
+      binds = stuck_binds(threshold, now, waiting_threshold)
+      class_bind = "$#{binds.size + 1}" if workflow_class
+      sql = <<~SQL
+        SELECT stuck.workflow_class, stuck.reason, COUNT(*) AS runs
+        FROM (#{stuck_runs_sql(waiting: !waiting_threshold.nil?, class_bind: class_bind)}) stuck
+        WHERE stuck.reason IS NOT NULL
+        GROUP BY stuck.workflow_class, stuck.reason
+      SQL
+      connection.exec_query(sql, "WorkflowEntry Stuck", binds + [workflow_class].compact).rows.to_h do |klass, reason, count|
+        [[klass, reason], count.to_i]
+      end
+    end
+
+    # The dashboard's needs-attention view (issue #542): the stuck runs of
+    # stuck_reason_counts grouped by (class, step, reason, error class), the
+    # biggest group first, at most +limit+ groups. first_seen_at/last_seen_at
+    # bound when the group's runs got stuck (paused_at, the overdue step's
+    # scheduled_at, or the waiting step's waiting_since); error_message is
+    # the newest one's. With +workflow_class+, that class's groups only.
+    def self.attention_groups(threshold:, now: Time.current, waiting_threshold: nil, limit: 200, workflow_class: nil)
+      binds = stuck_binds(threshold, now, waiting_threshold)
+      class_bind = "$#{binds.size + 1}" if workflow_class
+      sql = <<~SQL
+        SELECT stuck.workflow_class, stuck.step_name, stuck.reason, stuck.error_class, COUNT(*) AS runs,
+               MIN(stuck.since) AS first_seen_at, MAX(stuck.since) AS last_seen_at,
+               (ARRAY_AGG(stuck.error_message ORDER BY stuck.since DESC NULLS LAST, stuck.id DESC))[1] AS error_message
+        FROM (#{stuck_runs_sql(waiting: !waiting_threshold.nil?, class_bind: class_bind)}) stuck
+        WHERE stuck.reason IS NOT NULL
+        GROUP BY stuck.workflow_class, stuck.step_name, stuck.reason, stuck.error_class
+        ORDER BY runs DESC, last_seen_at DESC NULLS LAST, stuck.workflow_class
+        LIMIT #{Integer(limit)}
+      SQL
+      result = connection.exec_query(sql, "WorkflowEntry Attention", binds + [workflow_class].compact)
+      result.cast_values.map do |klass, step_name, reason, error_class, count, first_seen, last_seen, message|
+        { workflow_class: klass, step_name: step_name, reason: reason, error_class: error_class, count: count.to_i,
+          first_seen_at: first_seen, last_seen_at: last_seen, error_message: message }
+      end
+    end
+
+    # The workflow_ids of one attention group (+group+: workflow_class,
+    # step_name, reason, error_class; a nil error_class matches runs without
+    # one), oldest stuck first, at most +limit+: what a bulk action walks.
+    def self.attention_ids(group, threshold:, now: Time.current, waiting_threshold: nil, limit: 500)
+      binds = stuck_binds(threshold, now, waiting_threshold)
+      first = binds.size + 1
+      sql = <<~SQL
+        SELECT stuck.workflow_id
+        FROM (#{stuck_runs_sql(waiting: !waiting_threshold.nil?)}) stuck
+        WHERE stuck.workflow_class = $#{first} AND stuck.step_name = $#{first + 1} AND stuck.reason = $#{first + 2}
+          AND stuck.error_class IS NOT DISTINCT FROM $#{first + 3}
+        ORDER BY stuck.since NULLS LAST, stuck.id
+        LIMIT #{Integer(limit)}
+      SQL
+      values = binds + group.values_at(:workflow_class, :step_name, :reason, :error_class)
+      connection.exec_query(sql, "WorkflowEntry Attention Ids", values).rows.map(&:first)
+    end
+
+    # Each paused or ready run (and, when +waiting+, each waiting run) with
+    # its latest step row and why it is stuck (NULL when it is not). Binds
+    # $1 (the overdue cutoff) and, when +waiting+, $2 (the waiting cutoff);
+    # +class_bind+ names the placeholder that narrows it to one class.
+    def self.stuck_runs_sql(waiting:, class_bind: nil)
+      <<~SQL
+        SELECT w.id, w.workflow_id, w.workflow_class, COALESCE(s.step_name, w.current_step) AS step_name,
+               s.error_class, s.error_message, stuck_reason.reason,
+               CASE
+                 WHEN stuck_reason.reason IN ('stalled', 'paused_error', 'paused_manual') THEN w.paused_at
+                 WHEN stuck_reason.reason = 'overdue' THEN s.scheduled_at
+                 #{"WHEN stuck_reason.reason = 'waiting_overdue' THEN s.waiting_since" if waiting}
+               END AS since
+        FROM #{table_name} w
+        LEFT JOIN LATERAL (
+          SELECT step_name, status, error_class, error_message, scheduled_at#{", timeout_at, waiting_since" if waiting}
+          FROM #{WorkflowStep.table_name}
+          WHERE workflow_id = w.workflow_id ORDER BY id DESC LIMIT 1
+        ) s ON TRUE
+        CROSS JOIN LATERAL (
+          SELECT CASE
+                   WHEN w.status = 'paused' AND s.status = 'failed' AND s.error_class = 'Pgbus::Workflow::StepStalled' THEN 'stalled'
+                   WHEN w.status = 'paused' AND s.status = 'failed' THEN 'paused_error'
+                   WHEN w.status = 'paused' THEN 'paused_manual'
+                   WHEN w.status = 'ready' AND s.status = 'scheduled' AND s.scheduled_at < $1 THEN 'overdue'
+                   #{"WHEN w.status = 'waiting' AND s.status = 'waiting' AND s.timeout_at IS NULL AND s.waiting_since < $2 THEN 'waiting_overdue'" if waiting}
+                 END AS reason
+        ) stuck_reason
+        WHERE w.status IN ('paused', 'ready'#{", 'waiting'" if waiting})#{" AND w.workflow_class = #{class_bind}" if class_bind}
+      SQL
+    end
+    private_class_method :stuck_runs_sql
+
+    def self.stuck_binds(threshold, now, waiting_threshold)
+      waiting_threshold.nil? ? [now - threshold] : [now - threshold, now - waiting_threshold]
+    end
+    private_class_method :stuck_binds
+
+    # The Pgbus::Workflow subclass that started this run, or nil when it no
+    # longer exists.
+    def workflow_klass
+      klass = workflow_class.safe_constantize
+      klass if klass.is_a?(Class) && klass < Workflow
+    end
+
+    # True when the run started on a different step list than its class
+    # declares now (a step added, removed or reordered since).
+    def definition_outdated?
+      klass = workflow_klass
+      klass ? klass.definition_digest != definition_digest : false
+    end
+  end
+end

data/app/models/pgbus/workflow_event.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # The audit trail (pgbus_workflow_events): one row per transition and per
+  # operator action, with who did it (actor) and why (reason). Append-only:
+  # written by Pgbus::Workflow::Runner inside the transaction of the
+  # transition it records, read-only once inserted, removed with its run.
+  #
+  # Actors: "app" (an outside verb called without actor:), "system" (the
+  # StepJob, the sweep, crash detection), or what the caller passes
+  # ("dashboard:<name>", "mcp", "alice", ...).
+  class WorkflowEvent < BusRecord
+    self.table_name = "pgbus_workflow_events"
+
+    REASON_LIMIT = 1_000
+    DETAIL_LIMIT = 1_000
+
+    EVENTS = %w[
+      started step_scheduled step_started step_completed step_failed step_skipped step_reattempted
+      paused resumed cancelled finished run_now rescheduled rerun stall_detected late_completion_ignored
+      signal_received signal_claimed step_waiting wait_timed_out fan_out_opened fan_out_finished
+    ].freeze
+
+    def self.record!(workflow_id:, event:, actor:, reason: nil, step: nil, details: {})
+      create!(
+        workflow_id: workflow_id, event: event.to_s, actor: actor.to_s,
+        reason: reason && Workflow::ErrorCapture.truncate(reason.to_s, REASON_LIMIT),
+        step_id: step&.id, step_name: step&.step_name, attempt: step&.attempt,
+        details: cap(details), created_at: Time.current
+      )
+    end
+
+    # String values in details are capped, so one large error message cannot
+    # bloat the trail.
+    def self.cap(details)
+      details.to_h do |key, value|
+        [key.to_s, value.is_a?(String) ? Workflow::ErrorCapture.truncate(value, DETAIL_LIMIT) : value]
+      end
+    end
+
+    def readonly? = persisted? || super
+  end
+end

data/app/models/pgbus/workflow_signal.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  # One signal delivered to a workflow run (pgbus_workflow_signals, issue
+  # #541): pending (buffered, no step has taken it), claimed (a wait_for:
+  # step took it and runs with it as received_signal) and consumed (that
+  # step ended). Written only by Pgbus::Workflow::Signals under the run's row
+  # lock.
+  #
+  # +payload+ reads back the signal! payload with indifferent access
+  # (through ActiveJob::Arguments, as workflow arguments do).
+  class WorkflowSignal < BusRecord
+    self.table_name = "pgbus_workflow_signals"
+
+    STATUSES = %w[pending claimed consumed].freeze
+
+    # Inserts a signal. A second signal with the same (workflow_id,
+    # idempotency_key) inserts nothing and returns nil. Raw SQL rather than
+    # insert_all(unique_by:), which resolves the index through the schema
+    # cache (issue #401), as WorkflowEntry.insert_run! does.
+    def self.insert_signal!(workflow_id:, name:, payload:, idempotency_key:, now: Time.current)
+      sql = <<~SQL
+        INSERT INTO #{table_name} (workflow_id, name, payload, idempotency_key, status, created_at)
+        VALUES ($1, $2, $3, $4, 'pending', $5)
+        ON CONFLICT (workflow_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING
+        RETURNING id
+      SQL
+      result = connection.exec_query(sql, "WorkflowSignal Insert",
+                                     [workflow_id, name, JSON.generate(payload), idempotency_key, now])
+      id = result.rows.first&.first
+      id && find(id)
+    end
+
+    # The pending signals of a run (only +name+'s when given); iterate them
+    # oldest first with find_each, which batches by id.
+    def self.pending_for(workflow_id, name: nil)
+      scope = where(workflow_id: workflow_id, status: "pending")
+      name ? scope.where(name: name.to_s) : scope
+    end
+
+    # The signal an earlier attempt of +step_name+ claimed and did not
+    # consume (a reattempt, or a resume! after a failure, runs with it).
+    def self.held_by_step(workflow_id, step_name)
+      attempts = WorkflowStep.where(workflow_id: workflow_id, step_name: step_name.to_s).select(:id)
+      where(workflow_id: workflow_id, status: "claimed", claimed_by_step_id: attempts).order(:id).first
+    end
+
+    # pending → claimed, by +step_id+. Returns the number of rows moved.
+    def self.claim!(id, step_id:, now: Time.current)
+      where(id: id, status: "pending").update_all(status: "claimed", claimed_by_step_id: step_id, claimed_at: now)
+    end
+
+    # claimed → consumed. Returns the number of rows moved.
+    def self.consume!(id, now: Time.current)
+      where(id: id, status: "claimed").update_all(status: "consumed", consumed_at: now)
+    end
+
+    attr_writer :duplicate_delivery
+
+    # True on what signal! returned for a redelivery: a signal with the same
+    # idempotency key was already stored, and nothing else happened.
+    def duplicate_delivery? = @duplicate_delivery == true
+
+    def payload
+      return @payload if defined?(@payload)
+
+      @payload = Workflow::Payload.load(super) || {}.with_indifferent_access
+    end
+
+    def reload(...)
+      remove_instance_variable(:@payload) if defined?(@payload)
+      super
+    end
+  end
+end