pgbus 0.19.3 → 0.20.0

data/lib/pgbus/workflow/step_definition.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # One declared step: its name, its body (a block or the name of an
+    # instance method) and its options. Built and validated when the class
+    # body runs, so a bad option fails at load, not on the first run.
+    class StepDefinition
+      OPTIONS = %i[wait skip_if job_options timeout on_exception wait_for wait_timeout on_timeout].freeze
+      # What a wait_for: step does when its wait_timeout: passes first.
+      ON_TIMEOUT = %i[pause! cancel! skip! run].freeze
+      # The StepJob options a class (set_step_job_options) or a step
+      # (job_options:) may set. Anything else raises at class load.
+      JOB_OPTIONS = %i[queue priority wait wait_until].freeze
+
+      attr_reader :name, :body, :wait, :skip_if, :job_options, :timeout, :exception_policies,
+                  :wait_for, :wait_for_name, :wait_timeout, :on_timeout
+
+      def self.validate_job_options!(options)
+        raise ArgumentError, "job_options: must be a Hash, got #{options.inspect}" unless options.is_a?(Hash)
+
+        options.to_h do |key, value|
+          key = key.to_sym
+          unless JOB_OPTIONS.include?(key)
+            raise ArgumentError,
+                  "unknown job_options: #{key.inspect} — accepted: #{JOB_OPTIONS.map(&:inspect).join(", ")}"
+          end
+
+          [key, job_option_value(key, value)]
+        end.freeze
+      end
+
+      def self.job_option_value(key, value)
+        case key
+        when :queue
+          raise ArgumentError, "job_options: queue must be a queue name, got #{value.inspect}" if value.to_s.strip.empty?
+
+          -value.to_s
+        when :priority
+          return value if value.is_a?(Integer)
+
+          raise ArgumentError, "job_options: priority must be an Integer, got #{value.inspect}"
+        when :wait then seconds(value, "job_options: wait", positive: false)
+        when :wait_until
+          # Time, DateTime and TimeWithZone; not a String, which ActiveSupport
+          # also gives #to_time (nil for an unparseable one).
+          return value.to_time if value.respond_to?(:to_time) && !value.is_a?(String) && value.to_time
+
+          raise ArgumentError, "job_options: wait_until must be a Time, got #{value.inspect}"
+        else value
+        end
+      end
+
+      # Seconds as an Integer when whole, else a Float. Durations are
+      # accepted; nil stays nil.
+      def self.seconds(value, option, positive:)
+        return nil if value.nil?
+
+        seconds = duration?(value) ? value.to_f : value
+        unless seconds.is_a?(Numeric) && seconds.real? && seconds.finite? && (positive ? seconds.positive? : !seconds.negative?)
+          requirement = positive ? "a positive" : "a non-negative"
+          raise ArgumentError, "#{option}: must be #{requirement} number of seconds or a Duration, got #{value.inspect}"
+        end
+
+        seconds == seconds.to_i ? seconds.to_i : seconds.to_f
+      end
+
+      def self.duration?(value)
+        defined?(ActiveSupport::Duration) && value.is_a?(ActiveSupport::Duration)
+      end
+
+      def initialize(name:, body:, wait: nil, skip_if: nil, job_options: {}, timeout: nil, on_exception: nil,
+                     wait_for: nil, wait_timeout: nil, on_timeout: nil)
+        @name = -name.to_s
+        @body = body
+        @wait = self.class.seconds(wait, "wait", positive: false) || 0
+        @skip_if = validate_skip_if(skip_if)
+        @job_options = self.class.validate_job_options!(job_options.nil? ? {} : job_options)
+        @timeout = self.class.seconds(timeout, "timeout", positive: true)
+        @exception_policies = ExceptionPolicy.coerce_all(on_exception)
+        assign_wait(wait_for, wait_timeout, on_timeout)
+        freeze
+      end
+
+      def kind = "step"
+
+      # True for a wait_for: step: it parks until a matching signal arrives.
+      def waits_for_signal? = !wait_for.nil?
+
+      def removed? = false
+
+      # Runs the body on +workflow+; the return value is ignored.
+      def invoke(workflow)
+        body.is_a?(Symbol) ? workflow.send(body) : workflow.instance_exec(&body)
+        nil
+      end
+
+      def skip?(workflow)
+        return false if skip_if.nil?
+
+        result = skip_if.is_a?(Symbol) ? workflow.send(skip_if) : workflow.instance_exec(&skip_if)
+        result ? true : false
+      end
+
+      private
+
+      def assign_wait(wait_for, wait_timeout, on_timeout)
+        @wait_for = wait_for.nil? ? nil : SignalMatcher.coerce(wait_for)
+        @wait_for_name = @wait_for && SignalMatcher.label(@wait_for)
+        raise ArgumentError, "on_timeout: needs wait_for: and wait_timeout:" if !on_timeout.nil? && (@wait_for.nil? || wait_timeout.nil?)
+        raise ArgumentError, "wait_timeout: needs wait_for: (it bounds the wait for a signal)" if wait_timeout && @wait_for.nil?
+
+        @wait_timeout = self.class.seconds(wait_timeout, "wait_timeout", positive: true)
+        @on_timeout = @wait_timeout && normalize_on_timeout(on_timeout.nil? ? :pause! : on_timeout)
+      end
+
+      def normalize_on_timeout(value)
+        verb = value.to_s == "run" ? :run : :"#{value.to_s.delete_suffix("!")}!"
+        return verb if ON_TIMEOUT.include?(verb)
+
+        raise ArgumentError, "on_timeout: must be one of #{ON_TIMEOUT.map(&:inspect).join(", ")}, got #{value.inspect}"
+      end
+
+      def validate_skip_if(value)
+        return value if value.nil? || value.is_a?(Symbol) || value.is_a?(Proc)
+
+        raise ArgumentError, "skip_if: must be a lambda, a Proc or a method name, got #{value.inspect}"
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/step_execution.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # One StepJob delivery: claim the step row, check the guards, run the
+    # body in a fresh workflow instance, apply the outcome.
+    #
+    # 1. Claim, under the run's row lock: scheduled → in_progress (only once
+    #    due) and the run ready → performing. A row that is not claimable
+    #    (already running elsewhere, finished, early, or its run is paused or
+    #    gone) makes this delivery a no-op, except a redelivery of the very
+    #    message that claimed an in_progress row while the visibility
+    #    heartbeat is on: its worker died, and the stall action applies now.
+    # 2. Guards, still under the lock: the class or the step no longer
+    #    defined (pause with StepNotDefined, or skip a removed_step),
+    #    cancel_if, skip_if.
+    # 3. The body, outside any transaction, inside `catch` for the
+    #    flow-control verbs and `rescue StandardError` for the policies.
+    # 4. The outcome, under the lock again (StepOutcome).
+    #
+    # Whatever the step's outcome, the job itself completes normally: the
+    # step row and the pgbus.workflow.* events carry the truth.
+    class StepExecution
+      Claim = Data.define(:record, :step, :klass, :definition, :workflow, :guard_error)
+
+      def initialize(step_id, delivery: Pgbus::ActiveJob::Delivery.current, config: Pgbus.configuration)
+        @step_id = step_id
+        @delivery = delivery
+        @config = config
+      end
+
+      def call
+        step = WorkflowStep.find_by(id: @step_id)
+        return debug("step #{@step_id} no longer exists (its run was deleted)") unless step
+
+        StepContext.with(workflow_id: step.workflow_id, workflow_class: step.workflow_class, step_name: step.step_name,
+                         attempt: step.attempt) do
+          claim = claim(step)
+          run(claim) if claim
+        end
+        nil
+      end
+
+      private
+
+      attr_reader :config
+
+      # The worker tag and the app revision (which may call the app's
+      # app_revision callable or read a file) are resolved before the run's
+      # row lock is taken.
+      def claim(step)
+        @stamp = { worker: Runner.worker_tag, app_revision: Deploy::Revision.current(config) }
+        Runner.locked(step.workflow_id) do |record, notes|
+          step = WorkflowStep.find_by(id: step.id) if record
+          if record.nil? || step.nil?
+            debug("step #{@step_id}'s run no longer exists")
+          elsif step.status == "in_progress"
+            check_redelivery(record, step, notes)
+          elsif claimable?(record, step)
+            claim_step(record, step, notes)
+          else
+            debug("step #{step.id} is not claimable (step #{step.status}, run #{record.status}, due #{step.scheduled_at})")
+          end
+        end
+      end
+
+      def claimable?(record, step)
+        step.status == "scheduled" && States.legal?(:claim, record.status) && step.scheduled_at <= Time.current + 1
+      end
+
+      # A live worker keeps its message invisible (VisibilityHeartbeat), so
+      # the same message coming back for a row it claimed means that worker
+      # is gone. Without the heartbeat a long live step can outlive its
+      # visibility timeout, so the sweep's stall_at decides instead.
+      def check_redelivery(record, step, notes)
+        msg_id = @delivery && @delivery[:msg_id]
+        redelivered = !msg_id.nil? && step.msg_id == msg_id.to_i && @delivery[:read_ct].to_i > 1 && config.visibility_heartbeat
+        return debug("step #{step.id} is already in progress elsewhere") unless redelivered
+
+        StepOutcome.stall!(record, step, detection: "redelivery", notes: notes, config: config)
+        nil
+      end
+
+      def claim_step(record, step, notes)
+        now = Time.current
+        klass = record.workflow_klass
+        definition = klass&.workflow_steps&.fetch(step.step_name)
+        attrs = { stall_at: now + (definition&.timeout || config.workflow_stall_threshold), msg_id: @delivery&.dig(:msg_id),
+                  **@stamp }
+        return nil if WorkflowStep.claim!(step.id, now: now, **attrs).zero?
+
+        step.assign_attributes(status: "in_progress", started_at: now, **attrs)
+        Runner.transition!(record, :claim, current_step: step.step_name)
+        Runner.event!(record, "step_started", actor: Runner::SYSTEM_ACTOR, step: step,
+                                              details: attrs.slice(:worker, :app_revision, :msg_id).compact)
+        guard(record, step, klass, definition, notes)
+      end
+
+      # Returns the Claim to run, or nil when a guard already ended the
+      # attempt. A guard that raises is handed to the exception policies,
+      # like a body that raises.
+      def guard(record, step, klass, definition, notes)
+        return undefined!(record, step, notes, "#{record.workflow_class} is not a Pgbus::Workflow class any more") unless klass
+
+        unless definition
+          return skip!(record, step, notes, klass) if klass.workflow_steps.removed?(step.step_name)
+
+          return undefined!(record, step, notes, "#{klass.name} has no step #{step.step_name} " \
+                                                 "(declare `removed_step :#{step.step_name}` to skip it)")
+        end
+
+        workflow = klass.new(record)
+        workflow.send(:enter_step, StepContext::Attempt.new(step_id: step.id, step_name: step.step_name, attempt: step.attempt))
+        claim = Claim.new(record: record, step: step, klass: klass, definition: definition, workflow: workflow, guard_error: nil)
+        begin
+          verdict = workflow.send(:guarding) do
+            next FlowControl::Outcome.cancel if cancel?(klass, workflow)
+
+            FlowControl::Outcome.skip if definition.skip?(workflow)
+          end
+        rescue StandardError => e
+          return claim.with(guard_error: e)
+        end
+        return outcome!(record, step, notes, klass, verdict) if verdict
+
+        gate(record, step, definition, workflow, claim)
+      end
+
+      # A wait_for: step runs only with a signal (or once its wait timed
+      # out); without one it parks and this delivery ends. A matcher that
+      # raises is handed to the exception policies, like a guard.
+      def gate(record, step, definition, workflow, claim)
+        return claim unless definition.waits_for_signal?
+
+        Signals.gate(record, step, definition, workflow) ? claim : nil
+      rescue StandardError => e
+        claim.with(guard_error: e)
+      end
+
+      def cancel?(klass, workflow)
+        klass.cancel_conditions.any? do |condition|
+          condition.is_a?(Symbol) ? workflow.send(condition) : workflow.instance_exec(&condition)
+        end
+      end
+
+      def undefined!(record, step, notes, message)
+        StepOutcome.apply(record, step, FlowControl::Outcome.pause, notes: notes, error: StepNotDefined.new(message), klass: nil)
+        nil
+      end
+
+      def skip!(record, step, notes, klass) = outcome!(record, step, notes, klass, FlowControl::Outcome.skip)
+
+      def outcome!(record, step, notes, klass, outcome)
+        StepOutcome.apply(record, step, outcome, notes: notes, klass: klass, duration: 0.0)
+        nil
+      end
+
+      def run(claim)
+        started = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        outcome, error = claim.guard_error ? [nil, claim.guard_error] : execute(claim)
+        duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started
+        finish(claim, outcome, error, duration)
+      end
+
+      def execute(claim)
+        outcome = catch(FlowControl::THROW_TAG) do
+          claim.definition.invoke(claim.workflow)
+          nil
+        end
+        [outcome || FlowControl::Outcome.complete, nil]
+      rescue StandardError => e
+        [nil, e]
+      end
+
+      # The exception policy is decided before the run's lock is taken
+      # again: an imperative on_exception block is app code, and must not
+      # hold the row lock (and an open transaction) while it runs.
+      def finish(claim, outcome, error, duration)
+        return finish_late(claim, error) if error && !attempt_active?(claim)
+
+        decision = error && decide(claim, error)
+        outcome = decision.outcome if decision
+        Runner.locked(claim.record.workflow_id) do |record, notes|
+          next debug("step #{claim.step.id}'s run was deleted while the step ran") unless record
+          next park_fan_out(record, claim, outcome, notes) if outcome&.action == :fan_out
+
+          notes.report(decision.handler_error, handler_context(claim)) if decision&.handler_error
+          StepOutcome.apply(record, claim.step, outcome, notes: notes, error: error, output: claim.workflow.pending_output,
+                                                         report: decision ? decision.report : true, duration: duration,
+                                                         klass: claim.klass)
+        end
+      end
+
+      def park_fan_out(record, claim, outcome, notes)
+        FanOut.park(record, claim.step, outcome, notes: notes, output: claim.workflow.pending_output)
+      end
+
+      # While its body runs, a row leaves in_progress only for a terminal
+      # status (a parked row went waiting before any body ran), so an attempt
+      # that finds it gone has been declared stalled (or its run cancelled): late.
+      def attempt_active?(claim) = WorkflowStep.where(id: claim.step.id, status: "in_progress").exists?
+
+      # A late attempt that raised: its outcome is ignored either way, so an
+      # imperative on_exception block must not run (and cause side effects)
+      # for it. A declarative policy still names the outcome it would have
+      # applied and decides the report; an imperative one records none.
+      def finish_late(claim, error)
+        policy = policy_for(claim, error)
+        decision = policy.handler ? nil : decide(claim, error, policy: policy)
+        column = decision && StepOutcome::STEP_OUTCOME.fetch(decision.outcome.action)
+        report = decision ? decision.report : policy.report != :never
+        Runner.locked(claim.record.workflow_id) do |record, notes|
+          next debug("step #{claim.step.id}'s run was deleted while the step ran") unless record
+
+          StepOutcome.ignore_late_completion(record, claim.step, "failed", column, error)
+          notes.report(error, StepOutcome.error_context(record, claim.step, "late_completion_ignored")) if report
+        end
+      end
+
+      def policy_for(claim, error)
+        ExceptionPolicy.resolve(error, step_policies: claim.definition.exception_policies,
+                                       class_policies: claim.klass.exception_policies)
+      end
+
+      def decide(claim, error, policy: policy_for(claim, error))
+        step = claim.step
+        policy.decide(error, workflow: claim.workflow,
+                             consecutive_reattempts: -> { WorkflowStep.consecutive_reattempts(step.workflow_id, step.step_name) })
+      end
+
+      def handler_context(claim)
+        { action: "workflow_on_exception", workflow_class: claim.klass.name, workflow_id: claim.step.workflow_id,
+          step_name: claim.step.step_name, attempt: claim.step.attempt }
+      end
+
+      def debug(message)
+        Pgbus.logger.debug { "[Pgbus::Workflow] #{message} #{StepContext.log_tag}".rstrip }
+        nil
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/step_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Pgbus
+  class Workflow
+    # The job behind every step attempt: perform(step_id) runs one
+    # StepExecution. Enqueued only by Pgbus::WorkflowStep's after-commit
+    # callback, on config.workflow_queue unless the class
+    # (set_step_job_options) or the step (job_options:) says otherwise.
+    #
+    # No retry_on and no discard_on: a step's failure is the exception
+    # policies' business, and the job returns normally whatever the step's
+    # outcome. Redelivery is reserved for a worker that died mid-step, which
+    # the claim recognises. The visibility heartbeat must stay on for this
+    # class (never `pgbus_visibility_heartbeat false`): fast crash detection
+    # depends on a live worker keeping its message invisible.
+    class StepJob < ::ActiveJob::Base
+      # Always on, whatever ActiveJob::Base or the app set: the executor reads
+      # this, and a redelivery means a dead worker only while it is true.
+      def self.pgbus_visibility_heartbeat_enabled = true # rubocop:disable Naming/PredicateMethod -- JobMixin's reader
+
+      def perform(step_id)
+        StepExecution.new(step_id).call
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/step_outcome.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # Applies how a step attempt ended to its row and to the run, under the
+    # run's row lock: the StepJob after a body, a guard (cancel_if, skip_if,
+    # a removed or unknown step), crash detection and the sweep.
+    #
+    # The step row is written first, guarded on status = 'in_progress'. A
+    # row that is no longer in_progress was declared stalled (or the run was
+    # cancelled) while this attempt was still running: the outcome and its
+    # output are discarded and only a late_completion_ignored event is
+    # recorded. Otherwise the run moves by the States table: advance (the
+    # next step is scheduled), finish, reattempt, step_pause, step_cancel, or
+    # hold when an outside pause! got there first.
+    module StepOutcome
+      STEP_STATUS = { complete: "completed", finish: "completed", pause: "completed",
+                      skip: "skipped", cancel: "cancelled", reattempt: "failed" }.freeze
+      STEP_OUTCOME = { complete: "completed", finish: "finished_early", pause: "paused",
+                       skip: "skipped", cancel: "cancelled", reattempt: "reattempted" }.freeze
+
+      module_function
+
+      # Returns false when the attempt had lost its in_progress guard.
+      def apply(record, step, outcome, notes:, error: nil, output: nil, report: true, duration: nil, klass: record.workflow_klass) # rubocop:disable Naming/PredicateMethod
+        now = Time.current
+        status = error ? "failed" : STEP_STATUS.fetch(outcome.action)
+        column = STEP_OUTCOME.fetch(outcome.action)
+        captured = error && ErrorCapture.capture(error)
+        attrs = { status: status, outcome: column, finished_at: now, output: output, **captured.to_h }
+        if WorkflowStep.finish!(step.id, **attrs).zero?
+          ignore_late_completion(record, step, status, column, error)
+          notes.report(error, error_context(record, step, "late_completion_ignored")) if error && report
+          return false
+        end
+
+        Signals.settle(step, status, column, now: now)
+        step_events(record, step, outcome, column, error)
+        move_run(record, step, outcome, notes: notes, captured: captured, klass: klass, now: now)
+        notes.instrument("pgbus.workflow.step", step_payload(record, step, column, status, error, duration))
+        notes.report(error, error_context(record, step, column)) if error && report
+        true
+      end
+
+      # A step whose worker died (detection: "redelivery") or that outlived
+      # its stall_at (detection: "sweep"): config.workflow_stall_action
+      # applies, with Pgbus::Workflow::StepStalled recorded on the row.
+      def stall!(record, step, detection:, notes:, config: Pgbus.configuration)
+        why = if detection == "redelivery"
+                "its message came back while it was in_progress: the worker #{step.worker} died"
+              else
+                "it was still in_progress at its stall_at #{step.stall_at&.utc&.iso8601} (worker #{step.worker})"
+              end
+        error = StepStalled.new("#{step.step_name} attempt #{step.attempt} stalled: #{why}")
+        Runner.event!(record, "stall_detected", actor: Runner::SYSTEM_ACTOR, step: step,
+                                                details: { detection: detection, worker: step.worker,
+                                                           stall_at: step.stall_at&.utc&.iso8601(3) }.compact)
+        notes.instrument("pgbus.workflow.stall_detected",
+                         Runner.payload_for(record, step_name: step.step_name, attempt: step.attempt, detection: detection))
+        apply(record, step, stall_outcome(config.workflow_stall_action), notes: notes, error: error)
+      end
+
+      def stall_outcome(action)
+        case action
+        when :reattempt then FlowControl::Outcome.reattempt
+        when :cancel then FlowControl::Outcome.cancel
+        else FlowControl::Outcome.pause
+        end
+      end
+
+      def move_run(record, step, outcome, notes:, captured:, klass:, now:)
+        paused = record.status == "paused"
+        case outcome.action
+        when :complete, :skip
+          following = klass&.workflow_steps&.next_after(step.step_name)
+          following ? advance(record, following, paused: paused, now: now) : finish(record, notes)
+        when :finish then finish(record, notes)
+        when :reattempt
+          Runner.schedule!(record, step.step_name, at: now + (outcome.wait || 0), actor: Runner::SYSTEM_ACTOR)
+          Runner.transition!(record, paused ? :hold : :reattempt, current_step: step.step_name)
+        when :pause then pause(record, step, outcome, notes, captured: captured, paused: paused, now: now)
+        when :cancel then cancel(record, step, outcome, notes, now: now)
+        end
+      end
+
+      def advance(record, following, paused:, now:)
+        Runner.schedule!(record, following.name, at: now + following.wait, actor: Runner::SYSTEM_ACTOR)
+        Runner.transition!(record, paused ? :hold : :advance, current_step: following.name)
+      end
+
+      def finish(record, notes)
+        Runner.transition!(record, :finish, finished_at: Time.current)
+        Runner.event!(record, "finished", actor: Runner::SYSTEM_ACTOR)
+        notes.instrument("pgbus.workflow_finished", Runner.payload_for(record))
+      end
+
+      # The run row keeps the error that paused it (cleared by resume!).
+      def pause(record, step, outcome, notes, captured:, paused:, now:)
+        run_error = captured ? captured.except(:error_causes) : {}
+        return Runner.transition!(record, :hold, **run_error) if paused
+
+        error_class = captured&.fetch(:error_class)
+        Runner.transition!(record, :step_pause, paused_at: now, **run_error)
+        Runner.event!(record, "paused", actor: outcome.actor || Runner::SYSTEM_ACTOR, reason: outcome.reason, step: step,
+                                        details: { error_class: error_class }.compact)
+        notes.instrument("pgbus.workflow_paused", Runner.payload_for(record, step_name: step.step_name, error_class: error_class))
+      end
+
+      def cancel(record, step, outcome, notes, now:)
+        Runner.transition!(record, :step_cancel, cancelled_at: now)
+        WorkflowStep.cancel_open!(record.workflow_id, now: now)
+        Runner.event!(record, "cancelled", actor: outcome.actor || Runner::SYSTEM_ACTOR, reason: outcome.reason, step: step)
+        notes.instrument("pgbus.workflow_cancelled", Runner.payload_for(record, step_name: step.step_name))
+      end
+
+      def step_events(record, step, outcome, column, error)
+        actor = Runner::SYSTEM_ACTOR
+        if error
+          Runner.event!(record, "step_failed", actor: actor, step: step,
+                                               details: { error_class: error.class.name, error_message: error.message.to_s })
+        end
+        case outcome.action
+        when :complete, :finish, :pause
+          Runner.event!(record, "step_completed", actor: actor, step: step, details: { outcome: column }) unless error
+        when :skip then Runner.event!(record, "step_skipped", actor: outcome.actor || actor, reason: outcome.reason, step: step)
+        when :reattempt then Runner.event!(record, "step_reattempted", actor: actor, step: step, details: { wait: outcome.wait }.compact)
+        end
+      end
+
+      def ignore_late_completion(record, step, status, column, error)
+        details = { status: status, outcome: column, error_class: error&.class&.name }.compact
+        Runner.event!(record, "late_completion_ignored", actor: Runner::SYSTEM_ACTOR, step: step, details: details)
+        Pgbus.logger.warn do
+          "[Pgbus] Workflow #{record.workflow_class} #{record.workflow_id}: step #{step.step_name} attempt #{step.attempt} " \
+            "finished after it stopped being in_progress (stalled or cancelled); its outcome (#{column}) and output were discarded"
+        end
+      end
+
+      def step_payload(record, step, column, status, error, duration)
+        pickup = step.started_at && step.scheduled_at ? [step.started_at - step.scheduled_at, 0].max.to_f : nil
+        Runner.payload_for(record, step_name: step.step_name, attempt: step.attempt, outcome: column, status: status,
+                                   error_class: error&.class&.name, duration: duration, pickup_latency: pickup)
+      end
+
+      def error_context(record, step, column)
+        { workflow_class: record.workflow_class, workflow_id: record.workflow_id, step_name: step.step_name,
+          attempt: step.attempt, outcome: column }
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/steps.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # The ordered step list of one workflow class: StepDefinitions and
+    # RemovedStep gravestones, in declaration order. Immutable: every
+    # change (`step`, `removed_step`) builds a new frozen Steps that the
+    # class stores whole, so a subclass extends its parent's list without
+    # touching it and a reader never sees a list change under it
+    # (docs/concurrency.md).
+    class Steps
+      attr_reader :entries
+
+      def initialize(entries = [])
+        @entries = entries.dup.freeze
+        freeze
+      end
+
+      EMPTY = new
+
+      def add(definition, before: nil, after: nil)
+        raise ArgumentError, "use before_step: or after_step:, not both" if before && after
+        raise ArgumentError, "#{definition.name} is a removed step (removed_step); pick another name" if removed?(definition.name)
+        raise ArgumentError, "duplicate step #{definition.name}" if fetch(definition.name)
+
+        list = entries.dup
+        anchor = before || after
+        if anchor
+          index = position(anchor)
+          unless index
+            raise ArgumentError,
+                  "#{before ? "before_step" : "after_step"}: #{anchor.to_s.to_sym.inspect} — no step named #{anchor}"
+          end
+
+          list.insert(before ? index : index + 1, definition)
+        else
+          list << definition
+        end
+        self.class.new(list)
+      end
+
+      # Replaces a live step with a gravestone at its position, or appends a
+      # gravestone for a step that is already gone from the class.
+      def remove(name)
+        name = -name.to_s
+        raise ArgumentError, "#{name} is already a removed step" if removed?(name)
+
+        list = entries.dup
+        index = position(name)
+        gravestone = RemovedStep.new(name: name)
+        index ? list[index] = gravestone : list << gravestone
+        self.class.new(list)
+      end
+
+      def definitions = entries.reject(&:removed?)
+
+      def names = definitions.map(&:name)
+
+      def removed_names = entries.select(&:removed?).map(&:name)
+
+      def fetch(name)
+        name = name.to_s
+        definitions.find { |entry| entry.name == name }
+      end
+
+      def removed?(name)
+        name = name.to_s
+        entries.any? { |entry| entry.removed? && entry.name == name }
+      end
+
+      def first = definitions.first
+
+      # The next live step declared after +name+ (a step or a gravestone),
+      # or nil when +name+ is the last one or unknown.
+      def next_after(name)
+        index = position(name)
+        return nil unless index
+
+        entries.drop(index + 1).find { |entry| !entry.removed? }
+      end
+
+      def empty? = definitions.empty?
+
+      # Name for an anonymous `step { ... }`: step_<n>, the first n from
+      # entries.size + 1 not taken by a step or a gravestone.
+      def anonymous_name
+        number = entries.size + 1
+        number += 1 while entries.any? { |entry| entry.name == "step_#{number}" }
+        "step_#{number}"
+      end
+
+      # What definition_digest hashes: the ordered [name, kind] pairs.
+      def digest_pairs = entries.map { |entry| [entry.name, entry.kind] }
+
+      private
+
+      def position(name)
+        name = name.to_s
+        entries.index { |entry| entry.name == name }
+      end
+    end
+  end
+end