pgbus 0.19.3 → 0.20.0

data/lib/pgbus/web/workflow_timeline.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    # Where a workflow run's time went (issue #542), from its step rows and
+    # audit trail. Each attempt splits into
+    #
+    #   waited   created_at → scheduled_at   the step's wait: (or a reattempt's)
+    #   queued   scheduled_at → started_at   due, not yet claimed by a worker
+    #   ran      started_at → finished_at    the body (up to now while in progress)
+    #   waiting  waiting_since → …           parked on a signal or its fan-out batch
+    #
+    # and a paused run's pauses (paused → resumed, from the events) are their
+    # own segments, so a long gap explains itself. The show page's step table
+    # prints durations; the range-bar chart (charts.js renderTimeline) draws
+    # chart. Pure: rows in, hashes out.
+    module WorkflowTimeline
+      OPEN = %w[scheduled waiting in_progress].freeze
+      PAUSE_ENDS = %w[resumed cancelled finished].freeze
+      PAUSE_EVENTS = (%w[paused] + PAUSE_ENDS).freeze
+      RAN_SERIES = { "completed" => "ran_ok", "skipped" => "ran_ok", "failed" => "ran_failed",
+                     "in_progress" => "ran_open", "waiting" => "ran_open" }.freeze
+
+      module_function
+
+      # [{ kind:, from:, to: }] in time order. +step+ is a step hash (the
+      # data source's) with Time values.
+      def segments(step, now:)
+        created, scheduled, started, since = step.values_at(:created_at, :scheduled_at, :started_at, :waiting_since)
+        ending = step[:finished_at] || (OPEN.include?(step[:status].to_s) ? now : nil)
+        pieces = if since && scheduled && since < scheduled
+                   # A signal step woken by its signal: the row was scheduled
+                   # again at the wake and claimed a second time.
+                   [[:waited, created, since], [:waiting, since, scheduled], [:queued, scheduled, started],
+                    [:ran, started, ending]]
+                 elsif since
+                   [[:waited, created, scheduled], [:queued, scheduled, started], [:ran, started, since],
+                    [:waiting, since, ending]]
+                 else
+                   [[:waited, created, scheduled], [:queued, scheduled, started || ending], [:ran, started, ending]]
+                 end
+        clip(pieces, step[:finished_at])
+      end
+
+      # { waited:, queued:, ran:, waiting: } in seconds; nil for a kind the
+      # attempt never reached.
+      def durations(step, now:)
+        totals = segments(step, now: now).group_by { |segment| segment[:kind] }
+        %i[waited queued ran waiting].to_h do |kind|
+          pieces = totals[kind]
+          [kind, pieces&.sum { |segment| (segment[:to] - segment[:from]).to_f }]
+        end
+      end
+
+      # [{ from:, to: }]: each "paused" event up to the event that ended it,
+      # oldest first; a pause still open runs to now. +events+ in any order.
+      def paused_periods(events, status:, now:)
+        periods = []
+        opened = nil
+        events.sort_by { |event| event[:created_at] }.each do |event|
+          if event[:event] == "paused"
+            opened ||= event[:created_at]
+          elsif opened && PAUSE_ENDS.include?(event[:event])
+            periods << { from: opened, to: event[:created_at] }
+            opened = nil
+          end
+        end
+        periods << { from: opened, to: now } if opened && status.to_s == "paused"
+        periods
+      end
+
+      # The chart payload: one row per attempt (oldest first) and the paused
+      # periods. Times are epoch milliseconds; each segment carries its
+      # series (its colour: ran is coloured by how the attempt ended) and its
+      # duration in words for the tooltip.
+      def chart(steps, events, status:, now:)
+        rows = steps.sort_by { |step| [step[:created_at], step[:attempt].to_i] }.map do |step|
+          { label: "#{step[:step_name]} ##{step[:attempt]}", status: step[:status], outcome: step[:outcome],
+            error_class: step[:error_class],
+            segments: segments(step, now: now).map { |segment| chart_segment(segment, series_for(segment[:kind], step)) } }
+        end
+        paused = paused_periods(events, status: status, now: now).map do |period|
+          chart_segment(period.merge(kind: :paused), "paused")
+        end
+        { rows: rows, paused: paused }
+      end
+
+      def series_for(kind, step)
+        return kind.to_s unless kind == :ran
+
+        RAN_SERIES.fetch(step[:status].to_s, "ran_cancelled")
+      end
+
+      def chart_segment(segment, series)
+        { kind: segment[:kind].to_s, series: series, from: millis(segment[:from]), to: millis(segment[:to]),
+          duration: TimeFormat.duration(segment[:to] - segment[:from]) }
+      end
+
+      def millis(time) = (time.to_f * 1000).round
+
+      # Drops a piece without both ends or of no length, and ends every piece
+      # at the attempt's finished_at (a step cancelled before its scheduled_at).
+      def clip(pieces, finished)
+        pieces.filter_map do |kind, from, to|
+          next unless from && to
+
+          to = [to, finished].min if finished
+          { kind: kind, from: from, to: to } if to > from
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/workflow_window.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "time"
+require "active_support/core_ext/time"
+
+module Pgbus
+  module Web
+    # The window the workflow insights read (issue #549): 1 h, 24 h (the
+    # default), 7 d or 30 d, never longer than config.workflow_retention
+    # (finished and cancelled runs older than that are gone, so a longer
+    # window would undercount). Each length has its chart bucket, at most 30
+    # per window: 5 min, 1 h, 6 h, 1 d; a capped window takes the bucket of
+    # the shortest choice that covers it. Anything else raises Invalid (400
+    # on the page, an error on the MCP tool).
+    class WorkflowWindow
+      class Invalid < StandardError; end
+
+      # name => [seconds, bucket seconds]
+      CHOICES = Ractor.make_shareable({ "1h" => [3_600, 300], "24h" => [86_400, 3_600], "7d" => [604_800, 21_600],
+                                        "30d" => [2_592_000, 86_400] })
+      DEFAULT = "24h"
+
+      attr_reader :name, :seconds, :bucket_seconds, :now
+
+      # +now+ is fixed here, so every read and every link of one page share
+      # one window end.
+      def self.parse(value, retention: Pgbus.configuration.workflow_retention, now: Time.current)
+        name = value.nil? || value == "" ? DEFAULT : value
+        raise Invalid, "unknown window #{value.inspect} (one of #{CHOICES.keys.join(", ")})" unless name.is_a?(String) && CHOICES.key?(name)
+
+        new(name, retention: retention, now: now)
+      end
+
+      def initialize(name, retention:, now: Time.current)
+        @name = name
+        @now = now
+        requested = CHOICES.fetch(name).first
+        @seconds = retention ? [requested, retention.to_i].min : requested
+        @capped = @seconds < requested
+        @bucket_seconds = CHOICES.values.find { |length, _| length >= @seconds }.last
+        freeze
+      end
+
+      def capped? = @capped
+
+      def since(reference = now) = reference - seconds
+
+      def to_h = { name: name, seconds: seconds, bucket_seconds: bucket_seconds, capped: capped? }
+    end
+  end
+end

data/lib/pgbus/workflow/batch_finished_job.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Pgbus
+  class Workflow
+    # The on_complete callback of a fan_out step's batch (issue #541):
+    # perform(step_id) resumes the run once every member finished (FanOut).
+    # Sent on the step's StepJob queue (config.workflow_queue unless the
+    # class or step says otherwise). A callback is never a member of the
+    # batch it reports on, and the StepJob it leads to is sent outside any
+    # Pgbus.batch block, so neither joins a batch.
+    class BatchFinishedJob < ::ActiveJob::Base
+      queue_as { Pgbus.configuration.workflow_queue }
+
+      def perform(step_id)
+        FanOut.finished(step_id)
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/dsl.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # The class-body DSL. Every declaration builds a new frozen value and
+    # stores it whole in a class_attribute, so a subclass extends what its
+    # parent declared without changing the parent, and nothing is ever
+    # mutated after the class body ran (docs/concurrency.md).
+    module DSL
+      # step :name, wait:, skip_if:, timeout:, job_options:, on_exception:,
+      #      before_step:, after_step: do ... end
+      # step { ... }              # anonymous: step_1, step_2, ...
+      # step :name                # runs the instance method `name`
+      # step def name = ...       # the same, declared in one line
+      def step(name = nil, before_step: nil, after_step: nil, **options, &block)
+        raise ArgumentError, "a step needs a name or a block" if name.nil? && block.nil?
+
+        reject_unknown_options!(options)
+        name ||= _workflow_steps.anonymous_name
+        definition = StepDefinition.new(name: name, body: block || name.to_sym, **options)
+        self._workflow_steps = _workflow_steps.add(definition, before: before_step, after: after_step)
+        definition.name.to_sym
+      end
+
+      # fan_out :name, on_batch_failure: :pause!, **step_options do |b| ... end
+      #
+      # A step that adds a group of jobs with b.add and waits for them as one
+      # batch: the step parks as waiting, and the batch's on_complete resumes
+      # the run once every member finished. Any dead-lettered member applies
+      # on_batch_failure (:pause!, :cancel! or :skip!) with
+      # Pgbus::Workflow::FanOutFailed recorded. Takes every step option but
+      # wait_for:, wait_timeout: and on_timeout:. Without a block it runs the
+      # instance method of the same name, which gets the builder.
+      def fan_out(name = nil, on_batch_failure: :pause!, before_step: nil, after_step: nil, **options, &block)
+        raise ArgumentError, "a fan_out needs a name or a block" if name.nil? && block.nil?
+
+        name ||= _workflow_steps.anonymous_name
+        waits = options.keys & %i[wait_for wait_timeout on_timeout]
+        if waits.any?
+          raise ArgumentError, "fan_out #{name.to_s.to_sym.inspect} cannot take #{waits.first}: it waits for its batch, " \
+                               "not for a signal"
+        end
+
+        reject_unknown_options!(options, kind: "fan_out", accepted: FanOutDefinition::OPTIONS)
+        definition = FanOutDefinition.new(name: name, body: block || name.to_sym, on_batch_failure: on_batch_failure, **options)
+        self._workflow_steps = _workflow_steps.add(definition, before: before_step, after: after_step)
+        definition.name.to_sym
+      end
+
+      # A step that no longer exists. Declare it where the step used to be: a
+      # run still scheduled on it skips it and moves on to the step after.
+      def removed_step(*names)
+        names.each { |name| self._workflow_steps = _workflow_steps.remove(name) }
+      end
+
+      # Runs before every step under the workflow lock; truthy cancels the
+      # workflow. A block or a method name; several are OR-ed.
+      def cancel_if(method_name = nil, &block)
+        condition = block || method_name
+        raise ArgumentError, "cancel_if needs a block or a method name" unless condition.is_a?(Proc) || condition.is_a?(Symbol)
+
+        self._cancel_conditions = [*_cancel_conditions, condition].freeze
+      end
+
+      # A class-level exception policy (see ExceptionPolicy).
+      def on_exception(*matching, action: nil, **, &)
+        policy = ExceptionPolicy.new(action, matching: matching.flatten, **, &)
+        self._exception_policies = [*_exception_policies, policy].freeze
+      end
+
+      # StepJob options for every step of this class: queue, priority, wait,
+      # wait_until. A step's own job_options: win over these.
+      def set_step_job_options(**options)
+        self._step_job_options = _step_job_options.merge(StepDefinition.validate_job_options!(options)).freeze
+      end
+
+      def workflow_steps = _workflow_steps
+
+      def cancel_conditions = _cancel_conditions
+
+      def exception_policies = _exception_policies
+
+      def step_job_options = _step_job_options
+
+      # SHA256 of the ordered [name, kind] step list. Stored on every run at
+      # start, so a run that began on an older definition can be told apart.
+      def definition_digest = Registry.digest(_workflow_steps)
+
+      # The StepJob options for one step: the class's, then the step's own.
+      def step_job_options_for(step_name)
+        own = _workflow_steps.fetch(step_name)&.job_options || {}
+        _step_job_options.merge(own)
+      end
+
+      private
+
+      def reject_unknown_options!(options, kind: "step", accepted: StepDefinition::OPTIONS)
+        unknown = options.keys - accepted
+        return if unknown.empty?
+
+        raise ArgumentError, "unknown #{kind} option #{unknown.first.inspect} (accepted: #{accepted.join(", ")})"
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/error_capture.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # Turns an exception into the step row's error columns: what an operator
+    # needs to act on a failed step without opening the error service.
+    #
+    #   error_class      the class name
+    #   error_message    capped at MESSAGE_LIMIT characters
+    #   error_backtrace  cleaned through Rails.backtrace_cleaner when Rails is
+    #                    loaded, cut to config.workflow_error_backtrace_lines
+    #   error_causes     [{ "class", "message" }] along Exception#cause, at most
+    #                    MAX_CAUSES deep (a cyclic chain stops where it repeats)
+    module ErrorCapture
+      MESSAGE_LIMIT = 10_000
+      CAUSE_MESSAGE_LIMIT = 1_000
+      MAX_CAUSES = 5
+
+      module_function
+
+      def capture(error, backtrace_lines: Pgbus.configuration.workflow_error_backtrace_lines)
+        {
+          error_class: error.class.name,
+          error_message: truncate(error.message.to_s, MESSAGE_LIMIT),
+          error_backtrace: backtrace(error, backtrace_lines).join("\n"),
+          error_causes: causes(error)
+        }
+      end
+
+      # The columns cleared when a later attempt succeeds or a run resumes.
+      def cleared = { error_class: nil, error_message: nil, error_backtrace: nil }
+
+      def backtrace(error, lines)
+        raw = error.backtrace || []
+        cleaned = clean(raw)
+        (cleaned.empty? ? raw : cleaned).first(lines)
+      end
+
+      def causes(error)
+        seen = {}.compare_by_identity
+        seen[error] = true
+        chain = []
+        cause = error.cause
+        while cause && chain.size < MAX_CAUSES && !seen[cause]
+          seen[cause] = true
+          chain << { "class" => cause.class.name, "message" => truncate(cause.message.to_s, CAUSE_MESSAGE_LIMIT) }
+          cause = cause.cause
+        end
+        chain
+      end
+
+      def truncate(string, limit)
+        string.length > limit ? "#{string[0, limit - 1]}…" : string
+      end
+
+      def clean(lines)
+        return lines unless defined?(::Rails) && ::Rails.respond_to?(:backtrace_cleaner) && ::Rails.backtrace_cleaner
+
+        ::Rails.backtrace_cleaner.clean(lines)
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/exception_policy.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # What happens when a step body raises (issue #540). A policy matches an
+    # exception (by class, class name, or a list of either; none = every
+    # StandardError) and names an action:
+    #
+    #   on_exception Net::OpenTimeout, action: :reattempt!, wait: 15.seconds, max_reattempts: 5
+    #   on_exception PaymentDeclined, action: :cancel!, report: :never
+    #   on_exception action: :pause!                        # blanket
+    #   on_exception Timeout::Error do |error|              # imperative: must call a verb
+    #     error.message.include?("read") ? reattempt!(wait: 5) : skip!
+    #   end
+    #
+    # Resolution: the step's own policies, then the class's specific ones,
+    # then the class's blanket ones, then the default pause!. The error is
+    # recorded on the step row either way; +report+ decides whether it also
+    # goes to Pgbus::ErrorReporter (:always, :never, or :terminal_only: only
+    # when the action ends the retrying). max_reattempts counts consecutive
+    # reattempted attempts of the same step; once spent, terminal_action
+    # (default pause!) applies.
+    class ExceptionPolicy
+      ACTIONS = %i[pause! reattempt! cancel! skip!].freeze
+      TERMINAL_ACTIONS = %i[pause! cancel! skip!].freeze
+      REPORTS = %i[always never terminal_only].freeze
+
+      # +outcome+ is a FlowControl::Outcome; +report+ whether ErrorReporter
+      # hears of it; +handler_error+ is what an imperative block raised.
+      Decision = Data.define(:outcome, :report, :terminal, :handler_error)
+
+      attr_reader :action, :matching, :wait, :max_reattempts, :terminal_action, :report, :handler
+
+      class << self
+        # The step-level `on_exception:` option: a policy, an action
+        # (`:reattempt!`), a Hash of policy options, or an Array of those.
+        def coerce_all(value)
+          Array(value.is_a?(Hash) ? [value] : value).map { |entry| coerce(entry) }.freeze
+        end
+
+        def coerce(value)
+          case value
+          when ExceptionPolicy then value
+          when Symbol, String then new(value.to_sym)
+          when Hash
+            options = value.transform_keys(&:to_sym)
+            new(options.delete(:action), **options)
+          else
+            raise ArgumentError, "on_exception: takes a Pgbus::Workflow::ExceptionPolicy, an action, a Hash or an Array, " \
+                                 "got #{value.inspect}"
+          end
+        end
+
+        def resolve(error, step_policies: [], class_policies: [])
+          step_policies.find { |policy| policy.matches?(error) } ||
+            class_policies.find { |policy| policy.specific? && policy.matches?(error) } ||
+            class_policies.find { |policy| !policy.specific? && policy.matches?(error) } ||
+            DEFAULT
+        end
+      end
+
+      def initialize(action = nil, matching: nil, wait: nil, max_reattempts: nil, terminal_action: nil, report: :always, &handler)
+        raise ArgumentError, "on_exception takes action: or a block, not both" if action && handler
+        raise ArgumentError, "on_exception needs action: (#{ACTIONS.join(", ")}) or a block" unless action || handler
+
+        @action = action && normalize_action(action, ACTIONS, "action")
+        reattempt_only!(wait: wait, terminal_action: terminal_action)
+        @matching = normalize_matching(matching)
+        @wait = StepDefinition.seconds(wait, "wait", positive: false)
+        @max_reattempts = validate_max_reattempts(max_reattempts)
+        @terminal_action = normalize_action(terminal_action.nil? ? :pause! : terminal_action, TERMINAL_ACTIONS, "terminal_action")
+        @report = validate_report(report)
+        @handler = handler
+        freeze
+      end
+
+      def specific? = !matching.nil?
+
+      def matches?(error)
+        return error.is_a?(StandardError) if matching.nil?
+
+        matching.any? do |matcher|
+          matcher.is_a?(String) ? error.class.ancestors.any? { |ancestor| ancestor.name == matcher } : error.is_a?(matcher)
+        end
+      end
+
+      # +consecutive_reattempts+ is a callable (it reads the step rows), only
+      # called when max_reattempts needs it.
+      def decide(error, workflow:, consecutive_reattempts: -> { 0 })
+        return decide_with_handler(error, workflow) if handler
+        return decide_reattempt(consecutive_reattempts) if action == :reattempt!
+
+        Decision.new(outcome: FlowControl::Outcome.for(action), report: report != :never, terminal: true, handler_error: nil)
+      end
+
+      private
+
+      def decide_reattempt(consecutive_reattempts)
+        if max_reattempts && consecutive_reattempts.call >= max_reattempts
+          return Decision.new(outcome: FlowControl::Outcome.for(terminal_action), report: report != :never,
+                              terminal: true, handler_error: nil)
+        end
+
+        Decision.new(outcome: FlowControl::Outcome.reattempt(wait: wait), report: report == :always,
+                     terminal: false, handler_error: nil)
+      end
+
+      # The block runs on the workflow instance and must call a verb; one
+      # that returns without calling any pauses the workflow. A block that
+      # raises pauses it too, and its own error is handed back for reporting.
+      def decide_with_handler(error, workflow)
+        outcome = catch(FlowControl::THROW_TAG) do
+          workflow.instance_exec(error, &handler)
+          nil
+        end
+        terminal = !outcome&.reattempt?
+        Decision.new(outcome: outcome || FlowControl::Outcome.pause,
+                     report: report == :always || (report == :terminal_only && terminal),
+                     terminal: terminal, handler_error: nil)
+      rescue StandardError => e
+        Decision.new(outcome: FlowControl::Outcome.pause, report: true, terminal: true, handler_error: e)
+      end
+
+      def normalize_action(value, allowed, option)
+        verb = :"#{value.to_s.delete_suffix("!")}!"
+        return verb if allowed.include?(verb)
+
+        raise ArgumentError, "#{option}: must be one of #{allowed.map(&:inspect).join(", ")}, got #{value.inspect}"
+      end
+
+      def normalize_matching(matching)
+        return nil if matching.nil? || (matching.respond_to?(:empty?) && matching.empty?)
+
+        Array(matching).map do |matcher|
+          next -matcher.to_s if matcher.is_a?(String) || matcher.is_a?(Symbol)
+          next matcher if matcher.is_a?(Module)
+
+          raise ArgumentError, "matching: takes exception classes or class names, got #{matcher.inspect}"
+        end.freeze
+      end
+
+      # wait: and terminal_action: shape a reattempt!; anywhere else (another
+      # action, an imperative block, which passes its own wait) they would be
+      # silently ignored.
+      def reattempt_only!(**options)
+        given = options.compact.keys
+        return if given.empty? || action == :reattempt!
+
+        raise ArgumentError, "#{given.map { |key| "#{key}:" }.join(" and ")} only applies to action: :reattempt!"
+      end
+
+      def validate_max_reattempts(value)
+        return nil if value.nil?
+        raise ArgumentError, "max_reattempts: only applies to action: :reattempt!" unless action == :reattempt!
+        return value if value.is_a?(Integer) && value.positive?
+
+        raise ArgumentError, "max_reattempts: must be a positive Integer, got #{value.inspect}"
+      end
+
+      def validate_report(value)
+        return value if REPORTS.include?(value)
+
+        raise ArgumentError, "report: must be one of #{REPORTS.map(&:inspect).join(", ")}, got #{value.inspect}"
+      end
+    end
+
+    # What applies when no policy matches: pause!, reported.
+    ExceptionPolicy::DEFAULT = ExceptionPolicy.new(:pause!)
+  end
+end

data/lib/pgbus/workflow/fan_out.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # fan_out steps (issue #541): a group of jobs the workflow waits for as
+    # one batch.
+    #
+    #   fan_out :notify_vendors do |b|
+    #     b.add(*subject.vendors.map { |vendor| NotifyVendorJob.new(vendor.id) })
+    #   end
+    #
+    # 1. run (inside the StepJob, outside any transaction): the body gets a
+    #    Builder that records its b.add calls. A flow-control verb or an
+    #    exception in the body ends the step before any batch exists. When
+    #    the body returns, Pgbus.batch opens with on_complete
+    #    BatchFinishedJob.new(step_id) and the recorded adds.
+    # 2. park (under the run's lock): the step waits on the batch. A batch
+    #    that already finished (an empty one finishes at once, before
+    #    Pgbus.batch returns) is resolved here and then, since its callback
+    #    may have run before the step parked and found nothing to resume.
+    # 3. finished (BatchFinishedJob, under the lock): no dead-lettered member
+    #    completes the step and advances; any applies on_batch_failure with
+    #    FanOutFailed recorded. A step no longer waiting (skipped, cancelled,
+    #    already resolved by the park) is left alone.
+    #
+    # A reattempt or a resume! of a failed fan_out runs the whole body again
+    # and opens a fresh batch: member jobs must be idempotent.
+    module FanOut
+      # The `b` a fan_out body gets. Records its adds; the batch opens once
+      # the body returns. The workflow owns the batch's callbacks.
+      class Builder
+        CALLBACK_MESSAGE = "a fan_out step owns its batch's callbacks (on_complete resumes the workflow); " \
+                           "do the follow-up work in the next step"
+
+        def initialize
+          @adds = []
+        end
+
+        # Adds job instances (MyJob.new(args), optionally .set(...)). A block
+        # runs inside the batch, so a perform_later in it joins too.
+        def add(*jobs, &block)
+          jobs = Batch::Builder.job_instances(jobs)
+          @adds << [jobs, block]
+          jobs
+        end
+
+        def size = @adds.sum { |jobs, _block| jobs.size }
+
+        %i[on_complete on_success on_failure on_finish].each do |slot|
+          define_method(slot) { |*, **| raise ArgumentError, "#{slot}: #{CALLBACK_MESSAGE}" }
+        end
+
+        # Replays the recorded adds on a real batch builder.
+        def replay(batch_builder)
+          @adds.each { |jobs, block| batch_builder.add(*jobs, &block) }
+        end
+      end
+
+      module_function
+
+      def run(definition, workflow)
+        builder = Builder.new
+        definition.body.is_a?(Symbol) ? workflow.send(definition.body, builder) : workflow.instance_exec(builder, &definition.body)
+        batch = open_batch(workflow, workflow.send(:step_context), builder)
+        throw FlowControl::THROW_TAG, FlowControl::Outcome.fan_out(batch_id: batch.batch_id, size: batch.reload.total_jobs)
+      end
+
+      def open_batch(workflow, attempt, builder)
+        queue = workflow.class.step_job_options_for(attempt.step_name)[:queue] || Pgbus.configuration.workflow_queue
+        callback = BatchFinishedJob.new(attempt.step_id).set(queue: queue)
+        Pgbus.batch(description: "#{workflow.workflow_class}##{attempt.step_name} #{workflow.workflow_id}",
+                    properties: { workflow_id: workflow.workflow_id, step_id: attempt.step_id }) do |b|
+          b.on_complete(callback)
+          builder.replay(b)
+        end
+      end
+
+      # The StepJob's outcome for a fan_out whose batch is open. What the body
+      # recorded with output(...) is kept on the row until the step resolves.
+      def park(record, step, outcome, notes:, output: nil)
+        batch_id = outcome.details.fetch(:batch_id)
+        Runner.event!(record, "fan_out_opened", actor: Runner::SYSTEM_ACTOR, step: step, details: outcome.details)
+        unless Waiting.park!(record, step, now: Time.current, batch_id: batch_id, output: output)
+          StepOutcome.ignore_late_completion(record, step, "waiting", "fan_out", nil)
+          return
+        end
+
+        entry = BatchEntry.find_by(batch_id: batch_id)
+        resolve(record, step, entry, notes) if entry&.status == "finished"
+      end
+
+      # BatchFinishedJob#perform (and the sweep, for a lost one). Truthy when
+      # it resolved the step.
+      def finished(step_id)
+        step = WorkflowStep.find_by(id: step_id)
+        return log("step #{step_id} no longer exists (its run was deleted)") unless step
+
+        Runner.locked(step.workflow_id) do |record, notes|
+          step = record && WorkflowStep.find_by(id: step_id)
+          unless step&.status == "waiting" && step.batch_id
+            next log("step #{step_id} is not waiting on its batch (#{step&.status || "gone"}): nothing to resume")
+          end
+
+          entry = BatchEntry.find_by(batch_id: step.batch_id)
+          next resolve_vanished(record, step, notes) unless entry
+          next log("batch #{step.batch_id} of step #{step_id} has not finished") unless entry.status == "finished"
+
+          resolve(record, step, entry, notes)
+        end
+      end
+
+      # The sweep's repair for a lost callback: fan_out steps still waiting on
+      # a batch that finished more than +grace+ ago, or whose batch row is
+      # gone (batch retention ran before a long-delayed callback). Returns how
+      # many it resolved.
+      def resolve_lost(now:, grace:, limit: Sweep::BATCH_SIZE)
+        finished_batches = BatchEntry.finished.where(finished_at: ...(now - grace)).select(:batch_id)
+        waiting = WorkflowStep.where(status: "waiting").where.not(batch_id: nil)
+        lost = waiting.where(batch_id: finished_batches)
+                      .or(waiting.where(waiting_since: ...(now - grace)).where.not(batch_id: BatchEntry.select(:batch_id)))
+        lost.order(:id).limit(limit).pluck(:id).count { |step_id| finished(step_id) }
+      end
+
+      # The batch row is gone, so its outcome is unknown: the step pauses
+      # (whatever on_batch_failure says) with FanOutFailed, for a person to
+      # resume (a fresh batch) or skip.
+      def resolve_vanished(record, step, notes)
+        Runner.event!(record, "fan_out_finished", actor: Runner::SYSTEM_ACTOR, step: step,
+                                                  details: { batch_id: step.batch_id, vanished: true })
+        error = FanOutFailed.new("batch #{step.batch_id} no longer exists (deleted before its callback ran); its outcome is unknown")
+        Waiting.resolve!(record, step, FlowControl::Outcome.pause, notes: notes, error: error)
+      end
+
+      def resolve(record, step, entry, notes)
+        batch = Batch.find(entry.batch_id)
+        failed = batch.failed_jobs
+        total = batch.total_jobs
+        Runner.event!(record, "fan_out_finished", actor: Runner::SYSTEM_ACTOR, step: step,
+                                                  details: { batch_id: entry.batch_id, failed_jobs: failed, total_jobs: total })
+        return Waiting.resolve!(record, step, FlowControl::Outcome.complete, notes: notes) if failed.zero?
+
+        error = FanOutFailed.new("#{failed} of #{total} jobs dead-lettered (batch #{entry.batch_id})")
+        Waiting.resolve!(record, step, FlowControl::Outcome.for(failure_action(record, step)), notes: notes, error: error)
+      end
+
+      def failure_action(record, step)
+        definition = record.workflow_klass&.workflow_steps&.fetch(step.step_name)
+        definition.respond_to?(:on_batch_failure) ? definition.on_batch_failure : :pause!
+      end
+
+      def log(message)
+        Pgbus.logger.info { "[Pgbus::Workflow] fan_out: #{message}" }
+        nil
+      end
+    end
+  end
+end