pgbus 0.19.4 → 0.20.1

data/lib/pgbus/workflow/step_execution.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # One StepJob delivery: claim the step row, check the guards, run the
+    # body in a fresh workflow instance, apply the outcome.
+    #
+    # 1. Claim, under the run's row lock: scheduled → in_progress (only once
+    #    due) and the run ready → performing. A row that is not claimable
+    #    (already running elsewhere, finished, early, or its run is paused or
+    #    gone) makes this delivery a no-op, except a redelivery of the very
+    #    message that claimed an in_progress row while the visibility
+    #    heartbeat is on: its worker died, and the stall action applies now.
+    # 2. Guards, still under the lock: the class or the step no longer
+    #    defined (pause with StepNotDefined, or skip a removed_step),
+    #    cancel_if, skip_if.
+    # 3. The body, outside any transaction, inside `catch` for the
+    #    flow-control verbs and `rescue StandardError` for the policies.
+    # 4. The outcome, under the lock again (StepOutcome).
+    #
+    # Whatever the step's outcome, the job itself completes normally: the
+    # step row and the pgbus.workflow.* events carry the truth.
+    class StepExecution
+      Claim = Data.define(:record, :step, :klass, :definition, :workflow, :guard_error)
+
+      def initialize(step_id, delivery: Pgbus::ActiveJob::Delivery.current, config: Pgbus.configuration)
+        @step_id = step_id
+        @delivery = delivery
+        @config = config
+      end
+
+      def call
+        step = WorkflowStep.find_by(id: @step_id)
+        return debug("step #{@step_id} no longer exists (its run was deleted)") unless step
+
+        StepContext.with(workflow_id: step.workflow_id, workflow_class: step.workflow_class, step_name: step.step_name,
+                         attempt: step.attempt) do
+          claim = claim(step)
+          run(claim) if claim
+        end
+        nil
+      end
+
+      private
+
+      attr_reader :config
+
+      # The worker tag and the app revision (which may call the app's
+      # app_revision callable or read a file) are resolved before the run's
+      # row lock is taken.
+      def claim(step)
+        @stamp = { worker: Runner.worker_tag, app_revision: Deploy::Revision.current(config) }
+        Runner.locked(step.workflow_id) do |record, notes|
+          step = WorkflowStep.find_by(id: step.id) if record
+          if record.nil? || step.nil?
+            debug("step #{@step_id}'s run no longer exists")
+          elsif step.status == "in_progress"
+            check_redelivery(record, step, notes)
+          elsif claimable?(record, step)
+            claim_step(record, step, notes)
+          else
+            debug("step #{step.id} is not claimable (step #{step.status}, run #{record.status}, due #{step.scheduled_at})")
+          end
+        end
+      end
+
+      def claimable?(record, step)
+        step.status == "scheduled" && States.legal?(:claim, record.status) && step.scheduled_at <= Time.current + 1
+      end
+
+      # A live worker keeps its message invisible (VisibilityHeartbeat), so
+      # the same message coming back for a row it claimed means that worker
+      # is gone. Without the heartbeat a long live step can outlive its
+      # visibility timeout, so the sweep's stall_at decides instead.
+      def check_redelivery(record, step, notes)
+        msg_id = @delivery && @delivery[:msg_id]
+        redelivered = !msg_id.nil? && step.msg_id == msg_id.to_i && @delivery[:read_ct].to_i > 1 && config.visibility_heartbeat
+        return debug("step #{step.id} is already in progress elsewhere") unless redelivered
+
+        StepOutcome.stall!(record, step, detection: "redelivery", notes: notes, config: config)
+        nil
+      end
+
+      def claim_step(record, step, notes)
+        now = Time.current
+        klass = record.workflow_klass
+        definition = klass&.workflow_steps&.fetch(step.step_name)
+        attrs = { stall_at: now + (definition&.timeout || config.workflow_stall_threshold), msg_id: @delivery&.dig(:msg_id),
+                  **@stamp }
+        return nil if WorkflowStep.claim!(step.id, now: now, **attrs).zero?
+
+        step.assign_attributes(status: "in_progress", started_at: now, **attrs)
+        Runner.transition!(record, :claim, current_step: step.step_name)
+        Runner.event!(record, "step_started", actor: Runner::SYSTEM_ACTOR, step: step,
+                                              details: attrs.slice(:worker, :app_revision, :msg_id).compact)
+        guard(record, step, klass, definition, notes)
+      end
+
+      # Returns the Claim to run, or nil when a guard already ended the
+      # attempt. A guard that raises is handed to the exception policies,
+      # like a body that raises.
+      def guard(record, step, klass, definition, notes)
+        return undefined!(record, step, notes, "#{record.workflow_class} is not a Pgbus::Workflow class any more") unless klass
+
+        unless definition
+          return skip!(record, step, notes, klass) if klass.workflow_steps.removed?(step.step_name)
+
+          return undefined!(record, step, notes, "#{klass.name} has no step #{step.step_name} " \
+                                                 "(declare `removed_step :#{step.step_name}` to skip it)")
+        end
+
+        workflow = klass.new(record)
+        workflow.send(:enter_step, StepContext::Attempt.new(step_id: step.id, step_name: step.step_name, attempt: step.attempt))
+        claim = Claim.new(record: record, step: step, klass: klass, definition: definition, workflow: workflow, guard_error: nil)
+        begin
+          verdict = workflow.send(:guarding) do
+            next FlowControl::Outcome.cancel if cancel?(klass, workflow)
+
+            FlowControl::Outcome.skip if definition.skip?(workflow)
+          end
+        rescue StandardError => e
+          return claim.with(guard_error: e)
+        end
+        return outcome!(record, step, notes, klass, verdict) if verdict
+
+        gate(record, step, definition, workflow, claim)
+      end
+
+      # A wait_for: step runs only with a signal (or once its wait timed
+      # out); without one it parks and this delivery ends. A matcher that
+      # raises is handed to the exception policies, like a guard.
+      def gate(record, step, definition, workflow, claim)
+        return claim unless definition.waits_for_signal?
+
+        Signals.gate(record, step, definition, workflow) ? claim : nil
+      rescue StandardError => e
+        claim.with(guard_error: e)
+      end
+
+      def cancel?(klass, workflow)
+        klass.cancel_conditions.any? do |condition|
+          condition.is_a?(Symbol) ? workflow.send(condition) : workflow.instance_exec(&condition)
+        end
+      end
+
+      def undefined!(record, step, notes, message)
+        StepOutcome.apply(record, step, FlowControl::Outcome.pause, notes: notes, error: StepNotDefined.new(message), klass: nil)
+        nil
+      end
+
+      def skip!(record, step, notes, klass) = outcome!(record, step, notes, klass, FlowControl::Outcome.skip)
+
+      def outcome!(record, step, notes, klass, outcome)
+        StepOutcome.apply(record, step, outcome, notes: notes, klass: klass, duration: 0.0)
+        nil
+      end
+
+      def run(claim)
+        started = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        outcome, error = claim.guard_error ? [nil, claim.guard_error] : execute(claim)
+        duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started
+        finish(claim, outcome, error, duration)
+      end
+
+      def execute(claim)
+        outcome = catch(FlowControl::THROW_TAG) do
+          claim.definition.invoke(claim.workflow)
+          nil
+        end
+        [outcome || FlowControl::Outcome.complete, nil]
+      rescue StandardError => e
+        [nil, e]
+      end
+
+      # The exception policy is decided before the run's lock is taken
+      # again: an imperative on_exception block is app code, and must not
+      # hold the row lock (and an open transaction) while it runs.
+      def finish(claim, outcome, error, duration)
+        return finish_late(claim, error) if error && !attempt_active?(claim)
+
+        decision = error && decide(claim, error)
+        outcome = decision.outcome if decision
+        Runner.locked(claim.record.workflow_id) do |record, notes|
+          next debug("step #{claim.step.id}'s run was deleted while the step ran") unless record
+          next park_fan_out(record, claim, outcome, notes) if outcome&.action == :fan_out
+
+          notes.report(decision.handler_error, handler_context(claim)) if decision&.handler_error
+          StepOutcome.apply(record, claim.step, outcome, notes: notes, error: error, output: claim.workflow.pending_output,
+                                                         report: decision ? decision.report : true, duration: duration,
+                                                         klass: claim.klass)
+        end
+      end
+
+      def park_fan_out(record, claim, outcome, notes)
+        FanOut.park(record, claim.step, outcome, notes: notes, output: claim.workflow.pending_output)
+      end
+
+      # While its body runs, a row leaves in_progress only for a terminal
+      # status (a parked row went waiting before any body ran), so an attempt
+      # that finds it gone has been declared stalled (or its run cancelled): late.
+      def attempt_active?(claim) = WorkflowStep.where(id: claim.step.id, status: "in_progress").exists?
+
+      # A late attempt that raised: its outcome is ignored either way, so an
+      # imperative on_exception block must not run (and cause side effects)
+      # for it. A declarative policy still names the outcome it would have
+      # applied and decides the report; an imperative one records none.
+      def finish_late(claim, error)
+        policy = policy_for(claim, error)
+        decision = policy.handler ? nil : decide(claim, error, policy: policy)
+        column = decision && StepOutcome::STEP_OUTCOME.fetch(decision.outcome.action)
+        report = decision ? decision.report : policy.report != :never
+        Runner.locked(claim.record.workflow_id) do |record, notes|
+          next debug("step #{claim.step.id}'s run was deleted while the step ran") unless record
+
+          StepOutcome.ignore_late_completion(record, claim.step, "failed", column, error)
+          notes.report(error, StepOutcome.error_context(record, claim.step, "late_completion_ignored")) if report
+        end
+      end
+
+      def policy_for(claim, error)
+        ExceptionPolicy.resolve(error, step_policies: claim.definition.exception_policies,
+                                       class_policies: claim.klass.exception_policies)
+      end
+
+      def decide(claim, error, policy: policy_for(claim, error))
+        step = claim.step
+        policy.decide(error, workflow: claim.workflow,
+                             consecutive_reattempts: -> { WorkflowStep.consecutive_reattempts(step.workflow_id, step.step_name) })
+      end
+
+      def handler_context(claim)
+        { action: "workflow_on_exception", workflow_class: claim.klass.name, workflow_id: claim.step.workflow_id,
+          step_name: claim.step.step_name, attempt: claim.step.attempt }
+      end
+
+      def debug(message)
+        Pgbus.logger.debug { "[Pgbus::Workflow] #{message} #{StepContext.log_tag}".rstrip }
+        nil
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/step_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Pgbus
+  class Workflow
+    # The job behind every step attempt: perform(step_id) runs one
+    # StepExecution. Enqueued only by Pgbus::WorkflowStep's after-commit
+    # callback, on config.workflow_queue unless the class
+    # (set_step_job_options) or the step (job_options:) says otherwise.
+    #
+    # No retry_on and no discard_on: a step's failure is the exception
+    # policies' business, and the job returns normally whatever the step's
+    # outcome. Redelivery is reserved for a worker that died mid-step, which
+    # the claim recognises. The visibility heartbeat must stay on for this
+    # class (never `pgbus_visibility_heartbeat false`): fast crash detection
+    # depends on a live worker keeping its message invisible.
+    class StepJob < ::ActiveJob::Base
+      # Always on, whatever ActiveJob::Base or the app set: the executor reads
+      # this, and a redelivery means a dead worker only while it is true.
+      def self.pgbus_visibility_heartbeat_enabled = true # rubocop:disable Naming/PredicateMethod -- JobMixin's reader
+
+      def perform(step_id)
+        StepExecution.new(step_id).call
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/step_outcome.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # Applies how a step attempt ended to its row and to the run, under the
+    # run's row lock: the StepJob after a body, a guard (cancel_if, skip_if,
+    # a removed or unknown step), crash detection and the sweep.
+    #
+    # The step row is written first, guarded on status = 'in_progress'. A
+    # row that is no longer in_progress was declared stalled (or the run was
+    # cancelled) while this attempt was still running: the outcome and its
+    # output are discarded and only a late_completion_ignored event is
+    # recorded. Otherwise the run moves by the States table: advance (the
+    # next step is scheduled), finish, reattempt, step_pause, step_cancel, or
+    # hold when an outside pause! got there first.
+    module StepOutcome
+      STEP_STATUS = { complete: "completed", finish: "completed", pause: "completed",
+                      skip: "skipped", cancel: "cancelled", reattempt: "failed" }.freeze
+      STEP_OUTCOME = { complete: "completed", finish: "finished_early", pause: "paused",
+                       skip: "skipped", cancel: "cancelled", reattempt: "reattempted" }.freeze
+
+      module_function
+
+      # Returns false when the attempt had lost its in_progress guard.
+      def apply(record, step, outcome, notes:, error: nil, output: nil, report: true, duration: nil, klass: record.workflow_klass) # rubocop:disable Naming/PredicateMethod
+        now = Time.current
+        status = error ? "failed" : STEP_STATUS.fetch(outcome.action)
+        column = STEP_OUTCOME.fetch(outcome.action)
+        captured = error && ErrorCapture.capture(error)
+        attrs = { status: status, outcome: column, finished_at: now, output: output, **captured.to_h }
+        if WorkflowStep.finish!(step.id, **attrs).zero?
+          ignore_late_completion(record, step, status, column, error)
+          notes.report(error, error_context(record, step, "late_completion_ignored")) if error && report
+          return false
+        end
+
+        Signals.settle(step, status, column, now: now)
+        step_events(record, step, outcome, column, error)
+        move_run(record, step, outcome, notes: notes, captured: captured, klass: klass, now: now)
+        notes.instrument("pgbus.workflow.step", step_payload(record, step, column, status, error, duration))
+        notes.report(error, error_context(record, step, column)) if error && report
+        true
+      end
+
+      # A step whose worker died (detection: "redelivery") or that outlived
+      # its stall_at (detection: "sweep"): config.workflow_stall_action
+      # applies, with Pgbus::Workflow::StepStalled recorded on the row.
+      def stall!(record, step, detection:, notes:, config: Pgbus.configuration)
+        why = if detection == "redelivery"
+                "its message came back while it was in_progress: the worker #{step.worker} died"
+              else
+                "it was still in_progress at its stall_at #{step.stall_at&.utc&.iso8601} (worker #{step.worker})"
+              end
+        error = StepStalled.new("#{step.step_name} attempt #{step.attempt} stalled: #{why}")
+        Runner.event!(record, "stall_detected", actor: Runner::SYSTEM_ACTOR, step: step,
+                                                details: { detection: detection, worker: step.worker,
+                                                           stall_at: step.stall_at&.utc&.iso8601(3) }.compact)
+        notes.instrument("pgbus.workflow.stall_detected",
+                         Runner.payload_for(record, step_name: step.step_name, attempt: step.attempt, detection: detection))
+        apply(record, step, stall_outcome(config.workflow_stall_action), notes: notes, error: error)
+      end
+
+      def stall_outcome(action)
+        case action
+        when :reattempt then FlowControl::Outcome.reattempt
+        when :cancel then FlowControl::Outcome.cancel
+        else FlowControl::Outcome.pause
+        end
+      end
+
+      def move_run(record, step, outcome, notes:, captured:, klass:, now:)
+        paused = record.status == "paused"
+        case outcome.action
+        when :complete, :skip
+          following = klass&.workflow_steps&.next_after(step.step_name)
+          following ? advance(record, following, paused: paused, now: now) : finish(record, notes)
+        when :finish then finish(record, notes)
+        when :reattempt
+          Runner.schedule!(record, step.step_name, at: now + (outcome.wait || 0), actor: Runner::SYSTEM_ACTOR)
+          Runner.transition!(record, paused ? :hold : :reattempt, current_step: step.step_name)
+        when :pause then pause(record, step, outcome, notes, captured: captured, paused: paused, now: now)
+        when :cancel then cancel(record, step, outcome, notes, now: now)
+        end
+      end
+
+      def advance(record, following, paused:, now:)
+        Runner.schedule!(record, following.name, at: now + following.wait, actor: Runner::SYSTEM_ACTOR)
+        Runner.transition!(record, paused ? :hold : :advance, current_step: following.name)
+      end
+
+      def finish(record, notes)
+        Runner.transition!(record, :finish, finished_at: Time.current)
+        Runner.event!(record, "finished", actor: Runner::SYSTEM_ACTOR)
+        notes.instrument("pgbus.workflow_finished", Runner.payload_for(record))
+      end
+
+      # The run row keeps the error that paused it (cleared by resume!).
+      def pause(record, step, outcome, notes, captured:, paused:, now:)
+        run_error = captured ? captured.except(:error_causes) : {}
+        return Runner.transition!(record, :hold, **run_error) if paused
+
+        error_class = captured&.fetch(:error_class)
+        Runner.transition!(record, :step_pause, paused_at: now, **run_error)
+        Runner.event!(record, "paused", actor: outcome.actor || Runner::SYSTEM_ACTOR, reason: outcome.reason, step: step,
+                                        details: { error_class: error_class }.compact)
+        notes.instrument("pgbus.workflow_paused", Runner.payload_for(record, step_name: step.step_name, error_class: error_class))
+      end
+
+      def cancel(record, step, outcome, notes, now:)
+        Runner.transition!(record, :step_cancel, cancelled_at: now)
+        WorkflowStep.cancel_open!(record.workflow_id, now: now)
+        Runner.event!(record, "cancelled", actor: outcome.actor || Runner::SYSTEM_ACTOR, reason: outcome.reason, step: step)
+        notes.instrument("pgbus.workflow_cancelled", Runner.payload_for(record, step_name: step.step_name))
+      end
+
+      def step_events(record, step, outcome, column, error)
+        actor = Runner::SYSTEM_ACTOR
+        if error
+          Runner.event!(record, "step_failed", actor: actor, step: step,
+                                               details: { error_class: error.class.name, error_message: error.message.to_s })
+        end
+        case outcome.action
+        when :complete, :finish, :pause
+          Runner.event!(record, "step_completed", actor: actor, step: step, details: { outcome: column }) unless error
+        when :skip then Runner.event!(record, "step_skipped", actor: outcome.actor || actor, reason: outcome.reason, step: step)
+        when :reattempt then Runner.event!(record, "step_reattempted", actor: actor, step: step, details: { wait: outcome.wait }.compact)
+        end
+      end
+
+      def ignore_late_completion(record, step, status, column, error)
+        details = { status: status, outcome: column, error_class: error&.class&.name }.compact
+        Runner.event!(record, "late_completion_ignored", actor: Runner::SYSTEM_ACTOR, step: step, details: details)
+        Pgbus.logger.warn do
+          "[Pgbus] Workflow #{record.workflow_class} #{record.workflow_id}: step #{step.step_name} attempt #{step.attempt} " \
+            "finished after it stopped being in_progress (stalled or cancelled); its outcome (#{column}) and output were discarded"
+        end
+      end
+
+      def step_payload(record, step, column, status, error, duration)
+        pickup = step.started_at && step.scheduled_at ? [step.started_at - step.scheduled_at, 0].max.to_f : nil
+        Runner.payload_for(record, step_name: step.step_name, attempt: step.attempt, outcome: column, status: status,
+                                   error_class: error&.class&.name, duration: duration, pickup_latency: pickup)
+      end
+
+      def error_context(record, step, column)
+        { workflow_class: record.workflow_class, workflow_id: record.workflow_id, step_name: step.step_name,
+          attempt: step.attempt, outcome: column }
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/steps.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # The ordered step list of one workflow class: StepDefinitions and
+    # RemovedStep gravestones, in declaration order. Immutable: every
+    # change (`step`, `removed_step`) builds a new frozen Steps that the
+    # class stores whole, so a subclass extends its parent's list without
+    # touching it and a reader never sees a list change under it
+    # (docs/concurrency.md).
+    class Steps
+      attr_reader :entries
+
+      def initialize(entries = [])
+        @entries = entries.dup.freeze
+        freeze
+      end
+
+      EMPTY = new
+
+      def add(definition, before: nil, after: nil)
+        raise ArgumentError, "use before_step: or after_step:, not both" if before && after
+        raise ArgumentError, "#{definition.name} is a removed step (removed_step); pick another name" if removed?(definition.name)
+        raise ArgumentError, "duplicate step #{definition.name}" if fetch(definition.name)
+
+        list = entries.dup
+        anchor = before || after
+        if anchor
+          index = position(anchor)
+          unless index
+            raise ArgumentError,
+                  "#{before ? "before_step" : "after_step"}: #{anchor.to_s.to_sym.inspect} — no step named #{anchor}"
+          end
+
+          list.insert(before ? index : index + 1, definition)
+        else
+          list << definition
+        end
+        self.class.new(list)
+      end
+
+      # Replaces a live step with a gravestone at its position, or appends a
+      # gravestone for a step that is already gone from the class.
+      def remove(name)
+        name = -name.to_s
+        raise ArgumentError, "#{name} is already a removed step" if removed?(name)
+
+        list = entries.dup
+        index = position(name)
+        gravestone = RemovedStep.new(name: name)
+        index ? list[index] = gravestone : list << gravestone
+        self.class.new(list)
+      end
+
+      def definitions = entries.reject(&:removed?)
+
+      def names = definitions.map(&:name)
+
+      def removed_names = entries.select(&:removed?).map(&:name)
+
+      def fetch(name)
+        name = name.to_s
+        definitions.find { |entry| entry.name == name }
+      end
+
+      def removed?(name)
+        name = name.to_s
+        entries.any? { |entry| entry.removed? && entry.name == name }
+      end
+
+      def first = definitions.first
+
+      # The next live step declared after +name+ (a step or a gravestone),
+      # or nil when +name+ is the last one or unknown.
+      def next_after(name)
+        index = position(name)
+        return nil unless index
+
+        entries.drop(index + 1).find { |entry| !entry.removed? }
+      end
+
+      def empty? = definitions.empty?
+
+      # Name for an anonymous `step { ... }`: step_<n>, the first n from
+      # entries.size + 1 not taken by a step or a gravestone.
+      def anonymous_name
+        number = entries.size + 1
+        number += 1 while entries.any? { |entry| entry.name == "step_#{number}" }
+        "step_#{number}"
+      end
+
+      # What definition_digest hashes: the ordered [name, kind] pairs.
+      def digest_pairs = entries.map { |entry| [entry.name, entry.kind] }
+
+      private
+
+      def position(name)
+        name = name.to_s
+        entries.index { |entry| entry.name == name }
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/sweep.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # The dispatcher's workflow housekeeping.
+    #
+    # repair (every config.workflow_sweep_interval):
+    # - an in_progress step past its stall_at (started_at + the step's
+    #   timeout:, or workflow_stall_threshold) gets workflow_stall_action,
+    #   with StepStalled recorded. A dead worker is usually caught sooner, on
+    #   redelivery (StepExecution); this catches a live but hung step, and
+    #   every crash when the visibility heartbeat is off.
+    # - a scheduled step of a ready run, due more than workflow_stall_threshold
+    #   ago and not sent within it, is sent again by bumping enqueued_at (the
+    #   model callback re-fires). No queue inspection: a duplicate message is
+    #   a no-op at the claim.
+    # - a waiting step of a waiting run past its timeout_at (wait_timeout:)
+    #   gets its on_timeout (WaitTimeout), so a timeout fires within one
+    #   workflow_sweep_interval of its deadline.
+    # - a fan_out step still waiting on a batch that finished more than
+    #   workflow_stall_threshold ago lost its BatchFinishedJob (dead-lettered,
+    #   or never sent): the sweep resolves it as the callback would have.
+    # - pgbus.workflow.gauge: runs by class and status, and by class and stuck
+    #   reason (waiting_overdue: waiting longer than
+    #   workflow_waiting_visibility_threshold on a step without a timeout).
+    #
+    # cleanup (hourly): finished and cancelled runs older than
+    # workflow_retention are deleted in batches; their steps and events go
+    # with them (ON DELETE CASCADE).
+    module Sweep
+      BATCH_SIZE = 500
+
+      # The schema probe's memo (a true answer is kept for the life of the
+      # process), read and written under the lock; the probe itself runs
+      # outside it.
+      @schema_lock = Mutex.new
+      @tables_ready = false
+      @waiting_ready = false
+
+      class << self
+        def repair(config: Pgbus.configuration, now: Time.current)
+          return nil unless tables_ready?
+
+          threshold = config.workflow_stall_threshold
+          summary = { stalled: repair_stalled(now: now, config: config), resent: resend_overdue(now: now, threshold: threshold) }
+          waiting = waiting_ready?
+          summary[:timed_out] = waiting ? WaitTimeout.expire_due(now: now) : 0
+          summary[:fan_outs] = waiting ? FanOut.resolve_lost(now: now, grace: threshold) : 0
+          emit_gauge(now: now, threshold: threshold, waiting_threshold: (config.workflow_waiting_visibility_threshold if waiting))
+          summary
+        end
+
+        def cleanup(retention: Pgbus.configuration.workflow_retention, now: Time.current, batch_size: BATCH_SIZE)
+          return 0 unless retention&.positive? && tables_ready?
+
+          cutoff = now - retention
+          deleted = 0
+          loop do
+            ids = WorkflowEntry.stale(before: cutoff).order(:id).limit(batch_size).pluck(:id)
+            break if ids.empty?
+
+            deleted += WorkflowEntry.where(id: ids).delete_all
+          end
+          deleted
+        end
+
+        # True once the three workflow tables exist (pgbus:add_workflows).
+        def tables_ready?
+          return true if @schema_lock.synchronize { @tables_ready }
+
+          ready = begin
+            [WorkflowEntry, WorkflowStep, WorkflowEvent].all?(&:table_exists?)
+          rescue StandardError => e
+            Pgbus.logger.debug { "[Pgbus] Workflow table probe failed: #{e.class}: #{e.message}" }
+            false
+          end
+          @schema_lock.synchronize { @tables_ready = true } if ready
+          ready
+        end
+
+        # True once the waiting-step schema exists (pgbus:add_workflow_signals,
+        # or an add_workflows that carries it): the signals table and the
+        # step columns. Asked of the database through the connection, not the
+        # schema cache (which keeps a negative answer). A process that loaded
+        # the step model before the migration still needs a restart to see
+        # the new columns, as for any column added under a running app.
+        def waiting_ready?
+          return true if @schema_lock.synchronize { @waiting_ready }
+
+          ready = begin
+            connection = WorkflowStep.connection
+            connection.table_exists?(WorkflowSignal.table_name) && connection.column_exists?(WorkflowStep.table_name, :timeout_at)
+          rescue StandardError => e
+            Pgbus.logger.debug { "[Pgbus] Workflow waiting-step schema probe failed: #{e.class}: #{e.message}" }
+            false
+          end
+          @schema_lock.synchronize { @waiting_ready = true } if ready
+          ready
+        end
+
+        # Specs only.
+        def reset_tables_ready!
+          @schema_lock.synchronize do
+            @tables_ready = false
+            @waiting_ready = false
+          end
+        end
+
+        private
+
+        def repair_stalled(now:, config:)
+          WorkflowStep.stalled(now: now).order(:id).limit(BATCH_SIZE).pluck(:id, :workflow_id).count do |id, workflow_id|
+            stall(id, workflow_id, now: now, config: config)
+          end
+        end
+
+        # Re-checked under the run's lock: the step may have finished since
+        # the query.
+        def stall(id, workflow_id, now:, config:)
+          Runner.locked(workflow_id) do |record, notes|
+            step = record && WorkflowStep.find_by(id: id)
+            next false unless step&.status == "in_progress" && step.stall_at && step.stall_at < now
+
+            StepOutcome.stall!(record, step, detection: "sweep", notes: notes, config: config)
+            true
+          end
+        end
+
+        def resend_overdue(now:, threshold:)
+          WorkflowStep.overdue(threshold: threshold, now: now).order(:id).limit(BATCH_SIZE).to_a.each do |step|
+            step.update!(enqueued_at: now)
+          end.size
+        end
+
+        def emit_gauge(now:, threshold:, waiting_threshold: nil)
+          statuses = WorkflowEntry.status_counts
+          stuck = WorkflowEntry.stuck_reason_counts(threshold: threshold, now: now, waiting_threshold: waiting_threshold)
+          classes = (statuses.keys.map(&:first) + stuck.keys.map(&:first) + Workflow.registered_classes.map(&:name)).uniq.sort
+          Instrumentation.instrument(
+            "pgbus.workflow.gauge",
+            statuses: zero_filled(classes, States::WORKFLOW, statuses, :status),
+            stuck: zero_filled(classes, States::STUCK_REASONS, stuck, :reason)
+          )
+        end
+
+        # Every class × every value, so a count that drops to zero is
+        # reported as zero instead of keeping its last value.
+        def zero_filled(classes, values, counts, key)
+          classes.product(values).map do |klass, value|
+            { workflow_class: klass, key => value, count: counts.fetch([klass, value], 0) }
+          end
+        end
+      end
+    end
+  end
+end