@jameslovespancakes/pi-plus 1.0.11 → 1.0.13

package/src/domains/workflows/runtime/engine.ts

@@ -0,0 +1,579 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { Static, TSchema } from "typebox";
+import { bindParallel, bindPipeline, Semaphore } from "./concurrency.ts";
+import { WorkflowAgentLimiter } from "./agent-limits.ts";
+import { defaultAgentRetryScheduler, type AgentRetryScheduler } from "./agent-retry.ts";
+import { resolveWorkflowModelProfiles, type ResolvedWorkflowModelProfiles } from "./model-profiles.ts";
+import { abortReason, isWorkflowPauseError, linkAbortSignal, throwIfAborted } from "./cancellation.ts";
+import { createBudget } from "./budget.ts";
+import { runAgent, type AgentExecutionOptions, type RunContext } from "./agent-runner.ts";
+import { ProgressTracker } from "./progress.ts";
+import { createPerfRecorder, type PerfSink, type PerfSnapshot } from "./perf.ts";
+import { createWorkflowUsageRecorder, type WorkflowUsageSink } from "./usage.ts";
+import {
+  resolveWorkflowRunOptions,
+  type ResolvedWorkflowRunOptions,
+} from "./options.ts";
+import type { AgentOptions, IsolatedAgentResult, LoadedWorkflow, WorkflowApi, WorkflowProgressEvent, WorkflowRef, WorkflowRunOptions } from "./types.ts";
+import { WorkflowInspector } from "./ui/workflow-inspector.ts";
+import { WORKFLOW_VIEWER_OVERLAY_OPTIONS } from "./ui/workflow-viewer-layout.ts";
+import { createWorkflowJournal, createWorkflowRunId, pruneWorkflowJournals, workflowJournalPath } from "./journal.ts";
+import { WorktreeRegistry } from "./worktree.ts";
+import { runFinalizers } from "./finalizers.ts";
+import { unknownErrorMessage } from "./unknown-error.ts";
+import {
+  captureWorkflowResumeContext,
+  type AgentResumeBaseContext,
+} from "./resume-context.ts";
+import {
+  DurableWorkflowRun,
+  ProjectWorkflowRunStore,
+  workflowRunRecordPath,
+  type WorkflowRunStore,
+} from "./workflow-run-store.ts";
+import { createWorkflowRunRecord } from "./workflow-run-record.ts";
+import {
+  createProviderUsageLimitPauseRecord,
+  WorkflowProviderUsageLimitError,
+} from "./provider-usage-limit.ts";
+
+/** The workflow-facing slice of the progress tracker (satisfied by `ProgressTracker`). */
+export interface WorkflowProgress {
+  phase(title: string): void;
+  log(message: string): void;
+  event(event: WorkflowProgressEvent): void;
+}
+
+/** Engine state shared by the workflow and its individual agent calls. */
+export type WorkflowRunContext = RunContext;
+
+/** Per-context knobs threaded through `runWorkflowWithContext` (and re-derived for each sub-workflow). */
+export interface WorkflowContextOptions {
+  /** Controller whose `.signal` equals `rc.signal`; aborts in-flight `parallel` siblings on failure. */
+  abortController: AbortController;
+  /** Optional eager-submission cap for `parallel`. */
+  submissionLimit?: number;
+  /** Resolve a sub-workflow reference for `api.workflow()`. Omit to disable composition. */
+  resolveWorkflow?: (ref: WorkflowRef) => Promise<LoadedWorkflow>;
+  /** Nesting depth. Sub-workflows run at depth >= 1, where `api.workflow()` rejects. */
+  depth?: number;
+  /** Prefix applied to phase titles so sub-workflow phases nest as "<name> ▸ <title>". */
+  progressPrefix?: string;
+  /** Namespace applied to child structured progress keys/lanes/logs. */
+  progressNamespace?: string;
+}
+
+type Outcome<T> =
+  | { readonly ok: true; readonly value: T }
+  | { readonly ok: false; readonly error: unknown };
+
+export interface WorkflowEngineDependencies {
+  readonly worktrees?: WorktreeRegistry;
+  readonly retryScheduler?: AgentRetryScheduler;
+  readonly modelProfiles?: ResolvedWorkflowModelProfiles;
+}
+
+/**
+ * Run a workflow module: build the per-run primitives (binding agent/parallel/pipeline
+ * to a shared semaphore + progress tracker), invoke the workflow, return its result.
+ */
+export async function runWorkflow(
+  ctx: ExtensionContext,
+  mod: LoadedWorkflow,
+  args: string,
+  options: WorkflowRunOptions = {},
+  dependencies: WorkflowEngineDependencies = {},
+): Promise<unknown> {
+  return await runResolvedWorkflow(ctx, mod, args, resolveWorkflowRunOptions(options), dependencies);
+}
+
+/** Execute an options-normalized workflow without re-reading environment defaults. */
+export async function runResolvedWorkflow(
+  ctx: ExtensionContext,
+  mod: LoadedWorkflow,
+  args: string,
+  resolvedOptions: ResolvedWorkflowRunOptions,
+  dependencies: WorkflowEngineDependencies = {},
+): Promise<unknown> {
+  const runId = resolvedOptions.runId ?? createWorkflowRunId();
+  let durableProgress: DurableWorkflowRun | undefined;
+  const progress = new ProgressTracker(ctx, mod.meta.name, runId, (snapshot) => durableProgress?.updateProgress(snapshot));
+  const progressSource = {
+    snapshot: () => progress.snapshot(),
+    conversation: (agentId: number) => progress.conversation(agentId),
+    followUp: (agentId: number, message: string) => progress.followUp(agentId, message),
+    subscribe: (listener: () => void) => progress.subscribe(listener),
+  };
+  const perf = resolvedOptions.perfRecorder ?? createPerfRecorder(resolvedOptions.perf);
+  const usage = createWorkflowUsageRecorder((snapshot) => progress.updateUsage(snapshot));
+  const budget = createBudget(resolvedOptions.budget, usage);
+  const journalPath = workflowJournalPath(ctx.cwd, runId);
+  const resumePath = resolvedOptions.resumeFromRunId ? workflowJournalPath(ctx.cwd, resolvedOptions.resumeFromRunId) : undefined;
+  const runStore = new ProjectWorkflowRunStore(ctx.cwd);
+  let persistenceFailureReported = false;
+  const durableRun = new DurableWorkflowRun(
+    runStore,
+    createWorkflowRunRecord({
+      runId,
+      workflow: mod,
+      options: resolvedOptions,
+      progress: progress.snapshot(),
+      argumentsPresent: args.length > 0,
+    }),
+    (error) => {
+      if (persistenceFailureReported) return;
+      persistenceFailureReported = true;
+      progress.log(`run persistence failed: ${unknownErrorMessage(error)}`);
+    },
+  );
+  durableProgress = durableRun;
+  const worktrees = dependencies.worktrees ?? new WorktreeRegistry(ctx.cwd);
+  const runAbortController = new AbortController();
+  const unlinkContextAbortSignal = linkAbortSignal(ctx.signal, runAbortController);
+  const unlinkOptionAbortSignal = linkAbortSignal(resolvedOptions.signal, runAbortController);
+  const workflowOutcome = await captureOutcome(async () => {
+    await notifyLifecycleObserver(progress, "progress source callback", () => resolvedOptions.onProgressSource?.(progressSource));
+    if (resolvedOptions.inspect && ctx.hasUI && ctx.mode === "tui") {
+      let unsubscribe: (() => void) | undefined;
+      void ctx.ui
+        .custom<void>(
+          (tui, theme, _keybindings, done) => {
+            unsubscribe = progressSource.subscribe(() => tui.requestRender());
+            return new WorkflowInspector(
+              () => progress.snapshot(),
+              tui,
+              theme,
+              () => done(undefined),
+              undefined,
+              progressSource,
+            );
+          },
+          WORKFLOW_VIEWER_OVERLAY_OPTIONS,
+        )
+        .catch((error: unknown) => {
+          try {
+            progress.log(`inspector failed: ${unknownErrorMessage(error)}`);
+          } catch {
+            // Inspector reporting is detached and must never become another rejection.
+          }
+        })
+        .finally(() => unsubscribe?.());
+    }
+
+    const journal = await createWorkflowJournal({ resumePath, writePath: journalPath });
+    durableRun.transition({ state: "running", progress: progress.snapshot() });
+    await durableRun.flush().catch(() => undefined);
+    await notifyLifecycleObserver(progress, "run metadata callback", () =>
+      resolvedOptions.onRunMetadata?.({
+        runId,
+        resumedFromRunId: resolvedOptions.resumeFromRunId,
+        journalPath,
+        recordPath: workflowRunRecordPath(ctx.cwd, runId),
+      }),
+    );
+    progress.log(resolvedOptions.resumeFromRunId ? `run id: ${runId} (resuming from ${resolvedOptions.resumeFromRunId})` : `run id: ${runId}`);
+    const modelProfiles = dependencies.modelProfiles ?? resolveWorkflowModelProfiles({
+      cwd: ctx.cwd,
+      modelRegistry: ctx.modelRegistry,
+      hostModel: ctx.model,
+    });
+    const rc: WorkflowRunContext = {
+      cwd: ctx.cwd,
+      hostModel: ctx.model,
+      modelRegistry: ctx.modelRegistry,
+      semaphore: new Semaphore(resolvedOptions.concurrency),
+      agentLimiter: new WorkflowAgentLimiter(resolvedOptions.maxAgents),
+      agentTimeoutMs: resolvedOptions.agentTimeoutMs,
+      agentRetries: resolvedOptions.agentRetries,
+      pauseOnProviderUsageLimit: resolvedOptions.background !== undefined,
+      resumeEditedWorkflow: resolvedOptions.resumeEditedWorkflow,
+      retryScheduler: dependencies.retryScheduler ?? defaultAgentRetryScheduler,
+      modelProfiles,
+      progress,
+      signal: runAbortController.signal,
+      perf,
+      usage,
+      budget,
+      journal,
+      worktrees,
+    };
+
+    // perf.total_ms wraps the whole tree: sub-workflows run inside this span via api.workflow().
+    return perf.time("workflow.total_ms", () =>
+      runWorkflowWithContext(rc, progress, mod, args, {
+        abortController: runAbortController,
+        submissionLimit: resolvedOptions.parallelSubmissionLimit ?? (resolvedOptions.concurrency === null ? undefined : resolvedOptions.concurrency * 2),
+        resolveWorkflow: resolvedOptions.resolveWorkflow,
+        depth: 0,
+        progressPrefix: "",
+      }),
+    );
+  });
+
+  const workflowPause = workflowOutcome.ok
+    ? undefined
+    : backgroundPauseError(workflowOutcome.error, ctx.signal, resolvedOptions.signal);
+  const willPauseWorkflow = workflowPause !== undefined
+    && (!(workflowPause instanceof WorkflowProviderUsageLimitError) || resolvedOptions.background !== undefined);
+  const preserveFailedWorktrees = !workflowOutcome.ok
+    && !willPauseWorkflow
+    && !ctx.signal?.aborted
+    && !resolvedOptions.signal?.aborted;
+  if (preserveFailedWorktrees) worktrees.preserveRecoverable();
+  const finalizationOutcome = await captureOutcome(() =>
+    finalizeWorkflowRun({
+      cwd: ctx.cwd,
+      options: resolvedOptions,
+      perf,
+      usage,
+      progress,
+      runStore,
+      worktrees,
+      preserveWorktrees: preserveFailedWorktrees,
+      unlinkSignals: [unlinkContextAbortSignal, unlinkOptionAbortSignal],
+    }),
+  );
+  const retainedPaths = preserveFailedWorktrees ? worktrees.preservedPaths : [];
+  const terminalWorkflowError = workflowOutcome.ok
+    ? undefined
+    : withRetainedWorktrees(workflowOutcome.error, retainedPaths);
+
+  if (workflowOutcome.ok && finalizationOutcome.ok) {
+    durableRun.transition({
+      state: "completed",
+      progress: progress.snapshot(),
+      usage: usage.snapshot(),
+      result: workflowOutcome.value,
+    });
+  } else if (!workflowOutcome.ok) {
+    const error = finalizationOutcome.ok
+      ? terminalWorkflowError!
+      : combinedWorkflowError(terminalWorkflowError!, finalizationOutcome.error);
+    persistTerminalWorkflowError(error, workflowOutcome.error);
+  } else if (!finalizationOutcome.ok) {
+    persistTerminalWorkflowError(finalizationOutcome.error);
+  }
+  await durableRun.flush().catch(() => undefined);
+
+  function persistTerminalWorkflowError(error: unknown, pauseCandidate: unknown = error): void {
+    const pauseError = backgroundPauseError(pauseCandidate, ctx.signal, resolvedOptions.signal);
+    if (pauseError) {
+      if (pauseError instanceof WorkflowProviderUsageLimitError) {
+        if (resolvedOptions.background === undefined) {
+          durableRun.transition({
+            state: "failed",
+            progress: progress.snapshot(),
+            usage: usage.snapshot(),
+            error: pauseError,
+          });
+          return;
+        }
+        const providerPause = createProviderUsageLimitPauseRecord(
+          pauseError,
+          resolvedOptions,
+          mod.source.kind === "file" && args.length === 0,
+        );
+        durableRun.transition({
+          state: "paused",
+          progress: progress.snapshot(),
+          ...providerPause,
+        });
+        return;
+      }
+      durableRun.transition({
+        state: "paused",
+        progress: progress.snapshot(),
+        message: unknownErrorMessage(pauseError),
+      });
+      return;
+    }
+    durableRun.transition({
+      state: ctx.signal?.aborted || resolvedOptions.signal?.aborted ? "stopped" : "failed",
+      progress: progress.snapshot(),
+      usage: usage.snapshot(),
+      error,
+    });
+  }
+
+  if (workflowOutcome.ok) {
+    if (finalizationOutcome.ok) return workflowOutcome.value;
+    throw finalizationOutcome.error;
+  }
+  if (finalizationOutcome.ok) throw terminalWorkflowError;
+  throw combinedWorkflowError(terminalWorkflowError, finalizationOutcome.error);
+}
+
+function backgroundPauseError(error: unknown, ...signals: Array<AbortSignal | undefined>): unknown {
+  if (isWorkflowPauseError(error)) return error;
+  for (const signal of signals) {
+    if (!signal?.aborted) continue;
+    const reason = abortReason(signal);
+    if (isWorkflowPauseError(reason)) return reason;
+  }
+  return undefined;
+}
+
+async function captureOutcome<T>(operation: () => Promise<T>): Promise<Outcome<T>> {
+  try {
+    return { ok: true, value: await operation() };
+  } catch (error) {
+    return { ok: false, error };
+  }
+}
+
+interface WorkflowFinalizationInput {
+  readonly cwd: string;
+  readonly options: ResolvedWorkflowRunOptions;
+  readonly perf: PerfSink;
+  readonly usage: WorkflowUsageSink;
+  readonly progress: ProgressTracker;
+  readonly runStore: WorkflowRunStore;
+  readonly worktrees: WorktreeRegistry;
+  readonly preserveWorktrees: boolean;
+  readonly unlinkSignals: readonly (() => void)[];
+}
+
+async function finalizeWorkflowRun(input: WorkflowFinalizationInput): Promise<void> {
+  await runFinalizers(
+    [
+      {
+        name: "usage snapshot callback",
+        criticality: "best-effort",
+        run: () => input.options.onUsageSnapshot?.(input.usage.snapshot()),
+      },
+      {
+        name: "performance snapshot callback",
+        criticality: "best-effort",
+        run: async () => {
+          if (!input.options.perf) return;
+          const snapshot = input.perf.snapshot();
+          await input.options.onPerfSnapshot?.(snapshot);
+          input.progress.log(formatPerfSummary(snapshot));
+        },
+      },
+      ...input.unlinkSignals.map((run, index) => ({
+        name: `abort signal unlink ${index + 1}`,
+        criticality: "required" as const,
+        run,
+      })),
+      input.preserveWorktrees
+        ? {
+            name: "worktree cleanup and retention",
+            criticality: "required",
+            run: async () => {
+              await input.worktrees.removeUnpreserved();
+              for (const path of input.worktrees.preservedPaths) {
+                input.progress.log(`retained failed workflow worktree: ${path}`);
+              }
+            },
+          }
+        : {
+            name: "worktree cleanup",
+            criticality: "required",
+            run: () => input.worktrees.removeAll(),
+          },
+      { name: "journal pruning", criticality: "best-effort", run: () => pruneWorkflowJournals(input.cwd) },
+      { name: "run record pruning", criticality: "best-effort", run: () => input.runStore.prune() },
+      { name: "progress completion", criticality: "best-effort", run: () => input.progress.done() },
+      {
+        name: "progress snapshot callback",
+        criticality: "best-effort",
+        run: () => input.options.onProgressSnapshot?.(input.progress.snapshot()),
+      },
+      {
+        name: "progress source clear",
+        criticality: "best-effort",
+        run: () => input.options.onProgressSource?.(undefined),
+      },
+    ],
+    {
+      onBestEffortFailure: (failure) => {
+        try {
+          input.progress.log(`${failure.name} failed: ${unknownErrorMessage(failure.error)}`);
+        } catch {
+          // Observer failures must never replace the workflow outcome.
+        }
+      },
+    },
+  );
+}
+
+function withRetainedWorktrees(error: unknown, paths: readonly string[]): unknown {
+  if (paths.length === 0) return error;
+  const locations = paths.map((path) => `- ${path}`).join("\n");
+  return new Error(
+    `${unknownErrorMessage(error)}\n\nFailed workflow worktree${paths.length === 1 ? "" : "s"} retained for recovery:\n${locations}`,
+    { cause: error },
+  );
+}
+
+function combinedWorkflowError(workflowError: unknown, finalizationError: unknown): AggregateError {
+  return new AggregateError(
+    [workflowError, finalizationError],
+    `Workflow failed: ${unknownErrorMessage(workflowError)}; finalization also failed: ${unknownErrorMessage(finalizationError)}`,
+  );
+}
+
+function createWorkflowAgent(
+  rc: WorkflowRunContext,
+  scope: WorkflowScope,
+  mod: LoadedWorkflow,
+  resumeContext: AgentResumeBaseContext,
+): WorkflowApi["agent"] {
+  function agent<S extends TSchema>(
+    prompt: string,
+    opts: AgentOptions<S> & { schema: S; isolation: "worktree" },
+  ): Promise<IsolatedAgentResult<Static<S>>>;
+  function agent(prompt: string, opts: AgentOptions & { isolation: "worktree" }): Promise<IsolatedAgentResult<string>>;
+  function agent<S extends TSchema>(prompt: string, opts: AgentOptions<S> & { schema: S }): Promise<Static<S>>;
+  function agent(prompt: string, opts?: AgentOptions): Promise<string>;
+  function agent(prompt: string, agentOpts?: AgentOptions): Promise<unknown> {
+    const scopedOptions = scope.agentOptions(agentOpts);
+    const executionOptions: AgentExecutionOptions =
+      scopedOptions.isolation === "worktree" && mod.isolatedWorktreeBaseline !== undefined
+        ? { ...scopedOptions, worktreeBaseline: mod.isolatedWorktreeBaseline }
+        : scopedOptions;
+    return runAgent(rc, prompt, executionOptions, resumeContext);
+  }
+  return agent;
+}
+
+/**
+ * Build the `WorkflowApi` from an existing run context and invoke the module. Reused for both
+ * the top-level run and every `api.workflow()` sub-step, so children share the parent's semaphore,
+ * abort signal, and perf sink. No setup/teardown of its own — that belongs to `runWorkflow`.
+ */
+export async function runWorkflowWithContext(
+  rc: WorkflowRunContext,
+  progress: WorkflowProgress,
+  mod: LoadedWorkflow,
+  args: string,
+  opts: WorkflowContextOptions,
+): Promise<unknown> {
+  const depth = opts.depth ?? 0;
+  const scope = createWorkflowScope(progress, opts.progressPrefix ?? "", opts.progressNamespace);
+  const resumeContext: AgentResumeBaseContext = {
+    workflow: await captureWorkflowResumeContext(mod, rc.signal),
+  };
+
+  const agent = createWorkflowAgent(rc, scope, mod, resumeContext);
+
+  const workflow: WorkflowApi["workflow"] =
+    depth >= 1
+      ? async () => {
+          throw new Error("workflow() can only nest one level deep");
+        }
+      : async (ref, childArgs) => {
+          if (!opts.resolveWorkflow) throw new Error("sub-workflows are not enabled in this context");
+          const childMod = await opts.resolveWorkflow(ref);
+          // A child controller isolates the child's failures from the parent (so the parent can
+          // try/catch a failed sub-workflow), while still cancelling the child when the run aborts.
+          const childController = new AbortController();
+          const unlink = linkAbortSignal(rc.signal, childController);
+          try {
+            return await runWorkflowWithContext({ ...rc, signal: childController.signal }, progress, childMod, childArgs ?? "", {
+              abortController: childController,
+              submissionLimit: opts.submissionLimit,
+              resolveWorkflow: opts.resolveWorkflow,
+              depth: depth + 1,
+              progressPrefix: `${childMod.meta.name} ▸ `,
+              progressNamespace: childMod.meta.name,
+            });
+          } finally {
+            unlink();
+            scope.restorePhase();
+          }
+        };
+
+  const api: WorkflowApi = {
+    agent,
+    workflow,
+    parallel: bindParallel({
+      signal: rc.signal,
+      abortController: opts.abortController,
+      limit: opts.submissionLimit,
+    }),
+    pipeline: bindPipeline({ signal: rc.signal, abortController: opts.abortController }),
+    phase: scope.phase,
+    log: scope.log,
+    progress: scope.event,
+    args,
+    cwd: rc.cwd,
+    budget: rc.budget,
+    signal: rc.signal,
+  };
+
+  throwIfAborted(rc.signal);
+  const result = await mod.default(api);
+  throwIfAborted(rc.signal);
+  return result;
+}
+
+interface WorkflowScope {
+  agentOptions(opts: AgentOptions | undefined): AgentOptions;
+  phase(title: string): void;
+  log(message: string): void;
+  event(event: WorkflowProgressEvent): void;
+  restorePhase(): void;
+}
+
+function createWorkflowScope(progress: WorkflowProgress, prefix: string, namespace: string | undefined): WorkflowScope {
+  let currentPhase = `${prefix}Workflow`;
+  const display = (title: string) => `${prefix}${title}`;
+  return {
+    agentOptions(opts) {
+      const phase = opts?.phase ? display(opts.phase) : currentPhase;
+      return opts ? { ...opts, phase } : { phase };
+    },
+    phase(title) {
+      currentPhase = display(title);
+      progress.phase(currentPhase);
+    },
+    log(message) {
+      progress.log(namespace ? `${namespace}: ${message}` : message);
+    },
+    event(event) {
+      progress.event(namespace ? namespaceProgressEvent(namespace, event) : event);
+    },
+    restorePhase() {
+      progress.phase(currentPhase);
+    },
+  };
+}
+
+function namespaceProgressEvent(namespace: string, event: WorkflowProgressEvent): WorkflowProgressEvent {
+  switch (event.type) {
+    case "counter":
+      return { ...event, key: `${namespace}.${event.key}` };
+    case "counter_delta":
+      return { ...event, key: `${namespace}.${event.key}` };
+    case "lane_item":
+      return { ...event, lane: `${namespace} ▸ ${event.lane}` };
+    case "summary":
+      return { ...event, key: `${namespace}.${event.key}` };
+  }
+}
+
+function formatPerfSummary(snapshot: PerfSnapshot): string {
+  const parts = snapshot.aggregates
+    .filter((aggregate) => aggregate.count > 0)
+    .slice(0, 5)
+    .map((aggregate) => `${aggregate.name} ${Math.round(aggregate.total)}ms`);
+  return parts.length > 0 ? `perf: ${parts.join(", ")}` : "perf: no samples";
+}
+
+async function notifyLifecycleObserver(
+  progress: ProgressTracker,
+  name: string,
+  notify: () => void | Promise<void>,
+): Promise<void> {
+  try {
+    await notify();
+  } catch (error) {
+    try {
+      progress.log(`${name} failed: ${unknownErrorMessage(error)}`);
+    } catch {
+      // Lifecycle observers are advisory and must not replace the workflow outcome.
+    }
+  }
+}

package/src/domains/workflows/runtime/filesystem-error.ts

@@ -0,0 +1,4 @@
+/** Identify a missing filesystem path reported by Node's filesystem APIs. */
+export function isMissingPathError(error: unknown): boolean {
+  return error instanceof Error && "code" in error && error.code === "ENOENT";
+}

package/src/domains/workflows/runtime/finalizers.ts

@@ -0,0 +1,66 @@
+import { unknownErrorMessage } from "./unknown-error.ts";
+
+export type FinalizerCriticality = "required" | "best-effort";
+
+export interface Finalizer {
+  readonly name: string;
+  readonly criticality: FinalizerCriticality;
+  readonly run: () => void | Promise<void>;
+}
+
+export interface FinalizerFailure {
+  readonly name: string;
+  readonly criticality: FinalizerCriticality;
+  readonly error: unknown;
+}
+
+export interface RunFinalizersOptions {
+  readonly onBestEffortFailure?: (failure: FinalizerFailure) => void;
+}
+
+export class RequiredFinalizerError extends AggregateError {
+  readonly failures: readonly FinalizerFailure[];
+
+  constructor(failures: readonly FinalizerFailure[]) {
+    super(
+      failures.map((failure) => failure.error),
+      `Required workflow finalization failed: ${failures.map((failure) => `${failure.name} (${unknownErrorMessage(failure.error)})`).join(", ")}`,
+    );
+    this.name = "RequiredFinalizerError";
+    this.failures = failures;
+  }
+}
+
+/** Execute every finalizer once, reporting advisory failures and aggregating only required failures. */
+export async function runFinalizers(
+  finalizers: readonly Finalizer[],
+  options: RunFinalizersOptions = {},
+): Promise<readonly FinalizerFailure[]> {
+  const requiredFailures: FinalizerFailure[] = [];
+  const bestEffortFailures: FinalizerFailure[] = [];
+
+  for (const finalizer of finalizers) {
+    try {
+      await finalizer.run();
+    } catch (error) {
+      const failure: FinalizerFailure = {
+        name: finalizer.name,
+        criticality: finalizer.criticality,
+        error,
+      };
+      if (finalizer.criticality === "required") {
+        requiredFailures.push(failure);
+        continue;
+      }
+      bestEffortFailures.push(failure);
+      try {
+        options.onBestEffortFailure?.(failure);
+      } catch {
+        // Failure observers are advisory and must not change finalization semantics.
+      }
+    }
+  }
+
+  if (requiredFailures.length > 0) throw new RequiredFinalizerError(requiredFailures);
+  return bestEffortFailures;
+}