@prismatic-io/lux 0.0.1

package/src/orchestrator/run-execution.ts

@@ -0,0 +1,694 @@
+import { cp } from "node:fs/promises";
+import { dirname, isAbsolute, resolve } from "node:path";
+import { z } from "zod";
+import {
+  type AgentDriver,
+  type Answerer,
+  type AnswererContext,
+  type AnswererModelAccounting,
+  AnswererModelAccountingSchema,
+  AnswererModelCallAccountingSchema,
+  type AnswerHistoryEntry,
+  type Artifact,
+  type EvalCase,
+  emptyUsage,
+  type FixturesRef,
+  type GradingReport,
+  guardRunObserver,
+  inspectRunArtifact,
+  type LifecycleStage,
+  type OperationalFailure,
+  type PhaseDurations,
+  type ReadyState,
+  type Registry,
+  type ResolvedFixtures,
+  type Run,
+  type RunEvent,
+  type RunMetadata,
+  type RunProvenance,
+  type RunUsage,
+  summarizeToolOutcomes,
+  sumUsage,
+  type TokenUsage,
+  TokenUsageSchema,
+  valueHash,
+} from "../core/index.js";
+import {
+  createDriverSession,
+  type DriverSession,
+  type DriverSessionOutcome,
+} from "./driver-session.js";
+import { fixtureIdentity } from "./fixture-identity.js";
+import { gradeRun } from "./grading.js";
+import type { RunRequest, RunResult } from "./orchestrator.js";
+import { runProvenance } from "./provenance.js";
+import { type OpenedRunDir, openRunDir, writeFrozenCase } from "./run-dir.js";
+import {
+  executeRunLifecycle,
+  type RunFailurePhase,
+  type RunLifecycleEffects,
+} from "./run-lifecycle.js";
+import { isRunTraceLimit, type RunTraceLimitFailure } from "./trace-limits.js";
+
+/** Sentinel `casePath` for a pre-loaded case that has no file behind it. */
+export const INLINE_CASE_PATH = "<inline>";
+
+type RunDirCarrier = { luxRunDir?: string };
+type InterruptCounts = { ask: number; approve: number; custom: number };
+export type RunWithPluginsArgs = {
+  request: RunRequest;
+  casePath: string;
+  authoredCase: EvalCase;
+  evalCase: EvalCase;
+  validatedAssertionConfigs: readonly unknown[];
+  caseId: string;
+  driverName: string;
+  answererName: string;
+  answererConfig: unknown;
+  driverConfigHash: string;
+  answererConfigHash: string;
+  validatedAssertionsHash: string;
+  graderConfigHash: string;
+  evaluatorConfigHash: string;
+  luxVersion: string;
+  registry: Registry;
+  driver: AgentDriver;
+  answerer: Answerer;
+};
+
+type Collection = {
+  artifacts: Artifact[];
+  error?: string;
+};
+
+const errorMessage = (error: unknown): string =>
+  error instanceof Error ? error.message : String(error);
+
+const traceLimitError = (failure: RunTraceLimitFailure): Error & RunTraceLimitFailure =>
+  Object.assign(new Error(failure.reason), failure);
+
+const readAnswererModelAccounting = (answerer: Answerer): AnswererModelAccounting =>
+  AnswererModelAccountingSchema.parse(
+    answerer.modelAccounting?.() ?? {
+      calls: 0,
+      reportedCostUsd: 0,
+      unpricedModelCalls: 0,
+    },
+  );
+
+const readAnswererUsage = (answerer: Answerer): TokenUsage =>
+  TokenUsageSchema.parse(answerer.usage?.() ?? emptyUsage());
+
+/**
+ * Stage a case's fixtures into the driver's working directory. Relative paths
+ * resolve against fixturesRoot when set, otherwise against the case file.
+ */
+const stageFixtures = async (
+  ref: FixturesRef | undefined,
+  options: { fixturesRoot?: string; casePath: string; artifactsDir: string },
+): Promise<{ fixtures: ResolvedFixtures; hash: string } | undefined> => {
+  if (!ref) return undefined;
+  const path = typeof ref === "string" ? ref : ref.path;
+  if (!isAbsolute(path) && !options.fixturesRoot && options.casePath === INLINE_CASE_PATH) {
+    throw new Error(
+      "fixtures: an inline case has no file to resolve a relative fixtures path against; " +
+        "set fixturesRoot or use an absolute path",
+    );
+  }
+  const source = await fixtureIdentity(ref, options.fixturesRoot ?? dirname(options.casePath));
+  await cp(source.rootDir, options.artifactsDir, { recursive: true });
+  const staged = await fixtureIdentity(".", options.artifactsDir);
+  if (staged.hash !== source.hash) {
+    throw new Error(`fixtures: tree changed while staging: ${source.rootDir}`);
+  }
+  return { fixtures: { rootDir: source.rootDir }, hash: staged.hash };
+};
+
+export const runDirOfError = (error: unknown): string | null =>
+  error instanceof Error && typeof (error as RunDirCarrier).luxRunDir === "string"
+    ? ((error as RunDirCarrier).luxRunDir as string)
+    : null;
+
+const closeQuietly = async (close: () => Promise<void>, label: string): Promise<void> => {
+  try {
+    await close();
+  } catch (error) {
+    process.stderr.write(`[lux] warning: ${label} close failed: ${errorMessage(error)}\n`);
+  }
+};
+
+export const closeRunDirectory = async (
+  runDir: Pick<OpenedRunDir, "path" | "close">,
+  preservePrimaryError: boolean,
+): Promise<void> => {
+  if (preservePrimaryError) {
+    await closeQuietly(() => runDir.close(), "run dir");
+    return;
+  }
+  try {
+    await runDir.close();
+  } catch (error) {
+    if (error instanceof Error) (error as Error & RunDirCarrier).luxRunDir = runDir.path;
+    throw error;
+  }
+};
+
+const ReportedTokenCountSchema = z.number().int().nonnegative().catch(0);
+const SummaryUsageSchema = z.object({
+  usage: z.object({
+    input: ReportedTokenCountSchema,
+    output: ReportedTokenCountSchema,
+    cacheRead: ReportedTokenCountSchema,
+    cacheCreation: ReportedTokenCountSchema,
+  }),
+});
+const ReportedCostSchema = z.number().nonnegative();
+const SummaryModelAccountingSchema = z.object({
+  modelCalls: z.number().int().nonnegative(),
+  unpricedModelCalls: z.number().int().nonnegative().optional(),
+});
+
+const usageFromSummary = (summary: unknown): TokenUsage | null =>
+  SummaryUsageSchema.safeParse(summary).data?.usage ?? null;
+
+const costFromSummary = (summary: unknown): number | undefined => {
+  if (!summary || typeof summary !== "object") return undefined;
+  return ReportedCostSchema.safeParse((summary as { cost?: unknown }).cost).data;
+};
+
+const modelAccountingFromSummary = (
+  summary: unknown,
+): { modelCalls: number; unpricedModelCalls?: number | undefined } | null =>
+  SummaryModelAccountingSchema.safeParse(summary).data ?? null;
+
+const artifactMeasurements = async (
+  artifacts: readonly Artifact[],
+  runDir: string,
+): Promise<{ count: number; bytes?: number }> => {
+  const sizes = await Promise.all(
+    artifacts.map(async (artifact) => {
+      if (artifact.size !== undefined) return artifact.size;
+      const inspected = await inspectRunArtifact({ runDir, artifacts }, artifact.path);
+      return inspected.stats?.isFile() ? inspected.stats.size : null;
+    }),
+  );
+  const measurable = sizes.every((size) => size !== null);
+  return {
+    count: artifacts.length,
+    ...(measurable ? { bytes: sizes.reduce<number>((total, size) => total + (size ?? 0), 0) } : {}),
+  };
+};
+
+export const phaseDurations = (events: readonly RunEvent[]): PhaseDurations => {
+  const timestamps = new Map<LifecycleStage, number>();
+  for (const event of events) {
+    if (event.type === "lifecycle" && !timestamps.has(event.stage)) {
+      timestamps.set(event.stage, event.ts);
+    }
+  }
+  const span = (from: LifecycleStage, to: LifecycleStage): number => {
+    const start = timestamps.get(from);
+    const end = timestamps.get(to);
+    return start === undefined || end === undefined ? 0 : Math.max(0, end - start);
+  };
+  return {
+    agent: span("start-driver", "collect-start"),
+    collect: span("collect-start", "collect-end"),
+    grading: span("grade-start", "grade-end"),
+  };
+};
+
+const agentExitReason = (
+  outcome: DriverSessionOutcome,
+  collectionError: string | undefined,
+): RunMetadata["exitReason"] => {
+  if (collectionError !== undefined) return "error";
+  return outcome.kind === "done" ? "done" : outcome.error.exitReason;
+};
+
+const loopFailure = (
+  outcome: DriverSessionOutcome,
+  collectionError: string | undefined,
+): OperationalFailure | undefined => {
+  if (collectionError !== undefined) {
+    return { phase: "collection", reason: collectionError };
+  }
+  return outcome.kind === "error" ? outcome.operationalFailure : undefined;
+};
+
+const answererMetadata = (name: string, config: unknown): RunMetadata["answerer"] => {
+  if (!config || typeof config !== "object" || Array.isArray(config)) return { name };
+  const record = config as Record<string, unknown>;
+  return {
+    name,
+    ...(typeof record.provider === "string" ? { provider: record.provider } : {}),
+    ...(typeof record.model === "string" ? { model: record.model } : {}),
+  };
+};
+
+type GradingCompletionFailure = {
+  eventReason: string;
+  operationalFailure: OperationalFailure;
+};
+
+const gradingCompletionFailure = (
+  abortSignal: AbortSignal | undefined,
+  grading: GradingReport,
+): GradingCompletionFailure | undefined => {
+  if (abortSignal?.aborted) {
+    return {
+      eventReason: "run aborted during grading",
+      operationalFailure: { phase: "abort", reason: "run aborted" },
+    };
+  }
+  if (grading.incomplete?.phase !== "grading") return undefined;
+  const gradingError = grading.assertions.find(
+    (assertion) => assertion.errorMessage !== undefined,
+  )?.errorMessage;
+  const reason = gradingError ?? "grading did not complete";
+  return {
+    eventReason: `grading did not complete${gradingError ? `: ${gradingError}` : ""}`,
+    operationalFailure: { phase: "grading", reason },
+  };
+};
+
+class RunExecution implements RunLifecycleEffects<RunResult> {
+  private readonly args: RunWithPluginsArgs;
+  private readonly startedAt: Date;
+  private readonly provenance: Awaited<ReturnType<typeof runProvenance>>;
+  private readonly observer: ReturnType<typeof guardRunObserver>;
+  private readonly runDir: OpenedRunDir;
+  private readonly interruptCounts: InterruptCounts = { ask: 0, approve: 0, custom: 0 };
+  private readonly history: AnswerHistoryEntry[] = [];
+  private readonly subjectAttemptIds: string[] = [];
+  private toolCallCount = 0;
+  private loop: DriverSession | undefined;
+  private ready: ReadyState | undefined;
+  private fixturesHash: string | undefined;
+  private startAttempted = false;
+  private loopOutcome: DriverSessionOutcome | undefined;
+  private collection: Collection = { artifacts: [] };
+  private metadata: RunMetadata | undefined;
+  private usage: RunUsage | undefined;
+  private run: Run | undefined;
+
+  private constructor(
+    args: RunWithPluginsArgs,
+    startedAt: Date,
+    provenance: Awaited<ReturnType<typeof runProvenance>>,
+    observer: ReturnType<typeof guardRunObserver>,
+    runDir: OpenedRunDir,
+  ) {
+    this.args = args;
+    this.startedAt = startedAt;
+    this.provenance = provenance;
+    this.observer = observer;
+    this.runDir = runDir;
+  }
+
+  static async open(args: RunWithPluginsArgs): Promise<RunExecution> {
+    const startedAt = args.request.startedAt ?? new Date();
+    const provenance = await runProvenance(
+      args.casePath === INLINE_CASE_PATH ? null : args.casePath,
+    );
+    const observer = guardRunObserver(args.request.observer);
+    const runDir = await openRunDir({
+      runsRoot: resolve(args.request.runsRoot),
+      caseId: args.caseId,
+      driverName: args.driverName,
+      startedAt,
+      onEvent: observer?.event,
+    });
+    return new RunExecution(args, startedAt, provenance, observer, runDir);
+  }
+
+  async prepare(): Promise<void> {
+    this.lifecycle("load");
+    const context = this.answererContext();
+    this.lifecycle("prepare-answerer");
+    await this.args.answerer.prepare(context);
+    this.loop = createDriverSession({
+      driver: this.args.driver,
+      answerer: this.args.answerer,
+      context,
+      runDir: this.runDir,
+      interruptCounts: this.interruptCounts,
+      history: this.history,
+      ...(this.args.request.abortSignal ? { abortSignal: this.args.request.abortSignal } : {}),
+      onToolCall: () => {
+        this.toolCallCount++;
+      },
+    });
+  }
+
+  async start(): Promise<void> {
+    const { request, evalCase, casePath, driver } = this.args;
+    this.lifecycle("run-loop");
+    if (request.abortSignal?.aborted) return;
+    const staged = await stageFixtures(evalCase.fixtures, {
+      ...(request.fixturesRoot ? { fixturesRoot: request.fixturesRoot } : {}),
+      casePath,
+      artifactsDir: this.runDir.artifactsDir,
+    });
+    this.fixturesHash = staged?.hash;
+    if (request.abortSignal?.aborted) return;
+
+    this.lifecycle("start-driver");
+    await this.recordAgentAttemptStart();
+    this.startAttempted = true;
+    this.ready = await driver.start({
+      caseId: this.args.caseId,
+      prompt: evalCase.prompt,
+      fixtures: staged?.fixtures,
+      runsRoot: resolve(request.runsRoot),
+      runDir: this.runDir.path,
+      artifactsDir: this.runDir.artifactsDir,
+      abortSignal: request.abortSignal ?? new AbortController().signal,
+    });
+    this.append({ type: "ready", ...this.ready });
+  }
+
+  async recordFailure(phase: RunFailurePhase, error: unknown): Promise<void> {
+    const reason = errorMessage(error);
+    const operationalPhase = phase === "running" ? "driver-transport" : phase;
+    if (phase === "setup" && this.loop) {
+      this.loop.fail(`run setup failed: ${reason}`);
+    } else {
+      this.runDir.events.append({
+        type: "error",
+        exitReason: "error",
+        reason: `run ${phase} failed: ${reason}`,
+        ts: Date.now(),
+      });
+    }
+    await this.runDir.events.drain();
+    const metadata = this.metadata ?? this.fallbackFailureMetadata();
+    metadata.endedAt = new Date().toISOString();
+    metadata.exitReason = "error";
+    metadata.operationalFailure = { phase: operationalPhase, reason };
+    metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    await this.runDir.writeJson("run.json", metadata);
+    await this.args.request.recordPreGradingAccounting?.(metadata);
+    await this.args.request.recordModelAttemptsComplete?.(this.subjectAttemptIds);
+  }
+
+  async waitForAgent(): Promise<void> {
+    if (!this.loop)
+      throw new Error("run lifecycle entered running before preparing its event loop");
+    this.loopOutcome = await this.loop.outcome;
+  }
+
+  async collect(): Promise<void> {
+    this.lifecycle("collect-start");
+    this.collection = await this.collectArtifacts();
+    this.lifecycle("collect-end");
+    await this.runDir.events.drain();
+    const completed = await this.completedAgentRun();
+    this.metadata = completed.metadata;
+    this.usage = completed.usage;
+    this.run = completed.run;
+    await writeFrozenCase(this.runDir.path, this.args.authoredCase, this.args.casePath);
+    await this.runDir.writeJson("run.json", completed.metadata);
+    await this.runDir.writeJson("artifacts.json", this.collection.artifacts);
+    await this.args.request.recordPreGradingAccounting?.(completed.metadata);
+    await this.args.request.recordModelAttemptsComplete?.(this.subjectAttemptIds);
+  }
+
+  async grade(): Promise<RunResult> {
+    const metadata = this.require(this.metadata, "metadata");
+    const usage = this.require(this.usage, "usage");
+    const run = this.require(this.run, "run evidence");
+    this.lifecycle("grade-start");
+    const grading = await this.gradeEvidence(run);
+    const finalExitReason = this.applyGradingOutcome(metadata, grading);
+    this.lifecycle("grade-end");
+    metadata.endedAt = new Date().toISOString();
+    usage.grading = sumUsage(
+      grading.assertions.flatMap((assertion) => (assertion.usage ? [assertion.usage] : [])),
+    );
+    metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    await this.runDir.events.drain();
+    await this.runDir.writeJson("run.json", metadata);
+    await this.runDir.writeJson("artifacts.json", this.collection.artifacts);
+    await this.runDir.writeJson("grading.json", grading);
+    return { runDir: this.runDir.path, metadata, grading, exitReason: finalExitReason };
+  }
+
+  async close(primaryError: unknown | null): Promise<void> {
+    if (primaryError instanceof Error) {
+      (primaryError as Error & RunDirCarrier).luxRunDir = this.runDir.path;
+    }
+    if (primaryError !== null) {
+      if (!isRunTraceLimit(primaryError)) {
+        try {
+          this.lifecycle("close");
+        } catch (error) {
+          process.stderr.write(
+            `[lux] warning: could not record failed run cleanup: ${errorMessage(error)}\n`,
+          );
+        }
+      }
+      await closeRunDirectory(this.runDir, true);
+      return;
+    }
+    this.lifecycle("close");
+    await closeRunDirectory(this.runDir, false);
+  }
+
+  private lifecycle(stage: LifecycleStage): void {
+    this.append({ type: "lifecycle", stage, ts: Date.now() });
+  }
+
+  private append(event: RunEvent): void {
+    const failure = this.runDir.events.append(event);
+    if (failure) throw traceLimitError(failure);
+  }
+
+  private answererContext(): AnswererContext {
+    const { request, evalCase } = this.args;
+    const startAttempt = request.recordModelAttemptStart;
+    return {
+      persona: evalCase.persona ?? null,
+      prompt: evalCase.prompt,
+      history: this.history,
+      runDir: this.runDir.path,
+      ...(request.abortSignal ? { abortSignal: request.abortSignal } : {}),
+      ...(startAttempt
+        ? {
+            recordModelAttemptStart: async (_kind: "answerer") => {
+              const attemptId = await startAttempt("answerer");
+              if (!request.recordModelAttemptComplete) this.subjectAttemptIds.push(attemptId);
+              return attemptId;
+            },
+          }
+        : {}),
+      ...(request.recordModelAttemptComplete
+        ? {
+            recordModelAttemptComplete: async (attemptId, accounting) => {
+              await request.recordModelAttemptComplete?.(
+                attemptId,
+                AnswererModelCallAccountingSchema.parse(accounting),
+              );
+            },
+          }
+        : {}),
+    };
+  }
+
+  private async recordAgentAttemptStart(): Promise<void> {
+    const start = this.args.request.recordModelAttemptStart;
+    if (start) this.subjectAttemptIds.push(await start("agent"));
+  }
+
+  private fallbackFailureMetadata(): RunMetadata {
+    const failedAt = new Date();
+    const answererAccounting = readAnswererModelAccounting(this.args.answerer);
+    return {
+      schemaVersion: 1,
+      luxVersion: this.args.luxVersion,
+      ...(this.args.request.suiteId ? { suiteId: this.args.request.suiteId } : {}),
+      startedAt: this.startedAt.toISOString(),
+      agentEndedAt: failedAt.toISOString(),
+      endedAt: failedAt.toISOString(),
+      exitReason: "error",
+      driver: { name: this.args.driverName, ...(this.ready ?? { agentId: "start-failed" }) },
+      answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
+      usage: {
+        agent: emptyUsage(),
+        answerer: readAnswererUsage(this.args.answerer),
+        grading: emptyUsage(),
+      },
+      modelAccounting: {
+        agentCalls: this.startAttempted ? 1 : 0,
+        answererCalls: answererAccounting.calls,
+        reportedCostUsd: answererAccounting.reportedCostUsd,
+        unpricedModelCalls: (this.startAttempted ? 1 : 0) + answererAccounting.unpricedModelCalls,
+      },
+      interruptCounts: this.interruptCounts,
+      toolCallCount: this.toolCallCount,
+      toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
+      artifacts: { count: this.collection.artifacts.length },
+      provenance: this.runProvenance(),
+    };
+  }
+
+  private async collectArtifacts(): Promise<Collection> {
+    if (!this.ready) return { artifacts: [] };
+    try {
+      await this.args.driver.quiesce?.();
+      return { artifacts: await this.args.driver.collect() };
+    } catch (error) {
+      const reason = errorMessage(error);
+      this.runDir.events.append({
+        type: "error",
+        exitReason: "error",
+        reason: `artifact collection failed: ${reason}`,
+        ts: Date.now(),
+      });
+      return { artifacts: [], error: reason };
+    }
+  }
+
+  private async completedAgentRun(): Promise<{
+    metadata: RunMetadata;
+    usage: RunUsage;
+    run: Run;
+  }> {
+    const outcome = this.require(this.loopOutcome, "driver outcome");
+    const agentEndedAt = new Date();
+    const exitReason = agentExitReason(outcome, this.collection.error);
+    const summary = outcome.kind === "done" ? outcome.done.summary : undefined;
+    const driverCost = costFromSummary(summary);
+    const reportedModelAccounting = modelAccountingFromSummary(summary);
+    const answererAccounting = readAnswererModelAccounting(this.args.answerer);
+    const agentCalls = this.ready ? (reportedModelAccounting?.modelCalls ?? 1) : 0;
+    const operationalFailure = loopFailure(outcome, this.collection.error);
+    const usage: RunUsage = {
+      agent: usageFromSummary(summary) ?? emptyUsage(),
+      answerer: readAnswererUsage(this.args.answerer),
+      grading: emptyUsage(),
+    };
+    const metadata: RunMetadata = {
+      schemaVersion: 1,
+      luxVersion: this.args.luxVersion,
+      ...(this.args.request.suiteId ? { suiteId: this.args.request.suiteId } : {}),
+      startedAt: this.startedAt.toISOString(),
+      agentEndedAt: agentEndedAt.toISOString(),
+      endedAt: agentEndedAt.toISOString(),
+      exitReason,
+      driver: { name: this.args.driverName, ...(this.ready ?? { agentId: "not-started" }) },
+      answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
+      usage,
+      modelAccounting: {
+        agentCalls,
+        answererCalls: answererAccounting.calls,
+        reportedCostUsd: (driverCost ?? 0) + answererAccounting.reportedCostUsd,
+        unpricedModelCalls:
+          (reportedModelAccounting?.unpricedModelCalls ??
+            (driverCost === undefined ? agentCalls : 0)) + answererAccounting.unpricedModelCalls,
+      },
+      interruptCounts: this.interruptCounts,
+      toolCallCount: this.toolCallCount,
+      toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
+      artifacts: await artifactMeasurements(this.collection.artifacts, this.runDir.path),
+      ...(operationalFailure ? { operationalFailure } : {}),
+      provenance: this.runProvenance(),
+      ...(driverCost !== undefined ? { cost: { usd: driverCost } } : {}),
+    };
+    return {
+      metadata,
+      usage,
+      run: {
+        case: this.args.evalCase,
+        metadata,
+        events: this.runDir.recorded,
+        artifacts: this.collection.artifacts,
+        runDir: this.runDir.path,
+      },
+    };
+  }
+
+  private runProvenance(): RunProvenance {
+    const { request, authoredCase } = this.args;
+    return {
+      ...this.provenance,
+      caseHash: valueHash(authoredCase),
+      assertionsHash: valueHash(authoredCase.assertions ?? []),
+      validatedAssertionsHash: this.args.validatedAssertionsHash,
+      ...(this.fixturesHash ? { fixturesHash: this.fixturesHash } : {}),
+      ...(request.subjectHash ? { subjectHash: request.subjectHash } : {}),
+      ...(request.profileId ? { profileId: request.profileId } : {}),
+      ...(request.experimentId ? { experimentId: request.experimentId } : {}),
+      ...(request.variantId ? { variantId: request.variantId } : {}),
+      ...(request.candidateId ? { candidateId: request.candidateId } : {}),
+      driverConfigHash: this.args.driverConfigHash,
+      answererConfigHash: this.args.answererConfigHash,
+      graderConfigHash: this.args.graderConfigHash,
+      evaluatorConfigHash: this.args.evaluatorConfigHash,
+    };
+  }
+
+  private async gradeEvidence(run: Run): Promise<GradingReport> {
+    if (this.collection.error !== undefined) {
+      return {
+        schemaVersion: 1,
+        casePassed: null,
+        caseScore: null,
+        assertions: [],
+        incomplete: { exitReason: "error", phase: "collection" },
+      };
+    }
+    const request = this.args.request;
+    const modelAttempts =
+      request.recordModelAttemptStart && request.recordModelAttemptsComplete
+        ? {
+            start: request.recordModelAttemptStart,
+            complete: request.recordModelAttemptsComplete,
+          }
+        : undefined;
+    return gradeRun(
+      run,
+      this.args.registry,
+      this.observer,
+      request.abortSignal,
+      request.recordGradingAssertion,
+      modelAttempts,
+      this.args.validatedAssertionConfigs,
+    );
+  }
+
+  private applyGradingOutcome(
+    metadata: RunMetadata,
+    grading: GradingReport,
+  ): RunMetadata["exitReason"] {
+    const initialExitReason = metadata.exitReason;
+    const failure = gradingCompletionFailure(this.args.request.abortSignal, grading);
+    const finalExitReason = initialExitReason === "done" && failure ? "error" : initialExitReason;
+
+    if (finalExitReason !== initialExitReason && failure) {
+      this.runDir.events.append({
+        type: "error",
+        exitReason: "error",
+        reason: failure.eventReason,
+        ts: Date.now(),
+      });
+    }
+    metadata.exitReason = finalExitReason;
+    if (failure) metadata.operationalFailure = failure.operationalFailure;
+    if (finalExitReason !== "done") {
+      grading.incomplete ??= { exitReason: finalExitReason, phase: "run" };
+      if (grading.casePassed === true) grading.casePassed = false;
+    }
+    return finalExitReason;
+  }
+
+  private require<Value>(value: Value | undefined, label: string): Value {
+    if (value === undefined) {
+      throw new Error(`run lifecycle reached a phase without ${label}`);
+    }
+    return value;
+  }
+}
+
+export const executeRunWithPlugins = async (args: RunWithPluginsArgs): Promise<RunResult> =>
+  executeRunLifecycle(await RunExecution.open(args));