@prismatic-io/lux 0.0.1

package/src/orchestrator/orchestrator.ts

@@ -0,0 +1,919 @@
+import { basename, extname, isAbsolute, join, resolve } from "node:path";
+import {
+  type Answerer,
+  type AnswererModelCallAccounting,
+  type AnswererRef,
+  bindSubjectRoot,
+  type DriverRef,
+  type EvalCase,
+  type GradingAssertionEntry,
+  type GradingReport,
+  guardObserverCallback,
+  type HarnessModelConfig,
+  type LifecycleHooksPlugin,
+  mergeHarnessModelConfig,
+  type Registry,
+  type RunMetadata,
+  type RunObserver,
+  type SuiteLifecycleHookContext,
+  valueHash,
+} from "../core/index.js";
+import { evaluatorImplementationHash } from "./evaluator-identity.js";
+import {
+  assertionEvaluatorIdentities,
+  graderConfigHash,
+  validateAssertionPlan,
+} from "./grading.js";
+import { combineLifecycleFailures, setupLifecycleHooks } from "./lifecycle-hook-execution.js";
+import { loadEvalCase } from "./loader.js";
+import { LUX_VERSION } from "./package-version.js";
+import { writeRunBundleJson } from "./run-dir.js";
+import {
+  closeRunDirectory,
+  executeRunWithPlugins,
+  INLINE_CASE_PATH,
+  phaseDurations,
+  type RunWithPluginsArgs,
+  runDirOfError,
+} from "./run-execution.js";
+import {
+  executeSuiteLifecycle,
+  type SuiteLifecycleEffects,
+  type SuiteLifecycleFinalization,
+  validateSuiteConcurrency,
+} from "./suite-lifecycle.js";
+import { openSuiteRun } from "./suite-runs.js";
+import { buildSuiteSummary, writeSuiteSummary } from "./suite-summary.js";
+import { validatePluginConfig } from "./validate-plugin-config.js";
+
+const serializeEffect = (effect: () => Promise<void>): (() => Promise<void>) => {
+  let pending: Promise<void> = Promise.resolve();
+  return () => (pending = pending.catch(() => undefined).then(effect));
+};
+
+const errorMessage = (error: unknown, depth = 0): string => {
+  const message = error instanceof Error ? error.message : String(error);
+  if (!(error instanceof AggregateError) || depth >= 3) return message;
+  const details = [...error.errors].slice(0, 8).map((failure) => errorMessage(failure, depth + 1));
+  return details.length > 0 ? `${message}: ${details.join("; ")}` : message;
+};
+
+const causedByAbort = (error: unknown, signal: AbortSignal | undefined): boolean => {
+  if (!signal?.aborted || error instanceof AggregateError) return false;
+  const reason = signal.reason;
+  let current: unknown = error;
+  for (let depth = 0; depth < 8 && current !== undefined; depth += 1) {
+    if (current === reason) return true;
+    if (
+      current instanceof DOMException &&
+      current.name === "AbortError" &&
+      (reason === undefined || (reason instanceof DOMException && reason.name === "AbortError"))
+    ) {
+      return true;
+    }
+    current = current instanceof Error && "cause" in current ? current.cause : undefined;
+  }
+  return false;
+};
+
+const recordLifecycleCleanupFailure = async (result: RunResult, error: unknown): Promise<void> => {
+  const reason = errorMessage(error);
+  result.metadata.endedAt = new Date().toISOString();
+  result.metadata.exitReason = "error";
+  const primary = result.metadata.operationalFailure;
+  result.metadata.operationalFailure = primary
+    ? {
+        phase: primary.phase,
+        reason: `${primary.reason}; lifecycle cleanup also failed during close: ${reason}`,
+      }
+    : { phase: "close", reason };
+  result.exitReason = "error";
+  result.grading.casePassed = null;
+  await Promise.all([
+    writeRunBundleJson(join(result.runDir, "run.json"), result.metadata),
+    writeRunBundleJson(join(result.runDir, "grading.json"), result.grading),
+  ]);
+};
+
+export type OrchestratorOptions = {
+  registry: Registry;
+  /** Answerer used when a case names none. Defaults to "persona". */
+  defaultAnswerer?: AnswererRef;
+  /** Driver used when a case names none. */
+  defaultDriver?: DriverRef;
+  /** Shared provider/model defaults for built-in harness components. */
+  harness?: HarnessModelConfig;
+  /** Versioned project resource fixtures. */
+  lifecycleFixtures?: LifecycleHooksPlugin[] | undefined;
+};
+
+export type RunRequest = {
+  /** Path to a TS case file. Mutually exclusive with `case`. */
+  casePath?: string;
+  /** Pre-loaded EvalCase. Mutually exclusive with `casePath`. */
+  case?: EvalCase;
+  runsRoot: string;
+  /**
+   * Id to file the run under when the case declares none. Discovery derives
+   * it from the file's path relative to the cases root, so two id-less cases
+   * that share a basename (a/check.ts, b/check.ts) keep separate histories.
+   */
+  caseId?: string | undefined;
+  /** Root that relative `fixtures` dir paths resolve against. */
+  fixturesRoot?: string | undefined;
+  answererOverride?: string | undefined;
+  startedAt?: Date | undefined;
+  /** The `runSuite` invocation this case attempt belongs to. */
+  suiteId?: string | undefined;
+  /** Root substituted into values created with `subjectPath()`. */
+  subjectRoot?: string | undefined;
+  subjectHash?: string | undefined;
+  profileId?: string | undefined;
+  experimentId?: string | undefined;
+  variantId?: string | undefined;
+  candidateId?: string | undefined;
+  recordGradingAssertion?: (entry: GradingAssertionEntry, index: number) => void | Promise<void>;
+  /** Persists subject/answerer spend before any potentially long grading call. */
+  recordPreGradingAccounting?: (metadata: RunMetadata) => void | Promise<void>;
+  /** Durable model-attempt markers used by experiment ledgers. */
+  recordModelAttemptStart?: (kind: "agent" | "answerer" | "judge") => string | Promise<string>;
+  recordModelAttemptComplete?: (
+    attemptId: string,
+    accounting: AnswererModelCallAccounting,
+  ) => void | Promise<void>;
+  recordModelAttemptsComplete?: (attemptIds: readonly string[]) => void | Promise<void>;
+  abortSignal?: AbortSignal | undefined;
+  /** Watches the run as it happens. See {@link RunObserver}. */
+  observer?: RunObserver | undefined;
+};
+
+export type RunResult = {
+  runDir: string;
+  metadata: RunMetadata;
+  grading: GradingReport;
+  exitReason: RunMetadata["exitReason"];
+};
+
+export type SuiteCaseResult = {
+  caseId: string;
+  runDir: string | null;
+  exitReason: RunMetadata["exitReason"];
+  casePassed: boolean | null;
+  errorMessage?: string;
+  /** The full result, when the case got far enough to produce one. */
+  result?: RunResult;
+};
+
+/** Watches one case within a suite; returned by {@link SuiteObserver.caseStart}. */
+export type CaseObserver = RunObserver & {
+  /** The case settled. Receives the same entry that lands in `results`. */
+  end?(result: SuiteCaseResult): void;
+};
+
+/** A {@link RunObserver} that also sees case boundaries within a suite run. */
+export type SuiteObserver = RunObserver & {
+  /**
+   * Called when a worker picks the case up. Return a {@link CaseObserver} to
+   * receive that case's events under their own identity — the only way to
+   * tell cases apart when `concurrency > 1`. Returning nothing routes the
+   * case's events to this observer's own {@link RunObserver} methods.
+   */
+  // biome-ignore lint/suspicious/noConfusingVoidType: `| void` (not `| undefined`) lets a log-only caseStart implementation return nothing.
+  caseStart?(info: { caseId: string; index: number; total: number }): CaseObserver | void;
+};
+
+/**
+ * One case to run in a suite: its file and the id to display it under.
+ * `caseId` is the canonical id runs are filed under when the case declares
+ * none — display decorations (a `--loop` suffix) must not fork run history.
+ */
+export type SuiteItem = { path: string; id: string; caseId?: string | undefined };
+
+const resolveCaseId = (casePath: string, evalCase: EvalCase, fallback?: string): string => {
+  if (evalCase.id) return evalCase.id;
+  if (fallback) return fallback;
+  if (casePath === INLINE_CASE_PATH) return "inline";
+  const base = basename(casePath);
+  const ext = extname(base);
+  return ext ? base.slice(0, -ext.length) : base;
+};
+
+const driverNameOf = (ref: DriverRef): string => (typeof ref === "string" ? ref : ref.name);
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  Boolean(value) && typeof value === "object" && !Array.isArray(value);
+
+/** Merge plugin config recursively so nested Codex `config`/`env` overrides stay additive. */
+const mergePluginConfig = (base: unknown, local: unknown): unknown => {
+  if (!isRecord(base) || !isRecord(local)) return local ?? base;
+  const merged: Record<string, unknown> = { ...base };
+  for (const [key, value] of Object.entries(local)) {
+    merged[key] = key in merged ? mergePluginConfig(merged[key], value) : value;
+  }
+  return merged;
+};
+
+export const resolveDriverReference = (
+  evalCase: EvalCase,
+  fallback: DriverRef | undefined,
+): DriverRef => {
+  const local = evalCase.driver;
+  if (!local) {
+    if (!fallback) throw new Error("case names no driver and no defaultDriver is configured");
+    return fallback;
+  }
+  if (!fallback || driverNameOf(local) !== driverNameOf(fallback)) return local;
+
+  // Naming the same driver opts into project defaults. A case object only
+  // needs to carry the fields it wants to override (for example a timeout).
+  const baseConfig = typeof fallback === "object" ? fallback.config : undefined;
+  const localConfig = typeof local === "object" ? local.config : undefined;
+  return {
+    name: driverNameOf(local),
+    config: mergePluginConfig(baseConfig, localConfig),
+  };
+};
+
+export const resolveAnswererReference = (
+  evalCase: EvalCase,
+  fallback: AnswererRef,
+  harness: HarnessModelConfig | undefined,
+  override?: string,
+): AnswererRef => {
+  const ref = override ?? evalCase.answerer ?? fallback;
+  const name = typeof ref === "string" ? ref : ref.name;
+  if (!harness || name !== "persona") return ref;
+  return {
+    name: "persona",
+    config: mergeHarnessModelConfig(harness, typeof ref === "object" ? ref.config : undefined),
+  };
+};
+
+const answererNameOf = (ref: AnswererRef): string => (typeof ref === "string" ? ref : ref.name);
+
+const driverConfigOf = (ref: DriverRef): unknown =>
+  typeof ref === "object" ? ref.config : undefined;
+
+const answererConfigOf = (ref: AnswererRef): unknown =>
+  typeof ref === "object" ? ref.config : undefined;
+
+const EXPERIMENT_BEHAVIOR_SOURCE_ARGS = new Set([
+  "--continue",
+  "-c",
+  "--fork-session",
+  "--resume",
+  "-r",
+  "--add-dir",
+  "--append-system-prompt-file",
+  "--mcp-config",
+  "--plugin-dir",
+  "--settings",
+  "--setting-sources",
+  "--system-prompt-file",
+]);
+
+const CODEX_EXPERIMENT_SOURCE_ARGS = new Set([
+  "--add-dir",
+  "--cd",
+  "-C",
+  "--config",
+  "-c",
+  "--profile",
+  "-p",
+]);
+
+const bindSubprocessExperimentConfig = (config: Record<string, unknown>): unknown => {
+  if (typeof config.command !== "string" || !isAbsolute(config.command)) {
+    throw new Error(
+      "experiment subprocess commands must be absolute direct wrappers so Lux can fingerprint the invoked behavior",
+    );
+  }
+  if (typeof config.cwd === "string") {
+    throw new Error(
+      "experiment runs do not allow an explicit subprocess cwd; use the candidate subject and staged fixtures",
+    );
+  }
+  if (Array.isArray(config.args) && config.args.length > 0) {
+    throw new Error(
+      "experiment subprocesses do not allow args; put stable behavior in the fingerprinted direct wrapper",
+    );
+  }
+  if (isRecord(config.env) && Object.keys(config.env).length > 0) {
+    throw new Error(
+      "experiment subprocesses do not allow custom env; put stable behavior in the fingerprinted direct wrapper",
+    );
+  }
+  return { ...config, inheritEnv: false };
+};
+
+type AgentDriverName = "claude-code" | "codex";
+
+const validateAgentExperimentConfig = (
+  driverName: AgentDriverName,
+  config: Record<string, unknown>,
+): { driverEnv: Record<string, unknown>; extraArgs: string[] } => {
+  const driverEnv = isRecord(config.env) ? config.env : {};
+  const commandResolutionOverride = Object.keys(driverEnv).find((name) =>
+    ["PATH", "PATHEXT"].includes(name.toUpperCase()),
+  );
+  if (commandResolutionOverride) {
+    throw new Error(
+      `experiment runs do not allow driver env.${commandResolutionOverride}; command resolution must match the fingerprinted host executable`,
+    );
+  }
+  if (typeof config.cwd === "string") {
+    throw new Error(
+      `experiment runs do not allow an explicit ${driverName} cwd; use the candidate subject and staged fixtures`,
+    );
+  }
+
+  const extraArgs = Array.isArray(config.extraArgs)
+    ? config.extraArgs.filter((value): value is string => typeof value === "string")
+    : [];
+  const disallowedArgs =
+    driverName === "claude-code" ? EXPERIMENT_BEHAVIOR_SOURCE_ARGS : CODEX_EXPERIMENT_SOURCE_ARGS;
+  const behaviorSourceArg = extraArgs.find((arg) =>
+    [...disallowedArgs].some((name) => arg === name || arg.startsWith(`${name}=`)),
+  );
+  if (behaviorSourceArg) {
+    throw new Error(
+      `experiment runs do not allow behavior-bearing extraArgs '${behaviorSourceArg}'; use structured driver fields so Lux can fingerprint the source`,
+    );
+  }
+
+  const isolation = isRecord(config.isolation) ? config.isolation : {};
+  const structuredPaths = [
+    ...(driverName === "claude-code" && Array.isArray(config.pluginDirs) ? config.pluginDirs : []),
+    ...(Array.isArray(config.addDirs) ? config.addDirs : []),
+    ...(driverName === "claude-code" && Array.isArray(isolation.allowRead)
+      ? isolation.allowRead
+      : []),
+  ].filter((value): value is string => typeof value === "string");
+  const relativeSource = structuredPaths.find((path) => !isAbsolute(path));
+  if (relativeSource) {
+    throw new Error(
+      `experiment driver source paths must be absolute after subjectPath binding: ${relativeSource}`,
+    );
+  }
+  return { driverEnv, extraArgs };
+};
+
+const bindCodexExperimentConfig = (
+  config: Record<string, unknown>,
+  extraArgs: readonly string[],
+): unknown => {
+  if (typeof config.profile === "string") {
+    throw new Error(
+      "experiment runs do not allow a Codex profile; use inline driver config so Lux can fingerprint it",
+    );
+  }
+  if (config.ephemeral === false) {
+    throw new Error("experiment runs require ephemeral Codex sessions");
+  }
+  const inlineConfig = isRecord(config.config) ? config.config : {};
+  const externalConfigKey = ((): string | undefined => {
+    const visit = (value: Record<string, unknown>, parent = ""): string | undefined => {
+      for (const [key, nested] of Object.entries(value)) {
+        const path = parent ? `${parent}.${key}` : key;
+        if (/(?:mcp_servers|instructions_file|notify|project_doc)/i.test(key)) return path;
+        if (isRecord(nested)) {
+          const found = visit(nested, path);
+          if (found) return found;
+        }
+      }
+      return undefined;
+    };
+    return visit(inlineConfig);
+  })();
+  if (externalConfigKey) {
+    throw new Error(
+      `experiment runs do not allow path-bearing Codex config '${externalConfigKey}'`,
+    );
+  }
+  const appServer = config.interactionMode !== "exec";
+  return {
+    ...config,
+    ephemeral: true,
+    ignoreUserConfig: true,
+    ignoreRules: true,
+    ...(appServer && !config.isolation ? { isolation: { allowRead: [] } } : {}),
+    extraArgs,
+  };
+};
+
+const bindClaudeExperimentConfig = (
+  config: Record<string, unknown>,
+  extraArgs: readonly string[],
+  driverEnv: Record<string, unknown>,
+): unknown => {
+  if (config.isolation) return config;
+  return {
+    ...config,
+    extraArgs: [
+      ...extraArgs,
+      "--setting-sources",
+      "user",
+      "--strict-mcp-config",
+      "--settings",
+      JSON.stringify({ disableAllHooks: true }),
+    ],
+    env: {
+      ...driverEnv,
+      CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1",
+      CLAUDE_CODE_DISABLE_CLAUDE_MDS: "1",
+      CLAUDE_CODE_SKIP_PROMPT_HISTORY: "1",
+      CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1",
+      CLAUDE_CODE_AUTO_CONNECT_IDE: "false",
+      CLAUDE_CODE_DISABLE_OFFICIAL_MARKETPLACE_AUTOINSTALL: "1",
+      DISABLE_AUTOUPDATER: "1",
+    },
+  };
+};
+
+export const boundExperimentDriverConfig = (
+  driverName: string,
+  config: unknown,
+  experimentId: string | undefined,
+): unknown => {
+  if (!experimentId || !isRecord(config)) return config;
+  if (driverName === "subprocess") return bindSubprocessExperimentConfig(config);
+  if (driverName !== "claude-code" && driverName !== "codex") return config;
+  const { driverEnv, extraArgs } = validateAgentExperimentConfig(driverName, config);
+  return driverName === "codex"
+    ? bindCodexExperimentConfig(config, extraArgs)
+    : bindClaudeExperimentConfig(config, extraArgs, driverEnv);
+};
+
+const closeQuietly = async (close: () => Promise<void>, label: string): Promise<void> => {
+  try {
+    await close();
+  } catch (err) {
+    // Cleanup failures must not shadow the original error or stop other
+    // cleanup; surface them on stderr.
+    process.stderr.write(`[lux] warning: ${label} close failed: ${errorMessage(err)}\n`);
+  }
+};
+
+export class Orchestrator {
+  private evaluatorHash: Promise<string> | undefined;
+  readonly registry: Registry;
+  private readonly defaultAnswerer: AnswererRef;
+  private readonly defaultDriver: DriverRef | undefined;
+  private readonly harness: HarnessModelConfig | undefined;
+  private readonly lifecycleFixtures: readonly LifecycleHooksPlugin[];
+
+  constructor(options: OrchestratorOptions) {
+    this.registry = options.registry;
+    this.defaultAnswerer = options.defaultAnswerer ?? "persona";
+    this.defaultDriver = options.defaultDriver;
+    this.harness = options.harness;
+    this.lifecycleFixtures = [...(options.lifecycleFixtures ?? [])];
+  }
+
+  private evaluatorImplementationIdentity(): Promise<string> {
+    this.evaluatorHash ??= evaluatorImplementationHash();
+    return this.evaluatorHash;
+  }
+
+  async run(request: RunRequest): Promise<RunResult> {
+    if (this.lifecycleFixtures.length > 0 && !request.suiteId) {
+      throw new Error(
+        "Orchestrator.run() cannot execute configured lifecycle fixtures; use runSuite() for a single case",
+      );
+    }
+    if (!request.case && !request.casePath) {
+      throw new Error("RunRequest requires either 'case' or 'casePath'");
+    }
+    if (request.case && request.casePath) {
+      throw new Error("RunRequest allows only one of 'case' or 'casePath', not both");
+    }
+    const casePath = request.casePath ? resolve(request.casePath) : INLINE_CASE_PATH;
+    const authoredCase = request.case ?? (await loadEvalCase(casePath));
+    const evalCase = bindSubjectRoot(authoredCase, request.subjectRoot);
+    const caseId = resolveCaseId(casePath, evalCase, request.caseId);
+    const validatedAssertionConfigs = await validateAssertionPlan(
+      evalCase.assertions ?? [],
+      this.registry,
+    );
+
+    const driverRef = resolveDriverReference(evalCase, this.defaultDriver);
+    const driverName = driverNameOf(driverRef);
+    const answererRef = resolveAnswererReference(
+      evalCase,
+      this.defaultAnswerer,
+      this.harness,
+      request.answererOverride,
+    );
+    const answererName = answererNameOf(answererRef);
+
+    const driverPlugin = this.registry.getDriver(driverName);
+    if (!driverPlugin) {
+      throw new Error(`no driver registered for '${driverName}'`);
+    }
+    const answererPlugin = this.registry.getAnswerer(answererName);
+    if (!answererPlugin) {
+      throw new Error(`no answerer registered for '${answererName}'`);
+    }
+
+    // Validate both configs before instantiating either plugin so a
+    // schema failure can't leak a half-initialized driver/answerer.
+    const driverCfg = boundExperimentDriverConfig(
+      driverName,
+      validatePluginConfig(
+        driverPlugin.configSchema,
+        driverConfigOf(driverRef),
+        `driver '${driverName}'`,
+      ),
+      request.experimentId,
+    );
+    const answererCfg = validatePluginConfig(
+      answererPlugin.configSchema,
+      answererConfigOf(answererRef),
+      `answerer '${answererName}'`,
+    );
+
+    const driver = await driverPlugin.create(driverCfg);
+    let answerer: Answerer;
+    try {
+      answerer = await answererPlugin.create(answererCfg);
+    } catch (err) {
+      await closeQuietly(() => driver.close(), `driver '${driverName}'`);
+      throw err;
+    }
+
+    try {
+      const assertionImplementations = assertionEvaluatorIdentities(
+        authoredCase.assertions ?? [],
+        this.registry,
+      );
+      const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
+      const implementationHash = await this.evaluatorImplementationIdentity();
+      const graderHash = graderConfigHash({
+        luxVersion: LUX_VERSION,
+        implementationHash,
+        assertions: assertionImplementations,
+        validatedAssertionsHash,
+        graderContext: this.harness,
+      });
+      return await this.runWithPlugins({
+        request,
+        casePath,
+        authoredCase,
+        evalCase,
+        validatedAssertionConfigs,
+        caseId,
+        driverName,
+        answererName,
+        answererConfig: answererCfg,
+        driverConfigHash: valueHash(driverCfg),
+        answererConfigHash: valueHash(answererCfg),
+        validatedAssertionsHash,
+        graderConfigHash: graderHash,
+        evaluatorConfigHash: valueHash({
+          luxVersion: LUX_VERSION,
+          implementationHash,
+          harness: this.harness,
+          driver: { name: driverPlugin.name, create: driverPlugin.create },
+          answerer: { name: answererPlugin.name, create: answererPlugin.create },
+          assertions: assertionImplementations,
+          lifecycleFixtures: this.lifecycleFixtures,
+          validatedAssertionsHash,
+        }),
+        driver,
+        answerer,
+      });
+    } finally {
+      await closeQuietly(() => driver.close(), `driver '${driverName}'`);
+      await closeQuietly(() => answerer.close(), `answerer '${answererName}'`);
+    }
+  }
+
+  /**
+   * Run cases through a bounded pool of `concurrency` workers, continuing on
+   * per-case failures and aggregating results in input order. `items` is the
+   * exact run list — repetition (`--loop`) is the caller repeating entries.
+   */
+  async runSuite(
+    items: SuiteItem[],
+    opts: {
+      runsRoot: string;
+      fixturesRoot?: string | undefined;
+      answererOverride?: string | undefined;
+      observer?: SuiteObserver | undefined;
+      /** Aborts the cases in flight and stops the pool pulling new ones. */
+      abortSignal?: AbortSignal | undefined;
+      /** How many cases may run at once. Defaults to 1 (sequential). */
+      concurrency?: number | undefined;
+      subjectRoot?: string | undefined;
+      subjectHash?: string | undefined;
+      profileId?: string | undefined;
+      experimentId?: string | undefined;
+      experimentUnitId?: string | undefined;
+      experimentAttemptId?: string | undefined;
+      variantId?: string | undefined;
+      candidateId?: string | undefined;
+      recordGradingAssertion?: (
+        entry: GradingAssertionEntry,
+        index: number,
+      ) => void | Promise<void>;
+      recordPreGradingAccounting?: (metadata: RunMetadata) => void | Promise<void>;
+      recordModelAttemptStart?: (kind: "agent" | "answerer" | "judge") => string | Promise<string>;
+      recordModelAttemptComplete?: (
+        attemptId: string,
+        accounting: AnswererModelCallAccounting,
+      ) => void | Promise<void>;
+      recordModelAttemptsComplete?: (attemptIds: readonly string[]) => void | Promise<void>;
+    },
+  ): Promise<{ results: SuiteCaseResult[]; aborted: boolean; suiteDir: string }> {
+    const observer = opts.observer;
+    const requestedConcurrency = validateSuiteConcurrency(opts.concurrency);
+    const concurrency = Math.min(requestedConcurrency, Math.max(items.length, 1));
+    const suite = await openSuiteRun({
+      runsRoot: opts.runsRoot,
+      concurrency,
+      items,
+      ...(opts.profileId ? { profileId: opts.profileId } : {}),
+      ...(opts.experimentId ? { experimentId: opts.experimentId } : {}),
+      ...(opts.experimentUnitId ? { experimentUnitId: opts.experimentUnitId } : {}),
+      ...(opts.experimentAttemptId ? { experimentAttemptId: opts.experimentAttemptId } : {}),
+      ...(opts.variantId ? { variantId: opts.variantId } : {}),
+      ...(opts.candidateId ? { candidateId: opts.candidateId } : {}),
+      ...(opts.subjectHash ? { subjectHash: opts.subjectHash } : {}),
+      fixtures: this.lifecycleFixtures.map(({ name, pluginVersion }) => ({
+        name,
+        pluginVersion,
+      })),
+    });
+    const persistSuite = serializeEffect(suite.write);
+    const persistTerminalSuite = async (
+      lifecycleFinalization: SuiteLifecycleFinalization<SuiteCaseResult>,
+    ): Promise<void> => {
+      suite.manifest.aborted = lifecycleFinalization.status === "aborted";
+      suite.manifest.endedAt ??= new Date().toISOString();
+      suite.manifest.status =
+        lifecycleFinalization.status === "succeeded" ? "passed" : lifecycleFinalization.status;
+      if (lifecycleFinalization.error === null) {
+        delete suite.manifest.errorMessage;
+      } else {
+        suite.manifest.errorMessage = errorMessage(lifecycleFinalization.error);
+      }
+      await persistSuite();
+    };
+    const startCase = guardObserverCallback(observer?.caseStart?.bind(observer));
+    const caseHandles: Array<CaseObserver | undefined> = new Array(items.length);
+    const hookContext: SuiteLifecycleHookContext = {
+      suiteId: suite.manifest.suiteId,
+      suiteDir: suite.path,
+      runsRoot: resolve(opts.runsRoot),
+      concurrency,
+      items: items.map((item) => ({
+        id: item.id,
+        caseId: item.caseId ?? item.id,
+        path: resolve(item.path),
+      })),
+      ...(opts.abortSignal ? { abortSignal: opts.abortSignal } : {}),
+    };
+    let cleanupSuiteHooks: () => void | Promise<void> = async () => undefined;
+    if (!opts.abortSignal?.aborted) {
+      try {
+        cleanupSuiteHooks = await setupLifecycleHooks(
+          this.lifecycleFixtures,
+          "suite",
+          (hook) => hook.setupSuite?.(hookContext),
+          opts.abortSignal,
+        );
+      } catch (error) {
+        const aborted = causedByAbort(error, opts.abortSignal);
+        const failed: SuiteLifecycleFinalization<SuiteCaseResult> = {
+          status: aborted ? "aborted" : "failed",
+          items: items.map(() => ({ status: "pending", result: null })),
+          results: [],
+          error: aborted ? null : error,
+        };
+        await persistTerminalSuite(failed);
+        await writeSuiteSummary(
+          suite.path,
+          buildSuiteSummary(
+            suite.manifest,
+            failed.items.map(() => undefined),
+          ),
+        );
+        if (aborted) {
+          return { results: [], aborted: true, suiteDir: suite.path };
+        }
+        throw error;
+      }
+    }
+    const effects: SuiteLifecycleEffects<SuiteCaseResult> = {
+      itemStarted: async (index) => {
+        const suiteItem = suite.manifest.items[index];
+        if (suiteItem) suiteItem.status = "running";
+        await persistSuite();
+      },
+      runItem: async (index) => {
+        const item = items[index];
+        if (!item) throw new Error(`suite item ${index} is missing`);
+        const handle = startCase({ caseId: item.id, index, total: items.length }) ?? undefined;
+        caseHandles[index] = handle;
+        let cleanupCaseHooks: () => void | Promise<void>;
+        try {
+          cleanupCaseHooks = await setupLifecycleHooks(
+            this.lifecycleFixtures,
+            "case",
+            (hook) =>
+              hook.setupCase?.({
+                ...hookContext,
+                index,
+                item: hookContext.items[index] ?? {
+                  id: item.id,
+                  caseId: item.caseId ?? item.id,
+                  path: resolve(item.path),
+                },
+              }),
+            opts.abortSignal,
+          );
+        } catch (err) {
+          return {
+            caseId: item.id,
+            runDir: null,
+            exitReason: "error",
+            casePassed: null,
+            errorMessage: errorMessage(err),
+          };
+        }
+        let outcome: SuiteCaseResult;
+        try {
+          const result = await this.run({
+            casePath: item.path,
+            caseId: item.caseId ?? item.id,
+            runsRoot: opts.runsRoot,
+            fixturesRoot: opts.fixturesRoot,
+            answererOverride: opts.answererOverride,
+            observer: handle ?? observer,
+            abortSignal: opts.abortSignal,
+            suiteId: suite.manifest.suiteId,
+            subjectRoot: opts.subjectRoot,
+            subjectHash: opts.subjectHash,
+            profileId: opts.profileId,
+            experimentId: opts.experimentId,
+            variantId: opts.variantId,
+            candidateId: opts.candidateId,
+            ...(opts.recordGradingAssertion
+              ? { recordGradingAssertion: opts.recordGradingAssertion }
+              : {}),
+            ...(opts.recordPreGradingAccounting
+              ? { recordPreGradingAccounting: opts.recordPreGradingAccounting }
+              : {}),
+            ...(opts.recordModelAttemptStart
+              ? { recordModelAttemptStart: opts.recordModelAttemptStart }
+              : {}),
+            ...(opts.recordModelAttemptComplete
+              ? { recordModelAttemptComplete: opts.recordModelAttemptComplete }
+              : {}),
+            ...(opts.recordModelAttemptsComplete
+              ? { recordModelAttemptsComplete: opts.recordModelAttemptsComplete }
+              : {}),
+          });
+          outcome = {
+            caseId: item.id,
+            runDir: result.runDir,
+            exitReason: result.exitReason,
+            casePassed: result.grading.casePassed,
+            result,
+          };
+        } catch (err) {
+          outcome = {
+            caseId: item.id,
+            runDir: runDirOfError(err),
+            exitReason: "error",
+            casePassed: false,
+            errorMessage: errorMessage(err),
+          };
+        }
+        try {
+          await cleanupCaseHooks();
+        } catch (caughtCleanupError) {
+          let cleanupError = caughtCleanupError;
+          if (outcome.result) {
+            try {
+              await recordLifecycleCleanupFailure(outcome.result, caughtCleanupError);
+            } catch (persistenceError) {
+              cleanupError = combineLifecycleFailures(
+                caughtCleanupError,
+                persistenceError,
+                `case '${item.id}' lifecycle cleanup and run persistence failed`,
+              );
+            }
+          }
+          const failure = outcome.errorMessage
+            ? combineLifecycleFailures(
+                new Error(outcome.errorMessage),
+                cleanupError,
+                `case '${item.id}' and its lifecycle cleanup failed`,
+              )
+            : cleanupError;
+          outcome = {
+            ...outcome,
+            exitReason: "error",
+            casePassed: null,
+            errorMessage: errorMessage(failure),
+          };
+        }
+        return outcome;
+      },
+      itemCompleted: async (index, outcome) => {
+        const suiteItem = suite.manifest.items[index];
+        if (suiteItem) {
+          suiteItem.status = "completed";
+          suiteItem.runDir = outcome.runDir ? suite.relativeRunDir(outcome.runDir) : null;
+          suiteItem.exitReason = outcome.exitReason;
+          suiteItem.casePassed = outcome.casePassed;
+          if (outcome.errorMessage) suiteItem.errorMessage = outcome.errorMessage;
+        }
+        await persistSuite();
+        const handle = caseHandles[index];
+        guardObserverCallback(handle?.end?.bind(handle))(outcome);
+      },
+      resultFailed: (result) => result.exitReason !== "done" || result.casePassed === false,
+      finalize: async (lifecycleFinalization) => {
+        let terminalFinalization = lifecycleFinalization;
+        try {
+          await cleanupSuiteHooks();
+        } catch (cleanupError) {
+          terminalFinalization = {
+            ...lifecycleFinalization,
+            status: "failed",
+            error:
+              lifecycleFinalization.error === null
+                ? cleanupError
+                : combineLifecycleFailures(
+                    lifecycleFinalization.error,
+                    cleanupError,
+                    "suite execution and lifecycle cleanup failed",
+                  ),
+          };
+        }
+        let finalizationError = terminalFinalization.error;
+        try {
+          await persistTerminalSuite(terminalFinalization);
+        } catch (persistenceError) {
+          finalizationError =
+            finalizationError === null
+              ? persistenceError
+              : combineLifecycleFailures(
+                  finalizationError,
+                  persistenceError,
+                  "suite finalization and manifest persistence failed",
+                );
+        }
+        const slots = terminalFinalization.items.map((item) => item.result ?? undefined);
+        try {
+          await writeSuiteSummary(suite.path, buildSuiteSummary(suite.manifest, slots));
+        } catch (error) {
+          const summaryError = new Error(
+            `suite summary persistence failed: ${errorMessage(error)}`,
+            {
+              cause: error,
+            },
+          );
+          finalizationError =
+            finalizationError === null
+              ? summaryError
+              : combineLifecycleFailures(
+                  finalizationError,
+                  summaryError,
+                  "suite finalization and summary persistence failed",
+                );
+        }
+        if (finalizationError !== lifecycleFinalization.error) {
+          throw finalizationError;
+        }
+      },
+      recordFinalizationFailure: persistTerminalSuite,
+    };
+    const finalization = await executeSuiteLifecycle({
+      itemCount: items.length,
+      concurrency,
+      ...(opts.abortSignal ? { abortSignal: opts.abortSignal } : {}),
+      effects,
+    });
+    return {
+      results: [...finalization.results],
+      aborted: finalization.status === "aborted",
+      suiteDir: suite.path,
+    };
+  }
+
+  private runWithPlugins(
+    args: Omit<RunWithPluginsArgs, "luxVersion" | "registry">,
+  ): Promise<RunResult> {
+    return executeRunWithPlugins({
+      ...args,
+      luxVersion: LUX_VERSION,
+      registry: this.registry,
+    });
+  }
+}
+
+// Internal exports for testing
+export { closeQuietly, closeRunDirectory, phaseDurations, serializeEffect };