@prismatic-io/lux 0.0.1

package/src/orchestrator/experiment-runs.ts

@@ -0,0 +1,960 @@
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import {
+  appendFile,
+  link,
+  lstat,
+  mkdir,
+  open,
+  readdir,
+  readFile,
+  realpath,
+  rename,
+  unlink,
+  writeFile,
+} from "node:fs/promises";
+import { hostname } from "node:os";
+import { basename, join, relative, resolve, sep } from "node:path";
+import { z } from "zod";
+import {
+  type Candidate,
+  CandidateEvaluationSchema,
+  CandidateIdSchema,
+  CandidateSchema,
+  ConfidenceIntervalSchema,
+  type ExperimentCampaign,
+  ExperimentCampaignSchema,
+  emptyUsage,
+  TokenUsageSchema,
+  totalTokens,
+} from "../core/index.js";
+import { assertCandidateEnvelopeIntegrity, candidateEnvelopeHash } from "./candidate-integrity.js";
+import { safeReadJson } from "./fs-read.js";
+
+const EXPERIMENTS_DIR = ".experiments";
+const MANIFEST_FILE = "experiment.json";
+const EXECUTION_LOCK_FILE = ".execution.lock";
+const ACCOUNTING_JOURNAL_FILE = "accounting.jsonl";
+
+const ExperimentEvaluationSchema = CandidateEvaluationSchema.extend({
+  candidateId: CandidateIdSchema,
+  repetitions: z.number().int().positive(),
+  summaryPath: z.exactOptional(z.string()),
+  evidencePath: z.exactOptional(z.string()),
+});
+export type ExperimentEvaluation = z.infer<typeof ExperimentEvaluationSchema>;
+
+const ExperimentCandidateRefSchema = z.object({
+  id: CandidateIdSchema,
+  path: z.string(),
+  candidateHash: z.string(),
+  sourceBytes: z.number().int().nonnegative(),
+  /** Hash of the complete schema-normalized Candidate record, not only its components. */
+  candidateEnvelopeHash: z.string().optional(),
+});
+
+const ExperimentBudgetUsageSchema = z.object({
+  metricCalls: z.number().int().nonnegative(),
+  judgeCalls: z.number().int().nonnegative(),
+  unpricedModelCalls: z.number().int().nonnegative().optional(),
+  proposalCalls: z.number().int().nonnegative(),
+  usage: TokenUsageSchema,
+  costUsd: z.number().nonnegative(),
+});
+export type ExperimentBudgetUsage = z.infer<typeof ExperimentBudgetUsageSchema>;
+
+const AccountingDeltaSchema = ExperimentBudgetUsageSchema.partial();
+type AccountingDelta = z.infer<typeof AccountingDeltaSchema>;
+const AccountingJournalEntrySchema = z
+  .object({
+    id: z.string().min(1),
+    delta: AccountingDeltaSchema,
+    completedModelAttemptIds: z.array(z.string().min(1)).optional(),
+  })
+  .strict();
+
+export const ExperimentSelectionSchema = z.object({
+  decidedAt: z.string(),
+  /** Candidate population used for the final ranking; excludes discarded search-only candidates. */
+  decisionCandidateIds: z.array(CandidateIdSchema).optional(),
+  /** Optimize-only checkpoint naming the final promotion population. */
+  promotionFinalistCandidateIds: z.array(CandidateIdSchema).optional(),
+  rankedCandidateIds: z.array(CandidateIdSchema),
+  eligibleCandidateIds: z.array(CandidateIdSchema),
+  championCandidateId: z.exactOptional(CandidateIdSchema),
+  constraints: z.array(
+    z.object({
+      candidateId: CandidateIdSchema,
+      metric: z.string(),
+      profile: z.exactOptional(z.string()),
+      split: z.enum(["train", "validation", "test"]),
+      values: z.array(z.number()),
+      passed: z.boolean(),
+      interval: z.exactOptional(ConfidenceIntervalSchema),
+      note: z.exactOptional(z.string()),
+    }),
+  ),
+});
+export type ExperimentSelection = z.infer<typeof ExperimentSelectionSchema>;
+
+export const ExperimentRunSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    experimentId: z.string(),
+    campaignId: z.string(),
+    campaignPath: z.string(),
+    campaignHash: z.string(),
+    campaignSnapshot: ExperimentCampaignSchema,
+    resolvedProfiles: z.record(z.string(), z.unknown()),
+    evaluatorVersion: z.literal(1),
+    luxVersion: z.string(),
+    evaluationKind: z.enum(["agent", "grader"]).default("agent"),
+    evaluationHash: z.exactOptional(z.string()),
+    mode: z.enum(["experiment", "optimize"]),
+    status: z.enum(["running", "passed", "failed", "aborted"]),
+    startedAt: z.string(),
+    endedAt: z.string().nullable(),
+    /** Accumulated active execution time across completed process segments. */
+    activeTimeMs: z.number().nonnegative().default(0),
+    /** Start of the process segment that currently owns this running ledger. */
+    activeSegmentStartedAt: z.exactOptional(z.string()),
+    subjectRoot: z.string(),
+    baseSourceHash: z.string(),
+    /** Present after legacy candidate refs have been upgraded to full-envelope integrity. */
+    candidateIntegrityVersion: z.exactOptional(z.literal(1)),
+    candidates: z.array(ExperimentCandidateRefSchema),
+    evaluations: z.array(ExperimentEvaluationSchema),
+    /** Unit IDs whose observed model spend has been folded into budgetUsage. */
+    accountedEvaluationUnits: z.array(z.string()).default([]),
+    /**
+     * Number of durable accounting journal entries folded into budgetUsage.
+     * The append-only journal avoids rewriting an ever-growing manifest after
+     * every individual grader assertion.
+     */
+    accountingJournalEntries: z.number().int().nonnegative().default(0),
+    budgetUsage: ExperimentBudgetUsageSchema,
+    championCandidateId: z.exactOptional(CandidateIdSchema),
+    selection: z.exactOptional(ExperimentSelectionSchema),
+    errorMessage: z.exactOptional(z.string()),
+  })
+  .superRefine((manifest, ctx) => {
+    if (manifest.candidateIntegrityVersion !== 1) return;
+    for (const [index, candidate] of manifest.candidates.entries()) {
+      if (candidate.candidateEnvelopeHash !== undefined) continue;
+      ctx.addIssue({
+        code: "custom",
+        path: ["candidates", index, "candidateEnvelopeHash"],
+        message: "required for candidate integrity version 1",
+      });
+    }
+  });
+export type ExperimentRun = z.infer<typeof ExperimentRunSchema>;
+
+const timestampId = (date: Date): string => date.toISOString().replace(/[:.]/g, "-");
+
+const writeJsonAtomic = async (path: string, value: unknown): Promise<void> => {
+  const temporary = `${path}.${process.pid}.tmp`;
+  await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`);
+  await rename(temporary, path);
+};
+
+const persistJsonImmutable = async (path: string, value: unknown): Promise<void> => {
+  const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
+  try {
+    const handle = await open(temporary, "wx");
+    try {
+      await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`);
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    try {
+      // A hard link publishes the fully written inode without replacing an
+      // existing candidate. EEXIST is handled by comparing the persisted
+      // envelope in saveCandidate.
+      await link(temporary, path);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+    }
+  } finally {
+    await unlink(temporary).catch((error: NodeJS.ErrnoException) => {
+      if (error.code !== "ENOENT") throw error;
+    });
+  }
+};
+
+export type OpenedExperimentRun = {
+  path: string;
+  manifest: ExperimentRun;
+  write: () => Promise<void>;
+  recordAccounting: (id: string, delta: AccountingDelta) => Promise<boolean>;
+  /**
+   * Journal a provider attempt before it can spend. A start without a matching
+   * completion is charged as one unpriced call when the ledger resumes.
+   */
+  startModelAttempt: (kind: ModelAttemptKind) => Promise<string>;
+  /** Mark attempts complete only after their observed spend is journaled. */
+  completeModelAttempts: (attemptIds: readonly string[]) => Promise<void>;
+  /** Atomically journal observed spend and terminate its provider attempts. */
+  settleModelAttempts: (
+    id: string,
+    delta: AccountingDelta,
+    attemptIds: readonly string[],
+  ) => Promise<boolean>;
+  saveCandidate: (candidate: Candidate) => Promise<string>;
+  release: () => Promise<void>;
+};
+
+const ModelAttemptKindSchema = z.enum(["agent", "answerer", "judge", "proposal"]);
+type ModelAttemptKind = z.infer<typeof ModelAttemptKindSchema>;
+const MODEL_ATTEMPT_START_PREFIX = "model-attempt-start:";
+const MODEL_ATTEMPT_COMPLETE_PREFIX = "model-attempt-complete:";
+const MODEL_ATTEMPT_ABANDONED_PREFIX = "model-attempt-abandoned:";
+const ExecutionLockSchema = z
+  .object({
+    pid: z.number().int().positive(),
+    hostname: z.string().min(1),
+    acquiredAt: z.string().datetime(),
+  })
+  .strict();
+
+const modelAttemptParts = (
+  id: string,
+  prefix: string,
+): { kind: ModelAttemptKind; attemptId: string } | null => {
+  if (!id.startsWith(prefix)) return null;
+  const [kind, attemptId] = id.slice(prefix.length).split(":", 2);
+  const parsedKind = ModelAttemptKindSchema.safeParse(kind);
+  if (!parsedKind.success || !attemptId) return null;
+  return { kind: parsedKind.data, attemptId };
+};
+
+const modelAttemptId = (prefix: string, kind: ModelAttemptKind, attemptId: string): string =>
+  `${prefix}${kind}:${attemptId}`;
+
+const acquireExecutionLock = async (path: string): Promise<{ release: () => Promise<void> }> => {
+  const lockPath = join(path, EXECUTION_LOCK_FILE);
+  const acquire = async (
+    allowStaleRecovery: boolean,
+  ): Promise<Awaited<ReturnType<typeof open>>> => {
+    try {
+      const handle = await open(lockPath, "wx");
+      const owner = ExecutionLockSchema.parse({
+        pid: process.pid,
+        hostname: hostname(),
+        acquiredAt: new Date().toISOString(),
+      });
+      await handle.writeFile(`${JSON.stringify(owner)}\n`);
+      return handle;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      if (!allowStaleRecovery) {
+        throw new Error(`experiment is already owned by another process: ${path}`);
+      }
+      let rawOwner: unknown;
+      try {
+        rawOwner = JSON.parse(await readFile(lockPath, "utf8"));
+      } catch {
+        throw new Error(`experiment execution lock is present but unreadable: ${lockPath}`);
+      }
+      const parsedOwner = ExecutionLockSchema.safeParse(rawOwner);
+      if (!parsedOwner.success || parsedOwner.data.hostname !== hostname()) {
+        throw new Error(`experiment is already owned by another process: ${path}`);
+      }
+      const owner = parsedOwner.data;
+      try {
+        process.kill(owner.pid, 0);
+        throw new Error(`experiment is already owned by process ${owner.pid}: ${path}`);
+      } catch (ownerError) {
+        if ((ownerError as NodeJS.ErrnoException).code !== "ESRCH") throw ownerError;
+      }
+      await unlink(lockPath);
+      return acquire(false);
+    }
+  };
+  const handle = await acquire(true);
+  let released = false;
+  return {
+    release: async () => {
+      if (released) return;
+      released = true;
+      await handle.close();
+      await unlink(lockPath).catch((error: NodeJS.ErrnoException) => {
+        if (error.code !== "ENOENT") throw error;
+      });
+    },
+  };
+};
+
+const openedRun = (
+  path: string,
+  manifest: ExperimentRun,
+  release: () => Promise<void>,
+  accountedIds: Set<string>,
+): OpenedExperimentRun => {
+  const write = (): Promise<void> => {
+    if (manifest.status !== "running" && manifest.activeSegmentStartedAt) {
+      const segmentStartedAt = Date.parse(manifest.activeSegmentStartedAt);
+      const segmentEndedAt = manifest.endedAt ? Date.parse(manifest.endedAt) : Date.now();
+      if (Number.isFinite(segmentStartedAt) && Number.isFinite(segmentEndedAt)) {
+        manifest.activeTimeMs += Math.max(0, segmentEndedAt - segmentStartedAt);
+      }
+      delete manifest.activeSegmentStartedAt;
+    }
+    return writeJsonAtomic(join(path, MANIFEST_FILE), manifest);
+  };
+  const appendAccounting = async (
+    id: string,
+    rawDelta: AccountingDelta,
+    completedModelAttemptIds: readonly string[] = [],
+  ): Promise<boolean> => {
+    if (accountedIds.has(id)) return false;
+    const delta = AccountingDeltaSchema.parse(rawDelta);
+    const completed = [...new Set(completedModelAttemptIds)];
+    for (const attemptId of completed) {
+      if (!modelAttemptParts(attemptId, MODEL_ATTEMPT_START_PREFIX)) {
+        throw new Error(`invalid model attempt start id: ${attemptId}`);
+      }
+    }
+    const entry = AccountingJournalEntrySchema.parse({
+      id,
+      delta,
+      ...(completed.length > 0 ? { completedModelAttemptIds: completed } : {}),
+    });
+    await appendFile(join(path, ACCOUNTING_JOURNAL_FILE), `${JSON.stringify(entry)}\n`, "utf8");
+    accountedIds.add(id);
+    applyAccountingDelta(manifest.budgetUsage, delta);
+    manifest.accountingJournalEntries++;
+    return true;
+  };
+  const recordAccounting = (id: string, delta: AccountingDelta): Promise<boolean> =>
+    appendAccounting(id, delta);
+  const startModelAttempt = async (kind: ModelAttemptKind): Promise<string> => {
+    const attemptId = randomUUID();
+    await recordAccounting(
+      modelAttemptId(MODEL_ATTEMPT_START_PREFIX, kind, attemptId),
+      kind === "judge" ? { judgeCalls: 1 } : {},
+    );
+    return modelAttemptId(MODEL_ATTEMPT_START_PREFIX, kind, attemptId);
+  };
+  const completeModelAttempts = async (attemptIds: readonly string[]): Promise<void> => {
+    for (const startId of new Set(attemptIds)) {
+      const attempt = modelAttemptParts(startId, MODEL_ATTEMPT_START_PREFIX);
+      if (!attempt) throw new Error(`invalid model attempt start id: ${startId}`);
+      await recordAccounting(
+        modelAttemptId(MODEL_ATTEMPT_COMPLETE_PREFIX, attempt.kind, attempt.attemptId),
+        {},
+      );
+    }
+  };
+  const settleModelAttempts = (
+    id: string,
+    delta: AccountingDelta,
+    attemptIds: readonly string[],
+  ): Promise<boolean> => appendAccounting(id, delta, attemptIds);
+  const saveCandidate = async (candidate: Candidate): Promise<string> => {
+    if (manifest.candidateIntegrityVersion !== 1) {
+      throw new Error("legacy experiment candidate refs must be migrated before saving");
+    }
+    const verified = assertCandidateEnvelopeIntegrity(candidate);
+    const envelopeHash = candidateEnvelopeHash(verified);
+    const matches = manifest.candidates.filter((entry) => entry.id === verified.id);
+    if (matches.length > 1) {
+      throw new Error(`experiment candidate ID '${verified.id}' is duplicated`);
+    }
+    const existing = matches[0];
+    if (existing) {
+      const persisted = await verifiedCandidate(path, manifest, existing);
+      if (candidateEnvelopeHash(persisted) !== envelopeHash) {
+        throw new Error(
+          `experiment candidate '${verified.id}' already exists with a different envelope`,
+        );
+      }
+      // Upgrade a legacy v1 reference after proving its persisted candidate is
+      // equivalent across the complete schema-normalized envelope.
+      if (existing.candidateEnvelopeHash === undefined) {
+        existing.candidateEnvelopeHash = envelopeHash;
+        await write();
+      }
+      return containedPath(path, existing.path);
+    }
+
+    const candidatesDir = containedPath(path, "candidates");
+    const candidatesDirInfo = await lstat(candidatesDir);
+    if (candidatesDirInfo.isSymbolicLink() || !candidatesDirInfo.isDirectory()) {
+      throw new Error("experiment candidates directory must be a real directory");
+    }
+    if ((await realpath(candidatesDir)) !== resolve(await realpath(path), "candidates")) {
+      throw new Error("experiment candidates directory escapes its physical ledger");
+    }
+    const candidatePath = containedPath(path, `candidates/${verified.id}.json`);
+    const ref = {
+      id: verified.id,
+      path: relative(path, candidatePath),
+      candidateHash: verified.candidateHash,
+      sourceBytes: verified.sourceBytes,
+      candidateEnvelopeHash: envelopeHash,
+    };
+    await persistJsonImmutable(candidatePath, verified);
+    const persisted = await verifiedCandidate(path, manifest, ref);
+    if (candidateEnvelopeHash(persisted) !== envelopeHash) {
+      throw new Error(
+        `experiment candidate '${verified.id}' already exists with a different envelope`,
+      );
+    }
+    manifest.candidates.push(ref);
+    await write();
+    return candidatePath;
+  };
+  return {
+    path,
+    manifest,
+    write,
+    recordAccounting,
+    startModelAttempt,
+    completeModelAttempts,
+    settleModelAttempts,
+    saveCandidate,
+    release,
+  };
+};
+
+const applyAccountingDelta = (usage: ExperimentBudgetUsage, delta: AccountingDelta): void => {
+  usage.metricCalls += delta.metricCalls ?? 0;
+  usage.judgeCalls += delta.judgeCalls ?? 0;
+  usage.proposalCalls += delta.proposalCalls ?? 0;
+  usage.unpricedModelCalls = (usage.unpricedModelCalls ?? 0) + (delta.unpricedModelCalls ?? 0);
+  usage.usage = {
+    input: usage.usage.input + (delta.usage?.input ?? 0),
+    output: usage.usage.output + (delta.usage?.output ?? 0),
+    cacheRead: usage.usage.cacheRead + (delta.usage?.cacheRead ?? 0),
+    cacheCreation: usage.usage.cacheCreation + (delta.usage?.cacheCreation ?? 0),
+  };
+  usage.costUsd += delta.costUsd ?? 0;
+};
+
+const loadAccountingJournal = async (
+  path: string,
+  checkpointedEntries: number,
+): Promise<z.infer<typeof AccountingJournalEntrySchema>[]> => {
+  const journalPath = join(path, ACCOUNTING_JOURNAL_FILE);
+  if (!existsSync(journalPath)) return [];
+  const text = await readFile(journalPath, "utf8");
+  const lines = text.split("\n").filter((line) => line.trim().length > 0);
+  const entries: z.infer<typeof AccountingJournalEntrySchema>[] = [];
+  for (const [index, line] of lines.entries()) {
+    try {
+      let parsed: unknown;
+      try {
+        parsed = JSON.parse(line);
+      } catch {
+        throw new Error(`invalid accounting journal JSON at line ${index + 1}`);
+      }
+      const result = AccountingJournalEntrySchema.safeParse(parsed);
+      if (!result.success) {
+        throw new Error(`invalid accounting journal entry at line ${index + 1}`);
+      }
+      entries.push(result.data);
+    } catch (error) {
+      const isUncheckpointedTail = index >= checkpointedEntries && index === lines.length - 1;
+      if (!isUncheckpointedTail) throw error;
+      // appendFile can be torn by a hard kill. Only an invalid final entry
+      // beyond the manifest checkpoint is safe to discard.
+      await writeFile(
+        journalPath,
+        entries.length > 0 ? `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n` : "",
+      );
+      break;
+    }
+  }
+  return entries;
+};
+
+const containedPath = (rootPath: string, relativePath: string): string => {
+  const root = resolve(rootPath);
+  const path = resolve(root, relativePath);
+  if (path === root || !path.startsWith(`${root}${sep}`)) {
+    throw new Error(`experiment candidate path escapes its ledger: ${relativePath}`);
+  }
+  return path;
+};
+
+const verifiedCandidate = async (
+  experimentDir: string,
+  manifest: ExperimentRun,
+  ref: z.infer<typeof ExperimentCandidateRefSchema>,
+): Promise<Candidate> => {
+  const path = containedPath(experimentDir, ref.path);
+  const root = resolve(experimentDir);
+  const pathFromRoot = relative(root, path);
+  const components = pathFromRoot.split(sep);
+  let parent = root;
+  for (const component of components.slice(0, -1)) {
+    parent = join(parent, component);
+    const parentInfo = await lstat(parent).catch((error: NodeJS.ErrnoException) => {
+      if (error.code === "ENOENT") return null;
+      throw error;
+    });
+    if (!parentInfo || parentInfo.isSymbolicLink() || !parentInfo.isDirectory()) {
+      throw new Error(`experiment candidate '${ref.id}' has an invalid path component`);
+    }
+  }
+  const info = await lstat(path).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  });
+  if (!info || info.isSymbolicLink() || !info.isFile()) {
+    throw new Error(`experiment candidate '${ref.id}' is missing or invalid`);
+  }
+  const physicalRoot = await realpath(root);
+  const physicalPath = await realpath(path);
+  if (physicalPath !== resolve(physicalRoot, pathFromRoot)) {
+    throw new Error(`experiment candidate '${ref.id}' escapes the physical ledger`);
+  }
+  const parsedCandidate = await safeReadJson(path, CandidateSchema);
+  if (!parsedCandidate) throw new Error(`experiment candidate '${ref.id}' is missing or invalid`);
+  let candidate: Candidate;
+  try {
+    candidate = assertCandidateEnvelopeIntegrity(parsedCandidate);
+  } catch {
+    throw new Error(`experiment candidate '${ref.id}' failed its integrity check`);
+  }
+  if (
+    candidate.id !== ref.id ||
+    candidate.candidateHash !== ref.candidateHash ||
+    candidate.sourceBytes !== ref.sourceBytes ||
+    candidate.baseSourceHash !== manifest.baseSourceHash ||
+    (ref.candidateEnvelopeHash !== undefined &&
+      candidateEnvelopeHash(candidate) !== ref.candidateEnvelopeHash)
+  ) {
+    throw new Error(`experiment candidate '${ref.id}' failed its integrity check`);
+  }
+  return candidate;
+};
+
+const migrateLegacyCandidateIntegrity = async (
+  experimentDir: string,
+  manifest: ExperimentRun,
+): Promise<void> => {
+  if (manifest.candidateIntegrityVersion === 1) return;
+  for (const ref of manifest.candidates) {
+    ref.candidateEnvelopeHash = candidateEnvelopeHash(
+      await verifiedCandidate(experimentDir, manifest, ref),
+    );
+  }
+  manifest.candidateIntegrityVersion = 1;
+};
+
+export const openExperimentRun = async (options: {
+  runsRoot: string;
+  campaignId: string;
+  campaignPath: string;
+  campaignHash: string;
+  campaignSnapshot: ExperimentCampaign;
+  resolvedProfiles: Record<string, unknown>;
+  luxVersion: string;
+  subjectRoot: string;
+  baseSourceHash: string;
+  mode: ExperimentRun["mode"];
+  evaluationKind?: ExperimentRun["evaluationKind"];
+  evaluationHash?: string;
+  startedAt?: Date;
+}): Promise<OpenedExperimentRun> => {
+  const startedAt = options.startedAt ?? new Date();
+  const root = join(resolve(options.runsRoot), EXPERIMENTS_DIR);
+  await mkdir(root, { recursive: true });
+  const prefix = `${timestampId(startedAt)}-${options.campaignId.replace(/[^a-zA-Z0-9_-]+/g, "-")}`;
+  let experimentId = prefix;
+  let path = join(root, experimentId);
+  for (let suffix = 2; ; suffix++) {
+    try {
+      await mkdir(path);
+      break;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      experimentId = `${prefix}-${suffix}`;
+      path = join(root, experimentId);
+    }
+  }
+  await mkdir(join(path, "candidates"), { recursive: true });
+  const manifest: ExperimentRun = {
+    schemaVersion: 1,
+    experimentId,
+    campaignId: options.campaignId,
+    campaignPath: options.campaignPath,
+    campaignHash: options.campaignHash,
+    campaignSnapshot: options.campaignSnapshot,
+    resolvedProfiles: options.resolvedProfiles,
+    evaluatorVersion: 1,
+    luxVersion: options.luxVersion,
+    evaluationKind: options.evaluationKind ?? "agent",
+    ...(options.evaluationHash ? { evaluationHash: options.evaluationHash } : {}),
+    mode: options.mode,
+    status: "running",
+    startedAt: startedAt.toISOString(),
+    endedAt: null,
+    activeTimeMs: 0,
+    activeSegmentStartedAt: startedAt.toISOString(),
+    subjectRoot: options.subjectRoot,
+    baseSourceHash: options.baseSourceHash,
+    candidateIntegrityVersion: 1,
+    candidates: [],
+    evaluations: [],
+    accountedEvaluationUnits: [],
+    accountingJournalEntries: 0,
+    budgetUsage: {
+      metricCalls: 0,
+      judgeCalls: 0,
+      unpricedModelCalls: 0,
+      proposalCalls: 0,
+      usage: emptyUsage(),
+      costUsd: 0,
+    },
+  };
+  const ownership = await acquireExecutionLock(path);
+  const opened = openedRun(path, manifest, ownership.release, new Set());
+  await opened.write();
+  return opened;
+};
+
+export const loadExperimentRun = async (path: string): Promise<ExperimentRun | null> =>
+  safeReadJson(join(path, MANIFEST_FILE), ExperimentRunSchema);
+
+export const migrateExperimentCandidateIntegrity = async (path: string): Promise<ExperimentRun> => {
+  const absolute = resolve(path);
+  const ownership = await acquireExecutionLock(absolute);
+  try {
+    const manifest = await loadExperimentRun(absolute);
+    if (!manifest) throw new Error(`experiment manifest not found or invalid at ${absolute}`);
+    if (manifest.candidateIntegrityVersion !== 1) {
+      await migrateLegacyCandidateIntegrity(absolute, manifest);
+      await writeJsonAtomic(join(absolute, MANIFEST_FILE), manifest);
+    }
+    return manifest;
+  } finally {
+    await ownership.release();
+  }
+};
+
+export const experimentActiveElapsedMs = (
+  manifest: ExperimentRun,
+  nowMs: number = Date.now(),
+): number => {
+  const segmentStartedAt = manifest.activeSegmentStartedAt
+    ? Date.parse(manifest.activeSegmentStartedAt)
+    : Number.NaN;
+  return (
+    manifest.activeTimeMs +
+    (Number.isFinite(segmentStartedAt) ? Math.max(0, nowMs - segmentStartedAt) : 0)
+  );
+};
+
+export const resumeExperimentRun = async (path: string): Promise<OpenedExperimentRun> => {
+  const absolute = resolve(path);
+  const manifest = await loadExperimentRun(absolute);
+  if (!manifest) throw new Error(`experiment manifest not found or invalid at ${absolute}`);
+  if (manifest.status === "passed") {
+    throw new Error("passed experiments are immutable and cannot be resumed");
+  }
+  const ownership = await acquireExecutionLock(absolute);
+  try {
+    await migrateLegacyCandidateIntegrity(absolute, manifest);
+    const resumedAt = new Date();
+    if (manifest.activeSegmentStartedAt) {
+      const previousSegmentStart = Date.parse(manifest.activeSegmentStartedAt);
+      const previousSegmentEnd = manifest.endedAt
+        ? Date.parse(manifest.endedAt)
+        : resumedAt.getTime();
+      if (Number.isFinite(previousSegmentStart) && Number.isFinite(previousSegmentEnd)) {
+        // A running segment without a terminal checkpoint represents a hard
+        // kill. Charge the unknowable gap conservatively through recovery.
+        manifest.activeTimeMs += Math.max(0, previousSegmentEnd - previousSegmentStart);
+      }
+    } else if (manifest.activeTimeMs === 0) {
+      // Forward-compatible accounting for v1 ledgers written before active
+      // segments existed.
+      const legacyStart = Date.parse(manifest.startedAt);
+      const legacyEnd = manifest.endedAt ? Date.parse(manifest.endedAt) : resumedAt.getTime();
+      if (Number.isFinite(legacyStart) && Number.isFinite(legacyEnd)) {
+        manifest.activeTimeMs = Math.max(0, legacyEnd - legacyStart);
+      }
+    }
+    const journal = await loadAccountingJournal(absolute, manifest.accountingJournalEntries);
+    if (manifest.accountingJournalEntries > journal.length) {
+      throw new Error(
+        `accounting journal is truncated: manifest checkpoints ${manifest.accountingJournalEntries} entries but only ${journal.length} remain`,
+      );
+    }
+    const accountedIds = new Set(manifest.accountedEvaluationUnits);
+    for (const [index, entry] of journal.entries()) {
+      if (accountedIds.has(entry.id)) {
+        if (index >= manifest.accountingJournalEntries) {
+          throw new Error(`duplicate accounting journal id: ${entry.id}`);
+        }
+        continue;
+      }
+      accountedIds.add(entry.id);
+      if (index >= manifest.accountingJournalEntries) {
+        applyAccountingDelta(manifest.budgetUsage, entry.delta);
+      }
+    }
+    manifest.accountingJournalEntries = journal.length;
+    manifest.status = "running";
+    manifest.endedAt = null;
+    manifest.activeSegmentStartedAt = resumedAt.toISOString();
+    delete manifest.errorMessage;
+    const opened = openedRun(absolute, manifest, ownership.release, accountedIds);
+    const terminals = new Set(
+      journal.flatMap((entry) => {
+        const completed = modelAttemptParts(entry.id, MODEL_ATTEMPT_COMPLETE_PREFIX);
+        const abandoned = modelAttemptParts(entry.id, MODEL_ATTEMPT_ABANDONED_PREFIX);
+        return [
+          ...(completed ? [`${completed.kind}:${completed.attemptId}`] : []),
+          ...(abandoned ? [`${abandoned.kind}:${abandoned.attemptId}`] : []),
+          ...(entry.completedModelAttemptIds ?? []).flatMap((attemptId) => {
+            const settled = modelAttemptParts(attemptId, MODEL_ATTEMPT_START_PREFIX);
+            return settled ? [`${settled.kind}:${settled.attemptId}`] : [];
+          }),
+        ];
+      }),
+    );
+    for (const entry of journal) {
+      const started = modelAttemptParts(entry.id, MODEL_ATTEMPT_START_PREFIX);
+      if (!started || terminals.has(`${started.kind}:${started.attemptId}`)) continue;
+      // A hard-killed provider cannot report tokens or price. Charge one
+      // explicit unpriced attempt exactly once before any unit can rerun.
+      await opened.recordAccounting(
+        modelAttemptId(MODEL_ATTEMPT_ABANDONED_PREFIX, started.kind, started.attemptId),
+        { unpricedModelCalls: 1 },
+      );
+    }
+    await opened.write();
+    return opened;
+  } catch (error) {
+    await ownership.release();
+    throw error;
+  }
+};
+
+export const listExperimentRuns = async (
+  runsRoot: string,
+): Promise<Array<ExperimentRun & { experimentDir: string }>> => {
+  const root = join(resolve(runsRoot), EXPERIMENTS_DIR);
+  if (!existsSync(root)) return [];
+  const directories = (await readdir(root, { withFileTypes: true }))
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => join(root, entry.name));
+  const runs = await Promise.all(
+    directories.map(async (experimentDir) => ({
+      experimentDir,
+      experiment: await loadExperimentRun(experimentDir),
+    })),
+  );
+  return runs
+    .flatMap(({ experimentDir, experiment }) =>
+      experiment ? [{ ...experiment, experimentDir }] : [],
+    )
+    .sort((a, b) => b.experimentId.localeCompare(a.experimentId));
+};
+
+export const loadExperimentCandidate = async (
+  experimentDir: string,
+  candidateId: string,
+): Promise<Candidate | null> => {
+  const experiment = await loadExperimentRun(experimentDir);
+  const ref = experiment?.candidates.find((entry) => entry.id === candidateId);
+  return experiment && ref ? verifiedCandidate(experimentDir, experiment, ref) : null;
+};
+
+export const loadExperimentCandidates = async (experimentDir: string): Promise<Candidate[]> => {
+  const experiment = await loadExperimentRun(experimentDir);
+  if (!experiment) throw new Error(`experiment manifest not found or invalid at ${experimentDir}`);
+  return Promise.all(
+    experiment.candidates.map((candidate) =>
+      verifiedCandidate(experimentDir, experiment, candidate),
+    ),
+  );
+};
+
+const strictAccountingJournal = async (
+  experimentDir: string,
+): Promise<z.infer<typeof AccountingJournalEntrySchema>[]> => {
+  const path = join(resolve(experimentDir), ACCOUNTING_JOURNAL_FILE);
+  const info = await lstat(path).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  });
+  if (!info) return [];
+  if (info.isSymbolicLink() || !info.isFile()) {
+    throw new Error("experiment accounting journal must be a regular file");
+  }
+  return (await readFile(path, "utf8"))
+    .split("\n")
+    .filter((line) => line.trim().length > 0)
+    .map((line, index) => {
+      try {
+        return AccountingJournalEntrySchema.parse(JSON.parse(line));
+      } catch {
+        throw new Error(`invalid accounting journal entry at line ${index + 1}`);
+      }
+    });
+};
+
+export const verifyExperimentRunIntegrity = async (
+  experimentDir: string,
+  experiment: ExperimentRun,
+): Promise<Candidate[]> => {
+  const manifestPath = join(resolve(experimentDir), MANIFEST_FILE);
+  const manifestInfo = await lstat(manifestPath);
+  if (manifestInfo.isSymbolicLink() || !manifestInfo.isFile()) {
+    throw new Error("experiment manifest must be a regular file");
+  }
+  const candidateIds = experiment.candidates.map((candidate) => candidate.id);
+  const candidatePaths = experiment.candidates.map((candidate) => candidate.path);
+  if (new Set(candidateIds).size !== candidateIds.length) {
+    throw new Error("experiment candidate IDs must be unique");
+  }
+  if (new Set(candidatePaths).size !== candidatePaths.length) {
+    throw new Error("experiment candidate paths must be unique");
+  }
+  const candidates = await Promise.all(
+    experiment.candidates.map((candidate) =>
+      verifiedCandidate(experimentDir, experiment, candidate),
+    ),
+  );
+  const journal = await strictAccountingJournal(experimentDir);
+  if (journal.length !== experiment.accountingJournalEntries) {
+    throw new Error(
+      `accounting journal checkpoint mismatch: manifest=${experiment.accountingJournalEntries} journal=${journal.length}`,
+    );
+  }
+  const ids = new Set<string>();
+  const reconstructed: ExperimentBudgetUsage = {
+    metricCalls: 0,
+    judgeCalls: 0,
+    proposalCalls: 0,
+    unpricedModelCalls: 0,
+    usage: emptyUsage(),
+    costUsd: 0,
+  };
+  for (const entry of journal) {
+    if (ids.has(entry.id)) throw new Error(`duplicate accounting journal id: ${entry.id}`);
+    ids.add(entry.id);
+    applyAccountingDelta(reconstructed, entry.delta);
+  }
+  const actual = experiment.budgetUsage;
+  if (
+    reconstructed.metricCalls !== actual.metricCalls ||
+    reconstructed.judgeCalls !== actual.judgeCalls ||
+    reconstructed.proposalCalls !== actual.proposalCalls ||
+    (reconstructed.unpricedModelCalls ?? 0) !== (actual.unpricedModelCalls ?? 0) ||
+    reconstructed.usage.input !== actual.usage.input ||
+    reconstructed.usage.output !== actual.usage.output ||
+    reconstructed.usage.cacheRead !== actual.usage.cacheRead ||
+    reconstructed.usage.cacheCreation !== actual.usage.cacheCreation ||
+    reconstructed.costUsd !== actual.costUsd
+  ) {
+    throw new Error("experiment budget usage does not match its durable accounting journal");
+  }
+  return candidates;
+};
+
+export const isExperimentRun = (path: string): boolean => existsSync(join(path, MANIFEST_FILE));
+
+const conciseNumber = (value: number): string =>
+  Number.isInteger(value) ? String(value) : value.toFixed(4).replace(/0+$/, "").replace(/\.$/, "");
+
+const conciseText = (value: string, maximum = 160): string =>
+  value.length <= maximum ? value : `${value.slice(0, maximum - 1)}…`;
+
+export const formatExperimentRun = (path: string, experiment: ExperimentRun): string => {
+  const usage = experiment.budgetUsage.usage;
+  const tokens = totalTokens(usage);
+  const lines = [
+    `experimentDir: ${path}`,
+    `experiment:    ${experiment.experimentId}`,
+    `campaign:      ${experiment.campaignId}`,
+    `evaluation:    ${experiment.evaluationKind}`,
+    `mode:          ${experiment.mode}`,
+    `status:        ${experiment.status}`,
+    `started:       ${experiment.startedAt}`,
+    `ended:         ${experiment.endedAt ?? "—"}`,
+    `candidates:    ${experiment.candidates.length}`,
+    `evaluations:   ${experiment.evaluations.length}`,
+    `champion:      ${experiment.championCandidateId ?? "—"}`,
+    `spend:         metric=${experiment.budgetUsage.metricCalls} judge=${experiment.budgetUsage.judgeCalls} proposal=${experiment.budgetUsage.proposalCalls} unpriced=${experiment.budgetUsage.unpricedModelCalls ?? 0}`,
+    `tokens:        ${tokens} (input=${usage.input} output=${usage.output} cache-read=${usage.cacheRead} cache-create=${usage.cacheCreation})`,
+    `reported cost: $${experiment.budgetUsage.costUsd.toFixed(4)}`,
+  ];
+  if (experiment.errorMessage) lines.push(`error:         ${experiment.errorMessage}`);
+  if (experiment.selection) {
+    lines.push(
+      `ranking:       ${experiment.selection.rankedCandidateIds.join(" > ") || "—"}`,
+      `eligible:      ${experiment.selection.eligibleCandidateIds.join(", ") || "none"}`,
+      "gates:",
+    );
+    for (const constraint of experiment.selection.constraints) {
+      const scope = [
+        constraint.profile ? `profile=${constraint.profile}` : null,
+        `split=${constraint.split}`,
+      ]
+        .filter(Boolean)
+        .join(" ");
+      const interval = constraint.interval
+        ? ` ${constraint.interval.method}=${conciseNumber(constraint.interval.estimate)} [${conciseNumber(constraint.interval.lower)},${conciseNumber(constraint.interval.upper)}] n=${constraint.interval.n}`
+        : "";
+      const note = constraint.note ? ` note=${conciseText(constraint.note)}` : "";
+      lines.push(
+        `  ${constraint.passed ? "✓" : "✗"} ${constraint.candidateId} ${constraint.metric} ${scope} values=${constraint.values.map(conciseNumber).join(",") || "—"}${interval}${note}`,
+      );
+    }
+  }
+  if (experiment.evaluations.length > 0) lines.push("evidence:");
+  for (const evaluation of experiment.evaluations) {
+    const evaluationTokens = evaluation.usage ? totalTokens(evaluation.usage) : 0;
+    lines.push(
+      `  ${evaluation.candidateId} profile=${evaluation.profileId} split=${evaluation.split} repetitions=${evaluation.repetitions} pass=${conciseNumber(evaluation.passRate)} calls=${evaluation.metricCalls ?? "—"}/${evaluation.judgeCalls ?? "—"} tokens=${evaluationTokens} unpriced=${evaluation.unpricedModelCalls ?? 0} cost=$${(evaluation.costUsd ?? 0).toFixed(4)}`,
+      `    cases: ${
+        Object.entries(evaluation.casePassRates)
+          .sort(([a], [b]) => a.localeCompare(b))
+          .map(([id, passRate]) => `${id}=${conciseNumber(passRate)}`)
+          .join(", ") || "—"
+      }`,
+    );
+    for (const failure of evaluation.failedCases) {
+      lines.push(
+        `    failure: ${failure.caseId} score=${failure.score === null ? "—" : conciseNumber(failure.score)} ${conciseText(failure.feedback[0] ?? "no feedback")}`,
+      );
+    }
+  }
+  if (experiment.candidates.length > 0) lines.push("candidates:");
+  for (const candidate of experiment.candidates) {
+    lines.push(
+      `  ${candidate.id === experiment.championCandidateId ? "★" : " "} ${candidate.id}  ${candidate.sourceBytes} bytes  ${basename(candidate.path)}`,
+    );
+  }
+  return lines.join("\n");
+};
+
+export const formatExperimentRunsTable = (
+  experiments: Array<ExperimentRun & { experimentDir: string }>,
+): string => {
+  if (experiments.length === 0) return "(no experiment runs found)";
+  const rows = [
+    ["status", "experiment", "campaign", "mode", "candidates", "champion"],
+    ...experiments.map((experiment) => [
+      experiment.status.toUpperCase(),
+      basename(experiment.experimentDir),
+      experiment.campaignId,
+      experiment.mode,
+      String(experiment.candidates.length),
+      experiment.championCandidateId ?? "—",
+    ]),
+  ];
+  const widths = (rows[0] ?? []).map((_, column) =>
+    Math.max(...rows.map((row) => row[column]?.length ?? 0)),
+  );
+  return rows
+    .map((row) => row.map((cell, column) => cell.padEnd(widths[column] ?? 0)).join("  "))
+    .join("\n");
+};