@prismatic-io/lux 0.0.1

package/lib/orchestrator/experiment-runs.js

@@ -0,0 +1,796 @@
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { appendFile, link, lstat, mkdir, open, readdir, readFile, realpath, rename, unlink, writeFile, } from "node:fs/promises";
+import { hostname } from "node:os";
+import { basename, join, relative, resolve, sep } from "node:path";
+import { z } from "zod";
+import { CandidateEvaluationSchema, CandidateIdSchema, CandidateSchema, ConfidenceIntervalSchema, ExperimentCampaignSchema, emptyUsage, TokenUsageSchema, totalTokens, } from "../core/index.js";
+import { assertCandidateEnvelopeIntegrity, candidateEnvelopeHash } from "./candidate-integrity.js";
+import { safeReadJson } from "./fs-read.js";
+const EXPERIMENTS_DIR = ".experiments";
+const MANIFEST_FILE = "experiment.json";
+const EXECUTION_LOCK_FILE = ".execution.lock";
+const ACCOUNTING_JOURNAL_FILE = "accounting.jsonl";
+const ExperimentEvaluationSchema = CandidateEvaluationSchema.extend({
+    candidateId: CandidateIdSchema,
+    repetitions: z.number().int().positive(),
+    summaryPath: z.exactOptional(z.string()),
+    evidencePath: z.exactOptional(z.string()),
+});
+const ExperimentCandidateRefSchema = z.object({
+    id: CandidateIdSchema,
+    path: z.string(),
+    candidateHash: z.string(),
+    sourceBytes: z.number().int().nonnegative(),
+    /** Hash of the complete schema-normalized Candidate record, not only its components. */
+    candidateEnvelopeHash: z.string().optional(),
+});
+const ExperimentBudgetUsageSchema = z.object({
+    metricCalls: z.number().int().nonnegative(),
+    judgeCalls: z.number().int().nonnegative(),
+    unpricedModelCalls: z.number().int().nonnegative().optional(),
+    proposalCalls: z.number().int().nonnegative(),
+    usage: TokenUsageSchema,
+    costUsd: z.number().nonnegative(),
+});
+const AccountingDeltaSchema = ExperimentBudgetUsageSchema.partial();
+const AccountingJournalEntrySchema = z
+    .object({
+    id: z.string().min(1),
+    delta: AccountingDeltaSchema,
+    completedModelAttemptIds: z.array(z.string().min(1)).optional(),
+})
+    .strict();
+export const ExperimentSelectionSchema = z.object({
+    decidedAt: z.string(),
+    /** Candidate population used for the final ranking; excludes discarded search-only candidates. */
+    decisionCandidateIds: z.array(CandidateIdSchema).optional(),
+    /** Optimize-only checkpoint naming the final promotion population. */
+    promotionFinalistCandidateIds: z.array(CandidateIdSchema).optional(),
+    rankedCandidateIds: z.array(CandidateIdSchema),
+    eligibleCandidateIds: z.array(CandidateIdSchema),
+    championCandidateId: z.exactOptional(CandidateIdSchema),
+    constraints: z.array(z.object({
+        candidateId: CandidateIdSchema,
+        metric: z.string(),
+        profile: z.exactOptional(z.string()),
+        split: z.enum(["train", "validation", "test"]),
+        values: z.array(z.number()),
+        passed: z.boolean(),
+        interval: z.exactOptional(ConfidenceIntervalSchema),
+        note: z.exactOptional(z.string()),
+    })),
+});
+export const ExperimentRunSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    experimentId: z.string(),
+    campaignId: z.string(),
+    campaignPath: z.string(),
+    campaignHash: z.string(),
+    campaignSnapshot: ExperimentCampaignSchema,
+    resolvedProfiles: z.record(z.string(), z.unknown()),
+    evaluatorVersion: z.literal(1),
+    luxVersion: z.string(),
+    evaluationKind: z.enum(["agent", "grader"]).default("agent"),
+    evaluationHash: z.exactOptional(z.string()),
+    mode: z.enum(["experiment", "optimize"]),
+    status: z.enum(["running", "passed", "failed", "aborted"]),
+    startedAt: z.string(),
+    endedAt: z.string().nullable(),
+    /** Accumulated active execution time across completed process segments. */
+    activeTimeMs: z.number().nonnegative().default(0),
+    /** Start of the process segment that currently owns this running ledger. */
+    activeSegmentStartedAt: z.exactOptional(z.string()),
+    subjectRoot: z.string(),
+    baseSourceHash: z.string(),
+    /** Present after legacy candidate refs have been upgraded to full-envelope integrity. */
+    candidateIntegrityVersion: z.exactOptional(z.literal(1)),
+    candidates: z.array(ExperimentCandidateRefSchema),
+    evaluations: z.array(ExperimentEvaluationSchema),
+    /** Unit IDs whose observed model spend has been folded into budgetUsage. */
+    accountedEvaluationUnits: z.array(z.string()).default([]),
+    /**
+     * Number of durable accounting journal entries folded into budgetUsage.
+     * The append-only journal avoids rewriting an ever-growing manifest after
+     * every individual grader assertion.
+     */
+    accountingJournalEntries: z.number().int().nonnegative().default(0),
+    budgetUsage: ExperimentBudgetUsageSchema,
+    championCandidateId: z.exactOptional(CandidateIdSchema),
+    selection: z.exactOptional(ExperimentSelectionSchema),
+    errorMessage: z.exactOptional(z.string()),
+})
+    .superRefine((manifest, ctx) => {
+    if (manifest.candidateIntegrityVersion !== 1)
+        return;
+    for (const [index, candidate] of manifest.candidates.entries()) {
+        if (candidate.candidateEnvelopeHash !== undefined)
+            continue;
+        ctx.addIssue({
+            code: "custom",
+            path: ["candidates", index, "candidateEnvelopeHash"],
+            message: "required for candidate integrity version 1",
+        });
+    }
+});
+const timestampId = (date) => date.toISOString().replace(/[:.]/g, "-");
+const writeJsonAtomic = async (path, value) => {
+    const temporary = `${path}.${process.pid}.tmp`;
+    await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`);
+    await rename(temporary, path);
+};
+const persistJsonImmutable = async (path, value) => {
+    const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
+    try {
+        const handle = await open(temporary, "wx");
+        try {
+            await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`);
+            await handle.sync();
+        }
+        finally {
+            await handle.close();
+        }
+        try {
+            // A hard link publishes the fully written inode without replacing an
+            // existing candidate. EEXIST is handled by comparing the persisted
+            // envelope in saveCandidate.
+            await link(temporary, path);
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+        }
+    }
+    finally {
+        await unlink(temporary).catch((error) => {
+            if (error.code !== "ENOENT")
+                throw error;
+        });
+    }
+};
+const ModelAttemptKindSchema = z.enum(["agent", "answerer", "judge", "proposal"]);
+const MODEL_ATTEMPT_START_PREFIX = "model-attempt-start:";
+const MODEL_ATTEMPT_COMPLETE_PREFIX = "model-attempt-complete:";
+const MODEL_ATTEMPT_ABANDONED_PREFIX = "model-attempt-abandoned:";
+const ExecutionLockSchema = z
+    .object({
+    pid: z.number().int().positive(),
+    hostname: z.string().min(1),
+    acquiredAt: z.string().datetime(),
+})
+    .strict();
+const modelAttemptParts = (id, prefix) => {
+    if (!id.startsWith(prefix))
+        return null;
+    const [kind, attemptId] = id.slice(prefix.length).split(":", 2);
+    const parsedKind = ModelAttemptKindSchema.safeParse(kind);
+    if (!parsedKind.success || !attemptId)
+        return null;
+    return { kind: parsedKind.data, attemptId };
+};
+const modelAttemptId = (prefix, kind, attemptId) => `${prefix}${kind}:${attemptId}`;
+const acquireExecutionLock = async (path) => {
+    const lockPath = join(path, EXECUTION_LOCK_FILE);
+    const acquire = async (allowStaleRecovery) => {
+        try {
+            const handle = await open(lockPath, "wx");
+            const owner = ExecutionLockSchema.parse({
+                pid: process.pid,
+                hostname: hostname(),
+                acquiredAt: new Date().toISOString(),
+            });
+            await handle.writeFile(`${JSON.stringify(owner)}\n`);
+            return handle;
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+            if (!allowStaleRecovery) {
+                throw new Error(`experiment is already owned by another process: ${path}`);
+            }
+            let rawOwner;
+            try {
+                rawOwner = JSON.parse(await readFile(lockPath, "utf8"));
+            }
+            catch {
+                throw new Error(`experiment execution lock is present but unreadable: ${lockPath}`);
+            }
+            const parsedOwner = ExecutionLockSchema.safeParse(rawOwner);
+            if (!parsedOwner.success || parsedOwner.data.hostname !== hostname()) {
+                throw new Error(`experiment is already owned by another process: ${path}`);
+            }
+            const owner = parsedOwner.data;
+            try {
+                process.kill(owner.pid, 0);
+                throw new Error(`experiment is already owned by process ${owner.pid}: ${path}`);
+            }
+            catch (ownerError) {
+                if (ownerError.code !== "ESRCH")
+                    throw ownerError;
+            }
+            await unlink(lockPath);
+            return acquire(false);
+        }
+    };
+    const handle = await acquire(true);
+    let released = false;
+    return {
+        release: async () => {
+            if (released)
+                return;
+            released = true;
+            await handle.close();
+            await unlink(lockPath).catch((error) => {
+                if (error.code !== "ENOENT")
+                    throw error;
+            });
+        },
+    };
+};
+const openedRun = (path, manifest, release, accountedIds) => {
+    const write = () => {
+        if (manifest.status !== "running" && manifest.activeSegmentStartedAt) {
+            const segmentStartedAt = Date.parse(manifest.activeSegmentStartedAt);
+            const segmentEndedAt = manifest.endedAt ? Date.parse(manifest.endedAt) : Date.now();
+            if (Number.isFinite(segmentStartedAt) && Number.isFinite(segmentEndedAt)) {
+                manifest.activeTimeMs += Math.max(0, segmentEndedAt - segmentStartedAt);
+            }
+            delete manifest.activeSegmentStartedAt;
+        }
+        return writeJsonAtomic(join(path, MANIFEST_FILE), manifest);
+    };
+    const appendAccounting = async (id, rawDelta, completedModelAttemptIds = []) => {
+        if (accountedIds.has(id))
+            return false;
+        const delta = AccountingDeltaSchema.parse(rawDelta);
+        const completed = [...new Set(completedModelAttemptIds)];
+        for (const attemptId of completed) {
+            if (!modelAttemptParts(attemptId, MODEL_ATTEMPT_START_PREFIX)) {
+                throw new Error(`invalid model attempt start id: ${attemptId}`);
+            }
+        }
+        const entry = AccountingJournalEntrySchema.parse({
+            id,
+            delta,
+            ...(completed.length > 0 ? { completedModelAttemptIds: completed } : {}),
+        });
+        await appendFile(join(path, ACCOUNTING_JOURNAL_FILE), `${JSON.stringify(entry)}\n`, "utf8");
+        accountedIds.add(id);
+        applyAccountingDelta(manifest.budgetUsage, delta);
+        manifest.accountingJournalEntries++;
+        return true;
+    };
+    const recordAccounting = (id, delta) => appendAccounting(id, delta);
+    const startModelAttempt = async (kind) => {
+        const attemptId = randomUUID();
+        await recordAccounting(modelAttemptId(MODEL_ATTEMPT_START_PREFIX, kind, attemptId), kind === "judge" ? { judgeCalls: 1 } : {});
+        return modelAttemptId(MODEL_ATTEMPT_START_PREFIX, kind, attemptId);
+    };
+    const completeModelAttempts = async (attemptIds) => {
+        for (const startId of new Set(attemptIds)) {
+            const attempt = modelAttemptParts(startId, MODEL_ATTEMPT_START_PREFIX);
+            if (!attempt)
+                throw new Error(`invalid model attempt start id: ${startId}`);
+            await recordAccounting(modelAttemptId(MODEL_ATTEMPT_COMPLETE_PREFIX, attempt.kind, attempt.attemptId), {});
+        }
+    };
+    const settleModelAttempts = (id, delta, attemptIds) => appendAccounting(id, delta, attemptIds);
+    const saveCandidate = async (candidate) => {
+        if (manifest.candidateIntegrityVersion !== 1) {
+            throw new Error("legacy experiment candidate refs must be migrated before saving");
+        }
+        const verified = assertCandidateEnvelopeIntegrity(candidate);
+        const envelopeHash = candidateEnvelopeHash(verified);
+        const matches = manifest.candidates.filter((entry) => entry.id === verified.id);
+        if (matches.length > 1) {
+            throw new Error(`experiment candidate ID '${verified.id}' is duplicated`);
+        }
+        const existing = matches[0];
+        if (existing) {
+            const persisted = await verifiedCandidate(path, manifest, existing);
+            if (candidateEnvelopeHash(persisted) !== envelopeHash) {
+                throw new Error(`experiment candidate '${verified.id}' already exists with a different envelope`);
+            }
+            // Upgrade a legacy v1 reference after proving its persisted candidate is
+            // equivalent across the complete schema-normalized envelope.
+            if (existing.candidateEnvelopeHash === undefined) {
+                existing.candidateEnvelopeHash = envelopeHash;
+                await write();
+            }
+            return containedPath(path, existing.path);
+        }
+        const candidatesDir = containedPath(path, "candidates");
+        const candidatesDirInfo = await lstat(candidatesDir);
+        if (candidatesDirInfo.isSymbolicLink() || !candidatesDirInfo.isDirectory()) {
+            throw new Error("experiment candidates directory must be a real directory");
+        }
+        if ((await realpath(candidatesDir)) !== resolve(await realpath(path), "candidates")) {
+            throw new Error("experiment candidates directory escapes its physical ledger");
+        }
+        const candidatePath = containedPath(path, `candidates/${verified.id}.json`);
+        const ref = {
+            id: verified.id,
+            path: relative(path, candidatePath),
+            candidateHash: verified.candidateHash,
+            sourceBytes: verified.sourceBytes,
+            candidateEnvelopeHash: envelopeHash,
+        };
+        await persistJsonImmutable(candidatePath, verified);
+        const persisted = await verifiedCandidate(path, manifest, ref);
+        if (candidateEnvelopeHash(persisted) !== envelopeHash) {
+            throw new Error(`experiment candidate '${verified.id}' already exists with a different envelope`);
+        }
+        manifest.candidates.push(ref);
+        await write();
+        return candidatePath;
+    };
+    return {
+        path,
+        manifest,
+        write,
+        recordAccounting,
+        startModelAttempt,
+        completeModelAttempts,
+        settleModelAttempts,
+        saveCandidate,
+        release,
+    };
+};
+const applyAccountingDelta = (usage, delta) => {
+    usage.metricCalls += delta.metricCalls ?? 0;
+    usage.judgeCalls += delta.judgeCalls ?? 0;
+    usage.proposalCalls += delta.proposalCalls ?? 0;
+    usage.unpricedModelCalls = (usage.unpricedModelCalls ?? 0) + (delta.unpricedModelCalls ?? 0);
+    usage.usage = {
+        input: usage.usage.input + (delta.usage?.input ?? 0),
+        output: usage.usage.output + (delta.usage?.output ?? 0),
+        cacheRead: usage.usage.cacheRead + (delta.usage?.cacheRead ?? 0),
+        cacheCreation: usage.usage.cacheCreation + (delta.usage?.cacheCreation ?? 0),
+    };
+    usage.costUsd += delta.costUsd ?? 0;
+};
+const loadAccountingJournal = async (path, checkpointedEntries) => {
+    const journalPath = join(path, ACCOUNTING_JOURNAL_FILE);
+    if (!existsSync(journalPath))
+        return [];
+    const text = await readFile(journalPath, "utf8");
+    const lines = text.split("\n").filter((line) => line.trim().length > 0);
+    const entries = [];
+    for (const [index, line] of lines.entries()) {
+        try {
+            let parsed;
+            try {
+                parsed = JSON.parse(line);
+            }
+            catch {
+                throw new Error(`invalid accounting journal JSON at line ${index + 1}`);
+            }
+            const result = AccountingJournalEntrySchema.safeParse(parsed);
+            if (!result.success) {
+                throw new Error(`invalid accounting journal entry at line ${index + 1}`);
+            }
+            entries.push(result.data);
+        }
+        catch (error) {
+            const isUncheckpointedTail = index >= checkpointedEntries && index === lines.length - 1;
+            if (!isUncheckpointedTail)
+                throw error;
+            // appendFile can be torn by a hard kill. Only an invalid final entry
+            // beyond the manifest checkpoint is safe to discard.
+            await writeFile(journalPath, entries.length > 0 ? `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n` : "");
+            break;
+        }
+    }
+    return entries;
+};
+const containedPath = (rootPath, relativePath) => {
+    const root = resolve(rootPath);
+    const path = resolve(root, relativePath);
+    if (path === root || !path.startsWith(`${root}${sep}`)) {
+        throw new Error(`experiment candidate path escapes its ledger: ${relativePath}`);
+    }
+    return path;
+};
+const verifiedCandidate = async (experimentDir, manifest, ref) => {
+    const path = containedPath(experimentDir, ref.path);
+    const root = resolve(experimentDir);
+    const pathFromRoot = relative(root, path);
+    const components = pathFromRoot.split(sep);
+    let parent = root;
+    for (const component of components.slice(0, -1)) {
+        parent = join(parent, component);
+        const parentInfo = await lstat(parent).catch((error) => {
+            if (error.code === "ENOENT")
+                return null;
+            throw error;
+        });
+        if (!parentInfo || parentInfo.isSymbolicLink() || !parentInfo.isDirectory()) {
+            throw new Error(`experiment candidate '${ref.id}' has an invalid path component`);
+        }
+    }
+    const info = await lstat(path).catch((error) => {
+        if (error.code === "ENOENT")
+            return null;
+        throw error;
+    });
+    if (!info || info.isSymbolicLink() || !info.isFile()) {
+        throw new Error(`experiment candidate '${ref.id}' is missing or invalid`);
+    }
+    const physicalRoot = await realpath(root);
+    const physicalPath = await realpath(path);
+    if (physicalPath !== resolve(physicalRoot, pathFromRoot)) {
+        throw new Error(`experiment candidate '${ref.id}' escapes the physical ledger`);
+    }
+    const parsedCandidate = await safeReadJson(path, CandidateSchema);
+    if (!parsedCandidate)
+        throw new Error(`experiment candidate '${ref.id}' is missing or invalid`);
+    let candidate;
+    try {
+        candidate = assertCandidateEnvelopeIntegrity(parsedCandidate);
+    }
+    catch {
+        throw new Error(`experiment candidate '${ref.id}' failed its integrity check`);
+    }
+    if (candidate.id !== ref.id ||
+        candidate.candidateHash !== ref.candidateHash ||
+        candidate.sourceBytes !== ref.sourceBytes ||
+        candidate.baseSourceHash !== manifest.baseSourceHash ||
+        (ref.candidateEnvelopeHash !== undefined &&
+            candidateEnvelopeHash(candidate) !== ref.candidateEnvelopeHash)) {
+        throw new Error(`experiment candidate '${ref.id}' failed its integrity check`);
+    }
+    return candidate;
+};
+const migrateLegacyCandidateIntegrity = async (experimentDir, manifest) => {
+    if (manifest.candidateIntegrityVersion === 1)
+        return;
+    for (const ref of manifest.candidates) {
+        ref.candidateEnvelopeHash = candidateEnvelopeHash(await verifiedCandidate(experimentDir, manifest, ref));
+    }
+    manifest.candidateIntegrityVersion = 1;
+};
+export const openExperimentRun = async (options) => {
+    const startedAt = options.startedAt ?? new Date();
+    const root = join(resolve(options.runsRoot), EXPERIMENTS_DIR);
+    await mkdir(root, { recursive: true });
+    const prefix = `${timestampId(startedAt)}-${options.campaignId.replace(/[^a-zA-Z0-9_-]+/g, "-")}`;
+    let experimentId = prefix;
+    let path = join(root, experimentId);
+    for (let suffix = 2;; suffix++) {
+        try {
+            await mkdir(path);
+            break;
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+            experimentId = `${prefix}-${suffix}`;
+            path = join(root, experimentId);
+        }
+    }
+    await mkdir(join(path, "candidates"), { recursive: true });
+    const manifest = {
+        schemaVersion: 1,
+        experimentId,
+        campaignId: options.campaignId,
+        campaignPath: options.campaignPath,
+        campaignHash: options.campaignHash,
+        campaignSnapshot: options.campaignSnapshot,
+        resolvedProfiles: options.resolvedProfiles,
+        evaluatorVersion: 1,
+        luxVersion: options.luxVersion,
+        evaluationKind: options.evaluationKind ?? "agent",
+        ...(options.evaluationHash ? { evaluationHash: options.evaluationHash } : {}),
+        mode: options.mode,
+        status: "running",
+        startedAt: startedAt.toISOString(),
+        endedAt: null,
+        activeTimeMs: 0,
+        activeSegmentStartedAt: startedAt.toISOString(),
+        subjectRoot: options.subjectRoot,
+        baseSourceHash: options.baseSourceHash,
+        candidateIntegrityVersion: 1,
+        candidates: [],
+        evaluations: [],
+        accountedEvaluationUnits: [],
+        accountingJournalEntries: 0,
+        budgetUsage: {
+            metricCalls: 0,
+            judgeCalls: 0,
+            unpricedModelCalls: 0,
+            proposalCalls: 0,
+            usage: emptyUsage(),
+            costUsd: 0,
+        },
+    };
+    const ownership = await acquireExecutionLock(path);
+    const opened = openedRun(path, manifest, ownership.release, new Set());
+    await opened.write();
+    return opened;
+};
+export const loadExperimentRun = async (path) => safeReadJson(join(path, MANIFEST_FILE), ExperimentRunSchema);
+export const migrateExperimentCandidateIntegrity = async (path) => {
+    const absolute = resolve(path);
+    const ownership = await acquireExecutionLock(absolute);
+    try {
+        const manifest = await loadExperimentRun(absolute);
+        if (!manifest)
+            throw new Error(`experiment manifest not found or invalid at ${absolute}`);
+        if (manifest.candidateIntegrityVersion !== 1) {
+            await migrateLegacyCandidateIntegrity(absolute, manifest);
+            await writeJsonAtomic(join(absolute, MANIFEST_FILE), manifest);
+        }
+        return manifest;
+    }
+    finally {
+        await ownership.release();
+    }
+};
+export const experimentActiveElapsedMs = (manifest, nowMs = Date.now()) => {
+    const segmentStartedAt = manifest.activeSegmentStartedAt
+        ? Date.parse(manifest.activeSegmentStartedAt)
+        : Number.NaN;
+    return (manifest.activeTimeMs +
+        (Number.isFinite(segmentStartedAt) ? Math.max(0, nowMs - segmentStartedAt) : 0));
+};
+export const resumeExperimentRun = async (path) => {
+    const absolute = resolve(path);
+    const manifest = await loadExperimentRun(absolute);
+    if (!manifest)
+        throw new Error(`experiment manifest not found or invalid at ${absolute}`);
+    if (manifest.status === "passed") {
+        throw new Error("passed experiments are immutable and cannot be resumed");
+    }
+    const ownership = await acquireExecutionLock(absolute);
+    try {
+        await migrateLegacyCandidateIntegrity(absolute, manifest);
+        const resumedAt = new Date();
+        if (manifest.activeSegmentStartedAt) {
+            const previousSegmentStart = Date.parse(manifest.activeSegmentStartedAt);
+            const previousSegmentEnd = manifest.endedAt
+                ? Date.parse(manifest.endedAt)
+                : resumedAt.getTime();
+            if (Number.isFinite(previousSegmentStart) && Number.isFinite(previousSegmentEnd)) {
+                // A running segment without a terminal checkpoint represents a hard
+                // kill. Charge the unknowable gap conservatively through recovery.
+                manifest.activeTimeMs += Math.max(0, previousSegmentEnd - previousSegmentStart);
+            }
+        }
+        else if (manifest.activeTimeMs === 0) {
+            // Forward-compatible accounting for v1 ledgers written before active
+            // segments existed.
+            const legacyStart = Date.parse(manifest.startedAt);
+            const legacyEnd = manifest.endedAt ? Date.parse(manifest.endedAt) : resumedAt.getTime();
+            if (Number.isFinite(legacyStart) && Number.isFinite(legacyEnd)) {
+                manifest.activeTimeMs = Math.max(0, legacyEnd - legacyStart);
+            }
+        }
+        const journal = await loadAccountingJournal(absolute, manifest.accountingJournalEntries);
+        if (manifest.accountingJournalEntries > journal.length) {
+            throw new Error(`accounting journal is truncated: manifest checkpoints ${manifest.accountingJournalEntries} entries but only ${journal.length} remain`);
+        }
+        const accountedIds = new Set(manifest.accountedEvaluationUnits);
+        for (const [index, entry] of journal.entries()) {
+            if (accountedIds.has(entry.id)) {
+                if (index >= manifest.accountingJournalEntries) {
+                    throw new Error(`duplicate accounting journal id: ${entry.id}`);
+                }
+                continue;
+            }
+            accountedIds.add(entry.id);
+            if (index >= manifest.accountingJournalEntries) {
+                applyAccountingDelta(manifest.budgetUsage, entry.delta);
+            }
+        }
+        manifest.accountingJournalEntries = journal.length;
+        manifest.status = "running";
+        manifest.endedAt = null;
+        manifest.activeSegmentStartedAt = resumedAt.toISOString();
+        delete manifest.errorMessage;
+        const opened = openedRun(absolute, manifest, ownership.release, accountedIds);
+        const terminals = new Set(journal.flatMap((entry) => {
+            const completed = modelAttemptParts(entry.id, MODEL_ATTEMPT_COMPLETE_PREFIX);
+            const abandoned = modelAttemptParts(entry.id, MODEL_ATTEMPT_ABANDONED_PREFIX);
+            return [
+                ...(completed ? [`${completed.kind}:${completed.attemptId}`] : []),
+                ...(abandoned ? [`${abandoned.kind}:${abandoned.attemptId}`] : []),
+                ...(entry.completedModelAttemptIds ?? []).flatMap((attemptId) => {
+                    const settled = modelAttemptParts(attemptId, MODEL_ATTEMPT_START_PREFIX);
+                    return settled ? [`${settled.kind}:${settled.attemptId}`] : [];
+                }),
+            ];
+        }));
+        for (const entry of journal) {
+            const started = modelAttemptParts(entry.id, MODEL_ATTEMPT_START_PREFIX);
+            if (!started || terminals.has(`${started.kind}:${started.attemptId}`))
+                continue;
+            // A hard-killed provider cannot report tokens or price. Charge one
+            // explicit unpriced attempt exactly once before any unit can rerun.
+            await opened.recordAccounting(modelAttemptId(MODEL_ATTEMPT_ABANDONED_PREFIX, started.kind, started.attemptId), { unpricedModelCalls: 1 });
+        }
+        await opened.write();
+        return opened;
+    }
+    catch (error) {
+        await ownership.release();
+        throw error;
+    }
+};
+export const listExperimentRuns = async (runsRoot) => {
+    const root = join(resolve(runsRoot), EXPERIMENTS_DIR);
+    if (!existsSync(root))
+        return [];
+    const directories = (await readdir(root, { withFileTypes: true }))
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => join(root, entry.name));
+    const runs = await Promise.all(directories.map(async (experimentDir) => ({
+        experimentDir,
+        experiment: await loadExperimentRun(experimentDir),
+    })));
+    return runs
+        .flatMap(({ experimentDir, experiment }) => experiment ? [{ ...experiment, experimentDir }] : [])
+        .sort((a, b) => b.experimentId.localeCompare(a.experimentId));
+};
+export const loadExperimentCandidate = async (experimentDir, candidateId) => {
+    const experiment = await loadExperimentRun(experimentDir);
+    const ref = experiment?.candidates.find((entry) => entry.id === candidateId);
+    return experiment && ref ? verifiedCandidate(experimentDir, experiment, ref) : null;
+};
+export const loadExperimentCandidates = async (experimentDir) => {
+    const experiment = await loadExperimentRun(experimentDir);
+    if (!experiment)
+        throw new Error(`experiment manifest not found or invalid at ${experimentDir}`);
+    return Promise.all(experiment.candidates.map((candidate) => verifiedCandidate(experimentDir, experiment, candidate)));
+};
+const strictAccountingJournal = async (experimentDir) => {
+    const path = join(resolve(experimentDir), ACCOUNTING_JOURNAL_FILE);
+    const info = await lstat(path).catch((error) => {
+        if (error.code === "ENOENT")
+            return null;
+        throw error;
+    });
+    if (!info)
+        return [];
+    if (info.isSymbolicLink() || !info.isFile()) {
+        throw new Error("experiment accounting journal must be a regular file");
+    }
+    return (await readFile(path, "utf8"))
+        .split("\n")
+        .filter((line) => line.trim().length > 0)
+        .map((line, index) => {
+        try {
+            return AccountingJournalEntrySchema.parse(JSON.parse(line));
+        }
+        catch {
+            throw new Error(`invalid accounting journal entry at line ${index + 1}`);
+        }
+    });
+};
+export const verifyExperimentRunIntegrity = async (experimentDir, experiment) => {
+    const manifestPath = join(resolve(experimentDir), MANIFEST_FILE);
+    const manifestInfo = await lstat(manifestPath);
+    if (manifestInfo.isSymbolicLink() || !manifestInfo.isFile()) {
+        throw new Error("experiment manifest must be a regular file");
+    }
+    const candidateIds = experiment.candidates.map((candidate) => candidate.id);
+    const candidatePaths = experiment.candidates.map((candidate) => candidate.path);
+    if (new Set(candidateIds).size !== candidateIds.length) {
+        throw new Error("experiment candidate IDs must be unique");
+    }
+    if (new Set(candidatePaths).size !== candidatePaths.length) {
+        throw new Error("experiment candidate paths must be unique");
+    }
+    const candidates = await Promise.all(experiment.candidates.map((candidate) => verifiedCandidate(experimentDir, experiment, candidate)));
+    const journal = await strictAccountingJournal(experimentDir);
+    if (journal.length !== experiment.accountingJournalEntries) {
+        throw new Error(`accounting journal checkpoint mismatch: manifest=${experiment.accountingJournalEntries} journal=${journal.length}`);
+    }
+    const ids = new Set();
+    const reconstructed = {
+        metricCalls: 0,
+        judgeCalls: 0,
+        proposalCalls: 0,
+        unpricedModelCalls: 0,
+        usage: emptyUsage(),
+        costUsd: 0,
+    };
+    for (const entry of journal) {
+        if (ids.has(entry.id))
+            throw new Error(`duplicate accounting journal id: ${entry.id}`);
+        ids.add(entry.id);
+        applyAccountingDelta(reconstructed, entry.delta);
+    }
+    const actual = experiment.budgetUsage;
+    if (reconstructed.metricCalls !== actual.metricCalls ||
+        reconstructed.judgeCalls !== actual.judgeCalls ||
+        reconstructed.proposalCalls !== actual.proposalCalls ||
+        (reconstructed.unpricedModelCalls ?? 0) !== (actual.unpricedModelCalls ?? 0) ||
+        reconstructed.usage.input !== actual.usage.input ||
+        reconstructed.usage.output !== actual.usage.output ||
+        reconstructed.usage.cacheRead !== actual.usage.cacheRead ||
+        reconstructed.usage.cacheCreation !== actual.usage.cacheCreation ||
+        reconstructed.costUsd !== actual.costUsd) {
+        throw new Error("experiment budget usage does not match its durable accounting journal");
+    }
+    return candidates;
+};
+export const isExperimentRun = (path) => existsSync(join(path, MANIFEST_FILE));
+const conciseNumber = (value) => Number.isInteger(value) ? String(value) : value.toFixed(4).replace(/0+$/, "").replace(/\.$/, "");
+const conciseText = (value, maximum = 160) => value.length <= maximum ? value : `${value.slice(0, maximum - 1)}…`;
+export const formatExperimentRun = (path, experiment) => {
+    const usage = experiment.budgetUsage.usage;
+    const tokens = totalTokens(usage);
+    const lines = [
+        `experimentDir: ${path}`,
+        `experiment:    ${experiment.experimentId}`,
+        `campaign:      ${experiment.campaignId}`,
+        `evaluation:    ${experiment.evaluationKind}`,
+        `mode:          ${experiment.mode}`,
+        `status:        ${experiment.status}`,
+        `started:       ${experiment.startedAt}`,
+        `ended:         ${experiment.endedAt ?? "—"}`,
+        `candidates:    ${experiment.candidates.length}`,
+        `evaluations:   ${experiment.evaluations.length}`,
+        `champion:      ${experiment.championCandidateId ?? "—"}`,
+        `spend:         metric=${experiment.budgetUsage.metricCalls} judge=${experiment.budgetUsage.judgeCalls} proposal=${experiment.budgetUsage.proposalCalls} unpriced=${experiment.budgetUsage.unpricedModelCalls ?? 0}`,
+        `tokens:        ${tokens} (input=${usage.input} output=${usage.output} cache-read=${usage.cacheRead} cache-create=${usage.cacheCreation})`,
+        `reported cost: $${experiment.budgetUsage.costUsd.toFixed(4)}`,
+    ];
+    if (experiment.errorMessage)
+        lines.push(`error:         ${experiment.errorMessage}`);
+    if (experiment.selection) {
+        lines.push(`ranking:       ${experiment.selection.rankedCandidateIds.join(" > ") || "—"}`, `eligible:      ${experiment.selection.eligibleCandidateIds.join(", ") || "none"}`, "gates:");
+        for (const constraint of experiment.selection.constraints) {
+            const scope = [
+                constraint.profile ? `profile=${constraint.profile}` : null,
+                `split=${constraint.split}`,
+            ]
+                .filter(Boolean)
+                .join(" ");
+            const interval = constraint.interval
+                ? ` ${constraint.interval.method}=${conciseNumber(constraint.interval.estimate)} [${conciseNumber(constraint.interval.lower)},${conciseNumber(constraint.interval.upper)}] n=${constraint.interval.n}`
+                : "";
+            const note = constraint.note ? ` note=${conciseText(constraint.note)}` : "";
+            lines.push(`  ${constraint.passed ? "✓" : "✗"} ${constraint.candidateId} ${constraint.metric} ${scope} values=${constraint.values.map(conciseNumber).join(",") || "—"}${interval}${note}`);
+        }
+    }
+    if (experiment.evaluations.length > 0)
+        lines.push("evidence:");
+    for (const evaluation of experiment.evaluations) {
+        const evaluationTokens = evaluation.usage ? totalTokens(evaluation.usage) : 0;
+        lines.push(`  ${evaluation.candidateId} profile=${evaluation.profileId} split=${evaluation.split} repetitions=${evaluation.repetitions} pass=${conciseNumber(evaluation.passRate)} calls=${evaluation.metricCalls ?? "—"}/${evaluation.judgeCalls ?? "—"} tokens=${evaluationTokens} unpriced=${evaluation.unpricedModelCalls ?? 0} cost=$${(evaluation.costUsd ?? 0).toFixed(4)}`, `    cases: ${Object.entries(evaluation.casePassRates)
+            .sort(([a], [b]) => a.localeCompare(b))
+            .map(([id, passRate]) => `${id}=${conciseNumber(passRate)}`)
+            .join(", ") || "—"}`);
+        for (const failure of evaluation.failedCases) {
+            lines.push(`    failure: ${failure.caseId} score=${failure.score === null ? "—" : conciseNumber(failure.score)} ${conciseText(failure.feedback[0] ?? "no feedback")}`);
+        }
+    }
+    if (experiment.candidates.length > 0)
+        lines.push("candidates:");
+    for (const candidate of experiment.candidates) {
+        lines.push(`  ${candidate.id === experiment.championCandidateId ? "★" : " "} ${candidate.id}  ${candidate.sourceBytes} bytes  ${basename(candidate.path)}`);
+    }
+    return lines.join("\n");
+};
+export const formatExperimentRunsTable = (experiments) => {
+    if (experiments.length === 0)
+        return "(no experiment runs found)";
+    const rows = [
+        ["status", "experiment", "campaign", "mode", "candidates", "champion"],
+        ...experiments.map((experiment) => [
+            experiment.status.toUpperCase(),
+            basename(experiment.experimentDir),
+            experiment.campaignId,
+            experiment.mode,
+            String(experiment.candidates.length),
+            experiment.championCandidateId ?? "—",
+        ]),
+    ];
+    const widths = (rows[0] ?? []).map((_, column) => Math.max(...rows.map((row) => row[column]?.length ?? 0)));
+    return rows
+        .map((row) => row.map((cell, column) => cell.padEnd(widths[column] ?? 0)).join("  "))
+        .join("\n");
+};
+//# sourceMappingURL=experiment-runs.js.map