@deepthonk/core 0.1.4 → 0.2.0

package/dist/runner.js

@@ -1,19 +1,20 @@
+import { createHash, randomUUID } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import { fitBradleyTerry } from "./bradleyTerry.js";
+import { canonicalizeComparisons, fitBradleyTerry } from "./bradleyTerry.js";
 import { BudgetTracker } from "./budgetTracker.js";
+import { readValidatedPhaseReceiptUsage, runCheckpointedPhase, traceSchemaVersion, validatePhaseCommit, validatePhaseCommitIfPresent, writePhaseCommit } from "./checkpointStore.js";
 import { planBudget, validateProfile } from "./budget.js";
 import { BudgetExceededError, CancelledError, ConfigError, DeepThonkError } from "./errors.js";
 import { runArtifactFiles } from "./artifacts.js";
 import { parseJsonObject } from "./json.js";
-import { claimRunLock, emptyUsage, releaseRunLock } from "./lifecycle.js";
+import { claimRunLockOwnership, emptyUsage, releaseRunLock, verifyRunLockClaim } from "./lifecycle.js";
 import { makeKRegularPairs } from "./pairScheduler.js";
-import { runLimitedPhase } from "./phaseRunner.js";
 import { comparePrompt, finalizePrompt, generatePrompt, mutatePrompt } from "./prompts.js";
 import { createRng } from "./rng.js";
-import { assertNoPruneInProgress, buildPopulationMap, buildResumePlan, groupComparisons, persistPrunedTrace, pruneTraceToPlan, readResumeTrace, reconstructScores, replayBudgetUsage, resolveResumeRunId, resumeConfigError, toResumePlanStatus } from "./resume.js";
-import { runConfigSchema, builtInProfiles, phaseCompletedEventSchema } from "./schemas.js";
+import { assertNoPruneInProgress, assertNoLegacyRedactedBudget, applyUsageRecords, buildPopulationMap, buildResumePlan, groupComparisons, persistPrunedTrace, pruneTraceToPlan, readResumeTrace, reconstructScores, replayBudgetUsage, resolveResumeRunId, resumeConfigError, toResumePlanStatus, validateUsageRecords, validatedReceiptUsageIds } from "./resume.js";
+import { runConfigSchema, resolveRunConfigDefaults, builtInProfiles, phaseCompletedEventSchema } from "./schemas.js";
 import { aggregateCritiques } from "./critique.js";
 import { TraceStore } from "./traceStore.js";
 const compareOutputSchema = z.object({
@@ -26,25 +27,42 @@ const compareOutputSchema = z.object({
     selection_reason: z.string().default("")
 });
 export async function runDeepThonk(configInput, driver, control = {}, resumeState) {
-    const config = runConfigSchema.parse(configInput);
+    const config = resolveRunConfigDefaults(runConfigSchema.parse(configInput));
     validateProfile(config.profile);
     enforceBudget(config);
-    const runId = resumeState?.runId ?? `run_${new Date().toISOString().replace(/[:.]/g, "-")}_${Math.abs(config.seed)}`;
+    const runId = resumeState?.runId ??
+        config.runId ??
+        `run_${new Date().toISOString().replace(/[:.]/g, "-")}_${randomUUID()}`;
     const trace = new TraceStore(config.runDir);
     const rng = createRng(config.seed);
     const tracker = resumeState?.tracker ?? new BudgetTracker(config);
     const startedAt = resumeState?.startedAt ?? new Date().toISOString();
     let ownsLock = false;
-    if (!resumeState && !control.lockClaimed) {
-        const claimed = await claimRunLock(config.runDir, control.jobId ?? runId);
-        if (!claimed) {
-            throw new ConfigError(`Run directory is already claimed: ${config.runDir}`, {
-                code: "run.directory_locked",
-                retryable: false,
-                fix: "Wait for the active run to finish, or choose a fresh run directory."
-            });
+    let ownedLockClaimId;
+    let verifiedPreclaim = false;
+    if (!resumeState) {
+        if (control.lockClaim) {
+            verifiedPreclaim = await verifyRunLockClaim(config.runDir, control.lockClaim, control.jobId);
+            if (!verifiedPreclaim) {
+                throw new ConfigError("The supplied run lock claim does not own the current run.lock.", {
+                    code: "run.lock_not_owned",
+                    retryable: false,
+                    fix: "Claim the run directory with claimRunLockOwnership and pass that exact claim to the worker."
+                });
+            }
+        }
+        else {
+            const claimed = await claimRunLockOwnership(config.runDir, control.jobId ?? runId);
+            if (!claimed) {
+                throw new ConfigError(`Run directory is already claimed: ${config.runDir}`, {
+                    code: "run.directory_locked",
+                    retryable: false,
+                    fix: "Wait for the active run to finish, or choose a fresh run directory."
+                });
+            }
+            ownsLock = true;
+            ownedLockClaimId = claimed.claimId;
         }
-        ownsLock = true;
     }
     let stopped = false;
     let caughtError;
@@ -63,7 +81,7 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
         });
     };
     const assertNotCancelled = async (phase) => {
-        if (await control.shouldCancel?.()) {
+        if (control.signal?.aborted || (await control.shouldCancel?.())) {
             throw new CancelledError(`Run cancelled before ${phase}.`, {
                 code: "run.cancelled",
                 retryable: false,
@@ -89,6 +107,7 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
                 throw new BudgetExceededError(`Run already stopped before ${phase}.`, { code: "budget.stopped" });
             await assertNotCancelled(phase);
             await assertBudget(phase);
+            return tracker.reserveCall(phase);
         },
         async afterCall(phase) {
             await assertNotCancelled(phase);
@@ -97,7 +116,7 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
     };
     try {
         if (!resumeState) {
-            await trace.init({ ...config, version: await currentPackageVersion() }, runId);
+            await trace.init({ ...config, version: await currentPackageVersion() }, runId, { allowOwnedLock: ownsLock || verifiedPreclaim, pendingJobId: verifiedPreclaim ? control.jobId : undefined });
         }
         else {
             await trace.event({
@@ -115,9 +134,13 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
         }
         else {
             await assertNotCancelled("initial population");
-            population = await generateInitialPopulation(config, driver, trace, rng, runId, tracker, guards);
+            population = await generateInitialPopulation(config, driver, trace, rng, runId, tracker, guards, control.signal);
             await trace.writePopulation(0, population);
             await assertBudget("initial population");
+            await writePhaseCommit(config.runDir, "initial_generation", {
+                population,
+                usage: await phaseUsageRows(trace, "initial_generation")
+            });
             await markPhaseCompleted(trace, "initial_generation");
             await writeStatus("running", "population_completed", { generation: 0 });
         }
@@ -147,11 +170,17 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
                     pairs,
                     population,
                     tracker,
-                    guards
+                    guards,
+                    signal: control.signal
                 });
                 scores = fitBradleyTerry(population, comparisons, config.profile.lambda, gen);
                 await trace.writeScores(gen, scores);
                 await assertBudget(`generation ${gen} comparisons`);
+                await writePhaseCommit(config.runDir, `generation_judging:${gen}`, {
+                    comparisons,
+                    scores,
+                    usage: await phaseUsageRows(trace, `generation_judging:${gen}`)
+                });
                 await markPhaseCompleted(trace, "generation_judging", gen);
             }
             const ranked = rankPopulation(population, scores);
@@ -159,7 +188,9 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
             const discarded = bottomQuartile(ranked);
             const survivors = ranked.filter((candidate) => !discarded.has(candidate.id));
             const mutationParents = survivors.slice(0, config.profile.n - elites.length);
-            const critiquesByCandidate = aggregateCritiques(population, comparisons);
+            const critiquesByCandidate = aggregateCritiques(population, comparisons, {
+                aggregateChars: config.critiqueLimits.aggregateChars
+            });
             await assertNotCancelled(`generation ${gen} mutation`);
             await writeStatus("running", "generation_mutation", { generation: gen });
             await traceDiscardedCandidates(trace, ranked, new Set([...elites, ...mutationParents].map((candidate) => candidate.id)), discarded, gen);
@@ -171,7 +202,8 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
                 survivors: mutationParents,
                 critiquesByCandidate,
                 tracker,
-                guards
+                guards,
+                signal: control.signal
             });
             const eliteCopies = copyElites(elites, gen);
             for (const elite of eliteCopies) {
@@ -181,6 +213,10 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
             population = keepPopulationSize([...eliteCopies, ...mutants], config.profile.n);
             await trace.writePopulation(gen, population);
             await assertBudget(`generation ${gen} mutation`);
+            await writePhaseCommit(config.runDir, `generation_mutation:${gen}`, {
+                population,
+                usage: await phaseUsageRows(trace, `generation_mutation:${gen}`)
+            });
             await markPhaseCompleted(trace, "generation_mutation", gen);
             await writeStatus("running", "generation_completed", { generation: gen });
         }
@@ -193,8 +229,12 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
         else {
             await assertNotCancelled("final ranking");
             await writeStatus("running", "final_ranking", { generation: "final" });
-            const finalRng = phaseRng(config.seed, "final_judging");
-            const finalPairs = makeKRegularPairs(population.map((candidate) => candidate.id), config.profile.m, finalRng);
+            const finalRng = config.rank
+                ? createRng(config.rank.seed ?? hashSeed(`${config.seed}:final_judging:`))
+                : phaseRng(config.seed, "final_judging");
+            const finalPairs = config.rank
+                ? configuredFinalPairs(population.map((candidate) => candidate.id), config.rank, finalRng)
+                : makeKRegularPairs(population.map((candidate) => candidate.id), config.profile.m, finalRng);
             finalComparisons = await judgePairs({
                 config,
                 driver,
@@ -205,11 +245,17 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
                 pairs: finalPairs,
                 population,
                 tracker,
-                guards
+                guards,
+                signal: control.signal
             });
             finalScores = fitBradleyTerry(population, finalComparisons, config.profile.lambda, "final");
             await trace.writeScores("final", finalScores);
             await assertBudget("final ranking");
+            await writePhaseCommit(config.runDir, "final_judging", {
+                comparisons: finalComparisons,
+                scores: finalScores,
+                usage: await phaseUsageRows(trace, "final_judging")
+            });
             await markPhaseCompleted(trace, "final_judging");
         }
         const winner = rankPopulation(population, finalScores)[0];
@@ -217,7 +263,7 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
             throw new ConfigError("Run produced no winner.");
         await assertNotCancelled("finalization");
         await writeStatus("running", "finalizing", { generation: "final" });
-        const finalAnswer = await maybeFinalize(winner, config, driver, trace, tracker, guards);
+        const finalAnswer = await maybeFinalize(winner, config, driver, trace, tracker, guards, control.signal);
         await assertBudget("finalization");
         const completedAt = new Date().toISOString();
         const summary = {
@@ -236,6 +282,12 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
             completed_at: completedAt
         };
         await trace.writeSummary(summary, finalAnswer, winner.content);
+        await writePhaseCommit(config.runDir, "finalizing", {
+            summary,
+            finalAnswer,
+            winner: winner.content,
+            usage: await phaseUsageRows(trace, "finalizing")
+        });
         await markPhaseCompleted(trace, "finalizing");
         await trace.event({ type: "run.completed", winner_id: winner.id, completed_at: completedAt });
         if (resumeState)
@@ -259,9 +311,19 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
         throw error;
     }
     finally {
-        if (ownsLock) {
+        let closeError;
+        try {
+            await trace.close();
+        }
+        catch (error) {
+            if (caughtError !== undefined)
+                process.stderr.write(`deepthonk: failed to close trace handles: ${error.message}\n`);
+            else
+                closeError = error;
+        }
+        if (ownsLock || verifiedPreclaim) {
             try {
-                await releaseRunLock(config.runDir);
+                await releaseRunLock(config.runDir, ownsLock ? ownedLockClaimId : control.lockClaim?.claimId);
             }
             catch (error) {
                 if (caughtError !== undefined) {
@@ -272,27 +334,26 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
                 }
             }
         }
+        if (closeError !== undefined)
+            throw closeError;
     }
 }
 export async function resumeDeepThonk(runDir, driver, options = {}) {
-    const existingSummary = await readOptionalJson(runDir, runArtifactFiles.summary);
-    if (existingSummary) {
-        resumeConfigError("Run already has summary.json; nothing to resume.", "resume.already_complete", "Use deepthonk inspect/result to read the completed run.");
-    }
     const rawConfig = await readRequiredConfig(runDir);
+    assertNoLegacyRedactedBudget(rawConfig);
     const currentVersion = await currentPackageVersion();
     if (!sameMajorMinor(typeof rawConfig.version === "string" ? rawConfig.version : undefined, currentVersion)) {
         resumeConfigError(`Cannot resume run from version ${String(rawConfig.version ?? "missing")}; current package version is ${currentVersion}. Resume requires matching major.minor.`, "resume.version_mismatch", "Start a fresh run with the current DeepThonk version, or resume with a package version whose major.minor matches the trace.");
     }
     assertStoredOutputConfigComplete(rawConfig);
     const parsedConfig = runConfigSchema.parse(rawConfig);
-    const config = { ...parsedConfig, runDir };
+    const config = resolveRunConfigDefaults({ ...parsedConfig, runDir });
     validateProfile(config.profile);
     enforceBudget(config);
     assertResumeProviderMatches(rawConfig, config, driver, options);
     assertNoPruneInProgress(runDir);
     const lockRunId = resolveResumeRunId(config, undefined, []);
-    const claimed = await claimRunLock(runDir, lockRunId);
+    const claimed = await claimRunLockOwnership(runDir, lockRunId);
     if (!claimed) {
         throw new ConfigError(`Run directory is already claimed: ${runDir}`, {
             code: "run.directory_locked",
@@ -301,6 +362,7 @@ export async function resumeDeepThonk(runDir, driver, options = {}) {
         });
     }
     try {
+        const existingSummary = await readOptionalFinalSummary(runDir, rawConfig.traceSchemaVersion === traceSchemaVersion);
         const status = await readOptionalJson(runDir, runArtifactFiles.status);
         if ((status?.state === "running" || status?.state === "pending") && isLiveWorker(status.worker_pid)) {
             resumeConfigError(`Run is still in flight at phase ${status.phase}.`, "resume.in_flight", "Wait for the worker to finish, or cancel/stop it before resuming.");
@@ -308,16 +370,41 @@ export async function resumeDeepThonk(runDir, driver, options = {}) {
         const trace = await readResumeTrace(runDir);
         const runId = resolveResumeRunId(config, status, trace.events);
         const plan = buildResumePlan(config, trace.events);
-        if (plan.nextPhase.phase === "summary") {
-            resumeConfigError("Trace says finalizing completed, but summary.json is missing.", "resume.inconsistent_trace", "Inspect the run directory and restore summary.json, or start a fresh run.");
+        if (existingSummary && rawConfig.traceSchemaVersion !== traceSchemaVersion) {
+            resumeConfigError("Run already has summary.json; nothing to resume.", "resume.already_complete", "Use deepthonk inspect/result to read the completed run.");
+        }
+        if (existingSummary &&
+            rawConfig.traceSchemaVersion === traceSchemaVersion &&
+            plan.nextPhase.phase !== "finalizing" &&
+            plan.nextPhase.phase !== "summary") {
+            resumeConfigError(`summary.json exists before the finalizing phase boundary (${plan.nextPhase.phase}).`, "resume.trace_v2_integrity", "Restore the matching finalization artifacts or start a fresh run.");
         }
         const planStatus = toResumePlanStatus(runDir, runId, plan);
-        if (options.dryRun)
-            return planStatus;
-        const pruned = pruneTraceToPlan(trace, plan);
+        let receiptUsageIds;
+        if (rawConfig.traceSchemaVersion === traceSchemaVersion) {
+            const receiptUsage = await readValidatedPhaseReceiptUsage(runDir, phaseCursorKey(plan.nextPhase));
+            receiptUsageIds = validatedReceiptUsageIds(receiptUsage);
+        }
+        const pruned = pruneTraceToPlan(trace, plan, { dropUsageIds: receiptUsageIds });
+        assertResumeRunIdsAgree(runId, config, status, pruned);
         const populationByGeneration = buildPopulationMap(config, pruned.populations, pruned.candidates, plan);
         const comparisonsByGeneration = groupComparisons(pruned.comparisons);
-        const scoresByGeneration = reconstructScores(config, populationByGeneration, comparisonsByGeneration, pruned.scores, plan);
+        const scoresByGeneration = reconstructScores(config, populationByGeneration, comparisonsByGeneration, pruned.scores, plan, runId);
+        validateUsageRecords(pruned.usage);
+        if (rawConfig.traceSchemaVersion === traceSchemaVersion) {
+            await validateTraceV2CompletedPhases(runDir, config, plan.completed, populationByGeneration, comparisonsByGeneration, scoresByGeneration, pruned.usage);
+            if (plan.nextPhase.phase === "finalizing" && !plan.completed.has("finalizing")) {
+                await validatePhaseCommitIfPresent(runDir, "finalizing", await finalizingArtifacts(runDir, trace.usage));
+            }
+        }
+        if (plan.nextPhase.phase === "summary") {
+            if (existingSummary) {
+                resumeConfigError("Run already has a validated summary.json and finalizing commit; nothing to resume.", "resume.already_complete", "Use deepthonk inspect/result to read the completed run.");
+            }
+            resumeConfigError("Trace says finalizing completed, but summary.json is missing or invalid.", rawConfig.traceSchemaVersion === traceSchemaVersion ? "resume.trace_v2_integrity" : "resume.inconsistent_trace", "Inspect the run directory and restore the matching finalization artifacts, or start a fresh run.");
+        }
+        if (options.dryRun)
+            return planStatus;
         const tracker = replayBudgetUsage(config, pruned.usage);
         const startedAt = status?.started_at ?? new Date().toISOString();
         await new TraceStore(runDir).writeStatus({
@@ -343,7 +430,26 @@ export async function resumeDeepThonk(runDir, driver, options = {}) {
         });
     }
     finally {
-        await releaseRunLock(runDir);
+        await releaseRunLock(runDir, claimed?.claimId);
+    }
+}
+function assertResumeRunIdsAgree(resolvedRunId, config, status, trace) {
+    const observed = new Set();
+    if (config.runId)
+        observed.add(config.runId);
+    if (status?.run_id)
+        observed.add(status.run_id);
+    for (const event of trace.events) {
+        if (typeof event.run_id === "string")
+            observed.add(event.run_id);
+    }
+    for (const comparison of trace.comparisons) {
+        if (comparison.runId)
+            observed.add(comparison.runId);
+    }
+    observed.add(resolvedRunId);
+    if (observed.size > 1) {
+        resumeConfigError(`Resume artifacts disagree on run ID: ${[...observed].sort().join(", ")}.`, "resume.run_id_mismatch", "Restore artifacts from one run; do not combine trace files from different run directories.");
     }
 }
 async function markPhaseCompleted(trace, phase, generation) {
@@ -378,7 +484,7 @@ function resumeComparisons(resumeState, generation) {
             fix: "Inspect the run directory for missing comparison trace rows."
         });
     }
-    return comparisons;
+    return canonicalizeComparisons(comparisons);
 }
 function resumeScores(resumeState, generation, population, comparisons, lambda) {
     return resumeState?.scoresByGeneration.get(generation) ?? fitBradleyTerry(population, comparisons, lambda, generation);
@@ -399,6 +505,13 @@ function resumePhaseKey(phase, generation) {
         return `${phase}:${generation}`;
     return phase;
 }
+function phaseCursorKey(cursor) {
+    if ((cursor.phase === "generation_judging" || cursor.phase === "generation_mutation") &&
+        typeof cursor.generation === "number") {
+        return `${cursor.phase}:${cursor.generation}`;
+    }
+    return cursor.phase;
+}
 function providerLabel(driver) {
     const value = driver;
     if (typeof value.provider === "string")
@@ -425,10 +538,19 @@ function assertStoredOutputConfigComplete(rawConfig) {
     }
 }
 function assertResumeProviderMatches(rawConfig, config, driver, options) {
+    assertProviderReplayInternalConsistency(config);
     const runtimeProvider = options.provider ?? providerLabel(driver) ?? config.provider;
     if (runtimeProvider !== config.provider) {
         providerMismatch(`Cannot resume provider ${config.provider} with runtime provider ${runtimeProvider}.`);
     }
+    const expectedFingerprint = config.providerReplay?.routeFingerprint;
+    if (expectedFingerprint !== undefined && providerRouteFingerprint(driver) !== expectedFingerprint) {
+        providerMismatch("Cannot resume because the provider route fingerprint changed.");
+    }
+    if (config.providerReplay?.baseUrl !== undefined &&
+        normalizeBaseUrl(providerBaseUrl(driver)) !== normalizeBaseUrl(config.providerReplay.baseUrl)) {
+        providerMismatch(`Cannot resume base URL ${config.providerReplay.baseUrl} with runtime base URL ${providerBaseUrl(driver) ?? "missing"}.`);
+    }
     const expectedRoutes = expectedProviderRoutes(rawConfig, config);
     const actualRoutes = runtimeProviderRoutes(driver);
     for (const role of providerRoles) {
@@ -450,8 +572,92 @@ function assertResumeProviderMatches(rawConfig, config, driver, options) {
         }
     }
 }
+function assertProviderReplayInternalConsistency(config) {
+    const replay = config.providerReplay;
+    if (!replay)
+        return;
+    const expectedModels = {
+        generator: config.generatorModel,
+        mutator: config.mutatorModel,
+        judge: config.judgeModel,
+        finalizer: config.finalizerModel
+    };
+    if (replay.provider !== config.provider ||
+        replay.models.generator !== expectedModels.generator ||
+        replay.models.mutator !== expectedModels.mutator ||
+        replay.models.judge !== expectedModels.judge ||
+        replay.models.finalizer !== expectedModels.finalizer) {
+        resumeConfigError("Stored run provider/model fields disagree with providerReplay.", "resume.provider_replay_mismatch", "Restore config.json from the original run; do not edit provider or model fields independently.");
+    }
+    if (replay.routeFingerprint !== undefined && replay.routeFingerprint !== coreProviderReplayFingerprint(replay)) {
+        resumeConfigError("Stored providerReplay fingerprint does not match its route fields.", "resume.provider_replay_mismatch", "Restore the original providerReplay block instead of editing route fields.");
+    }
+}
+function coreProviderReplayFingerprint(replay) {
+    const roles = Object.fromEntries(providerRoles.flatMap((role) => {
+        const route = replay.roleProviders?.[role];
+        return route
+            ? [[role, routeFingerprintValue(route.provider, route.baseUrl, route.apiKeyEnv, route.model, route.supportsJsonMode)]]
+            : [];
+    }));
+    const value = {
+        base: routeFingerprintValue(replay.provider, replay.baseUrl, replay.apiKeyEnv, replay.models, replay.supportsJsonMode),
+        roles,
+        samplingPreferences: replay.samplingPreferences ?? null
+    };
+    return `sha256:${createHash("sha256").update(stableJson(value)).digest("hex")}`;
+}
+function routeFingerprintValue(provider, baseUrl, apiKeyEnv, model, supportsJsonMode) {
+    return Object.fromEntries(Object.entries({
+        provider,
+        baseUrl: normalizeFingerprintUrl(baseUrl),
+        apiKeyEnv,
+        model,
+        supportsJsonMode
+    }).filter(([, value]) => value !== undefined));
+}
+function normalizeFingerprintUrl(baseUrl) {
+    if (!baseUrl)
+        return undefined;
+    try {
+        const parsed = new URL(baseUrl);
+        parsed.hash = "";
+        parsed.hostname = parsed.hostname.toLowerCase();
+        parsed.pathname = parsed.pathname.replace(/\/+$/, "");
+        return parsed.toString().replace(/\/$/, "");
+    }
+    catch {
+        return baseUrl.replace(/\/+$/, "");
+    }
+}
+function stableJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(stableJson).join(",")}]`;
+    if (value && typeof value === "object") {
+        const entries = Object.entries(value)
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([key, inner]) => `${JSON.stringify(key)}:${stableJson(inner)}`);
+        return `{${entries.join(",")}}`;
+    }
+    return JSON.stringify(value) ?? "null";
+}
 const providerRoles = ["generator", "mutator", "judge", "finalizer"];
 function expectedProviderRoutes(rawConfig, config) {
+    const replayRoutes = config.providerReplay?.roleProviders;
+    if (replayRoutes) {
+        const routes = {};
+        for (const role of providerRoles) {
+            const route = replayRoutes[role];
+            if (!route)
+                continue;
+            routes[role] = {
+                provider: route.provider,
+                baseUrl: route.baseUrl,
+                model: route.model
+            };
+        }
+        return routes;
+    }
     const providers = rawConfig.providers;
     if (!isRecord(providers))
         return {};
@@ -486,6 +692,16 @@ function runtimeProviderRoutes(driver) {
     }
     return routes;
 }
+function providerRouteFingerprint(driver) {
+    const value = driver;
+    if (typeof value.routeFingerprint === "string")
+        return value.routeFingerprint;
+    if (typeof value.config?.routeFingerprint === "string")
+        return value.config.routeFingerprint;
+    if (value.baseDriver)
+        return providerRouteFingerprint(value.baseDriver);
+    return undefined;
+}
 function providerBaseUrl(driver) {
     const value = driver;
     if (typeof value.baseUrl === "string")
@@ -527,6 +743,20 @@ async function readRequiredConfig(runDir) {
     }
     return config;
 }
+async function readOptionalFinalSummary(runDir, traceV2) {
+    try {
+        return await readOptionalJson(runDir, runArtifactFiles.summary);
+    }
+    catch (error) {
+        if (!traceV2)
+            throw error;
+        throw new ConfigError(`Could not parse summary.json: ${error.message}`, {
+            code: "resume.trace_v2_integrity",
+            retryable: false,
+            fix: "Restore summary.json from the same completed run, or resume from the pre-finalization artifacts."
+        });
+    }
+}
 async function readOptionalJson(runDir, fileName) {
     try {
         return JSON.parse(await readFile(join(runDir, fileName), "utf8"));
@@ -537,6 +767,16 @@ async function readOptionalJson(runDir, fileName) {
         throw error;
     }
 }
+async function readOptionalText(runDir, fileName) {
+    try {
+        return await readFile(join(runDir, fileName), "utf8");
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return undefined;
+        throw error;
+    }
+}
 function isLiveWorker(pid) {
     if (!pid)
         return true;
@@ -544,8 +784,8 @@ function isLiveWorker(pid) {
         process.kill(pid, 0);
         return true;
     }
-    catch {
-        return false;
+    catch (error) {
+        return error.code !== "ESRCH";
     }
 }
 export function rankPopulation(population, scores) {
@@ -570,10 +810,81 @@ export function copyElites(elites, generation) {
         status: "elite",
         metadata: {
             ...candidate.metadata,
-            createdAt: new Date().toISOString()
+            createdAt: candidate.metadata.createdAt
         }
     }));
 }
+async function validateTraceV2CompletedPhases(runDir, config, completed, populations, comparisons, scores, usage) {
+    if (completed.has("initial_generation")) {
+        await validatePhaseCommit(runDir, "initial_generation", {
+            population: populations.get(0),
+            usage: usage.filter((record) => usagePhaseKey(record.phase) === "initial_generation")
+        });
+    }
+    for (let generation = 1; generation <= config.profile.t; generation += 1) {
+        if (completed.has(`generation_judging:${generation}`)) {
+            const phaseKey = `generation_judging:${generation}`;
+            await validatePhaseCommit(runDir, phaseKey, {
+                comparisons: comparisons.get(generation),
+                scores: scores.get(generation),
+                usage: usage.filter((record) => usagePhaseKey(record.phase) === phaseKey)
+            });
+        }
+        if (completed.has(`generation_mutation:${generation}`)) {
+            const phaseKey = `generation_mutation:${generation}`;
+            await validatePhaseCommit(runDir, phaseKey, {
+                population: populations.get(generation),
+                usage: usage.filter((record) => usagePhaseKey(record.phase) === phaseKey)
+            });
+        }
+    }
+    if (completed.has("final_judging")) {
+        await validatePhaseCommit(runDir, "final_judging", {
+            comparisons: comparisons.get("final"),
+            scores: scores.get("final"),
+            usage: usage.filter((record) => usagePhaseKey(record.phase) === "final_judging")
+        });
+    }
+    if (completed.has("finalizing")) {
+        await validatePhaseCommit(runDir, "finalizing", await finalizingArtifacts(runDir, usage));
+    }
+}
+async function finalizingArtifacts(runDir, usage) {
+    try {
+        return {
+            summary: await readOptionalJson(runDir, runArtifactFiles.summary),
+            finalAnswer: await readOptionalText(runDir, runArtifactFiles.final),
+            winner: await readOptionalText(runDir, runArtifactFiles.winner),
+            usage: usage.filter((record) => usagePhaseKey(record.phase) === "finalizing")
+        };
+    }
+    catch (error) {
+        throw new ConfigError(`Could not read finalization artifacts: ${error.message}`, {
+            code: "resume.trace_v2_integrity",
+            retryable: false,
+            fix: "Restore summary.json, final.txt, and winner.txt from the same completed run."
+        });
+    }
+}
+async function phaseUsageRows(trace, phaseKey) {
+    const usage = await trace.readJsonl(runArtifactFiles.usage).catch(() => []);
+    return usage.filter((record) => usagePhaseKey(record.phase) === phaseKey);
+}
+function usagePhaseKey(phase) {
+    if (phase === "gen0_initial")
+        return "initial_generation";
+    if (phase === "final_judge")
+        return "final_judging";
+    if (phase === "finalize")
+        return "finalizing";
+    const judge = /^gen(\d+)_judge$/.exec(phase);
+    if (judge)
+        return `generation_judging:${judge[1]}`;
+    const mutation = /^gen(\d+)_mutate$/.exec(phase);
+    if (mutation)
+        return `generation_mutation:${mutation[1]}`;
+    return undefined;
+}
 export function keepPopulationSize(population, n) {
     return population.slice(0, n);
 }
@@ -594,44 +905,92 @@ async function traceDiscardedCandidates(trace, ranked, carriedIds, bottomQuartil
         await trace.event({ type: "candidate.discarded", candidate_id: candidate.id, generation, reason: discardReason });
     }
 }
-async function generateInitialPopulation(config, driver, trace, rng, runId, tracker, guards) {
+async function generateInitialPopulation(config, driver, trace, rng, runId, tracker, guards, signal) {
     const jobs = Array.from({ length: config.profile.n }, (_, index) => ({
         index,
         id: `g0_c${index}_${rng.id("c")}`,
         prompt: generatePrompt(config.task, config.rubric, config.promptStyle, config.promptOverrides?.generate)
     }));
-    const candidates = await runLimitedPhase(jobs.map((job) => async () => {
-        await guards.beforeCall("initial population call");
-        const result = await driver.generate({
-            task: config.task,
-            rubric: config.rubric,
-            model: config.generatorModel,
-            temperature: config.profile.sampleTemperature,
-            candidateIndex: job.index,
-            prompt: job.prompt
-        });
-        const delta = tracker.record(result, { provider: config.provider, model: config.generatorModel });
-        await trace.writeUsage(buildUsageRecord({
-            phase: "gen0_initial",
-            role: "generator",
-            result,
-            delta,
-            provider: result.provider ?? config.provider,
-            model: result.model ?? config.generatorModel
-        }));
-        await guards.afterCall("initial population call");
-        const candidate = {
-            id: job.id,
-            generation: 0,
-            kind: "initial",
-            content: result.text,
-            status: "generated",
-            metadata: resultMetadata(result, config, job.prompt)
-        };
-        await trace.writeCandidate(candidate);
-        await trace.event({ type: "candidate.generated", run_id: runId, candidate_id: candidate.id });
-        return candidate;
-    }), config.concurrency.generate);
+    const promptRef = config.output.includePrompts && jobs[0] ? await trace.writePrompt(jobs[0].prompt) : undefined;
+    const candidates = await runCheckpointedPhase({
+        runDir: config.runDir,
+        phaseKey: "initial_generation",
+        jobs: jobs.map((job) => {
+            const receiptUsage = [];
+            return {
+                id: job.id,
+                input: {
+                    task: config.task,
+                    rubric: config.rubric,
+                    model: config.generatorModel,
+                    temperature: config.profile.sampleTemperature,
+                    candidateIndex: job.index,
+                    prompt: job.prompt,
+                    maxOutputTokens: config.modelOutputTokens?.generation ?? 4_096
+                },
+                receiptUsage: () => receiptUsage,
+                run: async (phaseSignal) => {
+                    const reservation = await guards.beforeCall("initial population call");
+                    let result;
+                    try {
+                        result = await driver.generate({
+                            task: config.task,
+                            rubric: config.rubric,
+                            model: config.generatorModel,
+                            temperature: config.profile.sampleTemperature,
+                            candidateIndex: job.index,
+                            prompt: job.prompt,
+                            signal: phaseSignal,
+                            maxOutputTokens: config.modelOutputTokens?.generation ?? 4_096
+                        });
+                    }
+                    catch (error) {
+                        await recordFailedInvocation({
+                            tracker,
+                            reservation,
+                            trace,
+                            phase: "gen0_initial",
+                            role: "generator",
+                            provider: config.provider,
+                            model: config.generatorModel,
+                            error
+                        });
+                        throw error;
+                    }
+                    const delta = tracker.record(result, { provider: config.provider, model: config.generatorModel }, reservation);
+                    const usageRecord = buildUsageRecord({
+                        phase: "gen0_initial",
+                        role: "generator",
+                        result,
+                        delta,
+                        provider: result.provider ?? config.provider,
+                        model: result.model ?? config.generatorModel
+                    });
+                    receiptUsage.push(usageRecord);
+                    await trace.writeUsage(usageRecord);
+                    await guards.afterCall("initial population call");
+                    const candidate = {
+                        id: job.id,
+                        generation: 0,
+                        kind: "initial",
+                        content: result.text,
+                        status: "generated",
+                        metadata: resultMetadata(result, config, promptRef)
+                    };
+                    return candidate;
+                }
+            };
+        }),
+        concurrency: config.concurrency.generate,
+        signal,
+        validateOutput: isCandidateCheckpoint,
+        onResult: async (candidate, context) => {
+            if (context.reused)
+                await restoreCheckpointUsage(context.receiptUsage, tracker, trace);
+            await trace.writeCandidate(candidate);
+            await trace.event({ type: "candidate.generated", run_id: runId, candidate_id: candidate.id });
+        }
+    });
     return candidates;
 }
 async function judgePairs(args) {
@@ -651,175 +1010,340 @@ async function judgePairs(args) {
             presentedA,
             presentedB,
             id: `${args.generation}_cmp_${pairIndex}_${args.rng.id("cmp")}`,
-            prompt: comparePrompt(args.config.task, presentedA, presentedB, args.config.rubric, args.config.promptStyle, args.config.promptOverrides?.compare)
+            prompt: comparePrompt(args.config.task, presentedA, presentedB, args.config.rubric, args.config.promptStyle, args.config.promptOverrides?.compare),
+            promptRef: undefined
         };
     });
-    const comparisons = await runLimitedPhase(jobs.map((job) => async () => {
-        let parsed;
-        let rawOutput;
-        let invalidJson = false;
-        let resultModel;
-        let resultProvider;
-        let resultRetryCount = 0;
-        let jsonParseFailures = 0;
-        let inputTokens = 0;
-        let inputCacheHitTokens = 0;
-        let inputCacheMissTokens = 0;
-        let outputTokens = 0;
-        let totalTokens = 0;
-        for (let attempt = 0; attempt <= args.config.retry.invalidJsonRetries; attempt += 1) {
-            await args.guards.beforeCall(`${args.generation} comparison call`);
-            const result = await args.driver.compare({
-                task: args.config.task,
-                rubric: args.config.rubric,
-                model: args.config.judgeModel,
-                temperature: args.config.profile.judgeTemperature,
-                candidateA: job.presentedA,
-                candidateB: job.presentedB,
-                prompt: job.prompt
-            });
-            const delta = args.tracker.record(result, { provider: args.config.provider, model: args.config.judgeModel });
-            await args.trace.writeUsage(buildUsageRecord({
-                phase: args.generation === "final" ? "final_judge" : `gen${args.generation}_judge`,
-                role: "judge",
-                result,
-                delta,
-                provider: result.provider ?? args.config.provider,
-                model: result.model ?? args.config.judgeModel
-            }));
-            await args.guards.afterCall(`${args.generation} comparison call`);
-            inputTokens += result.usage?.inputTokens ?? 0;
-            inputCacheHitTokens += result.usage?.inputCacheHitTokens ?? 0;
-            inputCacheMissTokens += result.usage?.inputCacheMissTokens ?? 0;
-            outputTokens += result.usage?.outputTokens ?? 0;
-            totalTokens += result.usage?.totalTokens ?? (result.usage?.inputTokens ?? 0) + (result.usage?.outputTokens ?? 0);
-            rawOutput = args.config.output.includeRawModelOutputs ? result.raw ?? result.text : undefined;
-            resultModel = result.model;
-            resultProvider = result.provider;
-            resultRetryCount += result.retryCount ?? 0;
-            try {
-                parsed = compareOutputSchema.parse(parseJsonObject(result.text));
-                invalidJson = false;
-                break;
-            }
-            catch {
-                invalidJson = true;
-                jsonParseFailures += 1;
-            }
-        }
-        if (invalidJson || !parsed) {
-            throw new ConfigError(`Judge produced ${jsonParseFailures} consecutive invalid-JSON responses (${args.config.retry.invalidJsonRetries + 1} attempts) for comparison ${job.id}. Refusing to synthesize a tie and pollute the ranking.`, {
-                code: "judge.persistent_invalid_json",
-                retryable: false,
-                fix: "The judge model is producing unparseable output. Inspect the raw response (set output.includeRawModelOutputs: true), switch judge models, or raise retry.invalidJsonRetries if the failures are transient."
+    const phaseKey = args.generation === "final" ? "final_judging" : `generation_judging:${args.generation}`;
+    const comparisons = await runCheckpointedPhase({
+        runDir: args.config.runDir,
+        phaseKey,
+        jobs: jobs.map((job) => {
+            const receiptUsage = [];
+            return {
+                id: job.id,
+                input: {
+                    task: args.config.task,
+                    rubric: args.config.rubric,
+                    model: args.config.judgeModel,
+                    temperature: args.config.profile.judgeTemperature,
+                    candidateA: job.presentedA,
+                    candidateB: job.presentedB,
+                    prompt: job.prompt,
+                    invalidJsonRetries: args.config.retry.invalidJsonRetries,
+                    maxOutputTokens: args.config.modelOutputTokens?.judge ?? 1_024
+                },
+                receiptUsage: () => receiptUsage,
+                run: async (phaseSignal) => {
+                    if (args.config.output.includePrompts && !job.promptRef)
+                        job.promptRef = await args.trace.writePrompt(job.prompt);
+                    let parsed;
+                    let rawOutput;
+                    let invalidJson = false;
+                    let resultModel;
+                    let resultProvider;
+                    let resultRetryCount = 0;
+                    let jsonParseFailures = 0;
+                    let inputTokens = 0;
+                    let inputCacheHitTokens = 0;
+                    let inputCacheMissTokens = 0;
+                    let outputTokens = 0;
+                    let totalTokens = 0;
+                    for (let attempt = 0; attempt <= args.config.retry.invalidJsonRetries; attempt += 1) {
+                        const reservation = await args.guards.beforeCall(`${args.generation} comparison call`);
+                        let result;
+                        try {
+                            result = await args.driver.compare({
+                                task: args.config.task,
+                                rubric: args.config.rubric,
+                                model: args.config.judgeModel,
+                                temperature: args.config.profile.judgeTemperature,
+                                candidateA: job.presentedA,
+                                candidateB: job.presentedB,
+                                prompt: job.prompt,
+                                signal: phaseSignal,
+                                maxOutputTokens: args.config.modelOutputTokens?.judge ?? 1_024
+                            });
+                        }
+                        catch (error) {
+                            await recordFailedInvocation({
+                                tracker: args.tracker,
+                                reservation,
+                                trace: args.trace,
+                                phase: args.generation === "final" ? "final_judge" : `gen${args.generation}_judge`,
+                                role: "judge",
+                                provider: args.config.provider,
+                                model: args.config.judgeModel,
+                                error
+                            });
+                            throw error;
+                        }
+                        const delta = args.tracker.record(result, { provider: args.config.provider, model: args.config.judgeModel }, reservation);
+                        const usageRecord = buildUsageRecord({
+                            phase: args.generation === "final" ? "final_judge" : `gen${args.generation}_judge`,
+                            role: "judge",
+                            result,
+                            delta,
+                            provider: result.provider ?? args.config.provider,
+                            model: result.model ?? args.config.judgeModel
+                        });
+                        receiptUsage.push(usageRecord);
+                        await args.trace.writeUsage(usageRecord);
+                        await args.guards.afterCall(`${args.generation} comparison call`);
+                        inputTokens += result.usage?.inputTokens ?? 0;
+                        inputCacheHitTokens += result.usage?.inputCacheHitTokens ?? 0;
+                        inputCacheMissTokens += result.usage?.inputCacheMissTokens ?? 0;
+                        outputTokens += result.usage?.outputTokens ?? 0;
+                        totalTokens += result.usage?.totalTokens ?? (result.usage?.inputTokens ?? 0) + (result.usage?.outputTokens ?? 0);
+                        rawOutput = args.config.output.includeRawModelOutputs ? result.raw ?? result.text : undefined;
+                        resultModel = result.model;
+                        resultProvider = result.provider;
+                        resultRetryCount += result.retryCount ?? 0;
+                        try {
+                            parsed = compareOutputSchema.parse(parseJsonObject(result.text));
+                            invalidJson = false;
+                            break;
+                        }
+                        catch {
+                            invalidJson = true;
+                            jsonParseFailures += 1;
+                        }
+                    }
+                    if (invalidJson || !parsed) {
+                        throw new ConfigError(`Judge produced ${jsonParseFailures} consecutive invalid-JSON responses (${args.config.retry.invalidJsonRetries + 1} attempts) for comparison ${job.id}. Refusing to synthesize a tie and pollute the ranking.`, {
+                            code: "judge.persistent_invalid_json",
+                            retryable: false,
+                            fix: "The judge model is producing unparseable output. Inspect the raw response (set output.includeRawModelOutputs: true), switch judge models, or raise retry.invalidJsonRetries if the failures are transient."
+                        });
+                    }
+                    const comparison = {
+                        id: job.id,
+                        runId: args.runId,
+                        generation: args.generation,
+                        candidateAId: job.presentedA.id,
+                        candidateBId: job.presentedB.id,
+                        presentedAOriginalId: job.presentedA.id,
+                        presentedBOriginalId: job.presentedB.id,
+                        winner: parsed.winner,
+                        confidence: parsed.confidence,
+                        critiqueForA: parsed.feedback_a ?? parsed.critique_for_A ?? "",
+                        critiqueForB: parsed.feedback_b ?? parsed.critique_for_B ?? "",
+                        selectionReason: parsed.selection_reason ?? "",
+                        rawOutput,
+                        model: resultModel,
+                        provider: resultProvider,
+                        metadata: compactMetadata({
+                            invalid_json_tie: invalidJson ? true : undefined,
+                            json_parse_failures: jsonParseFailures || undefined,
+                            model_call_count: jsonParseFailures + (parsed ? 1 : 0),
+                            input_tokens: inputTokens || undefined,
+                            input_cache_hit_tokens: inputCacheHitTokens || undefined,
+                            input_cache_miss_tokens: inputCacheMissTokens || undefined,
+                            output_tokens: outputTokens || undefined,
+                            total_tokens: totalTokens || undefined,
+                            scheduled_candidate_a_id: job.originalA.id,
+                            scheduled_candidate_b_id: job.originalB.id,
+                            provider_retry_count: resultRetryCount || undefined,
+                            promptRef: job.promptRef
+                        })
+                    };
+                    return comparison;
+                }
+            };
+        }),
+        concurrency: args.config.concurrency.judge,
+        signal: args.signal,
+        validateOutput: isComparisonCheckpoint,
+        onResult: async (comparison, context) => {
+            if (context.reused)
+                await restoreCheckpointUsage(context.receiptUsage, args.tracker, args.trace);
+            await args.trace.writeComparison(comparison);
+            await args.trace.event({
+                type: "comparison.completed",
+                run_id: args.runId,
+                comparison_id: comparison.id,
+                generation: args.generation
             });
         }
-        const comparison = {
-            id: job.id,
-            runId: args.runId,
-            generation: args.generation,
-            candidateAId: job.presentedA.id,
-            candidateBId: job.presentedB.id,
-            presentedAOriginalId: job.presentedA.id,
-            presentedBOriginalId: job.presentedB.id,
-            winner: parsed?.winner ?? "tie",
-            confidence: parsed?.confidence,
-            critiqueForA: parsed?.feedback_a ?? parsed?.critique_for_A ?? "Invalid comparison JSON; recorded as tie.",
-            critiqueForB: parsed?.feedback_b ?? parsed?.critique_for_B ?? "Invalid comparison JSON; recorded as tie.",
-            selectionReason: parsed?.selection_reason ?? "invalid_json_tie",
-            rawOutput,
-            model: resultModel,
-            provider: resultProvider,
-            metadata: compactMetadata({
-                invalid_json_tie: invalidJson ? true : undefined,
-                json_parse_failures: jsonParseFailures || undefined,
-                model_call_count: jsonParseFailures + (parsed ? 1 : 0),
-                input_tokens: inputTokens || undefined,
-                input_cache_hit_tokens: inputCacheHitTokens || undefined,
-                input_cache_miss_tokens: inputCacheMissTokens || undefined,
-                output_tokens: outputTokens || undefined,
-                total_tokens: totalTokens || undefined,
-                scheduled_candidate_a_id: job.originalA.id,
-                scheduled_candidate_b_id: job.originalB.id,
-                provider_retry_count: resultRetryCount || undefined,
-                prompt: args.config.output.includePrompts ? job.prompt : undefined
-            })
-        };
-        await args.trace.writeComparison(comparison);
-        await args.trace.event({
-            type: "comparison.completed",
-            run_id: args.runId,
-            comparison_id: comparison.id,
-            generation: args.generation
-        });
-        return comparison;
-    }), args.config.concurrency.judge);
+    });
     return comparisons;
 }
 async function mutateSurvivors(args) {
-    const mutants = await runLimitedPhase(args.survivors.map((candidate, index) => async () => {
-        const critique = args.critiquesByCandidate.get(candidate.id) ?? "";
-        const prompt = mutatePrompt(args.config.task, candidate, critique, args.config.rubric, args.config.promptStyle, args.config.promptOverrides?.mutate);
-        await args.guards.beforeCall(`generation ${args.generation} mutation call`);
-        const result = await args.driver.mutate({
-            task: args.config.task,
-            rubric: args.config.rubric,
-            model: args.config.mutatorModel,
-            temperature: args.config.profile.mutateTemperature,
-            candidate,
-            critique,
-            prompt
-        });
-        const delta = args.tracker.record(result, { provider: args.config.provider, model: args.config.mutatorModel });
-        await args.trace.writeUsage(buildUsageRecord({
-            phase: `gen${args.generation}_mutate`,
-            role: "mutator",
-            result,
-            delta,
-            provider: result.provider ?? args.config.provider,
-            model: result.model ?? args.config.mutatorModel
-        }));
-        await args.guards.afterCall(`generation ${args.generation} mutation call`);
-        const mutant = {
-            id: `g${args.generation}_m${index}_${candidate.id}`,
-            generation: args.generation,
-            parentId: candidate.id,
-            kind: "mutation",
-            content: result.text,
-            status: "mutated",
-            metadata: resultMetadata(result, args.config, prompt)
-        };
-        await args.trace.writeCandidate(mutant);
-        await args.trace.event({ type: "candidate.mutated", candidate_id: mutant.id, parent_id: mutant.parentId });
-        return mutant;
-    }), args.config.concurrency.mutate);
+    const mutants = await runCheckpointedPhase({
+        runDir: args.config.runDir,
+        phaseKey: `generation_mutation:${args.generation}`,
+        jobs: args.survivors.map((candidate, index) => {
+            const critique = args.critiquesByCandidate.get(candidate.id) ?? "";
+            const prompt = mutatePrompt(args.config.task, candidate, critique, args.config.rubric, args.config.promptStyle, args.config.promptOverrides?.mutate);
+            const receiptUsage = [];
+            let promptRef;
+            return {
+                id: `g${args.generation}_m${index}_${candidate.id}`,
+                input: {
+                    task: args.config.task,
+                    rubric: args.config.rubric,
+                    model: args.config.mutatorModel,
+                    temperature: args.config.profile.mutateTemperature,
+                    candidate,
+                    critique,
+                    prompt,
+                    maxOutputTokens: args.config.modelOutputTokens?.mutation ?? 4_096
+                },
+                receiptUsage: () => receiptUsage,
+                run: async (phaseSignal) => {
+                    if (args.config.output.includePrompts && !promptRef)
+                        promptRef = await args.trace.writePrompt(prompt);
+                    const reservation = await args.guards.beforeCall(`generation ${args.generation} mutation call`);
+                    let result;
+                    try {
+                        result = await args.driver.mutate({
+                            task: args.config.task,
+                            rubric: args.config.rubric,
+                            model: args.config.mutatorModel,
+                            temperature: args.config.profile.mutateTemperature,
+                            candidate,
+                            critique,
+                            prompt,
+                            signal: phaseSignal,
+                            maxOutputTokens: args.config.modelOutputTokens?.mutation ?? 4_096
+                        });
+                    }
+                    catch (error) {
+                        await recordFailedInvocation({
+                            tracker: args.tracker,
+                            reservation,
+                            trace: args.trace,
+                            phase: `gen${args.generation}_mutate`,
+                            role: "mutator",
+                            provider: args.config.provider,
+                            model: args.config.mutatorModel,
+                            error
+                        });
+                        throw error;
+                    }
+                    const delta = args.tracker.record(result, { provider: args.config.provider, model: args.config.mutatorModel }, reservation);
+                    const usageRecord = buildUsageRecord({
+                        phase: `gen${args.generation}_mutate`,
+                        role: "mutator",
+                        result,
+                        delta,
+                        provider: result.provider ?? args.config.provider,
+                        model: result.model ?? args.config.mutatorModel
+                    });
+                    receiptUsage.push(usageRecord);
+                    await args.trace.writeUsage(usageRecord);
+                    await args.guards.afterCall(`generation ${args.generation} mutation call`);
+                    const mutant = {
+                        id: `g${args.generation}_m${index}_${candidate.id}`,
+                        generation: args.generation,
+                        parentId: candidate.id,
+                        kind: "mutation",
+                        content: result.text,
+                        status: "mutated",
+                        metadata: resultMetadata(result, args.config, promptRef)
+                    };
+                    return mutant;
+                }
+            };
+        }),
+        concurrency: args.config.concurrency.mutate,
+        signal: args.signal,
+        validateOutput: isCandidateCheckpoint,
+        onResult: async (mutant, context) => {
+            if (context.reused)
+                await restoreCheckpointUsage(context.receiptUsage, args.tracker, args.trace);
+            await args.trace.writeCandidate(mutant);
+            await args.trace.event({ type: "candidate.mutated", candidate_id: mutant.id, parent_id: mutant.parentId });
+        }
+    });
     return mutants;
 }
-async function maybeFinalize(winner, config, driver, trace, tracker, guards) {
-    if (!config.finalizerModel || !driver.finalize)
-        return winner.content;
-    await guards.beforeCall("finalization call");
-    const result = await driver.finalize({
-        task: config.task,
-        rubric: config.rubric,
-        model: config.finalizerModel,
-        candidate: winner,
-        prompt: finalizePrompt(config.task, winner, config.rubric, config.promptStyle, config.promptOverrides?.finalize)
+async function maybeFinalize(winner, config, driver, trace, tracker, guards, signal) {
+    const enabled = Boolean(config.finalizerModel && driver.finalize);
+    const prompt = finalizePrompt(config.task, winner, config.rubric, config.promptStyle, config.promptOverrides?.finalize);
+    const promptRef = enabled && config.output.includePrompts ? await trace.writePrompt(prompt) : undefined;
+    let finalModel;
+    let finalProvider;
+    const receiptUsage = [];
+    const [finalized] = await runCheckpointedPhase({
+        runDir: config.runDir,
+        phaseKey: "finalizing",
+        jobs: [
+            {
+                id: `finalizer:${winner.id}`,
+                input: {
+                    enabled,
+                    task: config.task,
+                    rubric: config.rubric,
+                    model: config.finalizerModel,
+                    candidate: winner,
+                    prompt,
+                    maxOutputTokens: config.modelOutputTokens?.finalizer ?? 4_096
+                },
+                receiptUsage: () => receiptUsage,
+                run: async (phaseSignal) => {
+                    if (!enabled || !config.finalizerModel || !driver.finalize)
+                        return { answer: winner.content, promptRef: undefined };
+                    const reservation = await guards.beforeCall("finalization call");
+                    let result;
+                    try {
+                        result = await driver.finalize({
+                            task: config.task,
+                            rubric: config.rubric,
+                            model: config.finalizerModel,
+                            candidate: winner,
+                            prompt,
+                            signal: phaseSignal,
+                            maxOutputTokens: config.modelOutputTokens?.finalizer ?? 4_096
+                        });
+                    }
+                    catch (error) {
+                        await recordFailedInvocation({
+                            tracker,
+                            reservation,
+                            trace,
+                            phase: "finalize",
+                            role: "finalizer",
+                            provider: config.provider,
+                            model: config.finalizerModel,
+                            error
+                        });
+                        throw error;
+                    }
+                    const delta = tracker.record(result, { provider: config.provider, model: config.finalizerModel }, reservation);
+                    const usageRecord = buildUsageRecord({
+                        phase: "finalize",
+                        role: "finalizer",
+                        result,
+                        delta,
+                        provider: result.provider ?? config.provider,
+                        model: result.model ?? config.finalizerModel
+                    });
+                    receiptUsage.push(usageRecord);
+                    await trace.writeUsage(usageRecord);
+                    await guards.afterCall("finalization call");
+                    finalModel = result.model;
+                    finalProvider = result.provider;
+                    return { answer: result.text, promptRef };
+                }
+            }
+        ],
+        concurrency: 1,
+        signal,
+        validateOutput: (output) => Boolean(output &&
+            typeof output === "object" &&
+            typeof output.answer === "string"),
+        onResult: async (_output, context) => {
+            if (context.reused)
+                await restoreCheckpointUsage(context.receiptUsage, tracker, trace);
+            if (enabled) {
+                await trace.event({ type: "finalized", winner_id: winner.id, model: finalModel, provider: finalProvider, prompt_ref: promptRef });
+            }
+        }
     });
-    const delta = tracker.record(result, { provider: config.provider, model: config.finalizerModel });
-    await trace.writeUsage(buildUsageRecord({
-        phase: "finalize",
-        role: "finalizer",
-        result,
-        delta,
-        provider: result.provider ?? config.provider,
-        model: result.model ?? config.finalizerModel
-    }));
-    await guards.afterCall("finalization call");
-    await trace.event({ type: "finalized", winner_id: winner.id, model: result.model, provider: result.provider });
-    return result.text;
+    return finalized.answer;
 }
-function resultMetadata(result, config, prompt) {
+function resultMetadata(result, config, promptRef) {
     return compactMetadata({
         model: result.model,
         provider: result.provider,
@@ -831,13 +1355,65 @@ function resultMetadata(result, config, prompt) {
         latencyMs: result.latencyMs,
         retryCount: result.retryCount,
         rawOutput: config.output.includeRawModelOutputs ? result.raw : undefined,
-        prompt: config.output.includePrompts ? prompt : undefined,
+        promptRef: config.output.includePrompts ? promptRef : undefined,
         createdAt: new Date().toISOString()
     });
 }
 function compactMetadata(metadata) {
     return Object.fromEntries(Object.entries(metadata).filter(([, value]) => value !== undefined));
 }
+const restoredUsageByTracker = new WeakMap();
+async function restoreCheckpointUsage(usage, tracker, trace) {
+    if (usage === undefined)
+        return;
+    if (!Array.isArray(usage)) {
+        throw new ConfigError("Trace-v2 checkpoint usage must be an array of UsageRecord rows.", {
+            code: "resume.usage_invalid",
+            retryable: false
+        });
+    }
+    const ids = validatedReceiptUsageIds(usage);
+    const restored = restoredUsageByTracker.get(tracker) ?? new Set();
+    for (const id of ids) {
+        if (restored.has(id)) {
+            throw new ConfigError(`Trace-v2 usage ${id} was restored more than once.`, {
+                code: "resume.usage_invalid",
+                retryable: false
+            });
+        }
+        restored.add(id);
+    }
+    restoredUsageByTracker.set(tracker, restored);
+    const records = usage;
+    applyUsageRecords(tracker, records);
+    for (const record of records)
+        await trace.writeUsage(record);
+}
+function isCandidateCheckpoint(output) {
+    if (!output || typeof output !== "object" || Array.isArray(output))
+        return false;
+    const candidate = output;
+    return (typeof candidate.id === "string" &&
+        candidate.id.length > 0 &&
+        Number.isInteger(candidate.generation) &&
+        typeof candidate.content === "string" &&
+        typeof candidate.kind === "string" &&
+        Boolean(candidate.metadata && typeof candidate.metadata === "object" && typeof candidate.metadata.createdAt === "string"));
+}
+function isComparisonCheckpoint(output) {
+    if (!output || typeof output !== "object" || Array.isArray(output))
+        return false;
+    const comparison = output;
+    return (typeof comparison.id === "string" &&
+        typeof comparison.runId === "string" &&
+        (typeof comparison.generation === "number" || comparison.generation === "final") &&
+        typeof comparison.candidateAId === "string" &&
+        typeof comparison.candidateBId === "string" &&
+        (comparison.winner === "A" || comparison.winner === "B" || comparison.winner === "tie") &&
+        typeof comparison.critiqueForA === "string" &&
+        typeof comparison.critiqueForB === "string" &&
+        typeof comparison.selectionReason === "string");
+}
 function summaryProfile(profile) {
     return {
         n: profile.n,
@@ -878,6 +1454,7 @@ function summaryModels(config) {
 function buildUsageRecord(args) {
     return {
         schema_version: 1,
+        usage_id: randomUUID(),
         ts: new Date().toISOString(),
         phase: args.phase,
         role: args.role,
@@ -892,18 +1469,31 @@ function buildUsageRecord(args) {
         output_usd: args.delta.outputUsd,
         total_usd: args.delta.usd,
         latency_ms: args.result.latencyMs,
-        retry_count: args.result.retryCount
+        retry_count: args.result.retryCount,
+        outcome: args.outcome,
+        error_code: args.errorCode
     };
 }
+async function recordFailedInvocation(args) {
+    const delta = args.tracker.failCall(args.reservation);
+    await bestEffortTraceWrite("failed call usage", () => args.trace.writeUsage(buildUsageRecord({
+        phase: args.phase,
+        role: args.role,
+        result: { text: "" },
+        delta,
+        provider: args.provider,
+        model: args.model,
+        outcome: "failed",
+        errorCode: args.error instanceof DeepThonkError ? args.error.code : args.error instanceof Error ? args.error.name : "unknown_error"
+    })));
+}
 function enforceBudget(config) {
-    const plan = planBudget(config.profile);
-    // Each comparison call may fire up to invalidJsonRetries extra requests before the loop gives up.
-    // Include that worst-case headroom so a user who sets --max-calls = plan.calls isn't tripped by
-    // a single retry mid-run.
-    const judgeCalls = plan.per_generation_judge_calls * config.profile.t + plan.final_judge_calls;
-    const retryHeadroom = judgeCalls * config.retry.invalidJsonRetries;
-    const finalizerCalls = config.finalizerModel ? 1 : 0;
-    const plannedCalls = plan.calls + finalizerCalls + retryHeadroom;
+    const plan = planBudget(config.profile, {
+        invalidJsonRetries: config.retry.invalidJsonRetries,
+        includeFinalizer: Boolean(config.finalizerModel)
+    });
+    const explicitFinalCalls = config.rank ? configuredFinalPairCount(config.profile.n, config.rank) : plan.final_judge_calls;
+    const plannedCalls = plan.calls - plan.final_judge_calls + explicitFinalCalls + plan.finalizer_calls;
     if (config.budget?.maxCalls !== undefined && plannedCalls > config.budget.maxCalls) {
         throw new BudgetExceededError(`Planned call count ${plannedCalls} exceeds maxCalls ${config.budget.maxCalls}.`, {
             code: "budget.planned_calls_exceeded",
@@ -911,6 +1501,64 @@ function enforceBudget(config) {
         });
     }
 }
+function configuredFinalPairs(candidateIds, rank, rng) {
+    if (rank.mode === "k-regular") {
+        if (rank.k === undefined) {
+            throw new ConfigError("rank.mode k-regular requires rank.k.", { code: "rank.k_required", retryable: false });
+        }
+        const pairs = makeKRegularPairs(candidateIds, rank.k, rng);
+        assertRankCallLimit(pairs.length, rank.maxCalls);
+        return pairs;
+    }
+    const pairs = [];
+    for (let left = 0; left < candidateIds.length; left += 1) {
+        for (let right = left + 1; right < candidateIds.length; right += 1) {
+            pairs.push({ a: candidateIds[left], b: candidateIds[right] });
+        }
+    }
+    if (pairs.length > 100 && (rank.maxCalls === undefined || rank.maxCalls < pairs.length)) {
+        throw new ConfigError(`All-pairs final ranking requires explicit rank.maxCalls >= ${pairs.length}.`, {
+            code: "rank.max_calls_required",
+            retryable: false,
+            fix: "Raise rank.maxCalls or select k-regular mode."
+        });
+    }
+    assertRankCallLimit(pairs.length, rank.maxCalls);
+    return rng.shuffle(pairs);
+}
+function configuredFinalPairCount(n, rank) {
+    if (rank.mode === "all-pairs") {
+        const calls = (n * (n - 1)) / 2;
+        if (calls > 100 && (rank.maxCalls === undefined || rank.maxCalls < calls)) {
+            throw new ConfigError(`All-pairs final ranking requires explicit rank.maxCalls >= ${calls}.`, {
+                code: "rank.max_calls_required",
+                retryable: false
+            });
+        }
+        assertRankCallLimit(calls, rank.maxCalls);
+        return calls;
+    }
+    if (rank.k === undefined) {
+        throw new ConfigError("rank.mode k-regular requires rank.k.", { code: "rank.k_required", retryable: false });
+    }
+    if (rank.k >= n || (n * rank.k) % 2 !== 0) {
+        throw new ConfigError(`Invalid k-regular final ranking for n=${n}, k=${rank.k}.`, {
+            code: "rank.k_invalid",
+            retryable: false
+        });
+    }
+    const calls = (n * rank.k) / 2;
+    assertRankCallLimit(calls, rank.maxCalls);
+    return calls;
+}
+function assertRankCallLimit(calls, maxCalls) {
+    if (maxCalls !== undefined && calls > maxCalls) {
+        throw new ConfigError(`Final ranking requires ${calls} calls, exceeding rank.maxCalls ${maxCalls}.`, {
+            code: "rank.max_calls_exceeded",
+            retryable: false
+        });
+    }
+}
 function cloneUsage(usage) {
     return {
         ...emptyUsage(),