@deepthonk/core 0.1.4 → 0.1.5

package/dist/runner.js

@@ -1,3 +1,4 @@
+import { createHash, randomUUID } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
@@ -7,12 +8,12 @@ import { planBudget, validateProfile } from "./budget.js";
 import { BudgetExceededError, CancelledError, ConfigError, DeepThonkError } from "./errors.js";
 import { runArtifactFiles } from "./artifacts.js";
 import { parseJsonObject } from "./json.js";
-import { claimRunLock, emptyUsage, releaseRunLock } from "./lifecycle.js";
+import { claimRunLockOwnership, emptyUsage, releaseRunLock, verifyRunLockClaim } from "./lifecycle.js";
 import { makeKRegularPairs } from "./pairScheduler.js";
 import { runLimitedPhase } from "./phaseRunner.js";
 import { comparePrompt, finalizePrompt, generatePrompt, mutatePrompt } from "./prompts.js";
 import { createRng } from "./rng.js";
-import { assertNoPruneInProgress, buildPopulationMap, buildResumePlan, groupComparisons, persistPrunedTrace, pruneTraceToPlan, readResumeTrace, reconstructScores, replayBudgetUsage, resolveResumeRunId, resumeConfigError, toResumePlanStatus } from "./resume.js";
+import { assertNoPruneInProgress, assertNoLegacyRedactedBudget, buildPopulationMap, buildResumePlan, groupComparisons, persistPrunedTrace, pruneTraceToPlan, readResumeTrace, reconstructScores, replayBudgetUsage, resolveResumeRunId, resumeConfigError, toResumePlanStatus } from "./resume.js";
 import { runConfigSchema, builtInProfiles, phaseCompletedEventSchema } from "./schemas.js";
 import { aggregateCritiques } from "./critique.js";
 import { TraceStore } from "./traceStore.js";
@@ -29,22 +30,39 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
     const config = runConfigSchema.parse(configInput);
     validateProfile(config.profile);
     enforceBudget(config);
-    const runId = resumeState?.runId ?? `run_${new Date().toISOString().replace(/[:.]/g, "-")}_${Math.abs(config.seed)}`;
+    const runId = resumeState?.runId ??
+        config.runId ??
+        `run_${new Date().toISOString().replace(/[:.]/g, "-")}_${randomUUID()}`;
     const trace = new TraceStore(config.runDir);
     const rng = createRng(config.seed);
     const tracker = resumeState?.tracker ?? new BudgetTracker(config);
     const startedAt = resumeState?.startedAt ?? new Date().toISOString();
     let ownsLock = false;
-    if (!resumeState && !control.lockClaimed) {
-        const claimed = await claimRunLock(config.runDir, control.jobId ?? runId);
-        if (!claimed) {
-            throw new ConfigError(`Run directory is already claimed: ${config.runDir}`, {
-                code: "run.directory_locked",
-                retryable: false,
-                fix: "Wait for the active run to finish, or choose a fresh run directory."
-            });
+    let ownedLockClaimId;
+    let verifiedPreclaim = false;
+    if (!resumeState) {
+        if (control.lockClaim) {
+            verifiedPreclaim = await verifyRunLockClaim(config.runDir, control.lockClaim, control.jobId);
+            if (!verifiedPreclaim) {
+                throw new ConfigError("The supplied run lock claim does not own the current run.lock.", {
+                    code: "run.lock_not_owned",
+                    retryable: false,
+                    fix: "Claim the run directory with claimRunLockOwnership and pass that exact claim to the worker."
+                });
+            }
+        }
+        else {
+            const claimed = await claimRunLockOwnership(config.runDir, control.jobId ?? runId);
+            if (!claimed) {
+                throw new ConfigError(`Run directory is already claimed: ${config.runDir}`, {
+                    code: "run.directory_locked",
+                    retryable: false,
+                    fix: "Wait for the active run to finish, or choose a fresh run directory."
+                });
+            }
+            ownsLock = true;
+            ownedLockClaimId = claimed.claimId;
         }
-        ownsLock = true;
     }
     let stopped = false;
     let caughtError;
@@ -89,6 +107,7 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
                 throw new BudgetExceededError(`Run already stopped before ${phase}.`, { code: "budget.stopped" });
             await assertNotCancelled(phase);
             await assertBudget(phase);
+            return tracker.reserveCall(phase);
         },
         async afterCall(phase) {
             await assertNotCancelled(phase);
@@ -97,7 +116,7 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
     };
     try {
         if (!resumeState) {
-            await trace.init({ ...config, version: await currentPackageVersion() }, runId);
+            await trace.init({ ...config, version: await currentPackageVersion() }, runId, { allowOwnedLock: ownsLock || verifiedPreclaim, pendingJobId: verifiedPreclaim ? control.jobId : undefined });
         }
         else {
             await trace.event({
@@ -261,7 +280,7 @@ export async function runDeepThonk(configInput, driver, control = {}, resumeStat
     finally {
         if (ownsLock) {
             try {
-                await releaseRunLock(config.runDir);
+                await releaseRunLock(config.runDir, ownedLockClaimId);
             }
             catch (error) {
                 if (caughtError !== undefined) {
@@ -280,6 +299,7 @@ export async function resumeDeepThonk(runDir, driver, options = {}) {
         resumeConfigError("Run already has summary.json; nothing to resume.", "resume.already_complete", "Use deepthonk inspect/result to read the completed run.");
     }
     const rawConfig = await readRequiredConfig(runDir);
+    assertNoLegacyRedactedBudget(rawConfig);
     const currentVersion = await currentPackageVersion();
     if (!sameMajorMinor(typeof rawConfig.version === "string" ? rawConfig.version : undefined, currentVersion)) {
         resumeConfigError(`Cannot resume run from version ${String(rawConfig.version ?? "missing")}; current package version is ${currentVersion}. Resume requires matching major.minor.`, "resume.version_mismatch", "Start a fresh run with the current DeepThonk version, or resume with a package version whose major.minor matches the trace.");
@@ -292,7 +312,7 @@ export async function resumeDeepThonk(runDir, driver, options = {}) {
     assertResumeProviderMatches(rawConfig, config, driver, options);
     assertNoPruneInProgress(runDir);
     const lockRunId = resolveResumeRunId(config, undefined, []);
-    const claimed = await claimRunLock(runDir, lockRunId);
+    const claimed = await claimRunLockOwnership(runDir, lockRunId);
     if (!claimed) {
         throw new ConfigError(`Run directory is already claimed: ${runDir}`, {
             code: "run.directory_locked",
@@ -312,12 +332,13 @@ export async function resumeDeepThonk(runDir, driver, options = {}) {
             resumeConfigError("Trace says finalizing completed, but summary.json is missing.", "resume.inconsistent_trace", "Inspect the run directory and restore summary.json, or start a fresh run.");
         }
         const planStatus = toResumePlanStatus(runDir, runId, plan);
-        if (options.dryRun)
-            return planStatus;
         const pruned = pruneTraceToPlan(trace, plan);
+        assertResumeRunIdsAgree(runId, config, status, pruned);
         const populationByGeneration = buildPopulationMap(config, pruned.populations, pruned.candidates, plan);
         const comparisonsByGeneration = groupComparisons(pruned.comparisons);
-        const scoresByGeneration = reconstructScores(config, populationByGeneration, comparisonsByGeneration, pruned.scores, plan);
+        const scoresByGeneration = reconstructScores(config, populationByGeneration, comparisonsByGeneration, pruned.scores, plan, runId);
+        if (options.dryRun)
+            return planStatus;
         const tracker = replayBudgetUsage(config, pruned.usage);
         const startedAt = status?.started_at ?? new Date().toISOString();
         await new TraceStore(runDir).writeStatus({
@@ -343,7 +364,26 @@ export async function resumeDeepThonk(runDir, driver, options = {}) {
         });
     }
     finally {
-        await releaseRunLock(runDir);
+        await releaseRunLock(runDir, claimed?.claimId);
+    }
+}
+function assertResumeRunIdsAgree(resolvedRunId, config, status, trace) {
+    const observed = new Set();
+    if (config.runId)
+        observed.add(config.runId);
+    if (status?.run_id)
+        observed.add(status.run_id);
+    for (const event of trace.events) {
+        if (typeof event.run_id === "string")
+            observed.add(event.run_id);
+    }
+    for (const comparison of trace.comparisons) {
+        if (comparison.runId)
+            observed.add(comparison.runId);
+    }
+    observed.add(resolvedRunId);
+    if (observed.size > 1) {
+        resumeConfigError(`Resume artifacts disagree on run ID: ${[...observed].sort().join(", ")}.`, "resume.run_id_mismatch", "Restore artifacts from one run; do not combine trace files from different run directories.");
     }
 }
 async function markPhaseCompleted(trace, phase, generation) {
@@ -425,10 +465,19 @@ function assertStoredOutputConfigComplete(rawConfig) {
     }
 }
 function assertResumeProviderMatches(rawConfig, config, driver, options) {
+    assertProviderReplayInternalConsistency(config);
     const runtimeProvider = options.provider ?? providerLabel(driver) ?? config.provider;
     if (runtimeProvider !== config.provider) {
         providerMismatch(`Cannot resume provider ${config.provider} with runtime provider ${runtimeProvider}.`);
     }
+    const expectedFingerprint = config.providerReplay?.routeFingerprint;
+    if (expectedFingerprint !== undefined && providerRouteFingerprint(driver) !== expectedFingerprint) {
+        providerMismatch("Cannot resume because the provider route fingerprint changed.");
+    }
+    if (config.providerReplay?.baseUrl !== undefined &&
+        normalizeBaseUrl(providerBaseUrl(driver)) !== normalizeBaseUrl(config.providerReplay.baseUrl)) {
+        providerMismatch(`Cannot resume base URL ${config.providerReplay.baseUrl} with runtime base URL ${providerBaseUrl(driver) ?? "missing"}.`);
+    }
     const expectedRoutes = expectedProviderRoutes(rawConfig, config);
     const actualRoutes = runtimeProviderRoutes(driver);
     for (const role of providerRoles) {
@@ -450,8 +499,92 @@ function assertResumeProviderMatches(rawConfig, config, driver, options) {
         }
     }
 }
+function assertProviderReplayInternalConsistency(config) {
+    const replay = config.providerReplay;
+    if (!replay)
+        return;
+    const expectedModels = {
+        generator: config.generatorModel,
+        mutator: config.mutatorModel,
+        judge: config.judgeModel,
+        finalizer: config.finalizerModel
+    };
+    if (replay.provider !== config.provider ||
+        replay.models.generator !== expectedModels.generator ||
+        replay.models.mutator !== expectedModels.mutator ||
+        replay.models.judge !== expectedModels.judge ||
+        replay.models.finalizer !== expectedModels.finalizer) {
+        resumeConfigError("Stored run provider/model fields disagree with providerReplay.", "resume.provider_replay_mismatch", "Restore config.json from the original run; do not edit provider or model fields independently.");
+    }
+    if (replay.routeFingerprint !== undefined && replay.routeFingerprint !== coreProviderReplayFingerprint(replay)) {
+        resumeConfigError("Stored providerReplay fingerprint does not match its route fields.", "resume.provider_replay_mismatch", "Restore the original providerReplay block instead of editing route fields.");
+    }
+}
+function coreProviderReplayFingerprint(replay) {
+    const roles = Object.fromEntries(providerRoles.flatMap((role) => {
+        const route = replay.roleProviders?.[role];
+        return route
+            ? [[role, routeFingerprintValue(route.provider, route.baseUrl, route.apiKeyEnv, route.model, route.supportsJsonMode)]]
+            : [];
+    }));
+    const value = {
+        base: routeFingerprintValue(replay.provider, replay.baseUrl, replay.apiKeyEnv, replay.models, replay.supportsJsonMode),
+        roles,
+        samplingPreferences: replay.samplingPreferences ?? null
+    };
+    return `sha256:${createHash("sha256").update(stableJson(value)).digest("hex")}`;
+}
+function routeFingerprintValue(provider, baseUrl, apiKeyEnv, model, supportsJsonMode) {
+    return Object.fromEntries(Object.entries({
+        provider,
+        baseUrl: normalizeFingerprintUrl(baseUrl),
+        apiKeyEnv,
+        model,
+        supportsJsonMode
+    }).filter(([, value]) => value !== undefined));
+}
+function normalizeFingerprintUrl(baseUrl) {
+    if (!baseUrl)
+        return undefined;
+    try {
+        const parsed = new URL(baseUrl);
+        parsed.hash = "";
+        parsed.hostname = parsed.hostname.toLowerCase();
+        parsed.pathname = parsed.pathname.replace(/\/+$/, "");
+        return parsed.toString().replace(/\/$/, "");
+    }
+    catch {
+        return baseUrl.replace(/\/+$/, "");
+    }
+}
+function stableJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(stableJson).join(",")}]`;
+    if (value && typeof value === "object") {
+        const entries = Object.entries(value)
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([key, inner]) => `${JSON.stringify(key)}:${stableJson(inner)}`);
+        return `{${entries.join(",")}}`;
+    }
+    return JSON.stringify(value) ?? "null";
+}
 const providerRoles = ["generator", "mutator", "judge", "finalizer"];
 function expectedProviderRoutes(rawConfig, config) {
+    const replayRoutes = config.providerReplay?.roleProviders;
+    if (replayRoutes) {
+        const routes = {};
+        for (const role of providerRoles) {
+            const route = replayRoutes[role];
+            if (!route)
+                continue;
+            routes[role] = {
+                provider: route.provider,
+                baseUrl: route.baseUrl,
+                model: route.model
+            };
+        }
+        return routes;
+    }
     const providers = rawConfig.providers;
     if (!isRecord(providers))
         return {};
@@ -486,6 +619,16 @@ function runtimeProviderRoutes(driver) {
     }
     return routes;
 }
+function providerRouteFingerprint(driver) {
+    const value = driver;
+    if (typeof value.routeFingerprint === "string")
+        return value.routeFingerprint;
+    if (typeof value.config?.routeFingerprint === "string")
+        return value.config.routeFingerprint;
+    if (value.baseDriver)
+        return providerRouteFingerprint(value.baseDriver);
+    return undefined;
+}
 function providerBaseUrl(driver) {
     const value = driver;
     if (typeof value.baseUrl === "string")
@@ -544,8 +687,8 @@ function isLiveWorker(pid) {
         process.kill(pid, 0);
         return true;
     }
-    catch {
-        return false;
+    catch (error) {
+        return error.code !== "ESRCH";
     }
 }
 export function rankPopulation(population, scores) {
@@ -601,16 +744,32 @@ async function generateInitialPopulation(config, driver, trace, rng, runId, trac
         prompt: generatePrompt(config.task, config.rubric, config.promptStyle, config.promptOverrides?.generate)
     }));
     const candidates = await runLimitedPhase(jobs.map((job) => async () => {
-        await guards.beforeCall("initial population call");
-        const result = await driver.generate({
-            task: config.task,
-            rubric: config.rubric,
-            model: config.generatorModel,
-            temperature: config.profile.sampleTemperature,
-            candidateIndex: job.index,
-            prompt: job.prompt
-        });
-        const delta = tracker.record(result, { provider: config.provider, model: config.generatorModel });
+        const reservation = await guards.beforeCall("initial population call");
+        let result;
+        try {
+            result = await driver.generate({
+                task: config.task,
+                rubric: config.rubric,
+                model: config.generatorModel,
+                temperature: config.profile.sampleTemperature,
+                candidateIndex: job.index,
+                prompt: job.prompt
+            });
+        }
+        catch (error) {
+            await recordFailedInvocation({
+                tracker,
+                reservation,
+                trace,
+                phase: "gen0_initial",
+                role: "generator",
+                provider: config.provider,
+                model: config.generatorModel,
+                error
+            });
+            throw error;
+        }
+        const delta = tracker.record(result, { provider: config.provider, model: config.generatorModel }, reservation);
         await trace.writeUsage(buildUsageRecord({
             phase: "gen0_initial",
             role: "generator",
@@ -668,17 +827,33 @@ async function judgePairs(args) {
         let outputTokens = 0;
         let totalTokens = 0;
         for (let attempt = 0; attempt <= args.config.retry.invalidJsonRetries; attempt += 1) {
-            await args.guards.beforeCall(`${args.generation} comparison call`);
-            const result = await args.driver.compare({
-                task: args.config.task,
-                rubric: args.config.rubric,
-                model: args.config.judgeModel,
-                temperature: args.config.profile.judgeTemperature,
-                candidateA: job.presentedA,
-                candidateB: job.presentedB,
-                prompt: job.prompt
-            });
-            const delta = args.tracker.record(result, { provider: args.config.provider, model: args.config.judgeModel });
+            const reservation = await args.guards.beforeCall(`${args.generation} comparison call`);
+            let result;
+            try {
+                result = await args.driver.compare({
+                    task: args.config.task,
+                    rubric: args.config.rubric,
+                    model: args.config.judgeModel,
+                    temperature: args.config.profile.judgeTemperature,
+                    candidateA: job.presentedA,
+                    candidateB: job.presentedB,
+                    prompt: job.prompt
+                });
+            }
+            catch (error) {
+                await recordFailedInvocation({
+                    tracker: args.tracker,
+                    reservation,
+                    trace: args.trace,
+                    phase: args.generation === "final" ? "final_judge" : `gen${args.generation}_judge`,
+                    role: "judge",
+                    provider: args.config.provider,
+                    model: args.config.judgeModel,
+                    error
+                });
+                throw error;
+            }
+            const delta = args.tracker.record(result, { provider: args.config.provider, model: args.config.judgeModel }, reservation);
             await args.trace.writeUsage(buildUsageRecord({
                 phase: args.generation === "final" ? "final_judge" : `gen${args.generation}_judge`,
                 role: "judge",
@@ -722,11 +897,11 @@ async function judgePairs(args) {
             candidateBId: job.presentedB.id,
             presentedAOriginalId: job.presentedA.id,
             presentedBOriginalId: job.presentedB.id,
-            winner: parsed?.winner ?? "tie",
-            confidence: parsed?.confidence,
-            critiqueForA: parsed?.feedback_a ?? parsed?.critique_for_A ?? "Invalid comparison JSON; recorded as tie.",
-            critiqueForB: parsed?.feedback_b ?? parsed?.critique_for_B ?? "Invalid comparison JSON; recorded as tie.",
-            selectionReason: parsed?.selection_reason ?? "invalid_json_tie",
+            winner: parsed.winner,
+            confidence: parsed.confidence,
+            critiqueForA: parsed.feedback_a ?? parsed.critique_for_A ?? "",
+            critiqueForB: parsed.feedback_b ?? parsed.critique_for_B ?? "",
+            selectionReason: parsed.selection_reason ?? "",
             rawOutput,
             model: resultModel,
             provider: resultProvider,
@@ -760,17 +935,33 @@ async function mutateSurvivors(args) {
     const mutants = await runLimitedPhase(args.survivors.map((candidate, index) => async () => {
         const critique = args.critiquesByCandidate.get(candidate.id) ?? "";
         const prompt = mutatePrompt(args.config.task, candidate, critique, args.config.rubric, args.config.promptStyle, args.config.promptOverrides?.mutate);
-        await args.guards.beforeCall(`generation ${args.generation} mutation call`);
-        const result = await args.driver.mutate({
-            task: args.config.task,
-            rubric: args.config.rubric,
-            model: args.config.mutatorModel,
-            temperature: args.config.profile.mutateTemperature,
-            candidate,
-            critique,
-            prompt
-        });
-        const delta = args.tracker.record(result, { provider: args.config.provider, model: args.config.mutatorModel });
+        const reservation = await args.guards.beforeCall(`generation ${args.generation} mutation call`);
+        let result;
+        try {
+            result = await args.driver.mutate({
+                task: args.config.task,
+                rubric: args.config.rubric,
+                model: args.config.mutatorModel,
+                temperature: args.config.profile.mutateTemperature,
+                candidate,
+                critique,
+                prompt
+            });
+        }
+        catch (error) {
+            await recordFailedInvocation({
+                tracker: args.tracker,
+                reservation,
+                trace: args.trace,
+                phase: `gen${args.generation}_mutate`,
+                role: "mutator",
+                provider: args.config.provider,
+                model: args.config.mutatorModel,
+                error
+            });
+            throw error;
+        }
+        const delta = args.tracker.record(result, { provider: args.config.provider, model: args.config.mutatorModel }, reservation);
         await args.trace.writeUsage(buildUsageRecord({
             phase: `gen${args.generation}_mutate`,
             role: "mutator",
@@ -798,15 +989,31 @@ async function mutateSurvivors(args) {
 async function maybeFinalize(winner, config, driver, trace, tracker, guards) {
     if (!config.finalizerModel || !driver.finalize)
         return winner.content;
-    await guards.beforeCall("finalization call");
-    const result = await driver.finalize({
-        task: config.task,
-        rubric: config.rubric,
-        model: config.finalizerModel,
-        candidate: winner,
-        prompt: finalizePrompt(config.task, winner, config.rubric, config.promptStyle, config.promptOverrides?.finalize)
-    });
-    const delta = tracker.record(result, { provider: config.provider, model: config.finalizerModel });
+    const reservation = await guards.beforeCall("finalization call");
+    let result;
+    try {
+        result = await driver.finalize({
+            task: config.task,
+            rubric: config.rubric,
+            model: config.finalizerModel,
+            candidate: winner,
+            prompt: finalizePrompt(config.task, winner, config.rubric, config.promptStyle, config.promptOverrides?.finalize)
+        });
+    }
+    catch (error) {
+        await recordFailedInvocation({
+            tracker,
+            reservation,
+            trace,
+            phase: "finalize",
+            role: "finalizer",
+            provider: config.provider,
+            model: config.finalizerModel,
+            error
+        });
+        throw error;
+    }
+    const delta = tracker.record(result, { provider: config.provider, model: config.finalizerModel }, reservation);
     await trace.writeUsage(buildUsageRecord({
         phase: "finalize",
         role: "finalizer",
@@ -892,18 +1099,30 @@ function buildUsageRecord(args) {
         output_usd: args.delta.outputUsd,
         total_usd: args.delta.usd,
         latency_ms: args.result.latencyMs,
-        retry_count: args.result.retryCount
+        retry_count: args.result.retryCount,
+        outcome: args.outcome,
+        error_code: args.errorCode
     };
 }
+async function recordFailedInvocation(args) {
+    const delta = args.tracker.failCall(args.reservation);
+    await bestEffortTraceWrite("failed call usage", () => args.trace.writeUsage(buildUsageRecord({
+        phase: args.phase,
+        role: args.role,
+        result: { text: "" },
+        delta,
+        provider: args.provider,
+        model: args.model,
+        outcome: "failed",
+        errorCode: args.error instanceof DeepThonkError ? args.error.code : args.error instanceof Error ? args.error.name : "unknown_error"
+    })));
+}
 function enforceBudget(config) {
-    const plan = planBudget(config.profile);
-    // Each comparison call may fire up to invalidJsonRetries extra requests before the loop gives up.
-    // Include that worst-case headroom so a user who sets --max-calls = plan.calls isn't tripped by
-    // a single retry mid-run.
-    const judgeCalls = plan.per_generation_judge_calls * config.profile.t + plan.final_judge_calls;
-    const retryHeadroom = judgeCalls * config.retry.invalidJsonRetries;
-    const finalizerCalls = config.finalizerModel ? 1 : 0;
-    const plannedCalls = plan.calls + finalizerCalls + retryHeadroom;
+    const plan = planBudget(config.profile, {
+        invalidJsonRetries: config.retry.invalidJsonRetries,
+        includeFinalizer: Boolean(config.finalizerModel)
+    });
+    const plannedCalls = plan.calls + plan.finalizer_calls;
     if (config.budget?.maxCalls !== undefined && plannedCalls > config.budget.maxCalls) {
         throw new BudgetExceededError(`Planned call count ${plannedCalls} exceeds maxCalls ${config.budget.maxCalls}.`, {
             code: "budget.planned_calls_exceeded",