agent-hitch 0.2.6 → 0.2.7

package/dist/src/control-plane/eval-recovery.js

@@ -0,0 +1,181 @@
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import { SCHEMA_VERSION, atomicWriteJSON, credentialValuesFromEnv, readJSON, safeDiagnosticMessage, withFileLock } from "../foundation/index.js";
+import { EvalEventSink, parseEvalExecutionPlan, readExecutionLeases, recoverLocalDockerEvalLeases } from "../evals/index.js";
+import { idempotencyIndexPath, isTerminalControl, parseEvalControl, parseEvalSubmission, terminalControlState } from "./eval-records.js";
+export async function recoverPersistedEvals(input) {
+    const queue = [];
+    const entries = await readdir(input.evalsRoot, { withFileTypes: true });
+    for (const item of entries) {
+        if (!item.isDirectory() || !/^eval_[a-f0-9]{32}$/.test(item.name))
+            continue;
+        const evalId = item.name;
+        const directory = path.join(input.evalsRoot, evalId);
+        const submissionValue = await readJSON(path.join(directory, "submission.json"), null);
+        const controlValue = await readJSON(path.join(directory, "control.json"), null);
+        if (!submissionValue || !controlValue)
+            continue;
+        const submission = await parseEvalSubmission(submissionValue, evalId);
+        const credentialValues = credentialValuesFromEnv(submission.request.pass_env, input.credentialEnv ?? process.env);
+        const control = parseEvalControl(controlValue);
+        if (await repairIdempotencyIndex(input.root, submission)) {
+            const sink = new EvalEventSink(directory, evalId, input.onEvent);
+            await sink.open();
+            sink.emit({ type: "eval.idempotency-index.recovered", idempotency_key_hash: submission.idempotency_key_hash });
+            await sink.close();
+        }
+        const result = await readJSON(path.join(directory, "result.json"), null);
+        if (result) {
+            if (!isTerminalControl(control.state))
+                await updateControl(directory, (current) => ({ ...withoutAllocation(current), state: terminalControlState(result.status) }));
+            continue;
+        }
+        if (control.state === "queued") {
+            queue.push({ evalId, request: submission.request, ...(submission.execution ? { execution: submission.execution } : {}), directory, resumeExisting: false });
+            continue;
+        }
+        if (isTerminalControl(control.state))
+            continue;
+        const sink = new EvalEventSink(directory, evalId, input.onEvent);
+        await sink.open();
+        const leases = await readExecutionLeases(directory);
+        const localLeases = leases.filter((lease) => lease.provider === "local-docker");
+        const providerLeases = leases.filter((lease) => lease.provider !== "local-docker");
+        const recoveries = [await recoverLocalDockerEvalLeases({
+                root: input.root,
+                evalId,
+                evalDirectory: directory,
+                leases: localLeases,
+                env: input.credentialEnv ?? process.env,
+                cancelRequested: control.state === "cancelling",
+                emit: (event) => sink.emit(event),
+            })];
+        if (providerLeases.length > 0)
+            recoveries.push(input.recoverProviderLeases
+                ? await input.recoverProviderLeases({
+                    evalId, evalDirectory: directory, leases: providerLeases,
+                    cancelRequested: control.state === "cancelling", emit: (event) => sink.emit(event),
+                })
+                : { status: "ambiguous", recovered_lease_ids: [], code: "execution_state_ambiguous", message: "no recovery provider is available for active remote leases" });
+        const recovery = combinedRecovery(recoveries);
+        if (control.state === "cancelling") {
+            const now = new Date().toISOString();
+            await writeSyntheticResult(directory, evalId, submission.request, control, "cancelled", "cancelled", "eval cancellation was recovered after daemon restart", now);
+            await updateControl(directory, (current) => ({ ...withoutAllocation(current), state: "cancelled", error: { code: "cancelled", message: "eval cancellation was recovered after daemon restart" } }));
+            sink.emit({ type: "eval.recovered", status: "cancelled", code: "cancelled" });
+            await sink.close();
+            continue;
+        }
+        if (recovery.status === "ambiguous") {
+            const now = new Date().toISOString();
+            const code = recovery.code || "execution_state_ambiguous";
+            const message = safeDiagnosticMessage(recovery.message || "daemon restarted while eval execution state was ambiguous", credentialValues);
+            await writeSyntheticResult(directory, evalId, submission.request, control, "failed", code, message, now);
+            await updateControl(directory, (current) => ({ ...withoutAllocation(current), state: "failed", error: { code, message } }));
+            sink.emit({ type: "eval.recovered", status: "failed", code });
+            await sink.close();
+            continue;
+        }
+        const files = await resumableFiles(directory);
+        if (files === "inconsistent") {
+            const now = new Date().toISOString();
+            const message = "daemon restarted with incomplete persisted eval planning state";
+            await writeSyntheticResult(directory, evalId, submission.request, control, "failed", "execution_state_ambiguous", message, now);
+            await updateControl(directory, (current) => ({ ...withoutAllocation(current), state: "failed", error: { code: "execution_state_ambiguous", message } }));
+            sink.emit({ type: "eval.recovered", status: "failed", code: "execution_state_ambiguous" });
+        }
+        else {
+            queue.push({ evalId, request: submission.request, ...(submission.execution ? { execution: submission.execution } : {}), directory, resumeExisting: files === "complete" });
+            sink.emit({ type: "eval.recovered", status: "queued", recovery: files === "complete" ? "resume" : "restart-before-execution" });
+        }
+        await sink.close();
+    }
+    return queue;
+}
+async function repairIdempotencyIndex(root, submission) {
+    const keyHash = submission.idempotency_key_hash;
+    if (!keyHash)
+        return false;
+    return withFileLock(path.join(root, "locks", "eval-idempotency"), keyHash, async () => {
+        const file = idempotencyIndexPath(root, keyHash);
+        const expected = {
+            schema_version: "1",
+            eval_id: submission.eval_id,
+            submission_digest: submission.submission_digest,
+        };
+        const existing = await readJSON(file, null);
+        if (existing === null) {
+            await atomicWriteJSON(file, expected);
+            return true;
+        }
+        if (Object.keys(existing).length !== Object.keys(expected).length
+            || existing.schema_version !== expected.schema_version
+            || existing.eval_id !== expected.eval_id
+            || existing.submission_digest !== expected.submission_digest) {
+            throw new TypeError(`eval idempotency index conflicts with recovered submission: ${submission.eval_id}`);
+        }
+        return false;
+    }, { timeoutCode: "idempotency_locked", timeoutExitCode: 12 });
+}
+function combinedRecovery(results) {
+    const failure = results.find((result) => result.status === "ambiguous");
+    const recovered = results.flatMap((result) => result.recovered_lease_ids);
+    return failure
+        ? { status: "ambiguous", recovered_lease_ids: recovered, ...(failure.code ? { code: failure.code } : {}), ...(failure.message ? { message: failure.message } : {}) }
+        : { status: "resumable", recovered_lease_ids: recovered };
+}
+async function resumableFiles(directory) {
+    const [plan, executionPlan, progress, resolution] = await Promise.all([
+        readJSON(path.join(directory, "plan.json"), null),
+        readJSON(path.join(directory, "execution-plan.json"), null),
+        readJSON(path.join(directory, "progress.json"), null),
+        readJSON(path.join(directory, "resolution.json"), null),
+    ]);
+    const values = [plan, executionPlan, progress, resolution];
+    if (values.every((value) => value === null))
+        return "absent";
+    if (values.some((value) => value === null) || plan?.attempt_execution !== "harbor-task-slots-v1")
+        return "inconsistent";
+    try {
+        const parsed = parseEvalExecutionPlan(executionPlan);
+        return parsed.membership === "known"
+            && parsed.work_items.every((item) => item.task_ids.length === 1 && item.slots.length === 1 && item.logical_attempt !== null)
+            ? "complete"
+            : "inconsistent";
+    }
+    catch {
+        return "inconsistent";
+    }
+}
+async function writeSyntheticResult(directory, evalId, request, control, status, code, message, completedAt) {
+    if (await readJSON(path.join(directory, "result.json"), null))
+        return;
+    await atomicWriteJSON(path.join(directory, "result.json"), {
+        schema_version: SCHEMA_VERSION,
+        eval_id: evalId,
+        status,
+        exit_code: status === "cancelled" ? 9 : 12,
+        error: { code, message },
+        benchmark_id: request.benchmark_id,
+        benchmark_revision: request.benchmark_revision,
+        trials: [],
+        started_at: control.created_at,
+        completed_at: completedAt,
+    });
+}
+async function updateControl(directory, update) {
+    const current = parseEvalControl(await readJSON(path.join(directory, "control.json")));
+    await atomicWriteJSON(path.join(directory, "control.json"), parseEvalControl({
+        ...update(current),
+        schema_version: "1",
+        eval_id: current.eval_id,
+        generation: current.generation + 1,
+        created_at: current.created_at,
+        updated_at: new Date().toISOString(),
+    }));
+}
+function withoutAllocation(control) {
+    const { allocation_id: _allocationId, ...rest } = control;
+    return rest;
+}
+//# sourceMappingURL=eval-recovery.js.map

package/dist/src/control-plane/eval-recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-recovery.js","sourceRoot":"","sources":["../../../src/control-plane/eval-recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,uBAAuB,EAAE,QAAQ,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACjJ,OAAO,EAAE,aAAa,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAE7H,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAUzI,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAY3C;IACC,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAc,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,eAAe,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,IAAI,CAAC,CAAC;QACtG,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;QAChG,IAAI,CAAC,eAAe,IAAI,CAAC,YAAY;YAAE,SAAS;QAChD,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;QACtE,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;QAClH,MAAM,OAAO,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;QAC/C,IAAI,MAAM,sBAAsB,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YACjE,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kCAAkC,EAAE,oBAAoB,EAAE,UAAU,CAAC,oBAAoB,EAAE,CAAC,CAAC;YAC/G,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,CAAC;QACzG,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,MAAM,aAAa,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YACpK,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;YAC5J,SAAS;QACX,CAAC;QACD,IAAI,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,SAAS;QAC/C,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,cAAc,CAAC,CAAC;QAChF,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,cAAc,CAAC,CAAC;QACnF,MAAM,UAAU,GAA8B,CAAC,MAAM,4BAA4B,CAAC;gBAChF,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM;gBACN,aAAa,EAAE,SAAS;gBACxB,MAAM,EAAE,WAAW;gBACnB,GAAG,EAAE,KAAK,CAAC,aAAa,IAAI,OAAO,CAAC,GAAG;gBACvC,eAAe,EAAE,OAAO,CAAC,KAAK,KAAK,YAAY;gBAC/C,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;aAClC,CAAC,CAAC,CAAC;QACJ,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC;YAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB;gBACxE,CAAC,CAAC,MAAM,KAAK,CAAC,qBAAqB,CAAC;oBAClC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc;oBACxD,eAAe,EAAE,OAAO,CAAC,KAAK,KAAK,YAAY,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;iBACnF,CAAC;gBACF,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,OAAO,EAAE,4DAA4D,EAAE,CAAC,CAAC;QAChK,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;QAC9C,IAAI,OAAO,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,sDAAsD,EAAE,GAAG,CAAC,CAAC;YAClK,MAAM,aAAa,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,sDAAsD,EAAE,EAAE,CAAC,CAAC,CAAC;YACpM,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YAC9E,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,2BAA2B,CAAC;YAC1D,MAAM,OAAO,GAAG,qBAAqB,CAAC,QAAQ,CAAC,OAAO,IAAI,2DAA2D,EAAE,gBAAgB,CAAC,CAAC;YACzI,MAAM,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YACzG,MAAM,aAAa,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAC5H,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,KAAK,KAAK,cAAc,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,gEAAgE,CAAC;YACjF,MAAM,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,2BAA2B,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YAChI,MAAM,aAAa,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YACzJ,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC;QAC7F,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,KAAK,UAAU,EAAE,CAAC,CAAC;YAC3K,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,0BAA0B,EAAE,CAAC,CAAC;QAClI,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,IAAY,EAAE,UAA2D;IAC7G,MAAM,OAAO,GAAG,UAAU,CAAC,oBAAoB,CAAC;IAChD,IAAI,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC3B,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE;QACpF,MAAM,IAAI,GAAG,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG;YACf,cAAc,EAAE,GAAG;YACnB,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,iBAAiB,EAAE,UAAU,CAAC,iBAAiB;SAChD,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAiC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC5E,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACtC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM;eAC5D,QAAQ,CAAC,cAAc,KAAK,QAAQ,CAAC,cAAc;eACnD,QAAQ,CAAC,OAAO,KAAK,QAAQ,CAAC,OAAO;eACrC,QAAQ,CAAC,iBAAiB,KAAK,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YAC/D,MAAM,IAAI,SAAS,CAAC,+DAA+D,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3G,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,EAAE,EAAE,WAAW,EAAE,oBAAoB,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAkC;IAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAC1E,OAAO,OAAO;QACZ,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;QACpK,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,SAAiB;IAC7C,MAAM,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACpE,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC;QACjF,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,EAAE,IAAI,CAAC;QAC3E,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC;QACrE,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,IAAI,CAAC;KACxE,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC3D,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7D,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,IAAI,IAAI,EAAE,iBAAiB,KAAK,sBAAsB;QAAE,OAAO,cAAc,CAAC;IACxH,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,sBAAsB,CAAC,aAAa,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,UAAU,KAAK,OAAO;eAC/B,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC;YAC5H,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,cAAc,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,cAAc,CAAC;IAAC,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,SAAiB,EACjB,MAAc,EACd,OAAoB,EACpB,OAAsB,EACtB,MAA8B,EAC9B,IAAY,EACZ,OAAe,EACf,WAAmB;IAEnB,IAAI,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC;QAAE,OAAO;IACtE,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE;QACzD,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,MAAM;QACf,MAAM;QACN,SAAS,EAAE,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QAC1C,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;QACxB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,MAAM,EAAE,EAAE;QACV,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,YAAY,EAAE,WAAW;KAC1B,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,SAAiB,EAAE,MAAiD;IAC/F,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,gBAAgB,CAAC;QAC3E,GAAG,MAAM,CAAC,OAAO,CAAC;QAClB,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,UAAU,EAAE,OAAO,CAAC,UAAU,GAAG,CAAC;QAClC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAsB;IAC/C,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IAC1D,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/control-plane/eval-scheduler.js

@@ -0,0 +1,447 @@
+import { mkdir, rm } from "node:fs/promises";
+import path from "node:path";
+import { HitchError, atomicWriteJSON, credentialValuesFromEnv, ensureDir, hitchRootId, readJSON, safeDiagnosticMessage, sha256Bytes, sha256JSON, statePaths, withFileLock } from "../foundation/index.js";
+import { newEvalId, parseEvalExecutionPlan, readExecutionLeases, reapOwnedDockerResources, resolveLocalDatasetTaskIds, runEval, validateEvalId } from "../evals/index.js";
+import { scaleResources } from "./resources.js";
+import { CollisionLockManager } from "./collisions.js";
+import { assertExecutionPolicySupported, defaultEvalExecutionPolicy, idempotencyIndexPath, isTerminalControl, normalizeEvalSubmissionInput, parseEvalControl, reconcileIdempotencyKeys, terminalControlState, validateIdempotencyKey } from "./eval-records.js";
+import { WorkItemDispatcher } from "./work-dispatcher.js";
+import { workItemAdmission } from "./work-admission.js";
+import { localProviderStatusSnapshot, localWorkerSnapshot } from "./local-worker.js";
+import { recoverPersistedEvals } from "./eval-recovery.js";
+import { applyEvalPhase, applyEvalWorkItem, settleEvalWorkItems } from "./eval-control-work.js";
+import { EvalImageServices } from "./eval-image-services.js";
+import { writeSyntheticEvalResult } from "./synthetic-result.js";
+import { RemoteWorkCoordinator } from "./remote-work-coordinator.js";
+import { schedulerCapturePlan, schedulerQueuedEval } from "./scheduler-eval-entry.js";
+import { emitPersistedEvalEvent, updateEvalControl } from "./eval-control-state.js";
+export class EvalScheduler {
+    root;
+    evalsRoot;
+    resources;
+    trialResources;
+    workerId;
+    provider;
+    collisionDomainId;
+    executor;
+    onEvent;
+    unsubscribe;
+    unsubscribeCollisions;
+    collisions;
+    workItems;
+    dockerResourceReaper;
+    environmentImages;
+    remoteWork;
+    credentialEnv;
+    queue = [];
+    active = new Map();
+    completions = new Map();
+    accepting = true;
+    draining = false;
+    constructor({ root, resources, trialResources, executor = runEval, onEvent = () => { }, collisions = new CollisionLockManager(), workerId, provider = "local-docker", collisionDomainId, dockerResourceReaper = reapOwnedDockerResources, environmentImageResolver, environmentImageBuilder, remoteWorkers, remoteWorkerProtocol, credentialEnv = process.env }) {
+        this.root = root;
+        this.evalsRoot = statePaths(root).evals;
+        this.resources = resources;
+        this.trialResources = trialResources;
+        const rootHash = hitchRootId(root);
+        this.workerId = workerId || `worker_${rootHash}`;
+        this.provider = provider;
+        this.collisionDomainId = collisionDomainId || `local-docker-root:${rootHash}`;
+        this.executor = executor;
+        this.onEvent = onEvent;
+        this.collisions = collisions;
+        this.workItems = new WorkItemDispatcher({ resources, collisions });
+        this.dockerResourceReaper = dockerResourceReaper;
+        this.credentialEnv = credentialEnv;
+        this.environmentImages = new EvalImageServices({ root, provider, resources, onEvent: this.onEvent, ...(environmentImageResolver ? { resolver: environmentImageResolver } : {}), ...(environmentImageBuilder ? { builder: environmentImageBuilder } : {}) });
+        this.remoteWork = remoteWorkers && remoteWorkerProtocol
+            ? new RemoteWorkCoordinator({ root, registry: remoteWorkers, protocol: remoteWorkerProtocol, collisions })
+            : undefined;
+        this.unsubscribe = resources.subscribe(() => this.scheduleDrain());
+        this.unsubscribeCollisions = collisions.subscribe(() => this.scheduleDrain());
+    }
+    async initialize() {
+        await ensureDir(this.evalsRoot);
+        await this.recoverInterruptedEvals();
+        if (this.provider === "local-docker") {
+            try {
+                const report = await this.dockerResourceReaper({ root: this.root });
+                this.onEvent({ type: "docker.reaper.completed", root_id: report.root_id, scanned: report.scanned, deleted: report.deleted.length, issues: report.issues.length });
+            }
+            catch (error) {
+                this.onEvent({ type: "docker.reaper.failed", code: error.code || "docker_reaper_failed" });
+            }
+        }
+        this.scheduleDrain();
+    }
+    async submit(input, options = {}) {
+        if (!this.accepting)
+            throw new HitchError("daemon is shutting down", { code: "daemon_shutting_down", exitCode: 12 });
+        const normalized = await normalizeEvalSubmissionInput(input, { provider: this.provider, trialResources: this.trialResources });
+        const remote = normalized.execution.provider !== this.provider;
+        assertExecutionPolicySupported(normalized.execution, remote ? normalized.execution.provider : this.provider);
+        if (remote && !this.remoteWork)
+            throw new HitchError(`execution provider is unavailable: ${normalized.execution.provider}`, { code: "execution_provider_unavailable", exitCode: 10 });
+        if (remote && normalized.execution.build.mode !== "backend")
+            throw new HitchError("remote worker image prebuild is not available yet", { code: "remote_build_mode_unsupported", exitCode: 10 });
+        if (remote && await resolveLocalDatasetTaskIds(normalized.request.dataset) === null)
+            throw new HitchError("remote workers require enumerable task membership", { code: "remote_opaque_membership_unsupported", exitCode: 10 });
+        const modelCapturePlan = await schedulerCapturePlan({ request: normalized.request, execution: normalized.execution, localProvider: this.provider, localStatus: this.providerSnapshot(), ...(this.remoteWork ? { remoteWork: this.remoteWork } : {}) });
+        const idempotencyKey = reconcileIdempotencyKeys(normalized.idempotencyKey, options.idempotencyKey);
+        const canFit = remote
+            ? await this.remoteWork.canEverFit(normalized.execution.provider, normalized.execution.resources.default_trial)
+            : this.resources.canEverFit(normalized.execution.resources.default_trial);
+        if (!canFit) {
+            throw new HitchError("one eval trial exceeds the daemon resource capacity", {
+                code: "resource_request_unsatisfiable",
+                exitCode: 10,
+            });
+        }
+        const submissionDigest = sha256JSON({ request: normalized.request, execution: normalized.execution });
+        if (idempotencyKey !== undefined)
+            validateIdempotencyKey(idempotencyKey);
+        if (idempotencyKey) {
+            const keyHash = sha256Bytes(idempotencyKey);
+            return withFileLock(path.join(this.root, "locks", "eval-idempotency"), keyHash, async () => {
+                const indexPath = idempotencyIndexPath(this.root, keyHash);
+                const existing = await readJSON(indexPath, null);
+                if (existing) {
+                    if (existing.submission_digest !== submissionDigest || typeof existing.eval_id !== "string") {
+                        throw new HitchError("idempotency key was already used for a different eval request", {
+                            code: "idempotency_conflict",
+                            exitCode: 2,
+                        });
+                    }
+                    return validateEvalId(existing.eval_id);
+                }
+                const entry = await this.persistSubmission(normalized.request, normalized.execution, modelCapturePlan, submissionDigest, keyHash);
+                try {
+                    await atomicWriteJSON(indexPath, { schema_version: "1", eval_id: entry.evalId, submission_digest: submissionDigest });
+                }
+                catch (error) {
+                    await rm(entry.directory, { recursive: true, force: true });
+                    throw error;
+                }
+                return this.enqueue(entry);
+            }, { timeoutCode: "idempotency_locked", timeoutExitCode: 12 });
+        }
+        return this.enqueue(await this.persistSubmission(normalized.request, normalized.execution, modelCapturePlan, submissionDigest));
+    }
+    async persistSubmission(normalized, execution, modelCapturePlan, submissionDigest, keyHash) {
+        const evalId = newEvalId();
+        const directory = path.join(this.evalsRoot, evalId);
+        await mkdir(directory, { mode: 0o700 });
+        try {
+            const now = new Date().toISOString();
+            const submission = {
+                schema_version: "1",
+                eval_id: evalId,
+                request: normalized,
+                execution,
+                submission_digest: submissionDigest,
+                ...(keyHash ? { idempotency_key_hash: keyHash } : {}),
+                submitted_at: now,
+            };
+            const control = {
+                schema_version: "1",
+                eval_id: evalId,
+                generation: 0,
+                state: "queued",
+                requested_parallelism: execution.max_parallelism,
+                admitted_parallelism: 0,
+                active_leases: [],
+                queued_work_items: [],
+                terminal_work_items: [],
+                created_at: now,
+                updated_at: now,
+            };
+            await atomicWriteJSON(path.join(directory, "request.json"), normalized);
+            await atomicWriteJSON(path.join(directory, "submission.json"), submission);
+            await atomicWriteJSON(path.join(directory, "control.json"), control);
+            await this.emitPersisted(directory, evalId, { type: "eval.queued", requested_parallelism: execution.max_parallelism, model_capture: modelCapturePlan });
+            return this.queuedEval(evalId, normalized, execution, modelCapturePlan, directory);
+        }
+        catch (error) {
+            await rm(directory, { recursive: true, force: true });
+            throw error;
+        }
+    }
+    enqueue(entry) {
+        this.queue.push(entry);
+        this.scheduleDrain();
+        return entry.evalId;
+    }
+    async status(evalIdValue) {
+        const evalId = validateEvalId(evalIdValue);
+        const directory = path.join(this.evalsRoot, evalId);
+        const request = await readJSON(path.join(directory, "request.json"), null);
+        const submission = await readJSON(path.join(directory, "submission.json"), null);
+        const controlValue = await readJSON(path.join(directory, "control.json"), null);
+        if (!request || !controlValue)
+            return null;
+        const control = parseEvalControl(controlValue);
+        const persistedResult = await readJSON(path.join(directory, "result.json"), null);
+        const result = isTerminalControl(control.state) ? persistedResult : null;
+        const progress = await readJSON(path.join(directory, "progress.json"), null);
+        const active = this.active.get(evalId);
+        const workItems = this.workItems.evalSnapshot(evalId);
+        const admitted = active?.fineGrained ? workItems?.active ?? 0 : control.admitted_parallelism;
+        return {
+            request,
+            execution: submission?.execution ?? null,
+            control,
+            progress,
+            result,
+            leases: await readExecutionLeases(directory),
+            effective_parallelism: {
+                requested: control.requested_parallelism,
+                admitted,
+                running: active ? admitted : 0,
+            },
+        };
+    }
+    async cancel(evalIdValue) {
+        const evalId = validateEvalId(evalIdValue);
+        const queuedIndex = this.queue.findIndex((entry) => entry.evalId === evalId);
+        if (queuedIndex >= 0) {
+            const [entry] = this.queue.splice(queuedIndex, 1);
+            if (!entry)
+                return "not_cancellable";
+            const now = new Date().toISOString();
+            await this.emitPersisted(entry.directory, evalId, { type: "eval.cancel.requested", phase: "queued" });
+            await writeSyntheticEvalResult({ directory: entry.directory, evalId: entry.evalId, request: entry.request, status: "cancelled", code: "cancelled", message: "eval cancelled before launch", completedAt: now });
+            await this.updateControl(entry.directory, (control) => ({
+                ...control,
+                state: "cancelled",
+                admitted_parallelism: 0,
+                cancel_requested_at: now,
+            }));
+            await this.emitPersisted(entry.directory, evalId, { type: "eval.cancelled", phase: "queued" });
+            return "accepted";
+        }
+        const active = this.active.get(evalId);
+        if (active) {
+            const directory = path.join(this.evalsRoot, evalId);
+            const now = new Date().toISOString();
+            await this.emitPersisted(directory, evalId, { type: "eval.cancel.requested", phase: "running" });
+            const control = await this.updateControl(directory, (control) => isTerminalControl(control.state) ? control : ({
+                ...control,
+                state: "cancelling",
+                cancel_requested_at: control.cancel_requested_at || now,
+            }));
+            if (isTerminalControl(control.state))
+                return "terminal";
+            active.controller.abort();
+            return "accepted";
+        }
+        const status = await this.status(evalId).catch(() => null);
+        if (!status)
+            return "not_found";
+        return isTerminalControl(status.control.state) || status.result ? "terminal" : "not_cancellable";
+    }
+    snapshot() {
+        return {
+            queued: this.queue.length,
+            running: this.active.size,
+            accepting: this.accepting,
+            active: [...this.active.entries()].map(([evalId, active]) => ({
+                eval_id: evalId,
+                requested_parallelism: active.requestedParallelism,
+                admitted_parallelism: active.fineGrained ? this.workItems.evalSnapshot(evalId)?.active ?? 0 : active.admittedParallelism,
+                ...(active.lease ? { allocation_id: active.lease.allocation.allocation_id } : {}),
+                collision_keys: active.collisions?.keys ?? [],
+                scheduling: active.fineGrained ? "work-item-drr-v1" : "coarse-eval-v1",
+            })),
+            collisions: this.collisions.snapshot(),
+            work_items: this.workItems.snapshot(),
+        };
+    }
+    workerSnapshot() {
+        return localWorkerSnapshot({ workerId: this.workerId, provider: this.provider, collisionDomainId: this.collisionDomainId, accepting: this.accepting, resources: this.resources });
+    }
+    providerSnapshot() {
+        return localProviderStatusSnapshot({ workerId: this.workerId, provider: this.provider, collisionDomainId: this.collisionDomainId, accepting: this.accepting, resources: this.resources });
+    }
+    async shutdown() {
+        if (!this.accepting && this.completions.size === 0)
+            return;
+        this.accepting = false;
+        for (const entry of [...this.queue])
+            await this.cancel(entry.evalId);
+        for (const [evalId] of this.active)
+            await this.cancel(evalId);
+        await Promise.all([...this.completions.values()]);
+        this.workItems.close();
+        this.environmentImages.close();
+        this.unsubscribe();
+        this.unsubscribeCollisions();
+    }
+    scheduleDrain() {
+        if (this.draining)
+            return;
+        queueMicrotask(() => this.drain());
+    }
+    drain() {
+        if (this.draining || !this.accepting)
+            return;
+        this.draining = true;
+        try {
+            for (;;) {
+                const selected = this.selectRunnable();
+                if (!selected)
+                    break;
+                const { index, parallelism, lease, collisions, fineGrained } = selected;
+                const [entry] = this.queue.splice(index, 1);
+                if (!entry) {
+                    lease?.release();
+                    collisions?.release();
+                    continue;
+                }
+                this.start(entry, parallelism, fineGrained, lease, collisions);
+            }
+        }
+        finally {
+            this.draining = false;
+        }
+    }
+    selectRunnable() {
+        for (let index = 0; index < this.queue.length; index += 1) {
+            const entry = this.queue[index];
+            if (entry.fineGrained)
+                return { index, parallelism: entry.execution.max_parallelism, fineGrained: true };
+            const unit = entry.execution.resources.default_trial;
+            const parallelism = this.resources.maximumUnits(unit, entry.execution.max_parallelism);
+            if (parallelism < 1)
+                continue;
+            const collisions = this.collisions.tryAcquire(entry.evalId, entry.collisionKeys);
+            if (!collisions)
+                continue;
+            const lease = this.resources.tryAcquire(entry.evalId, "eval", scaleResources(unit, parallelism));
+            if (lease)
+                return { index, parallelism, fineGrained: false, lease, collisions };
+            collisions.release();
+        }
+        return null;
+    }
+    start(entry, parallelism, fineGrained, lease, collisions) {
+        const controller = new AbortController();
+        this.active.set(entry.evalId, { controller, requestedParallelism: entry.execution.max_parallelism, admittedParallelism: parallelism, fineGrained, ...(lease ? { lease } : {}), ...(collisions ? { collisions } : {}) });
+        const completion = this.execute(entry, parallelism, fineGrained, lease, controller)
+            .catch((error) => this.recordUnexpectedFailure(entry, error))
+            .finally(() => {
+            this.active.delete(entry.evalId);
+            this.completions.delete(entry.evalId);
+            lease?.release();
+            collisions?.release();
+            this.scheduleDrain();
+        });
+        this.completions.set(entry.evalId, completion);
+    }
+    async execute(entry, parallelism, fineGrained, lease, controller) {
+        const remote = entry.execution.provider !== this.provider;
+        const queuedControl = parseEvalControl(await readJSON(path.join(entry.directory, "control.json")));
+        await this.emitPersisted(entry.directory, entry.evalId, {
+            type: "eval.dispatch.started",
+            duration_ms: Math.max(0, Date.now() - Date.parse(queuedControl.created_at)),
+            requested_parallelism: entry.execution.max_parallelism,
+            admitted_parallelism: fineGrained ? 0 : parallelism,
+        });
+        await this.updateControl(entry.directory, (control) => ({
+            ...control,
+            state: "planning",
+            admitted_parallelism: fineGrained ? 0 : parallelism,
+            active_leases: [],
+            ...(lease ? { allocation_id: lease.allocation.allocation_id } : {}),
+        }));
+        const result = await this.executor({
+            evalId: entry.evalId,
+            request: entry.request,
+            normalizedRequest: entry.request,
+            maxConcurrentOverride: parallelism,
+            executionResources: entry.execution.resources.default_trial,
+            ...(entry.modelCapturePlan ? { modelCapturePlan: entry.modelCapturePlan } : {}),
+            executionResourceSource: "submission-default",
+            executionStrategy: "local-task-slots-v1",
+            environmentBuildMode: entry.execution.build.mode,
+            ...this.environmentImages.runOptions(),
+            executionWorker: {
+                workerId: remote ? "worker_remote_pool" : this.workerId,
+                provider: entry.execution.provider,
+                collisionDomainId: remote ? `remote-pool:${entry.execution.provider}` : this.collisionDomainId,
+                ...(lease ? { parentAllocationId: lease.allocation.allocation_id } : {}),
+            },
+            ...(fineGrained && !remote ? {
+                workItemAdmission: workItemAdmission({ dispatcher: this.workItems, request: entry.request, collisionDomainId: this.collisionDomainId }),
+            } : {}),
+            ...(remote && this.remoteWork ? { remoteWorkExecutor: this.remoteWork.execute } : {}),
+            precreated: true,
+            resumeExisting: entry.resumeExisting,
+            root: this.root,
+            env: this.credentialEnv,
+            signal: controller.signal,
+            onEvent: this.onEvent,
+            ...(remote ? {} : { dockerResourceReaper: this.dockerResourceReaper }),
+            onControlPhase: async (phase, work) => {
+                await this.updateControl(entry.directory, (control) => applyEvalPhase(control, phase, work?.queuedWorkItems, work?.terminalWorkItems));
+            },
+            onWorkItemState: async (workId, leaseId, state) => {
+                await this.updateControl(entry.directory, (control) => applyEvalWorkItem(control, workId, leaseId, state));
+            },
+        });
+        if (!await readJSON(path.join(entry.directory, "result.json"), null)) {
+            await atomicWriteJSON(path.join(entry.directory, "result.json"), result);
+        }
+        const state = terminalControlState(result.status);
+        await this.updateControl(entry.directory, (control) => {
+            const { allocation_id: _allocationId, ...released } = control;
+            return {
+                ...settleEvalWorkItems(released),
+                state,
+                admitted_parallelism: fineGrained ? 0 : parallelism,
+                ...(result.error ? { error: result.error } : {}),
+            };
+        });
+    }
+    async recordUnexpectedFailure(entry, error) {
+        const now = new Date().toISOString();
+        const cancelled = this.active.get(entry.evalId)?.controller.signal.aborted === true;
+        const code = cancelled ? "cancelled" : "control_plane_error";
+        const message = cancelled
+            ? "eval was cancelled"
+            : safeDiagnosticMessage(error, credentialValuesFromEnv(entry.request.pass_env, this.credentialEnv));
+        await writeSyntheticEvalResult({ directory: entry.directory, evalId: entry.evalId, request: entry.request, status: cancelled ? "cancelled" : "failed", code, message, completedAt: now });
+        await this.updateControl(entry.directory, (control) => {
+            const { allocation_id: _allocationId, ...released } = control;
+            return { ...settleEvalWorkItems(released), state: cancelled ? "cancelled" : "failed", error: { code, message } };
+        });
+        this.onEvent({ type: "eval.scheduler.error", eval_id: entry.evalId, code });
+    }
+    async recoverInterruptedEvals() {
+        const remoteWork = this.remoteWork;
+        const recovered = await recoverPersistedEvals({
+            root: this.root, evalsRoot: this.evalsRoot, onEvent: this.onEvent, credentialEnv: this.credentialEnv,
+            ...(remoteWork ? { recoverProviderLeases: (input) => remoteWork.recoverEvalLeases(input) } : {}),
+        });
+        for (const entry of recovered) {
+            const execution = entry.execution || defaultEvalExecutionPolicy(entry.request, { provider: this.provider, trialResources: this.trialResources, buildMode: "backend" });
+            assertExecutionPolicySupported(execution, execution.provider === this.provider ? this.provider : execution.provider);
+            if (execution.provider !== this.provider && !this.remoteWork)
+                throw new HitchError(`execution provider is unavailable: ${execution.provider}`, { code: "execution_provider_unavailable", exitCode: 10 });
+            const persistedPlan = entry.resumeExisting
+                ? parseEvalExecutionPlan(await readJSON(path.join(entry.directory, "execution-plan.json")))
+                : null;
+            const capturePlan = persistedPlan ? persistedPlan.model_capture : await schedulerCapturePlan({ request: entry.request, execution, localProvider: this.provider, localStatus: this.providerSnapshot(), ...(this.remoteWork ? { remoteWork: this.remoteWork } : {}) });
+            this.queue.push(await this.queuedEval(entry.evalId, entry.request, execution, capturePlan, entry.directory, entry.resumeExisting));
+        }
+    }
+    async queuedEval(evalId, request, execution, modelCapturePlan, directory, resumeExisting = false) {
+        return schedulerQueuedEval({ evalId, request, execution, ...(modelCapturePlan ? { modelCapturePlan } : {}), directory, collisionDomainId: this.collisionDomainId, resumeExisting });
+    }
+    async updateControl(directory, update) {
+        return updateEvalControl(directory, update);
+    }
+    async emitPersisted(directory, evalId, event) {
+        return emitPersistedEvalEvent(directory, evalId, this.onEvent, event);
+    }
+}
+//# sourceMappingURL=eval-scheduler.js.map