agent-hitch 0.2.5 → 0.2.7

package/dist/src/evals/service.js

@@ -1,42 +1,56 @@
 import path from "node:path";
-import { mkdir } from "node:fs/promises";
-import { prepareHarness, preparedArtifactDirectory, resolveHarness } from "../artifacts/index.js";
-import { HitchError, SCHEMA_VERSION, atomicWriteJSON, ensureDir, invalidInput, statePaths } from "../foundation/index.js";
+import { resolveHarness } from "../artifacts/index.js";
+import { HitchError, SCHEMA_VERSION, atomicWriteJSON, beginEvalEnvironmentImagePlanning, credentialValuesFromEnv, ensureDir, invalidInput, safeDiagnosticMessage, statePaths, withEnvironmentImageReferenceLock, writeEvalEnvironmentImageReferences } from "../foundation/index.js";
 import { parseHarnessReference } from "../revisions/index.js";
 import { buildLocalGitTransport, lockedHarnessRef, runHarborBackend, verifyLocalGitTransport } from "../backends/index.js";
 import { ensureControllerRuntime, writeRuntimeReference } from "../controller-runtime/index.js";
 import { importEvalTrialRun, importEvalTrialRuns, TrialBundlePendingError, validateEvalTrialReferences } from "./trial-import.js";
 import { EvalEventSink } from "./events.js";
 import { createEvalProgress, mergeEvalProgressTrial, writeEvalProgress } from "./progress.js";
+import { infrastructureFailureTrials, runInfrastructureRetries } from "./infrastructure-retry.js";
 import { newEvalId, resolveLocalDatasetTaskIds, validateEvalId, validateEvalRequest } from "./request.js";
 import { invalidTrialSlots } from "./rerun-slots.js";
-export async function runEval({ evalId = newEvalId(), request, root, env = process.env, harborExecutable, signal, onEvent, trialBundleGraceMs }) {
+import { prepareEvalDirectory } from "./directory.js";
+import { buildEvalExecutionPlan, DEFAULT_EVAL_TRIAL_RESOURCES } from "./execution-plan.js";
+import { planLocalEvalInputs } from "./local-eval-planning.js";
+import { resolvedImageMapping } from "./environment-image-planning.js";
+import { assertBackendTrialSet, attemptDirectoryName, localSourceBackendFailure, summarizeTrialRefs, transportSummary } from "./result-helpers.js";
+import { executePlannedHarborTasks } from "./planned-execution.js";
+import { assertEvalResumeState, executionPlanWorkState, loadEvalResumeState } from "./resume-state.js";
+import { modelCaptureDegradationEvent, resolveEvalModelCapturePlan } from "./model-capture-plan.js";
+import { finalizeEvalResult } from "./eval-finalization.js";
+import { prepareHostHarborArtifactForTest } from "./prepared-harness.js";
+import { prepareHarborArtifact } from "./harbor-artifact-builder.js";
+import { prepareEvalArtifactAssignments, preparedArtifactPlanFields } from "./eval-artifact-planning.js";
+import { startEvalModelCaptureRuntime } from "./model-capture-runtime.js";
+import { recoverPromotedEvalTrialPublications } from "./trial-publication-recovery.js";
+import { emitEvalPlanLifecycle } from "./eval-lifecycle-events.js";
+export async function runEval({ evalId = newEvalId(), request, root, env = process.env, harborExecutable, signal, onEvent, trialBundleGraceMs, precreated = false, normalizedRequest, maxConcurrentOverride, executionResources, executionResourceSource = "operator-default", executionStrategy = "legacy-attempt-shards", executionWorker, modelCapturePlan, workItemAdmission, remoteWorkExecutor, resumeExisting = false, onControlPhase, onWorkItemState, dockerResourceReaper, environmentBuildMode = "backend", environmentImageResolver, environmentImageBuilder, environmentImageManifestLoader, harborArtifactBuilder }) {
     if (!root)
         throw invalidInput("a Hitch state root is required for eval");
     evalId = validateEvalId(evalId);
-    const normalized = await validateEvalRequest(request);
-    const evalsDirectory = await ensureDir(statePaths(root).evals);
-    const evalDirectory = path.join(evalsDirectory, evalId);
-    try {
-        await mkdir(evalDirectory, { mode: 0o700 });
-    }
-    catch (error) {
-        if (error.code === "EEXIST") {
-            throw new HitchError(`eval ID already exists: ${evalId}`, { code: "eval_id_conflict", exitCode: 2 });
-        }
-        throw error;
+    const persistedRequest = normalizedRequest || await validateEvalRequest(request);
+    if (maxConcurrentOverride !== undefined && (!Number.isSafeInteger(maxConcurrentOverride) || maxConcurrentOverride < 1 || maxConcurrentOverride > persistedRequest.max_concurrent)) {
+        throw invalidInput("control-plane max concurrency override is invalid");
     }
-    const startedAt = new Date();
+    const normalized = maxConcurrentOverride === undefined ? persistedRequest : { ...persistedRequest, max_concurrent: maxConcurrentOverride };
+    const evalsDirectory = await ensureDir(statePaths(root).evals);
+    const evalDirectory = await prepareEvalDirectory({ evalsDirectory, evalId, request: persistedRequest, precreated });
+    let startedAt = new Date();
     const sink = new EvalEventSink(evalDirectory, evalId, onEvent);
     await sink.open();
-    await atomicWriteJSON(path.join(evalDirectory, "request.json"), normalized);
     let result;
     let trialRefs = [];
     let progress = null;
+    let captureRuntime;
     try {
+        const planningStartedAt = Date.now();
         sink.emit({ type: "eval.started", backend: normalized.backend, dataset: normalized.dataset });
+        sink.emit({ type: "eval.planning.started" });
+        await onControlPhase?.("planning");
         const requestedReference = parseHarnessReference(normalized.harness_ref);
         const resolvedRevision = await resolveHarness(normalized.harness_ref, { root, env });
+        await onControlPhase?.("preparing");
         let localTransport;
         if (resolvedRevision.source.type === "git" && resolvedRevision.source.registered !== true) {
             const selector = requestedReference.selector;
@@ -61,50 +75,6 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
                 status: "verified",
             });
         }
-        const selector = requestedReference.selector;
-        const verifiedLocalGitSource = localTransport && selector.type === "commit" && selector.source?.explicit
-            ? {
-                directory: selector.source.local_path,
-                commit: localTransport.manifest.commit,
-                tree: localTransport.manifest.tree,
-                resolutionIdentity: localTransport.manifest.resolution_identity,
-                payloadSha256: localTransport.manifest.payload_sha256,
-            }
-            : undefined;
-        const artifact = await prepareHarness(resolvedRevision, {
-            root,
-            env,
-            ...(signal ? { signal } : {}),
-            ...(verifiedLocalGitSource ? { verifiedLocalGitSource } : {}),
-        });
-        if (!artifact.artifact_integrity || !artifact.entrypoint_integrity) {
-            throw new HitchError("host-prepared harness artifact has no complete integrity metadata", {
-                code: "artifact_integrity_mismatch",
-                exitCode: 5,
-            });
-        }
-        const preparedArtifact = {
-            directory: preparedArtifactDirectory(root, artifact.artifact_id),
-            artifact_id: artifact.artifact_id,
-            artifact_integrity: artifact.artifact_integrity,
-            entrypoint_integrity: artifact.entrypoint_integrity,
-            harness_id: artifact.harness_id,
-            revision_identity: artifact.revision_identity,
-            adapter_version: artifact.adapter_version,
-            recipe_version: artifact.recipe_version,
-            platform: artifact.platform,
-            node_version: artifact.toolchain.node || process.version,
-            source_type: artifact.source_type,
-        };
-        sink.emit({
-            type: "eval.harness-artifact.prepared",
-            harness: artifact.harness_id,
-            artifact_id: artifact.artifact_id,
-            platform: artifact.platform,
-            cache_hit: artifact.cache_hit,
-        });
-        // Phase 2: the shared, read-only, SHA-256-addressed controller runtime
-        // cache replaces the per-eval Hitch runtime copy (spec §4).
         const controllerRuntime = await ensureControllerRuntime({ root });
         const runtimeRefFile = await writeRuntimeReference(evalDirectory, controllerRuntime);
         sink.emit({
@@ -114,10 +84,27 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             reference: runtimeRefFile,
         });
         const localTaskIds = await resolveLocalDatasetTaskIds(normalized.dataset);
+        const resume = resumeExisting ? await loadEvalResumeState(evalDirectory) : null;
+        await withEnvironmentImageReferenceLock(root, () => beginEvalEnvironmentImagePlanning(evalDirectory, evalId));
+        const localPlanning = await planLocalEvalInputs({ root, dataset: normalized.dataset, taskIds: localTaskIds, defaultResources: executionResources ?? DEFAULT_EVAL_TRIAL_RESOURCES, defaultSource: executionResourceSource, benchmarkId: normalized.benchmark_id, benchmarkRevision: normalized.benchmark_revision, buildMode: environmentBuildMode, harborTaskResourceInspector: path.join(controllerRuntime.directory, "payload", "integrations", "harbor", "hitch_harbor_task_resources.py"), ...(environmentImageResolver ? { resolver: environmentImageResolver } : {}), ...(environmentImageBuilder ? { builder: environmentImageBuilder } : {}), ...(resume ? { resumePlan: resume.executionPlan } : {}), ...(harborExecutable ? { harborExecutable } : {}), env, ...(signal ? { signal } : {}) });
+        await withEnvironmentImageReferenceLock(root, () => writeEvalEnvironmentImageReferences(evalDirectory, evalId, localPlanning.environmentImages));
+        const selectedArtifactBuilder = harborArtifactBuilder ?? (env.NODE_ENV === "test" && env.HITCH_TEST_HOST_ARTIFACT_BUILDER === "1" ? prepareHostHarborArtifactForTest : prepareHarborArtifact);
+        const prepared = await prepareEvalArtifactAssignments({ builder: selectedArtifactBuilder, root, resolvedRevision, requestedReference, controllerRuntime, ...(localTransport ? { localTransport } : {}), taskRuntimeContracts: localPlanning.taskRuntimeContracts, sink, env, ...(signal ? { signal } : {}) });
+        const preparedAssignments = prepared.assignments;
+        const preparedArtifact = prepared.primary;
+        const preparedArtifacts = prepared.artifactsById;
+        const multipleRuntimeContracts = preparedAssignments.length > 1;
+        const taskResources = localPlanning.taskResources;
         const plannedTasks = localTaskIds?.length ?? null;
         const plannedTrials = plannedTasks === null ? null : plannedTasks * normalized.attempts;
         if (plannedTrials !== null && !Number.isSafeInteger(plannedTrials))
             throw invalidInput("planned trial count exceeds the safe integer range");
+        if (resume)
+            startedAt = new Date(resume.progress.started_at);
+        const capture = resolveEvalModelCapturePlan({ requested: modelCapturePlan, resumed: resume?.executionPlan.model_capture, resuming: Boolean(resume) });
+        const activeCaptureRuntime = await startEvalModelCaptureRuntime({ plan: capture.plan, evalId, evalDirectory, env });
+        captureRuntime = activeCaptureRuntime;
+        capture.plan = activeCaptureRuntime.plan;
         const plan = {
             schema_version: SCHEMA_VERSION,
             eval_id: evalId,
@@ -134,20 +121,62 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             benchmark_id: normalized.benchmark_id,
             benchmark_revision: normalized.benchmark_revision,
             attempts: normalized.attempts,
-            attempt_execution: "harbor-attempt-shards-v1",
+            attempt_execution: (executionStrategy === "local-task-slots-v1" || multipleRuntimeContracts) && localTaskIds !== null
+                ? "harbor-task-slots-v1"
+                : "harbor-attempt-shards-v1",
             max_concurrent: normalized.max_concurrent,
+            ...(capture.persist ? { model_capture: capture.plan } : {}),
+            infrastructure_retries: normalized.infrastructure_retries,
+            infrastructure_retry_backoff_ms: normalized.infrastructure_retry_backoff_ms,
             ...(localTaskIds === null ? {} : { tasks: localTaskIds }),
             controller_runtime: {
                 runtime_id: controllerRuntime.runtime_id,
                 manifest_digest: controllerRuntime.manifest_digest,
             },
-            prepared_artifact: preparedArtifactSummary(preparedArtifact),
+            ...preparedArtifactPlanFields(preparedAssignments),
             ...(localTransport ? { local_source_transport: transportSummary(localTransport) } : {}),
-            created_at: new Date().toISOString(),
+            created_at: typeof resume?.plan.created_at === "string" ? resume.plan.created_at : new Date().toISOString(),
         };
-        await atomicWriteJSON(path.join(evalDirectory, "resolution.json"), resolvedRevision);
-        await atomicWriteJSON(path.join(evalDirectory, "plan.json"), plan);
-        progress = createEvalProgress({
+        const expectedExecutionPlan = buildEvalExecutionPlan({
+            evalId,
+            request: normalized,
+            candidate: {
+                revisionIdentity: resolvedRevision.identity,
+                artifactId: preparedArtifact.artifact_id,
+                artifactAssignments: preparedAssignments.map((entry) => ({
+                    taskIds: entry.taskIds,
+                    artifactId: entry.artifact.artifact_id,
+                    runtimeContract: {
+                        docker_platform: entry.runtimeContract.dockerPlatform,
+                        artifact_platform: entry.runtimeContract.artifactPlatform,
+                        node_version: entry.runtimeContract.nodeVersion,
+                    },
+                })),
+            },
+            tasks: localTaskIds,
+            maxParallelism: normalized.max_concurrent,
+            ...(executionResources ? { trialResources: executionResources } : {}),
+            ...(taskResources ? { taskResources } : {}),
+            ...(localPlanning.environmentImages.length > 0 ? { environmentImages: localPlanning.environmentImages } : {}),
+            ...(localPlanning.environmentImageFallbacks.length > 0 ? { environmentImageFallbacks: localPlanning.environmentImageFallbacks } : {}),
+            ...(executionWorker ? { provider: executionWorker.provider } : {}),
+            modelCapture: capture.persist ? capture.plan : null,
+            ...((executionStrategy === "local-task-slots-v1" || multipleRuntimeContracts) && localTaskIds !== null ? { workItemMode: "task-slots" } : {}),
+            createdAt: plan.created_at,
+        });
+        if (resume)
+            assertEvalResumeState({ state: resume, expectedPlan: plan, expectedExecutionPlan, expectedResolutionIdentity: resolvedRevision.identity, plannedTasks, plannedTrials });
+        else
+            await Promise.all([
+                atomicWriteJSON(path.join(evalDirectory, "resolution.json"), resolvedRevision),
+                atomicWriteJSON(path.join(evalDirectory, "plan.json"), plan),
+                atomicWriteJSON(path.join(evalDirectory, "execution-plan.json"), expectedExecutionPlan),
+            ]);
+        const executionPlan = resume?.executionPlan ?? expectedExecutionPlan;
+        const captureDegradation = modelCaptureDegradationEvent(executionPlan.model_capture);
+        if (captureDegradation)
+            sink.emit(captureDegradation);
+        progress = resume?.progress ?? createEvalProgress({
             evalId,
             benchmarkId: normalized.benchmark_id,
             benchmarkRevision: normalized.benchmark_revision,
@@ -155,7 +184,10 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             plannedTrials,
             startedAt: startedAt.toISOString(),
         });
-        await writeEvalProgress(evalDirectory, progress);
+        if (!resume)
+            await writeEvalProgress(evalDirectory, progress);
+        else
+            progress = (await recoverPromotedEvalTrialPublications({ root, evalDirectory, plan: executionPlan, progress, sink })).progress;
         sink.emit({
             type: "eval.planned",
             harness: resolvedRevision.harness_id,
@@ -163,7 +195,11 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             attempts: normalized.attempts,
             planned_tasks: plannedTasks,
             planned_trials: plannedTrials,
+            work_items: executionPlan.work_items.length,
+            membership: executionPlan.membership,
         });
+        emitEvalPlanLifecycle(sink, executionPlan, planningStartedAt);
+        await onControlPhase?.("running", executionPlanWorkState(executionPlan, progress));
         if (localTransport) {
             await verifyLocalGitTransport(localTransport, {
                 expected: {
@@ -176,126 +212,213 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             });
         }
         const backendRuns = [];
-        for (let logicalAttempt = 1; logicalAttempt <= normalized.attempts; logicalAttempt += 1) {
-            if (signal?.aborted)
-                break;
-            const backendDirectory = normalized.attempts === 1
-                ? path.join(evalDirectory, "harbor")
-                : path.join(evalDirectory, "harbor", attemptDirectoryName(logicalAttempt));
-            const harborJobDirectory = path.join(backendDirectory, "job");
-            const shardRefs = [];
-            const publish = async (ref) => {
-                if (ref.attempt !== logicalAttempt) {
-                    throw new HitchError(`Harbor returned attempt ${ref.attempt}, expected ${logicalAttempt}`, {
-                        code: "eval_trial_attempt_mismatch",
-                        exitCode: 12,
-                    });
-                }
-                if (!shardRefs.some((current) => current.trial_id === ref.trial_id))
-                    shardRefs.push(ref);
-                if (progress === null)
-                    throw new Error("eval progress was not initialized");
-                const previousGeneration = progress.generation;
-                progress = mergeEvalProgressTrial(progress, ref);
-                if (progress.generation === previousGeneration)
-                    return;
-                await validateEvalTrialReferences(root, evalId, [ref], {
-                    benchmarkId: normalized.benchmark_id,
-                    benchmarkRevision: normalized.benchmark_revision,
-                });
-                await writeEvalProgress(evalDirectory, progress);
-                sink.emit({
-                    type: "eval.trial.published",
-                    trial_id: ref.trial_id,
-                    task_id: ref.task_id,
-                    attempt: ref.attempt,
-                    run_id: ref.run_id,
-                    observation_status: ref.observation_status,
-                    settled_trials: progress.trials.length,
-                    generation: progress.generation,
-                });
-            };
-            const backendRun = await runHarborBackend({
+        const infrastructureRetryRuns = [];
+        const plannedTaskExecution = (executionStrategy === "local-task-slots-v1" || multipleRuntimeContracts) && localTaskIds !== null;
+        if (plannedTaskExecution) {
+            const execution = await executePlannedHarborTasks({
                 evalId,
                 evalDirectory,
-                backendDirectory,
-                logicalAttempt,
-                request: { ...normalized, attempts: 1 },
+                plan: executionPlan,
+                progress,
+                request: normalized,
                 root,
                 resolvedRevision,
-                runtimeDirectory: controllerRuntime.directory,
-                runtimeId: controllerRuntime.runtime_id,
+                controllerRuntime,
                 preparedArtifact,
+                preparedArtifacts,
                 ...(localTransport ? { localTransport } : {}),
                 env,
                 ...(harborExecutable !== undefined ? { harborExecutable } : {}),
                 ...(signal ? { signal } : {}),
                 ...(trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs }),
-                emit: (event) => sink.emit(event),
-                onTrialSettled: async (trial, context) => {
-                    try {
-                        const ref = await importEvalTrialRun({
-                            root,
-                            evalId,
-                            evalDirectory,
-                            harborJobDirectory,
-                            expectedAttempt: logicalAttempt,
-                            request: normalized,
-                            resolvedRevision,
-                            benchmarkId: normalized.benchmark_id,
-                            benchmarkRevision: normalized.benchmark_revision,
-                            runtimeId: controllerRuntime.runtime_id,
-                            requireCompleteMarker: true,
-                            allowMissingBundleDiagnostic: context.bundleWaitExpired,
-                        }, trial, shardRefs.length, shardRefs);
-                        await publish(ref);
-                        return true;
-                    }
-                    catch (error) {
-                        if (error instanceof TrialBundlePendingError)
-                            return false;
-                        throw error;
-                    }
+                sink,
+                worker: executionWorker || {
+                    workerId: `worker_process_${process.pid}`,
+                    provider: "local-docker",
+                    collisionDomainId: `local-process:${process.pid}`,
                 },
+                ...(workItemAdmission ? { admission: workItemAdmission } : {}),
+                ...(onWorkItemState ? { onWorkItemState } : {}),
+                ...(dockerResourceReaper ? { dockerResourceReaper } : {}),
+                ...(environmentImageManifestLoader ? { environmentImageManifestLoader } : {}),
+                ...(activeCaptureRuntime.exporter ? { interactionCaptureExporter: activeCaptureRuntime.exporter } : {}),
+                ...(remoteWorkExecutor ? { remoteWorkExecutor } : {}),
             });
-            backendRuns.push({ attempt: logicalAttempt, run: backendRun });
-            const terminalRefs = await importEvalTrialRuns({
-                root,
-                evalId,
-                evalDirectory,
-                harborJobDirectory,
-                expectedAttempt: logicalAttempt,
-                request: normalized,
-                resolvedRevision,
-                benchmarkId: normalized.benchmark_id,
-                benchmarkRevision: normalized.benchmark_revision,
-                runtimeId: controllerRuntime.runtime_id,
-                rawResult: backendRun.rawResult,
-            }, shardRefs);
-            for (const ref of terminalRefs)
-                await publish(ref);
-            assertBackendTrialSet(backendRun.rawResult, shardRefs);
-            await validateEvalTrialReferences(root, evalId, shardRefs, {
-                benchmarkId: normalized.benchmark_id,
-                benchmarkRevision: normalized.benchmark_revision,
-            });
-            if (backendRun.backend.process_exit_code !== 0 || backendRun.rawResult === null)
-                break;
-            if (localTransport && localSourceBackendFailure(backendRun.rawResult))
-                break;
+            progress = execution.progress;
+            backendRuns.push(...execution.backendRuns.map((entry) => ({
+                attempt: entry.attempt,
+                run: entry.run,
+                workId: entry.workId,
+                tasks: entry.tasks,
+                leaseId: entry.leaseId,
+            })));
+            infrastructureRetryRuns.push(...execution.infrastructureRetryRuns);
         }
+        else
+            for (let logicalAttempt = 1; logicalAttempt <= normalized.attempts; logicalAttempt += 1) {
+                if (signal?.aborted)
+                    break;
+                const backendDirectory = normalized.attempts === 1
+                    ? path.join(evalDirectory, "harbor")
+                    : path.join(evalDirectory, "harbor", attemptDirectoryName(logicalAttempt));
+                const harborJobDirectory = path.join(backendDirectory, "job");
+                const shardRefs = [];
+                const publish = async (ref) => {
+                    if (ref.attempt !== logicalAttempt) {
+                        throw new HitchError(`Harbor returned attempt ${ref.attempt}, expected ${logicalAttempt}`, {
+                            code: "eval_trial_attempt_mismatch",
+                            exitCode: 12,
+                        });
+                    }
+                    if (!shardRefs.some((current) => current.trial_id === ref.trial_id))
+                        shardRefs.push(ref);
+                    if (progress === null)
+                        throw new Error("eval progress was not initialized");
+                    const previousGeneration = progress.generation;
+                    progress = mergeEvalProgressTrial(progress, ref);
+                    if (progress.generation === previousGeneration)
+                        return;
+                    await validateEvalTrialReferences(root, evalId, [ref], {
+                        benchmarkId: normalized.benchmark_id,
+                        benchmarkRevision: normalized.benchmark_revision,
+                    });
+                    await writeEvalProgress(evalDirectory, progress);
+                    sink.emit({
+                        type: "eval.trial.published",
+                        trial_id: ref.trial_id,
+                        task_id: ref.task_id,
+                        attempt: ref.attempt,
+                        run_id: ref.run_id,
+                        observation_status: ref.observation_status,
+                        settled_trials: progress.trials.length,
+                        generation: progress.generation,
+                    });
+                };
+                const backendRun = await runHarborBackend({
+                    evalId,
+                    evalDirectory,
+                    backendDirectory,
+                    logicalAttempt,
+                    request: { ...normalized, attempts: 1 },
+                    root,
+                    resolvedRevision,
+                    runtimeDirectory: controllerRuntime.directory,
+                    runtimeId: controllerRuntime.runtime_id,
+                    preparedArtifact,
+                    ...(activeCaptureRuntime.exporter ? { modelProxy: activeCaptureRuntime.exporter.route } : {}),
+                    resolvedImages: resolvedImageMapping(executionPlan.work_items.find((item) => item.logical_attempt === logicalAttempt)?.image_refs ?? []),
+                    ...(executionResources ? { executionResources } : {}),
+                    env,
+                    ...(harborExecutable !== undefined ? { harborExecutable } : {}),
+                    ...(signal ? { signal } : {}),
+                    ...(trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs }),
+                    emit: (event) => sink.emit(event),
+                    onTrialSettled: async (trial, context) => {
+                        try {
+                            const ref = await importEvalTrialRun({
+                                root,
+                                evalId,
+                                evalDirectory,
+                                harborJobDirectory,
+                                expectedAttempt: logicalAttempt,
+                                request: normalized,
+                                resolvedRevision,
+                                benchmarkId: normalized.benchmark_id,
+                                benchmarkRevision: normalized.benchmark_revision,
+                                runtimeId: controllerRuntime.runtime_id,
+                                env,
+                                modelCapturePlan: activeCaptureRuntime.plan,
+                                ...(activeCaptureRuntime.exporter ? { interactionCaptureExporter: activeCaptureRuntime.exporter } : {}),
+                                requireCompleteMarker: true,
+                                allowMissingBundleDiagnostic: context.bundleWaitExpired,
+                            }, trial, shardRefs.length, shardRefs);
+                            await publish(ref);
+                            return true;
+                        }
+                        catch (error) {
+                            if (error instanceof TrialBundlePendingError)
+                                return false;
+                            throw error;
+                        }
+                    },
+                });
+                backendRuns.push({ attempt: logicalAttempt, run: backendRun });
+                const terminalRefs = await importEvalTrialRuns({
+                    root,
+                    evalId,
+                    evalDirectory,
+                    harborJobDirectory,
+                    expectedAttempt: logicalAttempt,
+                    request: normalized,
+                    resolvedRevision,
+                    benchmarkId: normalized.benchmark_id,
+                    benchmarkRevision: normalized.benchmark_revision,
+                    runtimeId: controllerRuntime.runtime_id,
+                    env,
+                    modelCapturePlan: activeCaptureRuntime.plan,
+                    ...(activeCaptureRuntime.exporter ? { interactionCaptureExporter: activeCaptureRuntime.exporter } : {}),
+                    rawResult: backendRun.rawResult,
+                }, shardRefs);
+                for (const ref of terminalRefs)
+                    await publish(ref);
+                assertBackendTrialSet(backendRun.rawResult, shardRefs);
+                await validateEvalTrialReferences(root, evalId, shardRefs, {
+                    benchmarkId: normalized.benchmark_id,
+                    benchmarkRevision: normalized.benchmark_revision,
+                });
+                if (backendRun.backend.process_exit_code !== 0 || backendRun.rawResult === null)
+                    break;
+                if (localTransport && localSourceBackendFailure(backendRun.rawResult))
+                    break;
+                if (progress === null)
+                    throw new Error("eval progress was not initialized");
+                const retries = await runInfrastructureRetries({
+                    evalId,
+                    evalDirectory,
+                    logicalAttempt,
+                    initialRefs: shardRefs,
+                    progress,
+                    request: normalized,
+                    root,
+                    resolvedRevision,
+                    controllerRuntime,
+                    preparedArtifact,
+                    resolvedImages: resolvedImageMapping(executionPlan.work_items.find((item) => item.logical_attempt === logicalAttempt)?.image_refs ?? []),
+                    ...(executionResources ? { executionResources } : {}),
+                    modelCapturePlan: activeCaptureRuntime.plan,
+                    ...(activeCaptureRuntime.exporter ? { interactionCaptureExporter: activeCaptureRuntime.exporter } : {}),
+                    ...(localTransport ? { localTransport } : {}),
+                    env,
+                    ...(harborExecutable !== undefined ? { harborExecutable } : {}),
+                    ...(signal ? { signal } : {}),
+                    ...(trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs }),
+                    sink,
+                    ...(localTransport ? { stopAfterResult: localSourceBackendFailure } : {}),
+                });
+                progress = retries.progress;
+                infrastructureRetryRuns.push(...retries.runs);
+            }
         if (progress === null)
             throw new Error("eval progress was not initialized");
+        await onControlPhase?.("finalizing");
         trialRefs = progress.trials;
         const cancelled = signal?.aborted === true;
         const localSourceFailure = localTransport
             ? backendRuns.some(({ run }) => localSourceBackendFailure(run.rawResult))
+                || infrastructureRetryRuns.some(({ run }) => localSourceBackendFailure(run.rawResult))
             : false;
-        const backendsSucceeded = backendRuns.length === normalized.attempts
-            && backendRuns.every(({ run }) => run.backend.process_exit_code === 0 && run.rawResult !== null);
+        const expectedBackendRuns = plannedTaskExecution ? executionPlan.work_items.length : normalized.attempts;
+        const backendsSucceeded = backendRuns.every(({ run }) => run.backend.process_exit_code === 0 && run.rawResult !== null)
+            && (plannedTaskExecution ? progress.trials.length === plannedTrials : backendRuns.length === expectedBackendRuns);
         const invalidTrials = localTaskIds === null
             ? trialRefs.filter((trial) => trial.observation_status !== "valid").map((trial) => ({ task_id: trial.task_id, attempt: trial.attempt }))
             : invalidTrialSlots(localTaskIds, normalized.attempts, progress);
+        const infrastructureFailures = infrastructureFailureTrials(trialRefs);
+        const infrastructureFailureSlots = infrastructureFailures.map((trial) => ({ task_id: trial.task_id, attempt: trial.attempt }));
+        const verifierRetriesExhausted = normalized.infrastructure_retries > 0 && infrastructureFailures.some((trial) => trial.invalid_reason === "verifier_infrastructure_failure");
+        const infrastructureErrorCode = verifierRetriesExhausted || (normalized.infrastructure_retries > 0 && infrastructureRetryRuns.length > 0)
+            ? "eval_infrastructure_retries_exhausted"
+            : "eval_has_infrastructure_failures";
         const succeeded = !cancelled && !localSourceFailure && backendsSucceeded && invalidTrials.length === 0;
         const singleBackend = backendRuns.length === 1 ? backendRuns[0].run : undefined;
         result = {
@@ -304,13 +427,39 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             status: cancelled ? "cancelled" : succeeded ? "succeeded" : "failed",
             exit_code: cancelled ? 9 : succeeded ? 0 : 13,
             ...(singleBackend ? { backend: singleBackend.backend, backend_summary: singleBackend.summary } : {}),
-            ...(normalized.attempts > 1 ? {
+            ...(plannedTaskExecution ? {
+                backend_work_items: backendRuns.map(({ attempt, workId, tasks, leaseId, run }) => ({
+                    work_id: workId,
+                    lease_id: leaseId,
+                    attempt,
+                    tasks,
+                    backend: run.backend,
+                    backend_summary: run.summary,
+                })),
+            } : normalized.attempts > 1 ? {
                 backend_runs: backendRuns.map(({ attempt, run }) => ({
                     attempt,
                     backend: run.backend,
                     backend_summary: run.summary,
                 })),
             } : {}),
+            infrastructure_retry_policy: {
+                max_retries: normalized.infrastructure_retries,
+                backoff_ms: normalized.infrastructure_retry_backoff_ms,
+                verifier_execution: "same_trial_verifier_only",
+                candidate_rerun_on_verifier_failure: false,
+            },
+            ...(infrastructureRetryRuns.length > 0 ? {
+                infrastructure_retry_runs: infrastructureRetryRuns.map(({ attempt, retry, tasks, triggers, refs, run, leaseId, workId }) => ({
+                    execution_kind: "physical-infrastructure-retry", ...(leaseId ? { lease_id: leaseId } : {}), ...(workId ? { work_id: workId } : {}), attempt,
+                    retry,
+                    tasks,
+                    trigger_trials: triggers,
+                    trials: refs,
+                    backend: run.backend,
+                    backend_summary: run.summary,
+                })),
+            } : {}),
             candidate: plan.candidate,
             dataset: normalized.dataset,
             benchmark_id: normalized.benchmark_id,
@@ -318,7 +467,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             generation: progress.generation,
             trials: trialRefs,
             summary: summarizeTrialRefs(trialRefs),
-            prepared_artifact: preparedArtifactSummary(preparedArtifact),
+            ...preparedArtifactPlanFields(preparedAssignments),
             ...(localTransport ? { local_source_transport: transportSummary(localTransport) } : {}),
             ...(succeeded ? {} : {
                 error: {
@@ -328,14 +477,18 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
                             ? "local_source_materialize_failed"
                             : !backendsSucceeded
                                 ? "harbor_failed"
-                                : "eval_has_invalid_tasks",
+                                : infrastructureFailures.length > 0
+                                    ? infrastructureErrorCode
+                                    : "eval_has_invalid_tasks",
                     message: cancelled
                         ? "eval was cancelled"
                         : localSourceFailure
                             ? "Harbor rejected the transported local Git source before candidate execution"
                             : !backendsSucceeded
-                                ? `Harbor attempt shards completed ${backendRuns.length}/${normalized.attempts}`
-                                : `eval has invalid or missing trials: ${invalidTrials.map(trial => `${trial.task_id}#${trial.attempt}`).join(", ")}`,
+                                ? `Harbor work items completed ${backendRuns.length}/${expectedBackendRuns}`
+                                : infrastructureFailures.length > 0
+                                    ? `${infrastructureErrorCode === "eval_infrastructure_retries_exhausted" ? "infrastructure retries exhausted" : "verifier infrastructure failure"}: ${infrastructureFailureSlots.map(trial => `${trial.task_id}#${trial.attempt}`).join(", ")}`
+                                    : `eval has invalid or missing trials: ${invalidTrials.map(trial => `${trial.task_id}#${trial.attempt}`).join(", ")}`,
                 },
             }),
             started_at: startedAt.toISOString(),
@@ -352,7 +505,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             exit_code: signal?.aborted ? 9 : typed ? error.exitCode : 12,
             error: {
                 code: signal?.aborted ? "cancelled" : typed ? error.code : "internal_error",
-                message: error?.message || String(error),
+                message: safeDiagnosticMessage(error, credentialValuesFromEnv(normalized.pass_env, env)),
             },
             benchmark_id: normalized.benchmark_id,
             benchmark_revision: normalized.benchmark_revision,
@@ -362,71 +515,8 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             completed_at: new Date().toISOString(),
         };
     }
-    await atomicWriteJSON(path.join(evalDirectory, "result.json"), result);
-    sink.emit({ type: result.status === "succeeded" ? "eval.completed" : "eval.failed", status: result.status, exit_code: result.exit_code, error: result.error });
-    await sink.close();
-    return result;
-}
-function assertBackendTrialSet(rawResult, refs) {
-    const trials = Array.isArray(rawResult?.trial_results) ? rawResult.trial_results : [];
-    const backendIds = new Set(trials.map((trial) => typeof trial.trial_name === "string" ? trial.trial_name : null).filter((value) => value !== null));
-    const refIds = new Set(refs.map((ref) => ref.trial_id));
-    if (backendIds.size !== trials.length || backendIds.size !== refIds.size || [...backendIds].some((id) => !refIds.has(id))) {
-        throw new Error("Harbor terminal trial set does not match published eval progress");
-    }
-}
-function attemptDirectoryName(attempt) {
-    return `attempt-${String(attempt).padStart(4, "0")}`;
-}
-export function summarizeTrialRefs(trials) {
-    const valid = trials.filter((trial) => trial.observation_status === "valid" && typeof trial.reward === "number");
-    const rewards = valid.map((trial) => trial.reward);
-    const aggregate = rewards.length
-        ? {
-            count: rewards.length,
-            mean: rewards.reduce((sum, reward) => sum + reward, 0) / rewards.length,
-            min: Math.min(...rewards),
-            max: Math.max(...rewards),
-        }
-        : null;
-    return {
-        n_trials: trials.length,
-        n_completed: valid.length,
-        n_invalid: trials.length - valid.length,
-        primary_reward: aggregate?.mean ?? null,
-        rewards: aggregate ? { reward: aggregate } : {},
-    };
-}
-function localSourceBackendFailure(rawResult) {
-    if (!rawResult)
-        return false;
-    // The bridge uses this fixed, non-secret marker when setup cannot verify or
-    // materialize the uploaded source. Do not surface provider exception text in
-    // the durable Hitch error, because it may contain backend diagnostics.
-    return JSON.stringify(rawResult).includes("hitch-local-source-materialize:");
-}
-function transportSummary(transport) {
-    const manifest = transport.manifest;
-    return {
-        kind: manifest.kind,
-        resolution_identity: manifest.resolution_identity,
-        commit: manifest.commit,
-        tree: manifest.tree,
-        payload_sha256: manifest.payload_sha256,
-        payload_bytes: manifest.payload_bytes,
-        object_count: manifest.object_count,
-        file_count: manifest.file_count,
-    };
-}
-function preparedArtifactSummary(artifact) {
-    return {
-        artifact_id: artifact.artifact_id,
-        artifact_integrity: artifact.artifact_integrity,
-        entrypoint_integrity: artifact.entrypoint_integrity,
-        harness_id: artifact.harness_id,
-        revision_identity: artifact.revision_identity,
-        platform: artifact.platform,
-        source_type: artifact.source_type,
-    };
+    if (captureRuntime)
+        await captureRuntime.close().catch((error) => sink.emit({ type: "interaction.capture.close-failed", code: error.code || "model_capture_close_failed" }));
+    return finalizeEvalResult(evalDirectory, sink, result);
 }
 //# sourceMappingURL=service.js.map