agent-hitch 0.2.5 → 0.2.7

package/dist/src/evals/recovery.js

@@ -0,0 +1,192 @@
+import path from "node:path";
+import { readHarborRawResult } from "../backends/index.js";
+import { HitchError, readJSON } from "../foundation/index.js";
+import { DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS, heartbeatExecutionLease, markExecutionLeaseLost, readExecutionLeases, reissueExecutionLease, releaseExecutionLease } from "./execution-leases.js";
+import { LocalDockerExecutionProvider, readLocalDockerProcessRecordByLease, waitForLocalDockerProcessTerminal } from "./local-docker-provider.js";
+import { loadEvalResumeState } from "./resume-state.js";
+import { mergeEvalProgressTrial, writeEvalProgress } from "./progress.js";
+import { assertBackendTrialSet } from "./result-helpers.js";
+import { importEvalTrialRuns, validateEvalTrialReferences } from "./trial-import.js";
+import { startEvalModelCaptureRuntime } from "./model-capture-runtime.js";
+import { readModelProxyRuntimeState } from "./model-proxy-runtime-state.js";
+export async function recoverLocalDockerEvalLeases(input) {
+    const emit = input.emit ?? (() => { });
+    const recovered = [];
+    let failure;
+    let captureRuntime;
+    try {
+        captureRuntime = await restoreModelCaptureForRecovery(input);
+        for (const lease of input.leases) {
+            if (!activeLease(lease))
+                continue;
+            try {
+                await recoverLease(input, lease, emit, captureRuntime);
+                recovered.push(lease.lease_id);
+            }
+            catch (error) {
+                const typed = error instanceof HitchError;
+                failure ??= {
+                    code: typed ? error.code : "execution_state_ambiguous",
+                    message: error?.message || String(error),
+                };
+                const current = (await readExecutionLeases(input.evalDirectory)).find((entry) => entry.lease_id === lease.lease_id);
+                if (current && activeLease(current))
+                    await markExecutionLeaseLost({
+                        evalDirectory: input.evalDirectory,
+                        leaseId: current.lease_id,
+                        expectedEpoch: current.epoch,
+                    });
+                emit({ type: "eval.lease.recovery-failed", lease_id: lease.lease_id, code: failure.code });
+                emit({ type: "eval.work.lost", work_id: lease.work_id, lease_id: lease.lease_id, code: failure.code });
+            }
+        }
+    }
+    catch (error) {
+        failure = { code: "execution_state_ambiguous", message: error?.message || String(error) };
+        for (const lease of input.leases.filter(activeLease)) {
+            const current = (await readExecutionLeases(input.evalDirectory)).find((entry) => entry.lease_id === lease.lease_id);
+            if (current && activeLease(current))
+                await markExecutionLeaseLost({ evalDirectory: input.evalDirectory, leaseId: current.lease_id, expectedEpoch: current.epoch });
+            emit({ type: "eval.lease.recovery-failed", lease_id: lease.lease_id, code: failure.code });
+            emit({ type: "eval.work.lost", work_id: lease.work_id, lease_id: lease.lease_id, code: failure.code });
+        }
+    }
+    finally {
+        await captureRuntime?.close().catch(() => undefined);
+    }
+    return failure
+        ? { status: "ambiguous", recovered_lease_ids: recovered, ...failure }
+        : { status: "resumable", recovered_lease_ids: recovered };
+}
+async function recoverLease(input, lease, emit, captureRuntime) {
+    if (Date.parse(lease.expires_at) <= Date.now())
+        emit({ type: "lease.expired", work_id: lease.work_id, lease_id: lease.lease_id, lease_epoch: lease.epoch, worker_id: lease.worker_id });
+    if (lease.provider !== "local-docker")
+        throw ambiguous(`unsupported recovery provider: ${lease.provider}`);
+    const provider = new LocalDockerExecutionProvider({
+        root: input.root,
+        workerId: lease.worker_id,
+        status: () => { throw new Error("recovery does not inspect provider capacity"); },
+    });
+    const record = await readLocalDockerProcessRecordByLease({ root: input.root, evalId: input.evalId, leaseId: lease.lease_id }).catch(() => {
+        throw ambiguous(`local provider has no durable process identity for ${lease.lease_id}`);
+    });
+    if (record.lease_epoch === lease.epoch - 1)
+        await provider.adoptLeaseEpoch(lease.lease_id, record.lease_epoch, lease.epoch);
+    else if (record.lease_epoch !== lease.epoch)
+        throw ambiguous(`local provider epoch does not match ${lease.lease_id}`);
+    const probe = await provider.recover(lease);
+    emit({ type: "eval.lease.recovery-probed", lease_id: lease.lease_id, lease_epoch: lease.epoch, state: probe.state });
+    if (probe.state === "released") {
+        await releaseExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
+        emit({ type: "lease.recovered", work_id: lease.work_id, lease_id: lease.lease_id, lease_epoch: lease.epoch, state: "already-released" });
+        return;
+    }
+    if (probe.state !== "running" && probe.state !== "terminal-uncollected") {
+        throw ambiguous(`local provider classified ${lease.lease_id} as ${probe.state}`);
+    }
+    const reissued = await reissueExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
+    await provider.adoptLeaseEpoch(lease.lease_id, lease.epoch, reissued.epoch);
+    emit({ type: "eval.lease.reissued", lease_id: lease.lease_id, previous_epoch: lease.epoch, lease_epoch: reissued.epoch });
+    let heartbeatFailure;
+    let heartbeatTail = Promise.resolve();
+    const timer = setInterval(() => {
+        heartbeatTail = heartbeatTail.then(async () => {
+            await heartbeatExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: reissued.epoch });
+        }).catch((error) => { heartbeatFailure ??= error; });
+    }, DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS);
+    timer.unref();
+    try {
+        if (input.cancelRequested && probe.state === "running")
+            await provider.cancel(lease.lease_id, reissued.epoch);
+        const terminal = await waitForLocalDockerProcessTerminal({ root: input.root, leaseId: lease.lease_id, epoch: reissued.epoch });
+        await heartbeatTail;
+        if (heartbeatFailure !== undefined)
+            throw heartbeatFailure;
+        try {
+            await collectRecoveredWork(input, reissued, terminal.backend_directory, emit, captureRuntime);
+        }
+        catch (error) {
+            if (!input.cancelRequested || error.code !== "recovery_collection_missing")
+                throw error;
+        }
+        if (!input.cancelRequested && terminal.process_exit_code !== null && terminal.process_exit_code !== 0) {
+            throw new HitchError(`recovered Harbor process exited with code ${terminal.process_exit_code}`, { code: "harbor_failed", exitCode: 13 });
+        }
+        await provider.release(lease.lease_id, reissued.epoch);
+        await releaseExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: reissued.epoch });
+        emit({ type: "lease.released", work_id: lease.work_id, lease_id: lease.lease_id, lease_epoch: reissued.epoch, worker_id: lease.worker_id });
+        emit({ type: "eval.lease.recovered", lease_id: lease.lease_id, lease_epoch: reissued.epoch, state: "released" });
+        emit({ type: "lease.recovered", lease_id: lease.lease_id, lease_epoch: reissued.epoch, state: "released" });
+    }
+    finally {
+        clearInterval(timer);
+        await heartbeatTail;
+    }
+}
+async function collectRecoveredWork(input, lease, backendDirectory, emit, captureRuntime) {
+    const state = await loadEvalResumeState(input.evalDirectory);
+    const work = state.executionPlan.work_items.find((item) => item.work_id === lease.work_id);
+    if (!work || work.logical_attempt === null || work.task_ids.length !== 1)
+        throw ambiguous("recovered lease does not match one planned task slot");
+    const jobDirectory = path.join(input.evalDirectory, backendDirectory, "job");
+    const rawResult = await readHarborRawResult(jobDirectory);
+    if (!rawResult)
+        throw new HitchError("recovered Harbor work item has no aggregate result", { code: "recovery_collection_missing", exitCode: 12 });
+    const resolvedRevision = await readJSON(path.join(input.evalDirectory, "resolution.json"));
+    const request = await readJSON(path.join(input.evalDirectory, "request.json"));
+    const runtime = state.plan.controller_runtime;
+    const runtimeId = typeof runtime?.runtime_id === "string" ? runtime.runtime_id : undefined;
+    const refs = await importEvalTrialRuns({
+        root: input.root,
+        evalId: input.evalId,
+        evalDirectory: input.evalDirectory,
+        harborJobDirectory: jobDirectory,
+        expectedAttempt: work.logical_attempt,
+        request,
+        resolvedRevision,
+        benchmarkId: state.progress.benchmark_id,
+        benchmarkRevision: state.progress.benchmark_revision,
+        ...(runtimeId ? { runtimeId } : {}),
+        env: input.env ?? process.env,
+        ...(state.executionPlan.model_capture ? { modelCapturePlan: captureRuntime?.plan ?? state.executionPlan.model_capture } : {}),
+        ...(captureRuntime?.exporter ? { interactionCaptureExporter: captureRuntime.exporter } : {}),
+        rawResult,
+    }, state.progress.trials);
+    const workRefs = refs.filter((ref) => ref.task_id === work.task_ids[0] && ref.attempt === work.logical_attempt);
+    assertBackendTrialSet(rawResult, workRefs);
+    await validateEvalTrialReferences(input.root, input.evalId, workRefs, {
+        benchmarkId: state.progress.benchmark_id,
+        benchmarkRevision: state.progress.benchmark_revision,
+    });
+    let progress = state.progress;
+    for (const ref of workRefs)
+        progress = mergeEvalProgressTrial(progress, ref);
+    if (progress.generation !== state.progress.generation)
+        await writeEvalProgress(input.evalDirectory, progress);
+    emit({ type: "eval.work-item.recovered", work_id: work.work_id, lease_id: lease.lease_id, trials: workRefs.length });
+}
+async function restoreModelCaptureForRecovery(input) {
+    if (!input.leases.some(activeLease))
+        return undefined;
+    const state = await loadEvalResumeState(input.evalDirectory);
+    const plan = state.executionPlan.model_capture;
+    if (!plan || plan.effective_mode !== "proxy" && plan.effective_mode !== "hybrid")
+        return undefined;
+    if (!await readModelProxyRuntimeState(input.evalDirectory, input.evalId, plan)) {
+        throw ambiguous("recoverable model proxy has no persisted endpoint identity");
+    }
+    const runtime = await startEvalModelCaptureRuntime({ plan, evalId: input.evalId, evalDirectory: input.evalDirectory, env: input.env ?? process.env });
+    if (!runtime.route || !runtime.exporter) {
+        await runtime.close().catch(() => undefined);
+        throw ambiguous("recoverable model proxy endpoint could not be restored");
+    }
+    return runtime;
+}
+function activeLease(lease) {
+    return lease.state === "offered" || lease.state === "accepted" || lease.state === "running" || lease.state === "releasing";
+}
+function ambiguous(message) {
+    return new HitchError(message, { code: "execution_state_ambiguous", exitCode: 12 });
+}
+//# sourceMappingURL=recovery.js.map

package/dist/src/evals/recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recovery.js","sourceRoot":"","sources":["../../../src/evals/recovery.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACjM,OAAO,EAAE,4BAA4B,EAAE,mCAAmC,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAClJ,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC1E,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAE1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC;AAS5E,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,KAQlD;IACC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACtC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,OAAsD,CAAC;IAC3D,IAAI,cAAmD,CAAC;IACxD,IAAI,CAAC;QACH,cAAc,GAAG,MAAM,8BAA8B,CAAC,KAAK,CAAC,CAAC;QAC7D,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;gBAAE,SAAS;YAClC,IAAI,CAAC;gBACH,MAAM,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;gBACvD,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,KAAK,YAAY,UAAU,CAAC;gBAC1C,OAAO,KAAK;oBACV,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,2BAA2B;oBACtD,OAAO,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACpD,CAAC;gBACF,MAAM,OAAO,GAAG,CAAC,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACpH,IAAI,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC;oBAAE,MAAM,sBAAsB,CAAC;wBAChE,aAAa,EAAE,KAAK,CAAC,aAAa;wBAClC,OAAO,EAAE,OAAO,CAAC,QAAQ;wBACzB,aAAa,EAAE,OAAO,CAAC,KAAK;qBAC7B,CAAC,CAAC;gBACH,IAAI,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3F,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YACzG,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,GAAG,EAAE,IAAI,EAAE,2BAA2B,EAAE,OAAO,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACrG,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;YACrD,MAAM,OAAO,GAAG,CAAC,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpH,IAAI,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC;gBAAE,MAAM,sBAAsB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,aAAa,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YACnK,IAAI,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3F,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,cAAc,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,OAAO;QACZ,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE;QACrE,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,KAAkH,EAClH,KAAuB,EACvB,IAA8C,EAC9C,cAAwC;IAExC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE;QAAE,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IACxL,IAAI,KAAK,CAAC,QAAQ,KAAK,cAAc;QAAE,MAAM,SAAS,CAAC,kCAAkC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3G,MAAM,QAAQ,GAAG,IAAI,4BAA4B,CAAC;QAChD,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,KAAK,CAAC,SAAS;QACzB,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC,CAAC;KAClF,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,mCAAmC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;QACvI,MAAM,SAAS,CAAC,sDAAsD,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1F,CAAC,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,WAAW,KAAK,KAAK,CAAC,KAAK,GAAG,CAAC;QAAE,MAAM,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;SACvH,IAAI,MAAM,CAAC,WAAW,KAAK,KAAK,CAAC,KAAK;QAAE,MAAM,SAAS,CAAC,uCAAuC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IACtH,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACrH,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;QAC/B,MAAM,qBAAqB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACzH,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzI,OAAO;IACT,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,sBAAsB,EAAE,CAAC;QACxE,MAAM,SAAS,CAAC,6BAA6B,KAAK,CAAC,QAAQ,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC1I,MAAM,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC5E,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAC1H,IAAI,gBAAyB,CAAC;IAC9B,IAAI,aAAa,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC5C,MAAM,uBAAuB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QAChI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACzC,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,IAAI,CAAC;QACH,IAAI,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;YAAE,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC9G,MAAM,QAAQ,GAAG,MAAM,iCAAiC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QAC/H,MAAM,aAAa,CAAC;QACpB,IAAI,gBAAgB,KAAK,SAAS;YAAE,MAAM,gBAAgB,CAAC;QAC3D,IAAI,CAAC;YAAC,MAAM,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,iBAAiB,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACpH,IAAI,CAAC,KAAK,CAAC,eAAe,IAAK,KAA2B,CAAC,IAAI,KAAK,6BAA6B;gBAAE,MAAM,KAAK,CAAC;QACjH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,QAAQ,CAAC,iBAAiB,KAAK,IAAI,IAAI,QAAQ,CAAC,iBAAiB,KAAK,CAAC,EAAE,CAAC;YACtG,MAAM,IAAI,UAAU,CAAC,6CAA6C,QAAQ,CAAC,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3I,CAAC;QACD,MAAM,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvD,MAAM,qBAAqB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QAC5H,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAC5I,IAAI,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACjH,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;IAC9G,CAAC;YAAS,CAAC;QACT,aAAa,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,aAAa,CAAC;IACtB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,KAAuF,EACvF,KAAuB,EACvB,gBAAwB,EACxB,IAA8C,EAC9C,cAAwC;IAExC,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3F,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,SAAS,CAAC,sDAAsD,CAAC,CAAC;IAClJ,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAC;IAC1D,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,UAAU,CAAC,oDAAoD,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAClJ,MAAM,gBAAgB,GAAG,MAAM,QAAQ,CAAmB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAC7G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAc,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,kBAAyD,CAAC;IACrF,MAAM,SAAS,GAAG,OAAO,OAAO,EAAE,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3F,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC;QACrC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,kBAAkB,EAAE,YAAY;QAChC,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,OAAO;QACP,gBAAgB;QAChB,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY;QACxC,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,kBAAkB;QACpD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnC,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;QAC7B,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,cAAc,EAAE,IAAI,IAAI,KAAK,CAAC,aAAa,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7H,GAAG,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,0BAA0B,EAAE,cAAc,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,SAAS;KACV,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,eAAe,CAAC,CAAC;IAChH,qBAAqB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,2BAA2B,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE;QACpE,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY;QACxC,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,kBAAkB;KACrD,CAAC,CAAC;IACH,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,QAAQ;QAAE,QAAQ,GAAG,sBAAsB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC7E,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,CAAC,QAAQ,CAAC,UAAU;QAAE,MAAM,iBAAiB,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IAC9G,IAAI,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;AACvH,CAAC;AAED,KAAK,UAAU,8BAA8B,CAAC,KAK7C;IACC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;QAAE,OAAO,SAAS,CAAC;IACtD,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,aAAa,CAAC;IAC/C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,cAAc,KAAK,OAAO,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IACnG,IAAI,CAAC,MAAM,0BAA0B,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;QAC/E,MAAM,SAAS,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,4BAA4B,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACtJ,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,SAAS,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,KAAuB;IAC1C,OAAO,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,WAAW,CAAC;AAC7H,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IAChC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;AACtF,CAAC"}

package/dist/src/evals/remote-infrastructure-retry.js

@@ -0,0 +1,99 @@
+import { setTimeout as delay } from "node:timers/promises";
+import { preparedArtifactForWorkItem } from "./work-item-artifacts.js";
+import { HitchError, sha256JSON } from "../foundation/index.js";
+import { retryableInfrastructureTrials } from "./infrastructure-retry.js";
+import { replaceInvalidEvalProgressTrial, writeEvalProgress } from "./progress.js";
+export async function runRemoteInfrastructureRetries(input) {
+    const executor = input.options.remoteWorkExecutor;
+    if (!executor)
+        throw new TypeError("remote work executor is unavailable");
+    let progress = input.progress;
+    let candidates = retryableInfrastructureTrials(input.initial.refs);
+    const runs = [];
+    for (let retry = 1; retry <= input.options.request.infrastructure_retries && candidates.length > 0; retry += 1) {
+        if (input.options.signal?.aborted)
+            break;
+        const triggers = [...candidates];
+        const backoffMs = input.options.request.infrastructure_retry_backoff_ms * retry;
+        input.options.sink.emit({
+            type: "eval.infrastructure-retry.scheduled", execution_kind: "physical-infrastructure-retry",
+            candidate_executes: true, attempt: input.item.logical_attempt, retry,
+            tasks: input.item.task_ids, backoff_ms: backoffMs, provider: input.item.provider,
+        });
+        if (backoffMs > 0)
+            await delay(backoffMs, undefined, input.options.signal ? { signal: input.options.signal } : undefined);
+        if (input.options.signal?.aborted)
+            break;
+        const work = retryWorkItem(input.item, retry);
+        const refs = [];
+        const publish = async (ref) => {
+            assertSelectedTrial(ref, input.item);
+            const existing = refs.find((entry) => entry.trial_id === ref.trial_id || entry.run_id === ref.run_id);
+            if (existing) {
+                if (JSON.stringify(existing) !== JSON.stringify(ref))
+                    throw new Error(`remote retry trial identity changed: ${ref.trial_id}`);
+                return;
+            }
+            refs.push(ref);
+            if (ref.observation_status === "valid") {
+                progress = replaceInvalidEvalProgressTrial(progress, ref);
+                await writeEvalProgress(input.options.evalDirectory, progress);
+            }
+            input.options.sink.emit({
+                type: ref.observation_status === "valid" ? "eval.infrastructure-retry.repaired" : "eval.infrastructure-retry.failed",
+                attempt: input.item.logical_attempt, retry, task_id: ref.task_id, trial_id: ref.trial_id,
+                run_id: ref.run_id, observation_status: ref.observation_status,
+                ...(ref.invalid_reason ? { invalid_reason: ref.invalid_reason } : {}), generation: progress.generation,
+            });
+        };
+        const completed = await executor({
+            evalId: input.options.evalId, evalDirectory: input.options.evalDirectory, root: input.options.root,
+            request: input.options.request, plan: input.options.plan, workItem: work,
+            resolvedRevision: input.options.resolvedRevision, preparedArtifact: preparedArtifactForWorkItem(input.options, work),
+            runtimeDirectory: input.options.controllerRuntime.directory, runtimeId: input.options.controllerRuntime.runtime_id,
+            ...(input.initial.environmentImages ? { environmentImages: input.initial.environmentImages } : {}),
+            ...(input.options.plan.model_capture ? { modelCapturePlan: input.options.plan.model_capture } : {}),
+            publicationMode: "replace-invalid",
+            ...(input.options.signal ? { signal: input.options.signal } : {}),
+            emit: (event) => input.options.sink.emit({ ...event, execution_kind: "physical-infrastructure-retry", infrastructure_retry: retry }),
+            publish,
+            onLeaseState: (leaseId, state) => input.options.onWorkItemState?.(work.work_id, leaseId, state) ?? Promise.resolve(),
+        });
+        assertPublishedRefs(completed.refs, refs, input.item);
+        candidates = retryableInfrastructureTrials(refs);
+        runs.push({
+            attempt: input.item.logical_attempt, retry, tasks: [...input.item.task_ids], triggers,
+            refs, run: completed.run, leaseId: completed.leaseId, workId: work.work_id,
+        });
+        input.options.sink.emit({
+            type: "eval.infrastructure-retry.completed", execution_kind: "physical-infrastructure-retry",
+            candidate_executes: true, attempt: input.item.logical_attempt, retry, tasks: input.item.task_ids,
+            repaired_tasks: refs.filter((ref) => ref.observation_status === "valid").map((ref) => ref.task_id).sort(),
+            remaining_tasks: candidates.map((ref) => ref.task_id).sort(), provider: input.item.provider,
+        });
+        if (completed.run.backend.process_exit_code !== 0 || completed.run.rawResult === null)
+            break;
+    }
+    return { progress, runs };
+}
+function assertSelectedTrial(ref, item) {
+    if (ref.attempt !== item.logical_attempt || !item.task_ids.includes(ref.task_id)) {
+        throw new HitchError(`remote infrastructure retry returned an unselected trial: ${ref.task_id}#${ref.attempt}`, {
+            code: "eval_infrastructure_retry_trial_mismatch", exitCode: 12,
+        });
+    }
+}
+function assertPublishedRefs(returned, published, item) {
+    for (const ref of returned)
+        assertSelectedTrial(ref, item);
+    if (returned.length !== published.length || returned.some((ref) => !published.some((entry) => JSON.stringify(entry) === JSON.stringify(ref)))) {
+        throw new HitchError("remote infrastructure retry returned refs that were not durably published", {
+            code: "eval_infrastructure_retry_result_mismatch", exitCode: 12,
+        });
+    }
+}
+function retryWorkItem(item, retry) {
+    const identity = sha256JSON({ work_id: item.work_id, execution_kind: "physical-infrastructure-retry", retry });
+    return { ...item, work_id: `work_${identity.slice("sha256:".length, "sha256:".length + 32)}` };
+}
+//# sourceMappingURL=remote-infrastructure-retry.js.map

package/dist/src/evals/remote-infrastructure-retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-infrastructure-retry.js","sourceRoot":"","sources":["../../../src/evals/remote-infrastructure-retry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEhE,OAAO,EAAE,6BAA6B,EAAE,MAAM,2BAA2B,CAAC;AAE1E,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEnF,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,KAKpD;IACC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAClD,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC1E,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC9B,IAAI,UAAU,GAAG,6BAA6B,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,MAAM,IAAI,GAA6B,EAAE,CAAC;IAC1C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/G,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACzC,MAAM,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;QACjC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,GAAG,KAAK,CAAC;QAChF,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YACtB,IAAI,EAAE,qCAAqC,EAAE,cAAc,EAAE,+BAA+B;YAC5F,kBAAkB,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK;YACpE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;SACjF,CAAC,CAAC;QACH,IAAI,SAAS,GAAG,CAAC;YAAE,MAAM,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC1H,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACzC,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAqB,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,KAAK,EAAE,GAAmB,EAAiB,EAAE;YAC3D,mBAAmB,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;YACtG,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC9H,OAAO;YACT,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACvC,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC1D,MAAM,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YACjE,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACtB,IAAI,EAAE,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC,kCAAkC;gBACpH,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACxF,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gBAC9D,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU;aACvG,CAAC,CAAC;QACL,CAAC,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;YAC/B,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI;YAClG,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI;YACxE,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,2BAA2B,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC;YACpH,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,UAAU;YAClH,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnG,eAAe,EAAE,iBAAiB;YAClC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,cAAc,EAAE,+BAA+B,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;YACpI,OAAO;YACP,YAAY,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE;SACrH,CAAC,CAAC;QACH,mBAAmB,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACtD,UAAU,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,IAAI,CAAC;YACR,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAyB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,QAAQ;YAC/F,IAAI,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO;SAC3E,CAAC,CAAC;QACH,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YACtB,IAAI,EAAE,qCAAqC,EAAE,cAAc,EAAE,+BAA+B;YAC5F,kBAAkB,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;YAChG,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;YACzG,eAAe,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;SAC5F,CAAC,CAAC;QACH,IAAI,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,KAAK,IAAI;YAAE,MAAM;IAC/F,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAmB,EAAE,IAAuB;IACvE,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,UAAU,CAAC,6DAA6D,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE;YAC9G,IAAI,EAAE,0CAA0C,EAAE,QAAQ,EAAE,EAAE;SAC/D,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAmC,EAAE,SAAoC,EAAE,IAAuB;IAC7H,KAAK,MAAM,GAAG,IAAI,QAAQ;QAAE,mBAAmB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9I,MAAM,IAAI,UAAU,CAAC,2EAA2E,EAAE;YAChG,IAAI,EAAE,2CAA2C,EAAE,QAAQ,EAAE,EAAE;SAChE,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAuB,EAAE,KAAa;IAC3D,MAAM,QAAQ,GAAG,UAAU,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,+BAA+B,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/G,OAAO,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,QAAQ,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACjG,CAAC"}

package/dist/src/evals/request.js

@@ -1,5 +1,7 @@
 export const DEFAULT_EVAL_TIMEOUT_MS = 15 * 60 * 1_000;
 export const DEFAULT_EVAL_SETUP_TIMEOUT_MS = 30 * 60 * 1_000;
+export const DEFAULT_INFRASTRUCTURE_RETRIES = 1;
+export const DEFAULT_INFRASTRUCTURE_RETRY_BACKOFF_MS = 1_000;
 const HARBOR_EVAL_BYPASS_ARG = {
     codex: "--dangerously-bypass-approvals-and-sandbox",
     opencode: "--dangerously-skip-permissions",
@@ -17,7 +19,8 @@ export async function validateEvalRequest(input) {
         throw invalidInput("eval request must be a JSON object");
     const allowed = new Set([
         "schema_version", "backend", "dataset", "harness_ref", "model", "attempts",
-        "max_concurrent", "timeout_ms", "setup_timeout_ms", "agent_args", "pass_env",
+        "max_concurrent", "infrastructure_retries", "infrastructure_retry_backoff_ms",
+        "timeout_ms", "setup_timeout_ms", "agent_args", "pass_env",
     ]);
     const unexpected = Object.keys(input).find((field) => !allowed.has(field));
     if (unexpected)
@@ -38,6 +41,8 @@ export async function validateEvalRequest(input) {
     const benchmark = await resolveBenchmarkReference(input.dataset.trim());
     const attempts = positiveInteger(input.attempts ?? 1, "attempts");
     const maxConcurrent = positiveInteger(input.max_concurrent ?? 4, "max_concurrent");
+    const infrastructureRetries = nonNegativeInteger(input.infrastructure_retries ?? DEFAULT_INFRASTRUCTURE_RETRIES, "infrastructure_retries");
+    const infrastructureRetryBackoff = nonNegativeNumber(input.infrastructure_retry_backoff_ms ?? DEFAULT_INFRASTRUCTURE_RETRY_BACKOFF_MS, "infrastructure_retry_backoff_ms");
     const timeout = nonNegativeNumber(input.timeout_ms ?? DEFAULT_EVAL_TIMEOUT_MS, "timeout_ms");
     const setupTimeout = nonNegativeNumber(input.setup_timeout_ms ?? DEFAULT_EVAL_SETUP_TIMEOUT_MS, "setup_timeout_ms");
     if (input.model !== undefined && typeof input.model !== "string")
@@ -60,6 +65,8 @@ export async function validateEvalRequest(input) {
         model: typeof input.model === "string" ? input.model : "",
         attempts,
         max_concurrent: maxConcurrent,
+        infrastructure_retries: infrastructureRetries,
+        infrastructure_retry_backoff_ms: infrastructureRetryBackoff,
         timeout_ms: timeout,
         setup_timeout_ms: setupTimeout,
         agent_args: agentArgs,
@@ -130,6 +137,12 @@ function positiveInteger(value, name) {
         throw invalidInput(`${name} must be a positive integer`);
     return parsed;
 }
+function nonNegativeInteger(value, name) {
+    const parsed = Number(value);
+    if (!Number.isSafeInteger(parsed) || parsed < 0)
+        throw invalidInput(`${name} must be a non-negative integer`);
+    return parsed;
+}
 function nonNegativeNumber(value, name) {
     const parsed = Number(value);
     if (!Number.isFinite(parsed) || parsed < 0)

package/dist/src/evals/request.js.map

@@ -1 +1 @@
-{"version":3,"file":"request.js","sourceRoot":"","sources":["../../../src/evals/request.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACvD,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAE7D,MAAM,sBAAsB,GAAqC;IAC/D,KAAK,EAAE,4CAA4C;IACnD,QAAQ,EAAE,gCAAgC;CAC3C,CAAC;AAEF,MAAM,UAAU,SAAS;IACvB,OAAO,QAAQ,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,EAAY,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;IACxF,OAAO,KAAe,CAAC;AACzB,CAAC;AAgBD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAuB;IAC/D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC;QACtB,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU;QAC1E,gBAAgB,EAAE,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU;KAC7E,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,IAAI,UAAU;QAAE,MAAM,YAAY,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,cAAc;QAAE,MAAM,YAAY,CAAC,+BAA+B,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7J,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,qDAAqD,CAAC,CAAC;IACxH,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IACzH,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACrI,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,MAAM,YAAY,CAAC,yEAAyE,CAAC,CAAC;IAChG,CAAC;IACD,gCAAgC,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;IAClE,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,IAAI,uBAAuB,EAAE,YAAY,CAAC,CAAC;IAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,gBAAgB,IAAI,6BAA6B,EAAE,kBAAkB,CAAC,CAAC;IACpH,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,wBAAwB,CAAC,CAAC;IAC/G,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxI,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QAClI,MAAM,YAAY,CAAC,sCAAsC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,CAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3F,MAAM,SAAS,GAAG,sBAAsB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAC/D,IAAI,SAAS,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9E,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACrC,WAAW,EAAE,SAAS,CAAC,SAAS;QAChC,KAAK,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACzD,QAAQ;QACR,cAAc,EAAE,aAAa;QAC7B,UAAU,EAAE,OAAO;QACnB,gBAAgB,EAAE,YAAY;QAC9B,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvF,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;KACjD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAAe;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACtC,OAAO;gBACL,YAAY,EAAE,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBAC7C,kBAAkB,EAAE,MAAM,eAAe,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;aAC1F,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,YAAY,CAAC,wEAAwE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,mCAAmC,CAAC,CAAC;IACjG,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,OAAe;IAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,IAAI,CAAC;QACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACpE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvF,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,IAAY;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACvG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,IAAY;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,gCAAgC,CAAC,CAAC;IACxG,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,gCAAgC,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAChG,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}
+{"version":3,"file":"request.js","sourceRoot":"","sources":["../../../src/evals/request.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACvD,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAC7D,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAChD,MAAM,CAAC,MAAM,uCAAuC,GAAG,KAAK,CAAC;AAE7D,MAAM,sBAAsB,GAAqC;IAC/D,KAAK,EAAE,4CAA4C;IACnD,QAAQ,EAAE,gCAAgC;CAC3C,CAAC;AAEF,MAAM,UAAU,SAAS;IACvB,OAAO,QAAQ,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,EAAY,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;IACxF,OAAO,KAAe,CAAC;AACzB,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAuB;IAC/D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC;QACtB,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU;QAC1E,gBAAgB,EAAE,wBAAwB,EAAE,iCAAiC;QAC7E,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU;KAC3D,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,IAAI,UAAU;QAAE,MAAM,YAAY,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,cAAc;QAAE,MAAM,YAAY,CAAC,+BAA+B,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7J,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,qDAAqD,CAAC,CAAC;IACxH,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IACzH,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACrI,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,MAAM,YAAY,CAAC,yEAAyE,CAAC,CAAC;IAChG,CAAC;IACD,gCAAgC,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;IAClE,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACnF,MAAM,qBAAqB,GAAG,kBAAkB,CAC9C,KAAK,CAAC,sBAAsB,IAAI,8BAA8B,EAC9D,wBAAwB,CACzB,CAAC;IACF,MAAM,0BAA0B,GAAG,iBAAiB,CAClD,KAAK,CAAC,+BAA+B,IAAI,uCAAuC,EAChF,iCAAiC,CAClC,CAAC;IACF,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,IAAI,uBAAuB,EAAE,YAAY,CAAC,CAAC;IAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,gBAAgB,IAAI,6BAA6B,EAAE,kBAAkB,CAAC,CAAC;IACpH,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,wBAAwB,CAAC,CAAC;IAC/G,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxI,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QAClI,MAAM,YAAY,CAAC,sCAAsC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,CAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3F,MAAM,SAAS,GAAG,sBAAsB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAC/D,IAAI,SAAS,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9E,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACrC,WAAW,EAAE,SAAS,CAAC,SAAS;QAChC,KAAK,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACzD,QAAQ;QACR,cAAc,EAAE,aAAa;QAC7B,sBAAsB,EAAE,qBAAqB;QAC7C,+BAA+B,EAAE,0BAA0B;QAC3D,UAAU,EAAE,OAAO;QACnB,gBAAgB,EAAE,YAAY;QAC9B,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvF,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;KACjD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAAe;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACtC,OAAO;gBACL,YAAY,EAAE,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBAC7C,kBAAkB,EAAE,MAAM,eAAe,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;aAC1F,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,YAAY,CAAC,wEAAwE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,mCAAmC,CAAC,CAAC;IACjG,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,OAAe;IAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,IAAI,CAAC;QACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACpE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvF,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,IAAY;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACvG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc,EAAE,IAAY;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IAC9G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,IAAY;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,gCAAgC,CAAC,CAAC;IACxG,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,gCAAgC,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAChG,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/src/evals/rerun-inputs.js

@@ -0,0 +1,89 @@
+import path from "node:path";
+import { loadPreparedArtifact, preparedArtifactDirectory } from "../artifacts/index.js";
+import { validateLocalGitTransportManifest, verifyLocalGitTransport } from "../backends/index.js";
+import { HitchError, readJSON } from "../foundation/index.js";
+import { loadHarborArtifact } from "./harbor-artifact-builder.js";
+export async function loadRerunResolvedRevision(evalDirectory, plan) {
+    const resolution = await readJSON(path.join(evalDirectory, "resolution.json"));
+    if (!resolution || resolution.identity !== requiredString(plan.candidate.revision_identity, "candidate revision identity")
+        || resolution.harness_id !== requiredString(plan.candidate.harness_id, "candidate harness id")) {
+        throw unavailable("eval resolution identity changed");
+    }
+    return resolution;
+}
+export async function loadRerunPreparedArtifacts(root, plan) {
+    const artifacts = new Map();
+    for (const summary of plan.preparedArtifacts) {
+        const artifact = await loadRerunPreparedArtifactSummary(root, summary);
+        artifacts.set(artifact.artifact_id, artifact);
+    }
+    if (artifacts.size === 0)
+        throw unavailable("eval has no prepared artifact assignments");
+    return artifacts;
+}
+async function loadRerunPreparedArtifactSummary(root, summary) {
+    const artifactId = requiredString(summary.artifact_id, "prepared artifact id");
+    if (summary.storage === "harbor-artifact-cache-v2") {
+        return loadHarborArtifact(root, {
+            artifact_id: artifactId,
+            artifact_integrity: requiredString(summary.artifact_integrity, "prepared artifact integrity"),
+            entrypoint_integrity: requiredString(summary.entrypoint_integrity, "prepared artifact entrypoint integrity"),
+            harness_id: requiredString(summary.harness_id, "prepared artifact harness id"),
+            revision_identity: requiredString(summary.revision_identity, "prepared artifact revision identity"),
+            platform: requiredString(summary.platform, "prepared artifact platform"),
+        });
+    }
+    const artifact = await loadPreparedArtifact(preparedArtifactDirectory(root, artifactId), {
+        artifact_id: artifactId,
+        artifact_integrity: requiredString(summary.artifact_integrity, "prepared artifact integrity"),
+        entrypoint_integrity: requiredString(summary.entrypoint_integrity, "prepared artifact entrypoint integrity"),
+        harness_id: requiredString(summary.harness_id, "prepared artifact harness id"),
+        revision_identity: requiredString(summary.revision_identity, "prepared artifact revision identity"),
+        platform: requiredString(summary.platform, "prepared artifact platform"),
+    });
+    return {
+        directory: preparedArtifactDirectory(root, artifact.artifact_id),
+        artifact_id: artifact.artifact_id,
+        artifact_integrity: requiredString(artifact.artifact_integrity, "artifact integrity"),
+        entrypoint_integrity: requiredString(artifact.entrypoint_integrity, "artifact entrypoint integrity"),
+        harness_id: artifact.harness_id,
+        revision_identity: artifact.revision_identity,
+        adapter_version: artifact.adapter_version,
+        recipe_version: artifact.recipe_version,
+        platform: artifact.platform,
+        node_version: artifact.toolchain.node || process.version,
+        source_type: artifact.source_type,
+    };
+}
+export async function loadRerunLocalTransport(evalDirectory, plan, resolution, env, signal) {
+    if (plan.localSourceTransport === undefined)
+        return undefined;
+    const directory = path.join(evalDirectory, "local-source");
+    const manifestPath = path.join(directory, "manifest.json");
+    const use = {
+        directory,
+        manifestPath,
+        payloadPath: path.join(directory, "payload.pack"),
+        resolutionPath: path.join(directory, "resolution.json"),
+        manifest: validateLocalGitTransportManifest(await readJSON(manifestPath)),
+    };
+    await verifyLocalGitTransport(use, {
+        expected: {
+            harnessId: resolution.harness_id,
+            resolutionIdentity: resolution.identity,
+            commit: requiredString(resolution.revision.commit, "local source commit"),
+        },
+        env: env ?? process.env,
+        ...(signal ? { signal } : {}),
+    });
+    return use;
+}
+function requiredString(value, label) {
+    if (typeof value !== "string" || value.length === 0)
+        throw unavailable(`${label} is missing`);
+    return value;
+}
+function unavailable(message) {
+    return new HitchError(message, { code: "eval_rerun_unavailable", exitCode: 2 });
+}
+//# sourceMappingURL=rerun-inputs.js.map

package/dist/src/evals/rerun-inputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rerun-inputs.js","sourceRoot":"","sources":["../../../src/evals/rerun-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAExF,OAAO,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAElG,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAQlE,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,aAAqB,EAAE,IAAoB;IACzF,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAmB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACjG,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,QAAQ,KAAK,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,6BAA6B,CAAC;WACrH,UAAU,CAAC,UAAU,KAAK,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,sBAAsB,CAAC,EAAE,CAAC;QACjG,MAAM,WAAW,CAAC,kCAAkC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAY,EAAE,IAAoB;IACjF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAqC,CAAC;IAC/D,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,gCAAgC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAChD,CAAC;IACD,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC;QAAE,MAAM,WAAW,CAAC,2CAA2C,CAAC,CAAC;IACzF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,gCAAgC,CAAC,IAAY,EAAE,OAAgC;IAC5F,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC;IAC/E,IAAI,OAAO,CAAC,OAAO,KAAK,0BAA0B,EAAE,CAAC;QACnD,OAAO,kBAAkB,CAAC,IAAI,EAAE;YAC9B,WAAW,EAAE,UAAU;YACvB,kBAAkB,EAAE,cAAc,CAAC,OAAO,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;YAC7F,oBAAoB,EAAE,cAAc,CAAC,OAAO,CAAC,oBAAoB,EAAE,wCAAwC,CAAC;YAC5G,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,8BAA8B,CAAC;YAC9E,iBAAiB,EAAE,cAAc,CAAC,OAAO,CAAC,iBAAiB,EAAE,qCAAqC,CAAC;YACnG,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,4BAA4B,CAAC;SACzE,CAAC,CAAC;IACL,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,yBAAyB,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE;QACvF,WAAW,EAAE,UAAU;QACvB,kBAAkB,EAAE,cAAc,CAAC,OAAO,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;QAC7F,oBAAoB,EAAE,cAAc,CAAC,OAAO,CAAC,oBAAoB,EAAE,wCAAwC,CAAC;QAC5G,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,8BAA8B,CAAC;QAC9E,iBAAiB,EAAE,cAAc,CAAC,OAAO,CAAC,iBAAiB,EAAE,qCAAqC,CAAC;QACnG,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,4BAA4B,CAAC;KACzE,CAAC,CAAC;IACH,OAAO;QACL,SAAS,EAAE,yBAAyB,CAAC,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC;QAChE,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,kBAAkB,EAAE,cAAc,CAAC,QAAQ,CAAC,kBAAkB,EAAE,oBAAoB,CAAC;QACrF,oBAAoB,EAAE,cAAc,CAAC,QAAQ,CAAC,oBAAoB,EAAE,+BAA+B,CAAC;QACpG,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,YAAY,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,OAAO,CAAC,OAAO;QACxD,WAAW,EAAE,QAAQ,CAAC,WAAW;KAClC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,aAAqB,EACrB,IAAoB,EACpB,UAA4B,EAC5B,GAAkC,EAClC,MAA+B;IAE/B,IAAI,IAAI,CAAC,oBAAoB,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAC3D,MAAM,GAAG,GAAyB;QAChC,SAAS;QACT,YAAY;QACZ,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC;QACjD,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC;QACvD,QAAQ,EAAE,iCAAiC,CAAC,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;KAC1E,CAAC;IACF,MAAM,uBAAuB,CAAC,GAAG,EAAE;QACjC,QAAQ,EAAE;YACR,SAAS,EAAE,UAAU,CAAC,UAAU;YAChC,kBAAkB,EAAE,UAAU,CAAC,QAAQ;YACvC,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,qBAAqB,CAAC;SAC1E;QACD,GAAG,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG;QACvB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9B,CAAC,CAAC;IACH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,WAAW,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IAC9F,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC"}

package/dist/src/evals/rerun-slots.js

@@ -3,7 +3,7 @@ import { evalTrialKey } from "./progress.js";
 export function selectRerunTasks(tasks, progress, selector, attempts = 1) {
     return uniqueTasks(selectRerunTrialSlots(tasks, attempts, progress, selector));
 }
-export function selectRerunTrialSlots(tasks, attempts, progress, selector) {
+export function selectRerunTrialSlots(tasks, attempts, progress, selector, options = {}) {
     const planned = new Set(tasks);
     if (planned.size !== tasks.length || tasks.some((task) => typeof task !== "string" || task.length === 0)) {
         throw unavailable("eval task plan is invalid");
@@ -12,7 +12,7 @@ export function selectRerunTrialSlots(tasks, attempts, progress, selector) {
         throw unavailable("eval attempts plan is invalid");
     const invalid = invalidTrialSlots(tasks, attempts, progress);
     if (selector.mode === "invalid")
-        return invalid;
+        return options.allowVerifierFailures ? invalid : rejectVerifierOnlyReruns(progress, invalid);
     const requested = [...new Set(selector.taskNames)].sort();
     if (requested.length === 0)
         throw invalidInput("eval rerun requires at least one --task");
@@ -24,7 +24,16 @@ export function selectRerunTrialSlots(tasks, attempts, progress, selector) {
         }
     }
     const requestedSet = new Set(requested);
-    return invalid.filter((slot) => requestedSet.has(slot.task_id));
+    const selected = invalid.filter((slot) => requestedSet.has(slot.task_id));
+    return options.allowVerifierFailures ? selected : rejectVerifierOnlyReruns(progress, selected);
+}
+function rejectVerifierOnlyReruns(progress, slots) {
+    const selected = new Set(slots.map(slotKey));
+    const verifierOnly = progress.trials.filter((trial) => selected.has(evalTrialKey(trial))
+        && (trial.invalid_reason === "verifier_infrastructure_failure" || trial.invalid_reason === "verifier_result_missing"));
+    if (verifierOnly.length === 0)
+        return slots;
+    throw new HitchError(`verifier-only retry is unavailable after the original trial environment closed: ${verifierOnly.map((trial) => `${trial.task_id}#${trial.attempt}`).join(", ")}; the Candidate Agent will not be rerun`, { code: "eval_verifier_only_rerun_unavailable", exitCode: 2 });
 }
 export function invalidTrialSlots(tasks, attempts, progress) {
     const byKey = new Map();

package/dist/src/evals/rerun-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"rerun-slots.js","sourceRoot":"","sources":["../../../src/evals/rerun-slots.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAW7C,MAAM,UAAU,gBAAgB,CAC9B,KAAwB,EACxB,QAAwB,EACxB,QAAuB,EACvB,QAAQ,GAAG,CAAC;IAEZ,OAAO,WAAW,CAAC,qBAAqB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,KAAwB,EACxB,QAAgB,EAChB,QAAwB,EACxB,QAAuB;IAEvB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACzG,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,WAAW,CAAC,+BAA+B,CAAC,CAAC;IACxG,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC7D,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IAChD,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IAC1F,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACxI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,UAAU,CAAC,+BAA+B,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAwB,EAAE,QAAgB,EAAE,QAAwB;IACpG,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,WAAW,CAAC,qCAAqC,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrH,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,QAAQ,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACxD,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACnF,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,QAAwB,EACxB,IAAoD,EACpD,OAAoB,EACpB,MAAc;IAEd,IAAI,QAAQ,CAAC,OAAO,KAAK,MAAM;QAAE,MAAM,WAAW,CAAC,mCAAmC,CAAC,CAAC;IACxF,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACjH,MAAM,WAAW,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClH,MAAM,WAAW,CAAC,2DAA2D,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACjE,MAAM,WAAW,CAAC,8CAA8C,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5G,CAAC;IACH,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAmB;IACzC,OAAO,GAAG,IAAI,CAAC,OAAO,SAAS,IAAI,CAAC,OAAO,EAAE,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA+B;IACvD,OAAO,CAAC,GAAG,KAAK,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;SACjE,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;AACtG,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAA+B;IACzD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;SACvC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC"}
+{"version":3,"file":"rerun-slots.js","sourceRoot":"","sources":["../../../src/evals/rerun-slots.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAW7C,MAAM,UAAU,gBAAgB,CAC9B,KAAwB,EACxB,QAAwB,EACxB,QAAuB,EACvB,QAAQ,GAAG,CAAC;IAEZ,OAAO,WAAW,CAAC,qBAAqB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,KAAwB,EACxB,QAAgB,EAChB,QAAwB,EACxB,QAAuB,EACvB,UAA+C,EAAE;IAEjD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACzG,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,WAAW,CAAC,+BAA+B,CAAC,CAAC;IACxG,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC7D,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9H,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IAC1F,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACxI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,UAAU,CAAC,+BAA+B,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,OAAO,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAwB,EAAE,KAAsB;IAChF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;WACnF,CAAC,KAAK,CAAC,cAAc,KAAK,iCAAiC,IAAI,KAAK,CAAC,cAAc,KAAK,yBAAyB,CAAC,CAAC,CAAC;IACzH,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,IAAI,UAAU,CAClB,mFAAmF,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,yCAAyC,EACvM,EAAE,IAAI,EAAE,sCAAsC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAC9D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAwB,EAAE,QAAgB,EAAE,QAAwB;IACpG,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,WAAW,CAAC,qCAAqC,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrH,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,QAAQ,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACxD,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACnF,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,QAAwB,EACxB,IAAoD,EACpD,OAAoB,EACpB,MAAc;IAEd,IAAI,QAAQ,CAAC,OAAO,KAAK,MAAM;QAAE,MAAM,WAAW,CAAC,mCAAmC,CAAC,CAAC;IACxF,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACjH,MAAM,WAAW,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClH,MAAM,WAAW,CAAC,2DAA2D,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACjE,MAAM,WAAW,CAAC,8CAA8C,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5G,CAAC;IACH,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAmB;IACzC,OAAO,GAAG,IAAI,CAAC,OAAO,SAAS,IAAI,CAAC,OAAO,EAAE,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA+B;IACvD,OAAO,CAAC,GAAG,KAAK,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;SACjE,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;AACtG,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAA+B;IACzD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;SACvC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC"}

package/dist/src/evals/rerun-types.js

@@ -0,0 +1,49 @@
+import { HitchError, invalidInput } from "../foundation/index.js";
+export const EVAL_RERUN_TYPES = [
+    "candidate-restart",
+    "candidate-resume",
+    "trajectory-replay",
+    "verifier-only",
+    "collect-only",
+];
+export function parseEvalRerunType(value) {
+    if (typeof value !== "string" || !EVAL_RERUN_TYPES.includes(value)) {
+        throw invalidInput(`eval rerun --type must be one of: ${EVAL_RERUN_TYPES.join(", ")}`);
+    }
+    return value;
+}
+export function evalRerunSemantics(type) {
+    switch (type) {
+        case "candidate-restart":
+            return { candidate_action: "restart", conversation_source: "original-instruction", sandbox_source: "clean", candidate_executes: true };
+        case "candidate-resume":
+            return { candidate_action: "resume", conversation_source: "native-session", sandbox_source: "checkpoint", candidate_executes: true };
+        case "trajectory-replay":
+            return { candidate_action: "replay", conversation_source: "canonical-trajectory", sandbox_source: "checkpoint", candidate_executes: true };
+        case "verifier-only":
+            return { candidate_action: "none", conversation_source: "none", sandbox_source: "retained", candidate_executes: false };
+        case "collect-only":
+            return { candidate_action: "none", conversation_source: "none", sandbox_source: "none", candidate_executes: false };
+    }
+}
+export function assertEvalRerunTypeSupported(type) {
+    if (type === "candidate-restart" || type === "collect-only")
+        return;
+    const unavailable = {
+        "candidate-resume": {
+            code: "eval_candidate_resume_unavailable",
+            message: "candidate-resume requires both a restorable sandbox checkpoint and adapter-native session resume; the current Harbor execution does not retain both",
+        },
+        "trajectory-replay": {
+            code: "eval_trajectory_replay_unavailable",
+            message: "trajectory-replay requires a verified trajectory, a restorable sandbox checkpoint, and explicit adapter replay support; trajectory evidence alone cannot restore tool or process state",
+        },
+        "verifier-only": {
+            code: "eval_verifier_only_rerun_unavailable",
+            message: "verifier-only requires the original retained sandbox; current verifier retries run only while that sandbox is live",
+        },
+    };
+    const failure = unavailable[type];
+    throw new HitchError(failure.message, { code: failure.code, exitCode: 2 });
+}
+//# sourceMappingURL=rerun-types.js.map