agent-hitch 0.2.6 → 0.2.8

package/dist/src/evals/recovery.js

@@ -0,0 +1,242 @@
+import path from "node:path";
+import { readHarborRawResult } from "../backends/index.js";
+import { HitchError, readJSON } from "../foundation/index.js";
+import { DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS, heartbeatExecutionLease, markExecutionLeaseLost, readExecutionLeases, reissueExecutionLease, releaseExecutionLease } from "./execution-leases.js";
+import { LocalDockerExecutionProvider, readLocalDockerProcessRecordByLease, waitForLocalDockerProcessTerminal } from "./local-docker-provider.js";
+import { loadEvalResumeState } from "./resume-state.js";
+import { mergeEvalProgressTrial, writeEvalProgress } from "./progress.js";
+import { replaceInvalidEvalProgressTrial } from "./progress.js";
+import { assertBackendTrialSet } from "./result-helpers.js";
+import { importEvalTrialRuns, validateEvalTrialReferences } from "./trial-import.js";
+import { startEvalModelCaptureRuntime } from "./model-capture-runtime.js";
+import { readModelProxyRuntimeState } from "./model-proxy-runtime-state.js";
+import { ensurePhysicalRetryDecision, readEvalRetryState, resolveRetryWork, transitionRetryDecision } from "./retry-state.js";
+import { classifyTrialFailure, physicalRetryAllowed } from "./failure-classifier.js";
+import { physicalRetryWorkItem } from "./physical-retry-work.js";
+import { retryBackoffMs } from "./retry-backoff.js";
+export async function recoverLocalDockerEvalLeases(input) {
+    const emit = input.emit ?? (() => { });
+    const recovered = [];
+    let failure;
+    let captureRuntime;
+    try {
+        captureRuntime = await restoreModelCaptureForRecovery(input);
+        for (const lease of input.leases) {
+            if (!activeLease(lease))
+                continue;
+            try {
+                await recoverLease(input, lease, emit, captureRuntime);
+                recovered.push(lease.lease_id);
+            }
+            catch (error) {
+                const typed = error instanceof HitchError;
+                failure ??= {
+                    code: typed ? error.code : "execution_state_ambiguous",
+                    message: error?.message || String(error),
+                };
+                const current = (await readExecutionLeases(input.evalDirectory)).find((entry) => entry.lease_id === lease.lease_id);
+                if (current && activeLease(current))
+                    await markExecutionLeaseLost({
+                        evalDirectory: input.evalDirectory,
+                        leaseId: current.lease_id,
+                        expectedEpoch: current.epoch,
+                    });
+                emit({ type: "eval.lease.recovery-failed", lease_id: lease.lease_id, code: failure.code });
+                emit({ type: "eval.work.lost", work_id: lease.work_id, lease_id: lease.lease_id, code: failure.code });
+            }
+        }
+    }
+    catch (error) {
+        failure = { code: "execution_state_ambiguous", message: error?.message || String(error) };
+        for (const lease of input.leases.filter(activeLease)) {
+            const current = (await readExecutionLeases(input.evalDirectory)).find((entry) => entry.lease_id === lease.lease_id);
+            if (current && activeLease(current))
+                await markExecutionLeaseLost({ evalDirectory: input.evalDirectory, leaseId: current.lease_id, expectedEpoch: current.epoch });
+            emit({ type: "eval.lease.recovery-failed", lease_id: lease.lease_id, code: failure.code });
+            emit({ type: "eval.work.lost", work_id: lease.work_id, lease_id: lease.lease_id, code: failure.code });
+        }
+    }
+    finally {
+        await captureRuntime?.close().catch(() => undefined);
+    }
+    return failure
+        ? { status: "ambiguous", recovered_lease_ids: recovered, ...failure }
+        : { status: "resumable", recovered_lease_ids: recovered };
+}
+async function recoverLease(input, lease, emit, captureRuntime) {
+    if (Date.parse(lease.expires_at) <= Date.now())
+        emit({ type: "lease.expired", work_id: lease.work_id, lease_id: lease.lease_id, lease_epoch: lease.epoch, worker_id: lease.worker_id });
+    if (lease.provider !== "local-docker")
+        throw ambiguous(`unsupported recovery provider: ${lease.provider}`);
+    const provider = new LocalDockerExecutionProvider({
+        root: input.root,
+        workerId: lease.worker_id,
+        status: () => { throw new Error("recovery does not inspect provider capacity"); },
+    });
+    const record = await readLocalDockerProcessRecordByLease({ root: input.root, evalId: input.evalId, leaseId: lease.lease_id }).catch(() => {
+        throw ambiguous(`local provider has no durable process identity for ${lease.lease_id}`);
+    });
+    if (record.lease_epoch === lease.epoch - 1)
+        await provider.adoptLeaseEpoch(lease.lease_id, record.lease_epoch, lease.epoch);
+    else if (record.lease_epoch !== lease.epoch)
+        throw ambiguous(`local provider epoch does not match ${lease.lease_id}`);
+    const probe = await provider.recover(lease);
+    emit({ type: "eval.lease.recovery-probed", lease_id: lease.lease_id, lease_epoch: lease.epoch, state: probe.state });
+    if (probe.state === "released") {
+        await releaseExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
+        emit({ type: "lease.recovered", work_id: lease.work_id, lease_id: lease.lease_id, lease_epoch: lease.epoch, state: "already-released" });
+        return;
+    }
+    if (probe.state !== "running" && probe.state !== "terminal-uncollected") {
+        throw ambiguous(`local provider classified ${lease.lease_id} as ${probe.state}`);
+    }
+    const reissued = await reissueExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
+    await provider.adoptLeaseEpoch(lease.lease_id, lease.epoch, reissued.epoch);
+    emit({ type: "eval.lease.reissued", lease_id: lease.lease_id, previous_epoch: lease.epoch, lease_epoch: reissued.epoch });
+    let heartbeatFailure;
+    let heartbeatTail = Promise.resolve();
+    const timer = setInterval(() => {
+        heartbeatTail = heartbeatTail.then(async () => {
+            await heartbeatExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: reissued.epoch });
+        }).catch((error) => { heartbeatFailure ??= error; });
+    }, DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS);
+    timer.unref();
+    try {
+        if (input.cancelRequested && probe.state === "running")
+            await provider.cancel(lease.lease_id, reissued.epoch);
+        const terminal = await waitForLocalDockerProcessTerminal({ root: input.root, leaseId: lease.lease_id, epoch: reissued.epoch });
+        await heartbeatTail;
+        if (heartbeatFailure !== undefined)
+            throw heartbeatFailure;
+        try {
+            await collectRecoveredWork(input, reissued, terminal.backend_directory, emit, captureRuntime);
+        }
+        catch (error) {
+            if (!input.cancelRequested || error.code !== "recovery_collection_missing")
+                throw error;
+        }
+        if (!input.cancelRequested && terminal.process_exit_code !== null && terminal.process_exit_code !== 0) {
+            throw new HitchError(`recovered Harbor process exited with code ${terminal.process_exit_code}`, { code: "harbor_failed", exitCode: 13 });
+        }
+        await provider.release(lease.lease_id, reissued.epoch);
+        await releaseExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: reissued.epoch });
+        emit({ type: "lease.released", work_id: lease.work_id, lease_id: lease.lease_id, lease_epoch: reissued.epoch, worker_id: lease.worker_id });
+        emit({ type: "eval.lease.recovered", lease_id: lease.lease_id, lease_epoch: reissued.epoch, state: "released" });
+        emit({ type: "lease.recovered", lease_id: lease.lease_id, lease_epoch: reissued.epoch, state: "released" });
+    }
+    finally {
+        clearInterval(timer);
+        await heartbeatTail;
+    }
+}
+async function collectRecoveredWork(input, lease, backendDirectory, emit, captureRuntime) {
+    const state = await loadEvalResumeState(input.evalDirectory);
+    const retryState = await readEvalRetryState(input.evalDirectory, input.evalId);
+    const dynamic = resolveRetryWork(state.executionPlan, retryState, lease.work_id);
+    const work = state.executionPlan.work_items.find((item) => item.work_id === lease.work_id) ?? dynamic?.item;
+    if (!work || work.logical_attempt === null || work.task_ids.length !== 1)
+        throw ambiguous("recovered lease does not match one planned task slot");
+    const existing = state.progress.trials.find((trial) => trial.task_id === work.task_ids[0] && trial.attempt === work.logical_attempt);
+    if (dynamic && existing?.observation_status === "valid") {
+        for (const decision of dynamic.decisions)
+            await settleRecoveredRetryDecision(input.evalDirectory, input.evalId, decision.decision_id, decision.state, "repaired");
+        emit({ type: "eval.retry.recovered", work_id: work.work_id, lease_id: lease.lease_id, state: "repaired", collection: "already-published" });
+        return;
+    }
+    const jobDirectory = path.join(input.evalDirectory, backendDirectory, "job");
+    const rawResult = await readHarborRawResult(jobDirectory);
+    if (!rawResult)
+        throw new HitchError("recovered Harbor work item has no aggregate result", { code: "recovery_collection_missing", exitCode: 12 });
+    const resolvedRevision = await readJSON(path.join(input.evalDirectory, "resolution.json"));
+    const request = await readJSON(path.join(input.evalDirectory, "request.json"));
+    const runtime = state.plan.controller_runtime;
+    const runtimeId = typeof runtime?.runtime_id === "string" ? runtime.runtime_id : undefined;
+    const refs = await importEvalTrialRuns({
+        root: input.root,
+        evalId: input.evalId,
+        evalDirectory: input.evalDirectory,
+        harborJobDirectory: jobDirectory,
+        expectedAttempt: work.logical_attempt,
+        request,
+        resolvedRevision,
+        benchmarkId: state.progress.benchmark_id,
+        benchmarkRevision: state.progress.benchmark_revision,
+        ...(runtimeId ? { runtimeId } : {}),
+        env: input.env ?? process.env,
+        ...(state.executionPlan.model_capture ? { modelCapturePlan: captureRuntime?.plan ?? state.executionPlan.model_capture } : {}),
+        ...(captureRuntime?.exporter ? { interactionCaptureExporter: captureRuntime.exporter } : {}),
+        ...(dynamic ? { publicationMode: "replace-invalid" } : {}),
+        rawResult,
+    }, dynamic ? [] : state.progress.trials);
+    const workRefs = refs.filter((ref) => ref.task_id === work.task_ids[0] && ref.attempt === work.logical_attempt);
+    assertBackendTrialSet(rawResult, workRefs);
+    await validateEvalTrialReferences(input.root, input.evalId, workRefs, {
+        benchmarkId: state.progress.benchmark_id,
+        benchmarkRevision: state.progress.benchmark_revision,
+    });
+    let progress = state.progress;
+    for (const ref of workRefs) {
+        if (dynamic) {
+            if (ref.observation_status === "valid")
+                progress = replaceInvalidEvalProgressTrial(progress, ref);
+        }
+        else
+            progress = mergeEvalProgressTrial(progress, ref);
+    }
+    if (progress.generation !== state.progress.generation)
+        await writeEvalProgress(input.evalDirectory, progress);
+    if (dynamic) {
+        const repaired = workRefs.some((ref) => ref.observation_status === "valid");
+        const retryable = workRefs.filter((ref) => physicalRetryAllowed(classifyTrialFailure(ref)));
+        const retryIndex = dynamic.decisions[0]?.retry_index ?? 0;
+        if (!repaired && retryable.length > 0 && retryIndex < state.executionPlan.retry_policy.infrastructure_retries) {
+            const origin = state.executionPlan.work_items.find((item) => item.slots.includes(dynamic.decisions[0].slot_id));
+            if (!origin)
+                throw ambiguous("recovered retry origin is absent from execution plan");
+            const nextWork = physicalRetryWorkItem(origin, retryIndex + 1, retryable);
+            for (const trigger of retryable)
+                await ensurePhysicalRetryDecision({
+                    evalDirectory: input.evalDirectory, evalId: input.evalId, item: origin, retryIndex: retryIndex + 1, trigger,
+                    notBefore: new Date(Date.now() + retryBackoffMs(state.executionPlan.retry_policy.infrastructure_retry_backoff_ms, retryIndex + 1, nextWork.work_id)).toISOString(),
+                });
+        }
+        const target = repaired ? "repaired" : retryIndex >= state.executionPlan.retry_policy.infrastructure_retries ? "exhausted" : "invalid";
+        for (const decision of dynamic.decisions)
+            await settleRecoveredRetryDecision(input.evalDirectory, input.evalId, decision.decision_id, decision.state, target);
+    }
+    emit({ type: "eval.work-item.recovered", work_id: work.work_id, lease_id: lease.lease_id, trials: workRefs.length });
+}
+async function settleRecoveredRetryDecision(evalDirectory, evalId, decisionId, current, target) {
+    if (current === target)
+        return;
+    if (current === "planned") {
+        await transitionRetryDecision({ evalDirectory, evalId, decisionId, state: "running" });
+        current = "running";
+    }
+    if (current === "running")
+        await transitionRetryDecision({ evalDirectory, evalId, decisionId, state: target });
+}
+async function restoreModelCaptureForRecovery(input) {
+    if (!input.leases.some(activeLease))
+        return undefined;
+    const state = await loadEvalResumeState(input.evalDirectory);
+    const plan = state.executionPlan.model_capture;
+    if (!plan || plan.effective_mode !== "proxy" && plan.effective_mode !== "hybrid")
+        return undefined;
+    if (!await readModelProxyRuntimeState(input.evalDirectory, input.evalId, plan)) {
+        throw ambiguous("recoverable model proxy has no persisted endpoint identity");
+    }
+    const runtime = await startEvalModelCaptureRuntime({ plan, evalId: input.evalId, evalDirectory: input.evalDirectory, env: input.env ?? process.env });
+    if (!runtime.route || !runtime.exporter) {
+        await runtime.close().catch(() => undefined);
+        throw ambiguous("recoverable model proxy endpoint could not be restored");
+    }
+    return runtime;
+}
+function activeLease(lease) {
+    return lease.state === "offered" || lease.state === "accepted" || lease.state === "running" || lease.state === "releasing";
+}
+function ambiguous(message) {
+    return new HitchError(message, { code: "execution_state_ambiguous", exitCode: 12 });
+}
+//# sourceMappingURL=recovery.js.map

package/dist/src/evals/recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recovery.js","sourceRoot":"","sources":["../../../src/evals/recovery.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACjM,OAAO,EAAE,4BAA4B,EAAE,mCAAmC,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAClJ,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAC1E,OAAO,EAAE,+BAA+B,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAE1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC;AAC5E,OAAO,EAAE,2BAA2B,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAC9H,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AASpD,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,KAQlD;IACC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACtC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,OAAsD,CAAC;IAC3D,IAAI,cAAmD,CAAC;IACxD,IAAI,CAAC;QACH,cAAc,GAAG,MAAM,8BAA8B,CAAC,KAAK,CAAC,CAAC;QAC7D,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;gBAAE,SAAS;YAClC,IAAI,CAAC;gBACH,MAAM,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;gBACvD,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,KAAK,YAAY,UAAU,CAAC;gBAC1C,OAAO,KAAK;oBACV,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,2BAA2B;oBACtD,OAAO,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACpD,CAAC;gBACF,MAAM,OAAO,GAAG,CAAC,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACpH,IAAI,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC;oBAAE,MAAM,sBAAsB,CAAC;wBAChE,aAAa,EAAE,KAAK,CAAC,aAAa;wBAClC,OAAO,EAAE,OAAO,CAAC,QAAQ;wBACzB,aAAa,EAAE,OAAO,CAAC,KAAK;qBAC7B,CAAC,CAAC;gBACH,IAAI,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3F,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YACzG,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,GAAG,EAAE,IAAI,EAAE,2BAA2B,EAAE,OAAO,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACrG,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;YACrD,MAAM,OAAO,GAAG,CAAC,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpH,IAAI,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC;gBAAE,MAAM,sBAAsB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,aAAa,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YACnK,IAAI,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3F,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,cAAc,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,OAAO;QACZ,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE;QACrE,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,KAAkH,EAClH,KAAuB,EACvB,IAA8C,EAC9C,cAAwC;IAExC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE;QAAE,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IACxL,IAAI,KAAK,CAAC,QAAQ,KAAK,cAAc;QAAE,MAAM,SAAS,CAAC,kCAAkC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3G,MAAM,QAAQ,GAAG,IAAI,4BAA4B,CAAC;QAChD,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,KAAK,CAAC,SAAS;QACzB,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC,CAAC;KAClF,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,mCAAmC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;QACvI,MAAM,SAAS,CAAC,sDAAsD,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1F,CAAC,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,WAAW,KAAK,KAAK,CAAC,KAAK,GAAG,CAAC;QAAE,MAAM,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;SACvH,IAAI,MAAM,CAAC,WAAW,KAAK,KAAK,CAAC,KAAK;QAAE,MAAM,SAAS,CAAC,uCAAuC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IACtH,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACrH,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;QAC/B,MAAM,qBAAqB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACzH,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzI,OAAO;IACT,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,sBAAsB,EAAE,CAAC;QACxE,MAAM,SAAS,CAAC,6BAA6B,KAAK,CAAC,QAAQ,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC1I,MAAM,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC5E,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAC1H,IAAI,gBAAyB,CAAC;IAC9B,IAAI,aAAa,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC5C,MAAM,uBAAuB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QAChI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACzC,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,IAAI,CAAC;QACH,IAAI,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;YAAE,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC9G,MAAM,QAAQ,GAAG,MAAM,iCAAiC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QAC/H,MAAM,aAAa,CAAC;QACpB,IAAI,gBAAgB,KAAK,SAAS;YAAE,MAAM,gBAAgB,CAAC;QAC3D,IAAI,CAAC;YAAC,MAAM,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,iBAAiB,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACpH,IAAI,CAAC,KAAK,CAAC,eAAe,IAAK,KAA2B,CAAC,IAAI,KAAK,6BAA6B;gBAAE,MAAM,KAAK,CAAC;QACjH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,QAAQ,CAAC,iBAAiB,KAAK,IAAI,IAAI,QAAQ,CAAC,iBAAiB,KAAK,CAAC,EAAE,CAAC;YACtG,MAAM,IAAI,UAAU,CAAC,6CAA6C,QAAQ,CAAC,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3I,CAAC;QACD,MAAM,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvD,MAAM,qBAAqB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QAC5H,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAC5I,IAAI,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACjH,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;IAC9G,CAAC;YAAS,CAAC;QACT,aAAa,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,aAAa,CAAC;IACtB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,KAAuF,EACvF,KAAuB,EACvB,gBAAwB,EACxB,IAA8C,EAC9C,cAAwC;IAExC,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,aAAa,EAAE,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,EAAE,IAAI,CAAC;IAC5G,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,SAAS,CAAC,sDAAsD,CAAC,CAAC;IAClJ,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,eAAe,CAAC,CAAC;IACrI,IAAI,OAAO,IAAI,QAAQ,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;QACxD,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS;YAAE,MAAM,4BAA4B,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAClK,IAAI,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAC5I,OAAO;IACT,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAC;IAC1D,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,UAAU,CAAC,oDAAoD,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAClJ,MAAM,gBAAgB,GAAG,MAAM,QAAQ,CAAmB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAC7G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAc,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,kBAAyD,CAAC;IACrF,MAAM,SAAS,GAAG,OAAO,OAAO,EAAE,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3F,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC;QACrC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,kBAAkB,EAAE,YAAY;QAChC,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,OAAO;QACP,gBAAgB;QAChB,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY;QACxC,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,kBAAkB;QACpD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnC,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;QAC7B,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,cAAc,EAAE,IAAI,IAAI,KAAK,CAAC,aAAa,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7H,GAAG,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,0BAA0B,EAAE,cAAc,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,iBAA0B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,SAAS;KACV,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,eAAe,CAAC,CAAC;IAChH,qBAAqB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,2BAA2B,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE;QACpE,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY;QACxC,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,kBAAkB;KACrD,CAAC,CAAC;IACH,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO;gBAAE,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACpG,CAAC;;YAAM,QAAQ,GAAG,sBAAsB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,CAAC,QAAQ,CAAC,UAAU;QAAE,MAAM,iBAAiB,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IAC9G,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC;QAC5E,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5F,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC;YAC9G,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YACjH,IAAI,CAAC,MAAM;gBAAE,MAAM,SAAS,CAAC,sDAAsD,CAAC,CAAC;YACrF,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,EAAE,SAAS,CAAC,CAAC;YAC1E,KAAK,MAAM,OAAO,IAAI,SAAS;gBAAE,MAAM,2BAA2B,CAAC;oBACjE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,CAAC,EAAE,OAAO;oBAC3G,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,aAAa,CAAC,YAAY,CAAC,+BAA+B,EAAE,UAAU,GAAG,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE;iBACnK,CAAC,CAAC;QACL,CAAC;QACD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,IAAI,KAAK,CAAC,aAAa,CAAC,YAAY,CAAC,sBAAsB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;QACvI,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS;YAAE,MAAM,4BAA4B,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAChK,CAAC;IACD,IAAI,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;AACvH,CAAC;AAED,KAAK,UAAU,4BAA4B,CACzC,aAAqB,EACrB,MAAc,EACd,UAAkB,EAClB,OAAiF,EACjF,MAA4C;IAE5C,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO;IAC/B,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,uBAAuB,CAAC,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QACvF,OAAO,GAAG,SAAS,CAAC;IACtB,CAAC;IACD,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,uBAAuB,CAAC,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;AACjH,CAAC;AAED,KAAK,UAAU,8BAA8B,CAAC,KAK7C;IACC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;QAAE,OAAO,SAAS,CAAC;IACtD,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,aAAa,CAAC;IAC/C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,cAAc,KAAK,OAAO,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IACnG,IAAI,CAAC,MAAM,0BAA0B,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;QAC/E,MAAM,SAAS,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,4BAA4B,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACtJ,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,SAAS,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,KAAuB;IAC1C,OAAO,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,WAAW,CAAC;AAC7H,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IAChC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;AACtF,CAAC"}

package/dist/src/evals/regrade-evidence.js

@@ -0,0 +1,77 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { lstat, readFile, readdir } from "node:fs/promises";
+import path from "node:path";
+import { validateRunObservation } from "../domain/index.js";
+import { atomicWriteJSON, sha256Bytes, sha256JSON, statePaths } from "../foundation/index.js";
+import { verifyResultBundleIndex } from "../runs/index.js";
+/** A bounded inventory includes empty directories and rejects symlinks. */
+export async function regradeTreeDigest(directory) {
+    const files = [];
+    let total = 0;
+    async function visit(relative) {
+        const absolute = path.join(directory, relative);
+        const info = await lstat(absolute);
+        if (info.isSymbolicLink() || (!info.isFile() && !info.isDirectory()))
+            throw new Error("regrade input contains a symlink or special file");
+        if (files.length >= 100_000 || (total += info.isFile() ? info.size : 0) > 4 * 1024 ** 3)
+            throw new Error("regrade input exceeds inventory limits");
+        if (info.isDirectory()) {
+            files.push({ path: relative, type: "directory" });
+            for (const name of (await readdir(absolute)).sort())
+                await visit(relative ? `${relative}/${name}` : name);
+        }
+        else {
+            const hash = createHash("sha256");
+            for await (const chunk of createReadStream(absolute))
+                hash.update(chunk);
+            files.push({ path: relative, bytes: info.size, mode: info.mode & 0o777, sha256: `sha256:${hash.digest("hex")}` });
+        }
+    }
+    await visit("");
+    return sha256JSON(files);
+}
+export async function sealRegradeAssessment(directory, record) {
+    const evidenceDigest = await regradeTreeDigest(path.join(directory, "evidence"));
+    const manifest = { ...record, schema_version: "1", kind: "verifier-only-assessment", candidate_executes: false, evidence_digest: evidenceDigest };
+    await atomicWriteJSON(path.join(directory, "assessment.json"), manifest);
+    return { id: path.basename(directory), digest: sha256Bytes(await readFile(path.join(directory, "assessment.json"))) };
+}
+export async function readRegradeObservation(root, evalId, trial) {
+    if (trial.run_group)
+        throw new Error("phase groups require their native phase assessment");
+    const reference = trial.assessment;
+    if (!reference || !/^assessment_[a-f0-9]{32}$/.test(reference.id) || !/^sha256:[a-f0-9]{64}$/.test(reference.digest))
+        throw new Error("invalid regrade assessment reference");
+    const directory = path.join(statePaths(root).evals, evalId, "assessments", reference.id);
+    const raw = await readFile(path.join(directory, "assessment.json"));
+    if (sha256Bytes(raw) !== reference.digest)
+        throw new Error("regrade assessment digest mismatch");
+    const record = JSON.parse(raw.toString());
+    const source = record.source;
+    if (record.schema_version !== "1" || record.kind !== "verifier-only-assessment" || record.candidate_executes !== false
+        || record.eval_id !== evalId || record.task_id !== trial.task_id || record.attempt !== trial.attempt
+        || source?.trial_id !== trial.trial_id || source.run_id !== trial.run_id)
+        throw new Error("regrade assessment source identity mismatch");
+    if (record.evidence_digest !== await regradeTreeDigest(path.join(directory, "evidence")))
+        throw new Error("regrade assessment evidence changed");
+    if (record.runtime_repair !== undefined) {
+        const repair = record.runtime_repair;
+        const receipt = JSON.parse(await readFile(path.join(directory, "evidence/runtime-repair.json"), "utf8"));
+        if (!repair || typeof repair !== "object" || repair.schema_version !== "1" || repair.kind !== "verifier-environment-runtime-repair"
+            || repair.path !== "integrations/harbor/hitch_harbor_environment.py"
+            || repair.source_runtime_id !== source.controller_runtime_id || repair.replacement_runtime_id !== record.controller_runtime_id
+            || ![repair.source_runtime_id, repair.replacement_runtime_id, repair.source_sha256, repair.replacement_sha256].every(value => typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value))
+            || !Number.isSafeInteger(repair.unchanged_file_count) || Number(repair.unchanged_file_count) < 0
+            || sha256JSON(repair) !== sha256JSON(receipt))
+            throw new Error("verifier runtime repair provenance mismatch");
+    }
+    else if (source.controller_runtime_id !== undefined && source.controller_runtime_id !== record.controller_runtime_id) {
+        throw new Error("verifier controller runtime changed without repair provenance");
+    }
+    const sourceIndex = await verifyResultBundleIndex(path.join(statePaths(root).runs, trial.run_id));
+    if (source.bundle_index_digest !== sha256JSON(sourceIndex))
+        throw new Error("regrade source run bundle changed");
+    return validateRunObservation(record.observation);
+}
+//# sourceMappingURL=regrade-evidence.js.map

package/dist/src/evals/regrade-evidence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"regrade-evidence.js","sourceRoot":"","sources":["../../../src/evals/regrade-evidence.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAC9F,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,SAAiB;IACvD,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,UAAU,KAAK,CAAC,QAAgB;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QAC1I,IAAI,KAAK,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACnJ,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YAClD,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE;gBAAE,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5G,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,QAAQ,CAAC;gBAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,KAAK,EAAE,MAAM,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QACpH,CAAC;IACH,CAAC;IACD,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,SAAiB,EAAE,MAA+B;IAC5F,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,EAAE,GAAG,MAAM,EAAE,cAAc,EAAE,GAAG,EAAE,IAAI,EAAE,0BAA0B,EAAE,kBAAkB,EAAE,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC;IAClJ,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,QAAQ,CAAC,CAAC;IACzE,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;AACxH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAY,EAAE,MAAc,EAAE,KAAqB;IAC9F,IAAI,KAAK,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC3F,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC;IACnC,IAAI,CAAC,SAAS,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC9K,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IACzF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACpE,IAAI,WAAW,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACjG,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAA4B,CAAC;IACrE,MAAM,MAAM,GAAG,MAAM,CAAC,MAA6C,CAAC;IACpE,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,KAAK,0BAA0B,IAAI,MAAM,CAAC,kBAAkB,KAAK,KAAK;WACjH,MAAM,CAAC,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO;WACjG,MAAM,EAAE,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAC3I,IAAI,MAAM,CAAC,eAAe,KAAK,MAAM,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACjJ,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAyC,CAAC;QAChE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,8BAA8B,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACzG,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,KAAK,qCAAqC;eAC9H,MAAM,CAAC,IAAI,KAAK,iDAAiD;eACjE,MAAM,CAAC,iBAAiB,KAAK,MAAM,CAAC,qBAAqB,IAAI,MAAM,CAAC,sBAAsB,KAAK,MAAM,CAAC,qBAAqB;eAC3H,CAAC,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;eAC5L,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC;eAC7F,UAAU,CAAC,MAAM,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAClH,CAAC;SAAM,IAAI,MAAM,CAAC,qBAAqB,KAAK,SAAS,IAAI,MAAM,CAAC,qBAAqB,KAAK,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACvH,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAClG,IAAI,MAAM,CAAC,mBAAmB,KAAK,UAAU,CAAC,WAAW,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACjH,OAAO,sBAAsB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;AACpD,CAAC"}

package/dist/src/evals/remote-infrastructure-retry.js

@@ -0,0 +1,165 @@
+import { setTimeout as delay } from "node:timers/promises";
+import { preparedArtifactForWorkItem } from "./work-item-artifacts.js";
+import { HitchError } from "../foundation/index.js";
+import { retryableInfrastructureTrials } from "./infrastructure-retry.js";
+import { replaceInvalidEvalProgressTrial, writeEvalProgress } from "./progress.js";
+import { physicalRetryWorkItem } from "./physical-retry-work.js";
+import { ensurePhysicalRetryDecision, transitionRetryDecision } from "./retry-state.js";
+import { retryBackoffMs } from "./retry-backoff.js";
+export async function runRemoteInfrastructureRetries(input) {
+    const executor = input.options.remoteWorkExecutor;
+    if (!executor)
+        throw new TypeError("remote work executor is unavailable");
+    let progress = input.progress;
+    let candidates = retryableInfrastructureTrials(input.initial.refs);
+    const runs = [];
+    const firstRetryIndex = input.firstRetryIndex ?? 1;
+    if (!Number.isSafeInteger(firstRetryIndex) || firstRetryIndex < 1)
+        throw new TypeError("first infrastructure retry index is invalid");
+    for (let retry = firstRetryIndex; retry <= input.options.request.infrastructure_retries && candidates.length > 0; retry += 1) {
+        if (input.options.signal?.aborted)
+            break;
+        const triggers = [...candidates];
+        const work = physicalRetryWorkItem(input.item, retry, triggers);
+        const backoffMs = retryBackoffMs(input.options.request.infrastructure_retry_backoff_ms, retry, work.work_id);
+        const notBefore = new Date(Date.now() + backoffMs).toISOString();
+        const decisions = await Promise.all(triggers.map((trigger) => ensurePhysicalRetryDecision({
+            evalDirectory: input.options.evalDirectory, evalId: input.options.evalId, item: input.item,
+            retryIndex: retry, trigger, notBefore,
+        })));
+        if (decisions.some((decision) => decision.retry_work_id !== work.work_id))
+            throw new Error(`persisted retry work identity conflicts with execution: ${work.work_id}`);
+        await input.options.onWorkItemQueued?.(work.work_id);
+        input.options.sink.emit({
+            type: "eval.retry.decision", execution_kind: "physical-infrastructure-retry", work_id: work.work_id,
+            decision_ids: decisions.map((decision) => decision.decision_id),
+            candidate_executes: true, attempt: input.item.logical_attempt, retry,
+            tasks: input.item.task_ids, backoff_ms: backoffMs, provider: input.item.provider,
+        });
+        const remainingBackoffMs = Math.max(0, Date.parse(decisions[0]?.not_before ?? notBefore) - Date.now());
+        input.metrics?.addBlocked("backoff", remainingBackoffMs);
+        if (remainingBackoffMs > 0)
+            await delay(remainingBackoffMs, undefined, input.options.signal ? { signal: input.options.signal } : undefined);
+        if (input.options.signal?.aborted)
+            break;
+        input.options.sink.emit({
+            type: "eval.retry.ready", work_id: work.work_id, decision_ids: decisions.map((decision) => decision.decision_id),
+            retry, tasks: input.item.task_ids, priority: input.item.scheduling?.remaining_path_ms ?? 0,
+        });
+        const refs = [];
+        let decisionsSettled = false;
+        const settleDecisions = async () => {
+            if (decisionsSettled || refs.length === 0)
+                return;
+            candidates = retryableInfrastructureTrials(refs);
+            if (retry < input.options.request.infrastructure_retries && candidates.length > 0) {
+                const nextWork = physicalRetryWorkItem(input.item, retry + 1, candidates);
+                const nextNotBefore = new Date(Date.now() + retryBackoffMs(input.options.request.infrastructure_retry_backoff_ms, retry + 1, nextWork.work_id)).toISOString();
+                await Promise.all(candidates.map((trigger) => ensurePhysicalRetryDecision({
+                    evalDirectory: input.options.evalDirectory, evalId: input.options.evalId, item: input.item,
+                    retryIndex: retry + 1, trigger, notBefore: nextNotBefore,
+                })));
+                await input.options.onWorkItemQueued?.(nextWork.work_id);
+            }
+            const retryState = refs.some((ref) => ref.observation_status === "valid")
+                ? "repaired"
+                : retry < input.options.request.infrastructure_retries && candidates.length > 0 ? "invalid" : "exhausted";
+            for (const decision of decisions)
+                await transitionRetryDecision({
+                    evalDirectory: input.options.evalDirectory, evalId: input.options.evalId, decisionId: decision.decision_id, state: retryState,
+                });
+            decisionsSettled = true;
+        };
+        const publish = async (ref) => {
+            assertSelectedTrial(ref, input.item);
+            const existing = refs.find((entry) => entry.trial_id === ref.trial_id || (entry.run_group ? entry.run_group.run_group_id === ref.run_group?.run_group_id : entry.run_id === ref.run_id));
+            if (existing) {
+                if (JSON.stringify(existing) !== JSON.stringify(ref))
+                    throw new Error(`remote retry trial identity changed: ${ref.trial_id}`);
+                return;
+            }
+            refs.push(ref);
+            if (ref.invalid_reason === "candidate_evidence_unavailable")
+                input.options.sink.emit({
+                    type: "eval.verifier.skipped", work_id: work.work_id, trial_id: ref.trial_id, task_id: ref.task_id,
+                    reason: "candidate_evidence_unavailable", candidate_executes: true, verifier_executes: false,
+                });
+            if (ref.observation_status === "valid") {
+                if (input.replaceProgressTrial) {
+                    await input.replaceProgressTrial(ref, work.work_id);
+                    progress = input.currentProgress?.() ?? progress;
+                }
+                else {
+                    progress = replaceInvalidEvalProgressTrial(progress, ref);
+                    await writeEvalProgress(input.options.evalDirectory, progress);
+                }
+            }
+            input.options.sink.emit({
+                type: ref.observation_status === "valid" ? "eval.infrastructure-retry.repaired" : "eval.infrastructure-retry.failed",
+                attempt: input.item.logical_attempt, retry, task_id: ref.task_id, trial_id: ref.trial_id,
+                run_id: ref.run_id, observation_status: ref.observation_status,
+                ...(ref.invalid_reason ? { invalid_reason: ref.invalid_reason } : {}), generation: (input.currentProgress?.() ?? progress).generation,
+            });
+        };
+        const completed = await executor({
+            evalId: input.options.evalId, evalDirectory: input.options.evalDirectory, root: input.options.root,
+            request: input.options.request, plan: input.options.plan, workItem: work,
+            resolvedRevision: input.options.resolvedRevision, preparedArtifact: preparedArtifactForWorkItem(input.options, work),
+            runtimeDirectory: input.options.controllerRuntime.directory, runtimeId: input.options.controllerRuntime.runtime_id,
+            ...(input.initial.environmentImages ? { environmentImages: input.initial.environmentImages } : {}),
+            ...(input.options.plan.model_capture ? { modelCapturePlan: input.options.plan.model_capture } : {}),
+            publicationMode: "replace-invalid",
+            ...(input.options.signal ? { signal: input.options.signal } : {}),
+            emit: (event) => input.options.sink.emit({ ...event, execution_kind: "physical-infrastructure-retry", infrastructure_retry: retry }),
+            publish,
+            onLeaseState: async (leaseId, state) => {
+                if (state === "running") {
+                    input.metrics?.startWork(work.work_id, "retry");
+                    for (const decision of decisions)
+                        await transitionRetryDecision({
+                            evalDirectory: input.options.evalDirectory, evalId: input.options.evalId, decisionId: decision.decision_id, state: "running",
+                        });
+                    input.options.sink.emit({ type: "eval.retry.admitted", work_id: work.work_id, decision_ids: decisions.map((decision) => decision.decision_id) });
+                }
+                if (state === "terminal") {
+                    await settleDecisions();
+                    input.metrics?.finishWork(work.work_id);
+                }
+                await input.options.onWorkItemState?.(work.work_id, leaseId, state);
+            },
+        });
+        assertPublishedRefs(completed.refs, refs, input.item);
+        await settleDecisions();
+        input.metrics?.finishWork(work.work_id);
+        runs.push({
+            attempt: input.item.logical_attempt, retry, tasks: [...input.item.task_ids], triggers,
+            refs, run: completed.run, leaseId: completed.leaseId, workId: work.work_id,
+        });
+        input.options.sink.emit({
+            type: "eval.infrastructure-retry.completed", execution_kind: "physical-infrastructure-retry",
+            candidate_executes: true, attempt: input.item.logical_attempt, retry, tasks: input.item.task_ids,
+            repaired_tasks: refs.filter((ref) => ref.observation_status === "valid").map((ref) => ref.task_id).sort(),
+            remaining_tasks: candidates.map((ref) => ref.task_id).sort(), provider: input.item.provider,
+        });
+        if (completed.run.backend.process_exit_code !== 0 || completed.run.rawResult === null)
+            break;
+    }
+    return { progress: input.currentProgress?.() ?? progress, runs };
+}
+function assertSelectedTrial(ref, item) {
+    if (ref.attempt !== item.logical_attempt || !item.task_ids.includes(ref.task_id)) {
+        throw new HitchError(`remote infrastructure retry returned an unselected trial: ${ref.task_id}#${ref.attempt}`, {
+            code: "eval_infrastructure_retry_trial_mismatch", exitCode: 12,
+        });
+    }
+}
+function assertPublishedRefs(returned, published, item) {
+    for (const ref of returned)
+        assertSelectedTrial(ref, item);
+    if (returned.length !== published.length || returned.some((ref) => !published.some((entry) => JSON.stringify(entry) === JSON.stringify(ref)))) {
+        throw new HitchError("remote infrastructure retry returned refs that were not durably published", {
+            code: "eval_infrastructure_retry_result_mismatch", exitCode: 12,
+        });
+    }
+}
+//# sourceMappingURL=remote-infrastructure-retry.js.map

package/dist/src/evals/remote-infrastructure-retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-infrastructure-retry.js","sourceRoot":"","sources":["../../../src/evals/remote-infrastructure-retry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,6BAA6B,EAAE,MAAM,2BAA2B,CAAC;AAE1E,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,2BAA2B,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AACxF,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGpD,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,KASpD;IACC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAClD,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC1E,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC9B,IAAI,UAAU,GAAG,6BAA6B,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,MAAM,IAAI,GAA6B,EAAE,CAAC;IAC1C,MAAM,eAAe,GAAG,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,eAAe,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IACtI,KAAK,IAAI,KAAK,GAAG,eAAe,EAAE,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC7H,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACzC,MAAM,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7G,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC;YACxF,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI;YAC1F,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS;SACtC,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACtK,MAAM,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YACtB,IAAI,EAAE,qBAAqB,EAAE,cAAc,EAAE,+BAA+B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO;YACnG,YAAY,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC/D,kBAAkB,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK;YACpE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;SACjF,CAAC,CAAC;QACH,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACvG,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;QACzD,IAAI,kBAAkB,GAAG,CAAC;YAAE,MAAM,KAAK,CAAC,kBAAkB,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC5I,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACzC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YACtB,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC;YAChH,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,IAAI,CAAC;SAC3F,CAAC,CAAC;QACH,MAAM,IAAI,GAAqB,EAAE,CAAC;QAClC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,MAAM,eAAe,GAAG,KAAK,IAAmB,EAAE;YAChD,IAAI,gBAAgB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAClD,UAAU,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClF,MAAM,QAAQ,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC1E,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,EAAE,KAAK,GAAG,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC9J,MAAM,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC;oBACxE,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI;oBAC1F,UAAU,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa;iBACzD,CAAC,CAAC,CAAC,CAAC;gBACL,MAAM,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC3D,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC;gBACvE,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;YAC5G,KAAK,MAAM,QAAQ,IAAI,SAAS;gBAAE,MAAM,uBAAuB,CAAC;oBAC9D,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,UAAU;iBAC9H,CAAC,CAAC;YACH,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,KAAK,EAAE,GAAmB,EAAiB,EAAE;YAC3D,mBAAmB,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,KAAK,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;YACzL,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC9H,OAAO;YACT,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,IAAI,GAAG,CAAC,cAAc,KAAK,gCAAgC;gBAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;oBACnF,IAAI,EAAE,uBAAuB,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO;oBAClG,MAAM,EAAE,gCAAgC,EAAE,kBAAkB,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK;iBAC7F,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACvC,IAAI,KAAK,CAAC,oBAAoB,EAAE,CAAC;oBAC/B,MAAM,KAAK,CAAC,oBAAoB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;oBACpD,QAAQ,GAAG,KAAK,CAAC,eAAe,EAAE,EAAE,IAAI,QAAQ,CAAC;gBACnD,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;oBAC1D,MAAM,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBACjE,CAAC;YACH,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBACtB,IAAI,EAAE,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC,kCAAkC;gBACpH,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACxF,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gBAC9D,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,KAAK,CAAC,eAAe,EAAE,EAAE,IAAI,QAAQ,CAAC,CAAC,UAAU;aACtI,CAAC,CAAC;QACL,CAAC,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;YAC/B,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI;YAClG,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI;YACxE,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,2BAA2B,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC;YACpH,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,UAAU;YAClH,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnG,eAAe,EAAE,iBAAiB;YAClC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,cAAc,EAAE,+BAA+B,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;YACpI,OAAO;YACP,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;gBACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;oBAChD,KAAK,MAAM,QAAQ,IAAI,SAAS;wBAAE,MAAM,uBAAuB,CAAC;4BAC9D,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,SAAS;yBAC7H,CAAC,CAAC;oBACH,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gBACnJ,CAAC;gBACD,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;oBACzB,MAAM,eAAe,EAAE,CAAC;oBACxB,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC1C,CAAC;gBACD,MAAM,KAAK,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YACtE,CAAC;SACF,CAAC,CAAC;QACH,mBAAmB,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACtD,MAAM,eAAe,EAAE,CAAC;QACxB,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC;YACR,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAyB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,QAAQ;YAC/F,IAAI,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO;SAC3E,CAAC,CAAC;QACH,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YACtB,IAAI,EAAE,qCAAqC,EAAE,cAAc,EAAE,+BAA+B;YAC5F,kBAAkB,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;YAChG,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;YACzG,eAAe,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;SAC5F,CAAC,CAAC;QACH,IAAI,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,SAAS,KAAK,IAAI;YAAE,MAAM;IAC/F,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,eAAe,EAAE,EAAE,IAAI,QAAQ,EAAE,IAAI,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAmB,EAAE,IAAuB;IACvE,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,UAAU,CAAC,6DAA6D,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE;YAC9G,IAAI,EAAE,0CAA0C,EAAE,QAAQ,EAAE,EAAE;SAC/D,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAmC,EAAE,SAAoC,EAAE,IAAuB;IAC7H,KAAK,MAAM,GAAG,IAAI,QAAQ;QAAE,mBAAmB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9I,MAAM,IAAI,UAAU,CAAC,2EAA2E,EAAE;YAChG,IAAI,EAAE,2CAA2C,EAAE,QAAQ,EAAE,EAAE;SAChE,CAAC,CAAC;IACL,CAAC;AACH,CAAC"}

package/dist/src/evals/request.js

@@ -38,12 +38,12 @@ export async function validateEvalRequest(input) {
         throw invalidInput("eval requires an immutable harness ref: version:<exact> or commit:<sha>");
     }
     assertExactLocalGitEvalReference(reference);
-    const benchmark = await resolveBenchmarkReference(input.dataset.trim());
+    const benchmark = await resolveDatasetReference(input.dataset.trim());
     const attempts = positiveInteger(input.attempts ?? 1, "attempts");
     const maxConcurrent = positiveInteger(input.max_concurrent ?? 4, "max_concurrent");
     const infrastructureRetries = nonNegativeInteger(input.infrastructure_retries ?? DEFAULT_INFRASTRUCTURE_RETRIES, "infrastructure_retries");
     const infrastructureRetryBackoff = nonNegativeNumber(input.infrastructure_retry_backoff_ms ?? DEFAULT_INFRASTRUCTURE_RETRY_BACKOFF_MS, "infrastructure_retry_backoff_ms");
-    const timeout = nonNegativeNumber(input.timeout_ms ?? DEFAULT_EVAL_TIMEOUT_MS, "timeout_ms");
+    const timeout = nonNegativeNumber(input.timeout_ms ?? (benchmark.manifest ? 0 : DEFAULT_EVAL_TIMEOUT_MS), "timeout_ms");
     const setupTimeout = nonNegativeNumber(input.setup_timeout_ms ?? DEFAULT_EVAL_SETUP_TIMEOUT_MS, "setup_timeout_ms");
     if (input.model !== undefined && typeof input.model !== "string")
         throw invalidInput("model must be a string");
@@ -54,6 +54,9 @@ export async function validateEvalRequest(input) {
         throw invalidInput("pass_env must be an array of strings");
     }
     const agentArgs = Array.isArray(input.agent_args) ? [...input.agent_args] : [];
+    if (benchmark.manifest) {
+        await assertStandardBenchmarkCandidate(input.dataset.trim(), benchmark.manifest, reference.harness_id, agentArgs);
+    }
     const bypassArg = HARBOR_EVAL_BYPASS_ARG[reference.harness_id];
     if (bypassArg && !agentArgs.includes(bypassArg))
         agentArgs.unshift(bypassArg);
@@ -76,13 +79,26 @@ export async function validateEvalRequest(input) {
     };
 }
 export async function resolveBenchmarkReference(dataset) {
+    const { benchmark_id, benchmark_revision } = await resolveDatasetReference(dataset);
+    return { benchmark_id, benchmark_revision };
+}
+async function resolveDatasetReference(dataset) {
     const raw = dataset.trim();
     const local = path.resolve(raw);
     try {
         if ((await stat(local)).isDirectory()) {
+            const manifest = await loadBenchmarkAdapterManifest(local);
+            if (manifest) {
+                return {
+                    benchmark_id: manifest.benchmark.id,
+                    benchmark_revision: manifest.dataset_digest,
+                    manifest,
+                };
+            }
             return {
                 benchmark_id: `local:${path.basename(local)}`,
                 benchmark_revision: await workspaceDigest(local, { excludedTopLevel: new Set([".git"]) }),
+                manifest: null,
             };
         }
     }
@@ -98,7 +114,7 @@ export async function resolveBenchmarkReference(dataset) {
     const revision = raw.slice(separator + 1);
     if (revision.toLowerCase() === "latest")
         throw invalidInput("dataset revision cannot be latest");
-    return { benchmark_id: benchmarkId, benchmark_revision: revision };
+    return { benchmark_id: benchmarkId, benchmark_revision: revision, manifest: null };
 }
 export async function resolveLocalDatasetTaskIds(dataset) {
     const local = path.resolve(dataset.trim());
@@ -155,4 +171,6 @@ import path from "node:path";
 import { SCHEMA_VERSION, invalidInput } from "../foundation/index.js";
 import { assertExactLocalGitEvalReference, parseHarnessReference } from "../revisions/index.js";
 import { workspaceDigest } from "../workspaces/index.js";
+import { loadBenchmarkAdapterManifest } from "./benchmark-adapter-manifest.js";
+import { assertStandardBenchmarkCandidate } from "./benchmark-candidate.js";
 //# sourceMappingURL=request.js.map