agent-hitch 0.2.9 → 0.2.10

package/dist/src/evals/remote-verifier-execution.js

@@ -0,0 +1,105 @@
+import { cp, mkdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { parse as parseTOML } from "smol-toml";
+import { parsePortableHarborRegradeConfig, restorePortableHarborRegradeConfig, runHarborRegrade } from "../backends/index.js";
+import { useControllerRuntimeDirectory } from "../controller-runtime/index.js";
+import { HitchError, atomicWriteJSON, readJSON, sha256JSON, statePaths } from "../foundation/index.js";
+import { dockerOwnershipLabelMap, dockerResourceOwnership } from "./docker-ownership.js";
+import { startDockerResourceObserver } from "./docker-resource-observer.js";
+import { runtimeResourcesForTask } from "./execution-plan-resources.js";
+import { readExecutionLeases } from "./execution-leases.js";
+import { regradeTreeDigest } from "./regrade-evidence.js";
+import { portableRemoteTrial } from "./remote-trial.js";
+import { parseRemoteVerifierSourceManifest, verifyRemoteVerifierSourceContents } from "./verifier-source.js";
+import { verifierRuntimeRepair } from "./verifier-runtime.js";
+/** Executes only the artifact regrade phase. The worker runner owns heartbeat, cancellation and proven resource release. */
+export async function runRemoteVerifierWork(input) {
+    const source = parseRemoteVerifierSourceManifest(input.sourceManifest);
+    const checkLease = async () => {
+        const current = (await readExecutionLeases(path.join(statePaths(input.root).evals, input.lease.eval_id))).find(lease => lease.lease_id === input.lease.lease_id);
+        const fields = ["lease_id", "eval_id", "work_id", "worker_id", "provider", "collision_domain_id", "epoch", "reservation", "resource_epochs"];
+        if (!current || current.state !== "running" || Date.parse(current.expires_at) <= Date.now()
+            || fields.some(field => sha256JSON(current[field] ?? null) !== sha256JSON(input.lease[field] ?? null)))
+            throw unavailable("remote verifier has no current execution lease");
+    };
+    await checkLease();
+    if (source.status !== "available" || !source.regrade_config_digest || !source.original_result_digest
+        || !/^sha256:[a-f0-9]{64}$/.test(input.candidateResultDigest) || sha256JSON(input.candidateResult) !== input.candidateResultDigest
+        || input.candidateResult.run_id !== source.run_id || input.candidateResult.status !== "succeeded"
+        || sha256JSON(portableRemoteTrial(input.sourceTrial)) !== source.source_result_digest || input.sourceTrial.config !== undefined
+        || input.sourceTrial.trial_name !== source.trial_id
+        || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/.test(input.trialName)
+        || !input.plan.slots.some(slot => slot.task_id === source.task_id) || input.plan.eval_id !== input.lease.eval_id
+        || input.plan.provider !== input.lease.provider)
+        throw unavailable("remote verifier source/work identity differs");
+    const metadata = input.sourceTrial.agent_result?.metadata;
+    if (metadata?.hitch_run_id !== source.run_id || metadata.controller_runtime_id !== source.controller_runtime_id)
+        throw unavailable("remote verifier candidate/runtime identity differs");
+    const task = parseTOML(await readFile(path.join(input.taskDirectory, "task.toml"), "utf8"));
+    const taskName = task.task?.name ?? path.basename(input.taskDirectory);
+    if (input.sourceTrial.task_name !== taskName)
+        throw unavailable("remote verifier task name differs");
+    const runtime = await useControllerRuntimeDirectory(input.runtimeDirectory, source.controller_runtime_id);
+    const verifierRuntime = await useControllerRuntimeDirectory(input.verifierRuntimeDirectory, input.verifierRuntimeId);
+    const repair = verifierRuntimeRepair(runtime, verifierRuntime);
+    const portable = parsePortableHarborRegradeConfig(await readJSON(path.join(input.sourceSnapshotDirectory, "regrade-config.json")));
+    if (sha256JSON(portable) !== source.regrade_config_digest || portable.source_config_digest !== source.source_config_digest)
+        throw unavailable("remote verifier config identity differs");
+    if (input.signal?.aborted)
+        throw input.signal.reason ?? new DOMException("Aborted", "AbortError");
+    await mkdir(input.directory, { recursive: false, mode: 0o700 });
+    const candidateDirectory = path.join(input.directory, "candidate");
+    await atomicWriteJSON(path.join(candidateDirectory, "result.json"), input.candidateResult);
+    const verifySource = () => verifyRemoteVerifierSourceContents({ manifest: source, snapshotDirectory: input.sourceSnapshotDirectory,
+        taskDirectory: input.taskDirectory, bundleDirectory: candidateDirectory });
+    await verifySource();
+    await checkLease();
+    if (input.signal?.aborted)
+        throw input.signal.reason ?? new DOMException("Aborted", "AbortError");
+    const replay = path.join(input.directory, "source-trial");
+    await cp(input.sourceSnapshotDirectory, replay, { recursive: true, force: false, errorOnExist: true, dereference: false });
+    if (await regradeTreeDigest(replay) !== source.snapshot_digest)
+        throw unavailable("remote verifier source changed while copying");
+    const ownership = dockerResourceOwnership(input.root, input.lease, source.task_id);
+    const { sourceConfig, regradeConfig } = restorePortableHarborRegradeConfig({ portable, sourceConfigDigest: source.source_config_digest,
+        taskDirectory: input.taskDirectory, sourceDirectory: replay, outputDirectory: path.join(input.directory, "trials"),
+        trialName: input.trialName, sourceResult: input.sourceTrial, ownershipLabels: dockerOwnershipLabelMap(ownership) });
+    await atomicWriteJSON(path.join(replay, "config.json"), sourceConfig);
+    // Harbor's TrialResult requires config for decoding; the private original stays on its original worker.
+    await atomicWriteJSON(path.join(replay, "result.json"), { ...input.sourceTrial, config: sourceConfig });
+    const replayDigest = await regradeTreeDigest(replay);
+    const limits = runtimeResourcesForTask(input.plan, source.task_id, input.lease.reservation);
+    const environment = regradeConfig.environment, kwargs = environment.kwargs;
+    if (environment.cpu_enforcement_policy !== "limit" || environment.memory_enforcement_policy !== "limit"
+        || environment.override_cpus !== limits.mainLimits.cpu_millis / 1000 || environment.override_memory_mb !== limits.mainLimits.memory_bytes / 1024 ** 2
+        || (kwargs.hitch_main_gpu_count ?? 0) !== (limits.mainLimits.gpu_count ?? 0)
+        || environment.override_gpus != null && environment.override_gpus !== (limits.mainLimits.gpu_count ?? 0)
+        || sha256JSON(kwargs.hitch_service_resource_limits ?? {}) !== sha256JSON(limits.sidecarLimits))
+        throw unavailable("remote verifier resource settings differ from its lease plan");
+    const observer = startDockerResourceObserver({ ownership, workerId: input.lease.worker_id, collisionDomainId: input.lease.collision_domain_id,
+        reservation: input.lease.reservation, mainLimits: limits.mainLimits, sidecarLimits: limits.sidecarLimits, env: input.env,
+        ...(input.signal ? { signal: input.signal } : {}) });
+    let outcome;
+    try {
+        outcome = await runHarborRegrade({ root: input.root, directory: input.directory, config: regradeConfig,
+            runtimeDirectory: verifierRuntime.directory, trustedResult: input.candidateResult, env: input.env,
+            ...(input.processHooks ? { processHooks: input.processHooks } : {}),
+            ...(input.harborExecutable ? { harborExecutable: input.harborExecutable } : {}), ...(input.signal ? { signal: input.signal } : {}) });
+    }
+    finally {
+        await atomicWriteJSON(path.join(input.directory, "execution.json"), await observer.stop());
+    }
+    await verifySource();
+    await checkLease();
+    if (input.signal?.aborted)
+        throw input.signal.reason ?? new DOMException("Aborted", "AbortError");
+    if (await regradeTreeDigest(replay) !== replayDigest || outcome.trial.trial_name !== input.trialName || outcome.trial.task_name !== taskName
+        || sha256JSON(outcome.trial.agent_result) !== sha256JSON(input.sourceTrial.agent_result))
+        throw unavailable("remote verifier mutated its source candidate");
+    return { trial: portableRemoteTrial(outcome.trial), backend: outcome.backend, execution: await readJSON(path.join(input.directory, "execution.json")),
+        source_manifest_digest: sha256JSON(source), candidate_result_digest: input.candidateResultDigest,
+        config_digest: sha256JSON(regradeConfig), controller_runtime_id: verifierRuntime.runtime_id,
+        ...(repair ? { runtime_repair: repair } : {}), trial_directory: path.join(input.directory, "trials", input.trialName) };
+}
+function unavailable(message) { return new HitchError(message, { code: "eval_verifier_only_unavailable", exitCode: 2 }); }
+//# sourceMappingURL=remote-verifier-execution.js.map

package/dist/src/evals/remote-verifier-execution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-verifier-execution.js","sourceRoot":"","sources":["../../../src/evals/remote-verifier-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,gCAAgC,EAAE,kCAAkC,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC9H,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAE/E,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACvG,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AACzF,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,iCAAiC,EAAE,kCAAkC,EAAE,MAAM,sBAAsB,CAAC;AAC7G,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,4HAA4H;AAC5H,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAQ3C;IACC,MAAM,MAAM,GAAG,iCAAiC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACvE,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE;QAC5B,MAAM,OAAO,GAAG,CAAC,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjK,MAAM,MAAM,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,qBAAqB,EAAE,OAAO,EAAE,aAAa,EAAE,iBAAiB,CAAU,CAAC;QACtJ,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE;eACtF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;YAAE,MAAM,WAAW,CAAC,gDAAgD,CAAC,CAAC;IAChL,CAAC,CAAC;IACF,MAAM,UAAU,EAAE,CAAC;IACnB,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,MAAM,CAAC,qBAAqB,IAAI,CAAC,MAAM,CAAC,sBAAsB;WAC/F,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,KAAK,CAAC,qBAAqB;WAC/H,KAAK,CAAC,eAAe,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,MAAM,KAAK,WAAW;WAC9F,UAAU,CAAC,mBAAmB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,KAAK,MAAM,CAAC,oBAAoB,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,SAAS;WAC5H,KAAK,CAAC,WAAW,CAAC,UAAU,KAAK,MAAM,CAAC,QAAQ;WAChD,CAAC,oCAAoC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;WAC3D,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,CAAC,OAAO;WAC7G,KAAK,CAAC,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,CAAC,QAAQ;QAAE,MAAM,WAAW,CAAC,8CAA8C,CAAC,CAAC;IACrH,MAAM,QAAQ,GAAI,KAAK,CAAC,WAAW,CAAC,YAAmE,EAAE,QAAQ,CAAC;IAClH,IAAI,QAAQ,EAAE,YAAY,KAAK,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,qBAAqB,KAAK,MAAM,CAAC,qBAAqB;QAAE,MAAM,WAAW,CAAC,oDAAoD,CAAC,CAAC;IACzL,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5F,MAAM,QAAQ,GAAI,IAAI,CAAC,IAA4C,EAAE,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAChH,IAAI,KAAK,CAAC,WAAW,CAAC,SAAS,KAAK,QAAQ;QAAE,MAAM,WAAW,CAAC,mCAAmC,CAAC,CAAC;IACrG,MAAM,OAAO,GAAG,MAAM,6BAA6B,CAAC,KAAK,CAAC,gBAAgB,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAC1G,MAAM,eAAe,GAAG,MAAM,6BAA6B,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrH,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,gCAAgC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,uBAAuB,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC;IACnI,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,qBAAqB,IAAI,QAAQ,CAAC,oBAAoB,KAAK,MAAM,CAAC,oBAAoB;QAAE,MAAM,WAAW,CAAC,yCAAyC,CAAC,CAAC;IACzL,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;QAAE,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAClG,MAAM,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACnE,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,aAAa,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;IAC3F,MAAM,YAAY,GAAG,GAAG,EAAE,CAAC,kCAAkC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,CAAC,uBAAuB;QAChI,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC7E,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,UAAU,EAAE,CAAC;IACnB,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;QAAE,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAC1D,MAAM,EAAE,CAAC,KAAK,CAAC,uBAAuB,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3H,IAAI,MAAM,iBAAiB,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,eAAe;QAAE,MAAM,WAAW,CAAC,8CAA8C,CAAC,CAAC;IAClI,MAAM,SAAS,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IACnF,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,kCAAkC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,CAAC,oBAAoB;QACpI,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,CAAC;QAClH,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE,eAAe,EAAE,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACtH,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,YAAY,CAAC,CAAC;IACtE,wGAAwG;IACxG,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;IACxG,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC5F,MAAM,WAAW,GAAG,aAAa,CAAC,WAAsC,EAAE,MAAM,GAAG,WAAW,CAAC,MAAiC,CAAC;IACjI,IAAI,WAAW,CAAC,sBAAsB,KAAK,OAAO,IAAI,WAAW,CAAC,yBAAyB,KAAK,OAAO;WAClG,WAAW,CAAC,aAAa,KAAK,MAAM,CAAC,UAAU,CAAC,UAAU,GAAG,IAAI,IAAI,WAAW,CAAC,kBAAkB,KAAK,MAAM,CAAC,UAAU,CAAC,YAAY,GAAG,IAAI,IAAI,CAAC;WAClJ,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,CAAC;WACzE,WAAW,CAAC,aAAa,IAAI,IAAI,IAAI,WAAW,CAAC,aAAa,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,CAAC;WACrG,UAAU,CAAC,MAAM,CAAC,6BAA6B,IAAI,EAAE,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC;QAAE,MAAM,WAAW,CAAC,8DAA8D,CAAC,CAAC;IACpL,MAAM,QAAQ,GAAG,2BAA2B,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,iBAAiB,EAAE,KAAK,CAAC,KAAK,CAAC,mBAAmB;QAC3I,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG;QACxH,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACvD,IAAI,OAAqD,CAAC;IAC1D,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,gBAAgB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,aAAa;YACpG,gBAAgB,EAAE,eAAe,CAAC,SAAS,EAAE,aAAa,EAAE,KAAK,CAAC,eAAe,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG;YACjG,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC1I,CAAC;YAAS,CAAC;QAAC,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,gBAAgB,CAAC,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;IACzG,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,UAAU,EAAE,CAAC;IACnB,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;QAAE,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAClG,IAAI,MAAM,iBAAiB,CAAC,MAAM,CAAC,KAAK,YAAY,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,SAAS,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,KAAK,QAAQ;WACvI,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,YAAY,CAAC;QAAE,MAAM,WAAW,CAAC,8CAA8C,CAAC,CAAC;IAC9J,OAAO,EAAE,KAAK,EAAE,mBAAmB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;QACnJ,sBAAsB,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,uBAAuB,EAAE,KAAK,CAAC,qBAAqB;QAChG,aAAa,EAAE,UAAU,CAAC,aAAa,CAAC,EAAE,qBAAqB,EAAE,eAAe,CAAC,UAAU;QAC3F,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;AAC5H,CAAC;AACD,SAAS,WAAW,CAAC,OAAe,IAAgB,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,gCAAgC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/evals/remote-verifier-journal.js

@@ -0,0 +1,21 @@
+import path from "node:path";
+import { HitchError, readJSON, sha256JSON } from "../foundation/index.js";
+import { assertRemoteVerifierWork } from "./remote-verifier-contract.js";
+export function remoteVerifierWorkIdentity(input) {
+    assertRemoteVerifierWork({ verifier: input.verifier, plan: input.plan, work: input.work, physical: input.physical,
+        runtimeId: input.verifier.source_manifest.controller_runtime_id });
+    if (input.physical.kind !== "verifier-only" || input.request.training_binding)
+        throw ambiguous();
+    return { schema_version: "2", execution_kind: "verifier-only", rerun_id: input.physical.rerun_id, source_work_id: input.physical.source_work_id,
+        work_id: input.work.work_id, plan_digest: sha256JSON(input.plan), request_digest: sha256JSON(input.request), verifier_work_digest: sha256JSON(input.verifier) };
+}
+/** A scoring dispatch must already have an immutable journal for crash recovery. */
+export async function assertRemoteVerifierDispatch(input) {
+    const identity = remoteVerifierWorkIdentity(input);
+    const file = path.join(input.evalDirectory, "reruns", identity.rerun_id, "remote-work", `${identity.work_id}.json`);
+    const record = await readJSON(file, null);
+    if (!record || record.state !== "dispatching" || record.result || sha256JSON(record.identity) !== sha256JSON(identity))
+        throw ambiguous();
+}
+function ambiguous() { return new HitchError("remote verifier dispatch requires its frozen durable work identity", { code: "execution_state_ambiguous", exitCode: 12 }); }
+//# sourceMappingURL=remote-verifier-journal.js.map

package/dist/src/evals/remote-verifier-journal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-verifier-journal.js","sourceRoot":"","sources":["../../../src/evals/remote-verifier-journal.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAC1E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,UAAU,0BAA0B,CAAC,KAE1C;IACC,wBAAwB,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ;QAC/G,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,qBAAqB,EAAE,CAAC,CAAC;IACrE,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,eAAe,IAAI,KAAK,CAAC,OAAO,CAAC,gBAAgB;QAAE,MAAM,SAAS,EAAE,CAAC;IACjG,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,cAAc;QAC7I,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,oBAAoB,EAAE,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;AACpK,CAAC;AAED,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,KAAmF;IACpI,MAAM,QAAQ,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,aAAa,EAAE,GAAG,QAAQ,CAAC,OAAO,OAAO,CAAC,CAAC;IACpH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAgE,IAAI,EAAE,IAAI,CAAC,CAAC;IACzG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,KAAK,aAAa,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC;QAAE,MAAM,SAAS,EAAE,CAAC;AAC5I,CAAC;AACD,SAAS,SAAS,KAAiB,OAAO,IAAI,UAAU,CAAC,oEAAoE,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/evals/remote-verifier-outcome.js

@@ -0,0 +1,51 @@
+import { sha256JSON } from "../foundation/index.js";
+import { parseExecutionEvidence } from "./execution-evidence.js";
+import { runtimeResourcesForTask } from "./execution-plan-resources.js";
+const SHA = /^sha256:[a-f0-9]{64}$/;
+export function parseRemoteVerifierOutcome(value) {
+    if (!object(value))
+        throw invalid();
+    const keys = ["trial", "backend", "execution", "config_digest", "controller_runtime_id", ...(value.runtime_repair === undefined ? [] : ["runtime_repair"])];
+    if (Object.keys(value).length !== keys.length || Object.keys(value).some(key => !keys.includes(key))
+        || !object(value.trial) || Object.hasOwn(value.trial, "config")
+        || value.trial.agent_setup != null || value.trial.agent_execution != null
+        || typeof value.trial.trial_name !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/.test(value.trial.trial_name)
+        || !object(value.backend) || Object.keys(value.backend).sort().join(",") !== "process_exit_code,signal"
+        || !(value.backend.process_exit_code === null || Number.isSafeInteger(value.backend.process_exit_code)
+            && Number(value.backend.process_exit_code) >= 0 && Number(value.backend.process_exit_code) <= 255)
+        || !(value.backend.signal === null || typeof value.backend.signal === "string" && /^SIG[A-Z0-9]{1,24}$/.test(value.backend.signal))
+        || ![value.config_digest, value.controller_runtime_id].every(v => typeof v === "string" && SHA.test(v))
+        || value.runtime_repair !== undefined && !validRepair(value.runtime_repair, value.controller_runtime_id))
+        throw invalid();
+    return { ...structuredClone(value), execution: parseExecutionEvidence(value.execution) };
+}
+export function assertRemoteVerifierOutcome(input) {
+    const o = parseRemoteVerifierOutcome(input.outcome), v = input.verifier, e = o.execution, l = input.lease;
+    const limits = runtimeResourcesForTask(input.plan, v.source_ref.task_id, input.work.reservation);
+    if (input.work.eval_id !== l.eval_id || input.work.work_id !== l.work_id || input.work.provider !== l.provider
+        || e.eval_id !== l.eval_id || e.work_id !== l.work_id || e.lease_id !== l.lease_id || e.lease_epoch !== l.epoch
+        || e.worker_id !== l.worker_id || e.provider !== l.provider || e.collision_domain_id !== l.collision_domain_id
+        || e.task_id !== v.source_ref.task_id || sha256JSON(e.reservation) !== sha256JSON(input.work.reservation)
+        || sha256JSON(l.reservation) !== sha256JSON(e.reservation)
+        || sha256JSON(e.enforced.main_limits) !== sha256JSON(limits.mainLimits) || sha256JSON(e.enforced.sidecar_limits) !== sha256JSON(limits.sidecarLimits)
+        || o.controller_runtime_id !== v.verifier_runtime_id || o.trial.task_name !== v.source_trial.task_name
+        || o.trial.trial_name !== remoteVerifierTrialName(v.assessment_id)
+        || sha256JSON(o.trial.agent_result) !== sha256JSON(v.source_trial.agent_result))
+        throw invalid();
+}
+export function remoteVerifierTrialName(assessmentId) {
+    if (!/^assessment_[a-f0-9]{32}$/.test(assessmentId))
+        throw invalid();
+    return `regrade__${assessmentId}`;
+}
+function object(value) { return !!value && typeof value === "object" && !Array.isArray(value); }
+function validRepair(value, runtimeId) {
+    const keys = ["schema_version", "kind", "source_runtime_id", "replacement_runtime_id", "path", "source_sha256", "replacement_sha256", "unchanged_file_count"];
+    return object(value) && Object.keys(value).length === keys.length && Object.keys(value).every(key => keys.includes(key))
+        && value.schema_version === "1" && value.kind === "verifier-environment-runtime-repair"
+        && value.path === "integrations/harbor/hitch_harbor_environment.py" && value.replacement_runtime_id === runtimeId
+        && [value.source_runtime_id, value.replacement_runtime_id, value.source_sha256, value.replacement_sha256].every(v => typeof v === "string" && SHA.test(v))
+        && Number.isSafeInteger(value.unchanged_file_count) && Number(value.unchanged_file_count) >= 0;
+}
+function invalid() { return new TypeError("remote verifier outcome differs from its scoring work, candidate or execution lease"); }
+//# sourceMappingURL=remote-verifier-outcome.js.map

package/dist/src/evals/remote-verifier-outcome.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-verifier-outcome.js","sourceRoot":"","sources":["../../../src/evals/remote-verifier-outcome.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAExE,MAAM,GAAG,GAAG,uBAAuB,CAAC;AACpC,MAAM,UAAU,0BAA0B,CAAC,KAAc;IACvD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,MAAM,OAAO,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,uBAAuB,EAAE,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAC5J,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;WAC/F,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC;WAC5D,KAAK,CAAC,KAAK,CAAC,WAAW,IAAI,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,eAAe,IAAI,IAAI;WACtE,OAAO,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,oCAAoC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC;WAChH,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,0BAA0B;WACpG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,KAAK,IAAI,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC;eACjG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,IAAI,GAAG,CAAC;WACjG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;WAChI,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;WACpG,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,qBAAqB,CAAC;QAAE,MAAM,OAAO,EAAE,CAAC;IAC5H,OAAO,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,sBAAsB,CAAC,KAAK,CAAC,SAAS,CAAC,EAAwC,CAAC;AACjI,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,KAE3C;IACC,MAAM,CAAC,GAAG,0BAA0B,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC;IAC1G,MAAM,MAAM,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACjG,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ;WACzG,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,KAAK;WAC5G,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,mBAAmB,KAAK,CAAC,CAAC,mBAAmB;WAC3G,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,UAAU,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC;WACtG,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;WACvD,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC;WAClJ,CAAC,CAAC,qBAAqB,KAAK,CAAC,CAAC,mBAAmB,IAAI,CAAC,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,CAAC,YAAY,CAAC,SAAS;WACnG,CAAC,CAAC,KAAK,CAAC,UAAU,KAAK,uBAAuB,CAAC,CAAC,CAAC,aAAa,CAAC;WAC/D,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC;QAAE,MAAM,OAAO,EAAE,CAAC;AACrG,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,YAAoB;IAC1D,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,YAAY,CAAC;QAAE,MAAM,OAAO,EAAE,CAAC;IACrE,OAAO,YAAY,YAAY,EAAE,CAAC;AACpC,CAAC;AACD,SAAS,MAAM,CAAC,KAAc,IAAsC,OAAO,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAC3I,SAAS,WAAW,CAAC,KAAc,EAAE,SAAkB;IACrD,MAAM,IAAI,GAAG,CAAC,gBAAgB,EAAE,MAAM,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,EAAE,eAAe,EAAE,oBAAoB,EAAE,sBAAsB,CAAC,CAAC;IAC9J,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;WACnH,KAAK,CAAC,cAAc,KAAK,GAAG,IAAI,KAAK,CAAC,IAAI,KAAK,qCAAqC;WACpF,KAAK,CAAC,IAAI,KAAK,iDAAiD,IAAI,KAAK,CAAC,sBAAsB,KAAK,SAAS;WAC9G,CAAC,KAAK,CAAC,iBAAiB,EAAE,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;WACvJ,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;AACnG,CAAC;AACD,SAAS,OAAO,KAAgB,OAAO,IAAI,SAAS,CAAC,qFAAqF,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/evals/remote-verifier-rerun.js

@@ -0,0 +1,134 @@
+import path from "node:path";
+import { useControllerRuntimeById } from "../controller-runtime/index.js";
+import { HitchError, atomicWriteJSON, readJSON, sha256JSON, statePaths } from "../foundation/index.js";
+import { validateEvalId } from "./request.js";
+import { EvalEventSink } from "./events.js";
+import { readExecutionLeases } from "./execution-leases.js";
+import { parseEvalExecutionPlan } from "./execution-plan.js";
+import { replaceInvalidEvalProgressTrial, writeEvalProgress } from "./progress.js";
+import { readRegradeObservation } from "./regrade-evidence.js";
+import { remoteVerifierWorkIdentity } from "./remote-verifier-journal.js";
+import { assertRemoteVerifierPublication, freezeRemoteVerifierSelection } from "./remote-verifier-selection.js";
+import { loadRerunPreparedArtifacts, loadRerunResolvedRevision, parseRerunPlan } from "./rerun-inputs.js";
+import { validateEvalTrialReferences } from "./trial-import.js";
+import { frozenRerunBenchmark } from "./verifier-only-rerun.js";
+import { finishVerifierRerun } from "./verifier-rerun-result.js";
+import { verifierRuntimeRepair } from "./verifier-runtime.js";
+import { preparedArtifactForWorkItem } from "./work-item-artifacts.js";
+/** Execute only scoring, with a frozen source selection and a durable per-work journal. */
+export async function remoteVerifierOnlyEvalRerun(input) {
+    const evalId = validateEvalId(input.evalId);
+    const execution = parseEvalExecutionPlan(await readJSON(path.join(input.evalDirectory, "execution-plan.json")));
+    const benchmark = await frozenRerunBenchmark(input.evalDirectory);
+    if (!benchmark || execution.provider === "local-docker" || execution.membership !== "known" || execution.eval_id !== input.evalId
+        || benchmark.id !== input.request.benchmark_id || benchmark.revision !== input.request.benchmark_revision || input.request.training_binding
+        || !input.remoteWorkExecutor || input.executionWorker?.provider !== execution.provider)
+        throw ambiguous("remote scoring requires the frozen benchmark, provider and executor");
+    const plan = parseRerunPlan(await readJSON(path.join(input.evalDirectory, "plan.json")), input.evalId, input.request);
+    const resolution = await loadRerunResolvedRevision(input.evalDirectory, plan), artifacts = await loadRerunPreparedArtifacts(input.root, plan);
+    const runtimeId = String(plan.controllerRuntime.runtime_id);
+    const runtime = await useControllerRuntimeById(statePaths(input.root), runtimeId.replace(/^sha256:/, ""));
+    if (runtime.runtime_id !== runtimeId || runtime.manifest_digest !== plan.controllerRuntime.manifest_digest)
+        throw ambiguous("source controller runtime changed");
+    const verifierRuntime = input.verifierRuntimeId ? await useControllerRuntimeById(statePaths(input.root), input.verifierRuntimeId.slice(7)) : runtime;
+    verifierRuntimeRepair(runtime, verifierRuntime);
+    const selection = await freezeRemoteVerifierSelection({ ...input, plan: execution, slots: input.selectedTrials,
+        taskRoot: benchmark.tasks, sourceRuntimeId: runtimeId, verifierRuntimeId: verifierRuntime.runtime_id });
+    const artifactFor = (work) => {
+        if (!work.artifact_id && artifacts.size !== 1)
+            throw ambiguous("scoring source has no unique frozen artifact");
+        return preparedArtifactForWorkItem({ preparedArtifact: artifacts.values().next().value, preparedArtifacts: artifacts }, work);
+    };
+    for (const item of selection)
+        artifactFor(item.work);
+    let progress = input.progress;
+    const repaired = [], sources = [];
+    for (const item of selection) {
+        input.signal?.throwIfAborted();
+        const { work, physical, source, record: { descriptor, slot } } = item;
+        const identity = remoteVerifierWorkIdentity({ verifier: descriptor, physical, plan: execution, request: input.request, work });
+        const file = path.join(input.rerunDirectory, "remote-work", `${work.work_id}.json`);
+        const previous = await readJSON(file, null);
+        const released = async (leaseId) => {
+            if (!(await readExecutionLeases(input.evalDirectory)).some(lease => lease.lease_id === leaseId && lease.work_id === work.work_id
+                && lease.provider === execution.provider && lease.state === "released"))
+                throw ambiguous("scoring work requires confirmed release of its own lease");
+        };
+        if (previous && sha256JSON(previous.identity) !== sha256JSON(identity))
+            throw ambiguous("scoring work journal identity changed");
+        if (previous) {
+            if (previous.state === "not-started" && !previous.result)
+                await released(previous.lease_id);
+            else if (previous.state === "completed" && previous.result)
+                await released(previous.result.leaseId);
+            else
+                throw ambiguous("scoring work was dispatched and requires lease reconciliation");
+        }
+        const published = [];
+        const publish = async (ref) => {
+            assertRemoteVerifierPublication(descriptor.source_ref, ref, descriptor.assessment_id);
+            if (published.length && sha256JSON(published[0]) !== sha256JSON(ref))
+                throw ambiguous("scoring work changed its acknowledged result");
+            if (published.length)
+                return;
+            if (ref.observation_status === "valid") {
+                await validateEvalTrialReferences(input.root, input.evalId, [ref], { benchmarkId: benchmark.id, benchmarkRevision: benchmark.revision });
+                progress = replaceInvalidEvalProgressTrial(progress, ref);
+                await writeEvalProgress(input.evalDirectory, progress);
+            }
+            published.push(ref);
+        };
+        let result = previous?.state === "completed" ? previous.result : undefined;
+        if (!result) {
+            await atomicWriteJSON(file, { identity, state: "dispatching" });
+            const sink = new EvalEventSink(path.join(input.rerunDirectory, "remote-work", work.work_id), evalId);
+            await sink.open();
+            result = await input.remoteWorkExecutor({ root: input.root, evalId, evalDirectory: input.evalDirectory,
+                request: input.request, plan: execution, workItem: work, physicalExecution: physical,
+                preparedArtifact: artifactFor(work), resolvedRevision: resolution, runtimeDirectory: runtime.directory, runtimeId,
+                verifierOnly: { descriptor, sourceSnapshotDirectory: source.sourceSnapshotDirectory, verifierRuntimeDirectory: verifierRuntime.directory },
+                publicationMode: "replace-invalid", ...(input.signal ? { signal: input.signal } : {}), publish,
+                emit: event => sink.emit({ ...event, rerun_id: input.rerunId, execution_kind: "verifier-only", source_work_id: physical.source_work_id }),
+                onLeaseState: async (leaseId, state) => {
+                    const current = await readJSON(file);
+                    if (sha256JSON(current.identity) !== sha256JSON(identity) || current.lease_id && current.lease_id !== leaseId
+                        || current.result && current.result.leaseId !== leaseId)
+                        throw ambiguous("scoring lease callback changed journal ownership");
+                    // The coordinator collects before release. Its terminal callback must never erase the result.
+                    if (state === "terminal" && ["collected", "completed"].includes(current.state) && current.result)
+                        return;
+                    if (state === "running" && current.state !== "dispatching" || state === "terminal" && !["running", "terminal"].includes(current.state))
+                        throw ambiguous("scoring lease callback regressed durable state");
+                    await atomicWriteJSON(file, { ...current, lease_id: leaseId, state });
+                },
+            }).finally(() => sink.close());
+            await released(result.leaseId);
+            if (sha256JSON(result.refs) !== sha256JSON(published))
+                throw ambiguous("scoring result differs from acknowledged references");
+            const current = await readJSON(file);
+            if (current.result && sha256JSON(current.result) !== sha256JSON(result))
+                throw ambiguous("scoring executor changed its collected result");
+            await atomicWriteJSON(file, { identity, state: "completed", lease_id: result.leaseId, result });
+        }
+        if (!result.refs.length && result.run.rawResult === null && !result.assessments?.length) {
+            throw new HitchError("remote scoring failed before producing an assessment", { code: "eval_rerun_harbor_failed", exitCode: 13 });
+        }
+        const ref = result.refs[0], assessment = result.assessments?.[0];
+        if (result.refs.length !== 1 || result.assessments?.length !== 1 || !ref || assessment?.id !== descriptor.assessment_id)
+            throw ambiguous("scoring result has no unique assessment");
+        assertRemoteVerifierPublication(descriptor.source_ref, ref, descriptor.assessment_id);
+        await readRegradeObservation(input.root, input.evalId, { ...descriptor.source_ref, assessment });
+        const manifest = await readJSON(path.join(input.evalDirectory, "assessments", assessment.id, "assessment.json"));
+        if (manifest.rerun_id !== input.rerunId || manifest.remote_verifier_work_digest !== sha256JSON(descriptor)
+            || ref.observation_status === "valid" && sha256JSON(ref.assessment) !== sha256JSON(assessment))
+            throw ambiguous("scoring assessment belongs to different work");
+        await publish(ref);
+        if (ref.observation_status === "valid")
+            repaired.push(slot);
+        sources.push({ source_trial_id: descriptor.source_ref.trial_id, source_run_id: descriptor.source_ref.run_id, source_work_id: source.execution.work_id,
+            source_backend_directory: path.relative(input.evalDirectory, path.dirname(path.dirname(source.trialDirectory))).split(path.sep).join("/") });
+    }
+    return finishVerifierRerun({ ...input, progress, repaired, sources, remote: true });
+}
+function ambiguous(message) { return new HitchError(message, { code: "execution_state_ambiguous", exitCode: 12 }); }
+//# sourceMappingURL=remote-verifier-rerun.js.map

package/dist/src/evals/remote-verifier-rerun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-verifier-rerun.js","sourceRoot":"","sources":["../../../src/evals/remote-verifier-rerun.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACvG,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACnF,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAC1E,OAAO,EAAE,+BAA+B,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAC;AAChH,OAAO,EAAE,0BAA0B,EAAE,yBAAyB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAI1G,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAEhE,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AAIvE,2FAA2F;AAC3F,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,KAAkD;IAClG,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,sBAAsB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC;IAChH,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAClE,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,QAAQ,KAAK,cAAc,IAAI,SAAS,CAAC,UAAU,KAAK,OAAO,IAAI,SAAS,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM;WAC5H,SAAS,CAAC,EAAE,KAAK,KAAK,CAAC,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,QAAQ,KAAK,KAAK,CAAC,OAAO,CAAC,kBAAkB,IAAI,KAAK,CAAC,OAAO,CAAC,gBAAgB;WACxI,CAAC,KAAK,CAAC,kBAAkB,IAAI,KAAK,CAAC,eAAe,EAAE,QAAQ,KAAK,SAAS,CAAC,QAAQ;QAAE,MAAM,SAAS,CAAC,qEAAqE,CAAC,CAAC;IACjL,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACtH,MAAM,UAAU,GAAG,MAAM,yBAAyB,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,MAAM,0BAA0B,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9I,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,wBAAwB,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1G,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,CAAC,eAAe,KAAK,IAAI,CAAC,iBAAiB,CAAC,eAAe;QAAE,MAAM,SAAS,CAAC,mCAAmC,CAAC,CAAC;IACjK,MAAM,eAAe,GAAG,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,wBAAwB,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACrJ,qBAAqB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,6BAA6B,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,cAAc;QAC5G,QAAQ,EAAE,SAAS,CAAC,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,iBAAiB,EAAE,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC;IAC1G,MAAM,WAAW,GAAG,CAAC,IAAyC,EAAE,EAAE;QAChE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC;YAAE,MAAM,SAAS,CAAC,8CAA8C,CAAC,CAAC;QAC/G,OAAO,2BAA2B,CAAC,EAAE,gBAAgB,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,EAAE,iBAAiB,EAAE,SAAS,EAAE,EAAE,IAAI,CAAC,CAAC;IACjI,CAAC,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,SAAS;QAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC9B,MAAM,QAAQ,GAAoB,EAAE,EAAE,OAAO,GAA4C,EAAE,CAAC;IAC5F,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QAC/B,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC;QACtE,MAAM,QAAQ,GAAG,0BAA0B,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/H,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,OAAO,OAAO,CAAC,CAAC;QACpF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAiB,IAAI,EAAE,IAAI,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,KAAK,EAAE,OAA2B,EAAiB,EAAE;YACpE,IAAI,CAAC,CAAC,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO;mBAC3H,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,CAAC;gBAAE,MAAM,SAAS,CAAC,0DAA0D,CAAC,CAAC;QACzJ,CAAC,CAAC;QACF,IAAI,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC;YAAE,MAAM,SAAS,CAAC,uCAAuC,CAAC,CAAC;QACjI,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,QAAQ,CAAC,KAAK,KAAK,aAAa,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAAE,MAAM,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;iBACvF,IAAI,QAAQ,CAAC,KAAK,KAAK,WAAW,IAAI,QAAQ,CAAC,MAAM;gBAAE,MAAM,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;;gBAC/F,MAAM,SAAS,CAAC,+DAA+D,CAAC,CAAC;QACxF,CAAC;QACD,MAAM,SAAS,GAA0C,EAAE,CAAC;QAC5D,MAAM,OAAO,GAAG,KAAK,EAAE,GAAkD,EAAiB,EAAE;YAC1F,+BAA+B,CAAC,UAAU,CAAC,UAAU,EAAE,GAAG,EAAE,UAAU,CAAC,aAAa,CAAC,CAAC;YACtF,IAAI,SAAS,CAAC,MAAM,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,GAAG,CAAC;gBAAE,MAAM,SAAS,CAAC,8CAA8C,CAAC,CAAC;YACtI,IAAI,SAAS,CAAC,MAAM;gBAAE,OAAO;YAC7B,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACvC,MAAM,2BAA2B,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,iBAAiB,EAAE,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACzI,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC1D,MAAM,iBAAiB,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YACzD,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC,CAAC;QACF,IAAI,MAAM,GAAG,QAAQ,EAAE,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,eAAe,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;YACrG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,MAAM,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa;gBACpG,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ;gBACpF,gBAAgB,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,gBAAgB,EAAE,UAAU,EAAE,gBAAgB,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS;gBACjH,YAAY,EAAE,EAAE,UAAU,EAAE,uBAAuB,EAAE,MAAM,CAAC,uBAAuB,EAAE,wBAAwB,EAAE,eAAe,CAAC,SAAS,EAAE;gBAC1I,eAAe,EAAE,iBAAiB,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO;gBAC9F,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,cAAc,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC;gBACzI,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;oBACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAU,IAAI,CAAC,CAAC;oBAC9C,IAAI,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO;2BACxG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,KAAK,OAAO;wBAAE,MAAM,SAAS,CAAC,kDAAkD,CAAC,CAAC;oBAC/H,8FAA8F;oBAC9F,IAAI,KAAK,KAAK,UAAU,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM;wBAAE,OAAO;oBACzG,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,aAAa,IAAI,KAAK,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;wBAAE,MAAM,SAAS,CAAC,gDAAgD,CAAC,CAAC;oBAC1M,MAAM,eAAe,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBACxE,CAAC;aACF,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/B,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC/B,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,SAAS,CAAC;gBAAE,MAAM,SAAS,CAAC,qDAAqD,CAAC,CAAC;YAC9H,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAU,IAAI,CAAC,CAAC;YAC9C,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC;gBAAE,MAAM,SAAS,CAAC,+CAA+C,CAAC,CAAC;YAC1I,MAAM,eAAe,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC;YACxF,MAAM,IAAI,UAAU,CAAC,sDAAsD,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACnI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,UAAU,EAAE,EAAE,KAAK,UAAU,CAAC,aAAa;YAAE,MAAM,SAAS,CAAC,yCAAyC,CAAC,CAAC;QACpL,+BAA+B,CAAC,UAAU,CAAC,UAAU,EAAE,GAAG,EAAE,UAAU,CAAC,aAAa,CAAC,CAAC;QACtF,MAAM,sBAAsB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,GAAG,UAAU,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC;QACjG,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAA0B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC1I,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,2BAA2B,KAAK,UAAU,CAAC,UAAU,CAAC;eACrG,GAAG,CAAC,kBAAkB,KAAK,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,UAAU,CAAC,UAAU,CAAC;YAAE,MAAM,SAAS,CAAC,8CAA8C,CAAC,CAAC;QAClK,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QACnB,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,EAAE,eAAe,EAAE,UAAU,CAAC,UAAU,CAAC,QAAQ,EAAE,aAAa,EAAE,UAAU,CAAC,UAAU,CAAC,MAAO,EAAE,cAAc,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YACpJ,wBAAwB,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACjJ,CAAC;IACD,OAAO,mBAAmB,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AACtF,CAAC;AACD,SAAS,SAAS,CAAC,OAAe,IAAgB,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/evals/remote-verifier-selection.js

@@ -0,0 +1,61 @@
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import { HitchError, atomicWriteJSON, readJSON, sha256JSON } from "../foundation/index.js";
+import { verifierOnlyWorkItem } from "./physical-work-plan.js";
+import { assertRemoteVerifierWork, parseRemoteVerifierWork } from "./remote-verifier-contract.js";
+import { loadRemoteVerifierSource } from "./remote-verifier-source.js";
+export async function freezeRemoteVerifierSelection(input) {
+    const identity = { schema_version: "2", eval_id: input.evalId, rerun_id: input.rerunId,
+        plan_digest: sha256JSON(input.plan), request_digest: sha256JSON(input.request), selected_slots_digest: sha256JSON(input.slots),
+        source_runtime_id: input.sourceRuntimeId, verifier_runtime_id: input.verifierRuntimeId };
+    const file = path.join(input.rerunDirectory, "verifier-selection.json");
+    const saved = await readJSON(file, null);
+    if (saved && (sha256JSON(saved.identity) !== sha256JSON(identity) || !Array.isArray(saved.items) || saved.items.length !== input.slots.length))
+        throw ambiguous();
+    if (!saved && (await readdir(path.join(input.rerunDirectory, "remote-work")).catch(error => {
+        if (error.code === "ENOENT")
+            return [];
+        throw error;
+    })).length)
+        throw ambiguous();
+    const items = [];
+    for (const [index, slot] of input.slots.entries()) {
+        const frozen = saved?.items[index];
+        if (saved && (!frozen || sha256JSON(frozen.slot) !== sha256JSON(slot)))
+            throw ambiguous();
+        const original = frozen ? parseRemoteVerifierWork(frozen.descriptor).source_ref
+            : input.progress.trials.find(ref => ref.task_id === slot.task_id && ref.attempt === slot.attempt);
+        if (!original)
+            throw ambiguous();
+        const source = await loadRemoteVerifierSource({ ...input, sourceRef: original, provider: input.plan.provider,
+            taskDirectory: path.join(input.taskRoot, slot.task_id), benchmarkId: input.request.benchmark_id, benchmarkRevision: input.request.benchmark_revision });
+        const logical = input.plan.slots.find(s => s.task_id === slot.task_id && s.attempt === slot.attempt);
+        const planned = logical && input.plan.work_items.find(w => w.slots.includes(logical.slot_id));
+        if (!planned)
+            throw ambiguous();
+        const assessmentId = `assessment_${sha256JSON({ rerun_id: input.rerunId, source_ref: original,
+            source_manifest: source.manifest, verifier_runtime_id: input.verifierRuntimeId }).slice(7, 39)}`;
+        const descriptor = parseRemoteVerifierWork({ schema_version: "2", kind: "verifier-only", assessment_id: assessmentId,
+            source_ref: original, source_manifest: source.manifest, source_trial: source.sourceTrial, candidate_result: source.candidateResult,
+            candidate_result_digest: source.candidateResultDigest, canonical_bundle_digest: source.canonicalBundleDigest, verifier_runtime_id: input.verifierRuntimeId });
+        const work = verifierOnlyWorkItem(planned, input.rerunId, assessmentId);
+        const physical = { schema_version: "2", kind: "verifier-only", source_work_id: planned.work_id, rerun_id: input.rerunId, assessment_id: assessmentId };
+        assertRemoteVerifierWork({ verifier: descriptor, plan: input.plan, work, physical, runtimeId: input.sourceRuntimeId });
+        const record = { slot, source_work_id: planned.work_id, work_id: work.work_id, descriptor };
+        if (frozen && sha256JSON(frozen) !== sha256JSON(record))
+            throw ambiguous();
+        items.push({ record, work, physical, source });
+    }
+    // Seal the entire selection before the first offer, including undispatched slots.
+    if (!saved)
+        await atomicWriteJSON(file, { identity, items: items.map(item => item.record) });
+    return items;
+}
+export function assertRemoteVerifierPublication(original, ref, assessmentId) {
+    if (ref.task_id !== original.task_id || ref.attempt !== original.attempt || ref.trial_id !== original.trial_id || ref.run_id !== original.run_id
+        || ref.observation_status === "valid" && ref.assessment?.id !== assessmentId
+        || ref.observation_status !== "valid" && sha256JSON(ref) !== sha256JSON(original))
+        throw ambiguous();
+}
+function ambiguous() { return new HitchError("remote verifier selection or original candidate changed", { code: "execution_state_ambiguous", exitCode: 12 }); }
+//# sourceMappingURL=remote-verifier-selection.js.map

package/dist/src/evals/remote-verifier-selection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-verifier-selection.js","sourceRoot":"","sources":["../../../src/evals/remote-verifier-selection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAClG,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAGvE,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAInD;IACC,MAAM,QAAQ,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO;QACpF,WAAW,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,qBAAqB,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC;QAC9H,iBAAiB,EAAE,KAAK,CAAC,eAAe,EAAE,mBAAmB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC;IAC3F,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,yBAAyB,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiJ,IAAI,EAAE,IAAI,CAAC,CAAC;IACzL,IAAI,KAAK,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,MAAM,SAAS,EAAE,CAAC;IAClK,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;QACzF,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAClE,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC,CAAC,MAAM;QAAE,MAAM,SAAS,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,CAAC;YAAE,MAAM,SAAS,EAAE,CAAC;QAC1F,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,UAAU;YAC7E,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC;QACpG,IAAI,CAAC,QAAQ;YAAE,MAAM,SAAS,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;YAC1G,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC1J,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC;QACrG,MAAM,OAAO,GAAG,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAC9F,IAAI,CAAC,OAAO;YAAE,MAAM,SAAS,EAAE,CAAC;QAChC,MAAM,YAAY,GAAG,cAAc,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ;YAC3F,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,mBAAmB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QACnG,MAAM,UAAU,GAAG,uBAAuB,CAAC,EAAE,cAAc,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY;YAClH,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,WAAW,EAAE,gBAAgB,EAAE,MAAM,CAAC,eAAe;YAClI,uBAAuB,EAAE,MAAM,CAAC,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,CAAC,qBAAqB,EAAE,mBAAmB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAChK,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,EAAE,cAAc,EAAE,GAAY,EAAE,IAAI,EAAE,eAAwB,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;QACzK,wBAAwB,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;QACvH,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;QAC5F,IAAI,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC;YAAE,MAAM,SAAS,EAAE,CAAC;QAC3E,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,kFAAkF;IAClF,IAAI,CAAC,KAAK;QAAE,MAAM,eAAe,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7F,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,QAAwB,EAAE,GAAmB,EAAE,YAAoB;IACjH,IAAI,GAAG,CAAC,OAAO,KAAK,QAAQ,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,KAAK,QAAQ,CAAC,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;WAC3I,GAAG,CAAC,kBAAkB,KAAK,OAAO,IAAI,GAAG,CAAC,UAAU,EAAE,EAAE,KAAK,YAAY;WACzE,GAAG,CAAC,kBAAkB,KAAK,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC;QAAE,MAAM,SAAS,EAAE,CAAC;AACzG,CAAC;AACD,SAAS,SAAS,KAAiB,OAAO,IAAI,UAAU,CAAC,yDAAyD,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/evals/remote-verifier-source.js

@@ -0,0 +1,33 @@
+import path from "node:path";
+import { HitchError, readJSON, sha256JSON, statePaths } from "../foundation/index.js";
+import { loadRunRecord, verifyResultBundleIndex } from "../runs/index.js";
+import { parseExecutionEvidence } from "./execution-evidence.js";
+import { parseEvalTrialRef } from "./progress.js";
+import { validateEvalTrialReferences } from "./trial-import.js";
+import { verifyImportedRemoteVerifierSource } from "./verifier-source-import.js";
+/** Locate imported artifacts using the canonical candidate's actual execution, including candidate-restart work. */
+export async function loadRemoteVerifierSource(input) {
+    const original = parseEvalTrialRef(input.sourceRef);
+    if (original.run_group || original.assessment || original.observation_status !== "invalid"
+        || !["verifier_infrastructure_failure", "verifier_result_missing"].includes(original.invalid_reason ?? "")
+        || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/.test(original.trial_id))
+        throw invalid();
+    await validateEvalTrialReferences(input.root, input.evalId, [original], { benchmarkId: input.benchmarkId, benchmarkRevision: input.benchmarkRevision });
+    const runDirectory = path.join(statePaths(input.root).runs, original.run_id);
+    const bundle = await verifyResultBundleIndex(runDirectory), candidate = await loadRunRecord(runDirectory, { verifyTrajectory: true });
+    if (candidate.record.status !== "succeeded" || candidate.record_status !== "valid" || candidate.trajectory_status !== "valid")
+        throw invalid();
+    const execution = parseExecutionEvidence(await readJSON(path.join(runDirectory, "execution.json")));
+    if (execution.eval_id !== input.evalId || execution.provider !== input.provider || execution.task_id !== original.task_id)
+        throw invalid();
+    const trialDirectory = path.join(input.evalDirectory, "harbor", "work-items", execution.work_id, `epoch-${String(execution.lease_epoch).padStart(6, "0")}`, "job", original.trial_id);
+    const manifest = await verifyImportedRemoteVerifierSource({ root: input.root, runId: original.run_id, trialDirectory, taskDirectory: input.taskDirectory });
+    if (!manifest.original_result_digest || !manifest.regrade_config_digest)
+        throw invalid();
+    const sourceTrial = await readJSON(path.join(trialDirectory, "result.json"));
+    const candidateResult = await readJSON(path.join(runDirectory, "result.json"));
+    return { original, manifest, sourceTrial, candidateResult, execution, trialDirectory, sourceSnapshotDirectory: path.join(trialDirectory, "verifier-source"),
+        canonicalBundleDigest: sha256JSON(bundle), candidateResultDigest: sha256JSON(candidateResult) };
+}
+function invalid() { return new HitchError("remote verifier source has no intact original candidate and import receipt", { code: "eval_verifier_only_unavailable", exitCode: 2 }); }
+//# sourceMappingURL=remote-verifier-source.js.map

package/dist/src/evals/remote-verifier-source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-verifier-source.js","sourceRoot":"","sources":["../../../src/evals/remote-verifier-source.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,kCAAkC,EAAE,MAAM,6BAA6B,CAAC;AAEjF,oHAAoH;AACpH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,KAG9C;IACC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACpD,IAAI,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,kBAAkB,KAAK,SAAS;WACrF,CAAC,CAAC,iCAAiC,EAAE,yBAAyB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,cAAc,IAAI,EAAE,CAAC;WACvG,CAAC,oCAAoC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,MAAM,OAAO,EAAE,CAAC;IACpF,MAAM,2BAA2B,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACxJ,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,YAAY,CAAC,EAAE,SAAS,GAAG,MAAM,aAAa,CAAC,YAAY,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IACtI,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,SAAS,CAAC,aAAa,KAAK,OAAO,IAAI,SAAS,CAAC,iBAAiB,KAAK,OAAO;QAAE,MAAM,OAAO,EAAE,CAAC;IAC/I,MAAM,SAAS,GAAG,sBAAsB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC;IACpG,IAAI,SAAS,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,IAAI,SAAS,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,SAAS,CAAC,OAAO,KAAK,QAAQ,CAAC,OAAO;QAAE,MAAM,OAAO,EAAE,CAAC;IAC3I,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,CAAC,OAAO,EAC7F,SAAS,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvF,MAAM,QAAQ,GAAG,MAAM,kCAAkC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;IAC5J,IAAI,CAAC,QAAQ,CAAC,sBAAsB,IAAI,CAAC,QAAQ,CAAC,qBAAqB;QAAE,MAAM,OAAO,EAAE,CAAC;IACzF,MAAM,WAAW,GAAG,MAAM,QAAQ,CAA0B,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC,CAAC;IACtG,MAAM,eAAe,GAAG,MAAM,QAAQ,CAA0B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC;IACxG,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,cAAc,EAAE,uBAAuB,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,iBAAiB,CAAC;QACzJ,qBAAqB,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,qBAAqB,EAAE,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;AACpG,CAAC;AACD,SAAS,OAAO,KAAiB,OAAO,IAAI,UAAU,CAAC,4EAA4E,EAAE,EAAE,IAAI,EAAE,gCAAgC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/evals/remote-worker-cleanup.js

@@ -0,0 +1,46 @@
+import { HitchError, sha256JSON } from "../foundation/index.js";
+import { parseRemoteExecutionAdmission, parseRemoteExecutionOwnership, parseRemoteWorkerHostIdentity } from "./remote-execution-ownership.js";
+export function parseRemoteWorkerCleanupChallenge(value, offer) {
+    const record = exact(value, ["schema_version", "cleanup_id", "nonce", "worker_id", "generation", "issued_at", "expires_at", "admission"]);
+    const admission = parseRemoteExecutionAdmission(record.admission, offer);
+    if (record.schema_version !== "2" || typeof record.cleanup_id !== "string" || !/^cleanup_[a-f0-9]{32}$/.test(record.cleanup_id)
+        || typeof record.nonce !== "string" || !/^[a-f0-9]{32}$/.test(record.nonce) || record.worker_id !== offer.worker_id
+        || !Number.isSafeInteger(record.generation) || record.generation <= offer.generation
+        || !timestamp(record.issued_at) || !timestamp(record.expires_at) || Date.parse(record.expires_at) <= Date.parse(record.issued_at)
+        || Date.parse(record.expires_at) - Date.parse(record.issued_at) > 5 * 60_000
+        || !offer.accepted_at || !offer.accept_receipt_digest)
+        throw invalid();
+    return { ...record, admission };
+}
+export function parseRemoteWorkerCleanupReceipt(value, offer) {
+    const record = exact(value, ["schema_version", "challenge", "observed_at", "observation"]);
+    const challenge = parseRemoteWorkerCleanupChallenge(record.challenge, offer);
+    const reboot = record.schema_version === "3";
+    const observation = exact(record.observation, ["ownership", "execution_process", "worker_status", "docker_resources_empty", reboot ? "host_identity" : "process_group_empty"]);
+    const ownership = parseRemoteExecutionOwnership(observation.ownership, offer);
+    if (record.schema_version !== "2" && !reboot || !timestamp(record.observed_at) || Date.parse(record.observed_at) < Date.parse(challenge.issued_at)
+        || Date.parse(record.observed_at) > Date.parse(challenge.expires_at)
+        || sha256JSON(ownership) !== challenge.admission.ownership_digest
+        || sha256JSON(observation.execution_process) !== sha256JSON(challenge.admission.execution_process)
+        || observation.docker_resources_empty !== true)
+        throw invalid();
+    if (reboot) {
+        const host = parseRemoteWorkerHostIdentity(observation.host_identity);
+        if (ownership.schema_version !== "3" || observation.worker_status !== "previous-boot"
+            || host.platform !== ownership.host_identity.platform || host.host_id !== ownership.host_identity.host_id
+            || host.boot_id === ownership.host_identity.boot_id)
+            throw invalid();
+    }
+    else if (!["terminal", "identity-mismatch"].includes(String(observation.worker_status)) || observation.process_group_empty !== true)
+        throw invalid();
+    return { ...record, challenge, observation: { ...observation, ownership } };
+}
+function exact(value, fields) {
+    if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length !== fields.length
+        || Object.keys(value).some(key => !fields.includes(key)))
+        throw invalid();
+    return value;
+}
+function timestamp(value) { return typeof value === "string" && Number.isFinite(Date.parse(value)); }
+function invalid() { return new HitchError("worker generation cleanup does not match the original execution", { code: "remote_worker_cleanup_ambiguous", exitCode: 12 }); }
+//# sourceMappingURL=remote-worker-cleanup.js.map

package/dist/src/evals/remote-worker-cleanup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-worker-cleanup.js","sourceRoot":"","sources":["../../../src/evals/remote-worker-cleanup.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,6BAA6B,EAAE,6BAA6B,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAE9I,MAAM,UAAU,iCAAiC,CAAC,KAAc,EAAE,KAAwB;IACxF,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,gBAAgB,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC;IAC1I,MAAM,SAAS,GAAG,6BAA6B,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACzE,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;WAC1H,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;WAChH,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,IAAK,MAAM,CAAC,UAAqB,IAAI,KAAK,CAAC,UAAU;WAC7F,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC;WAC9H,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM;WACzE,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,qBAAqB;QAAE,MAAM,OAAO,EAAE,CAAC;IACzE,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAA+C,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,KAAc,EAAE,KAAwB;IACtF,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,gBAAgB,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC,CAAC;IAC3F,MAAM,SAAS,GAAG,iCAAiC,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,KAAK,GAAG,CAAC;IAC7C,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,WAAW,EAAE,mBAAmB,EAAE,eAAe,EAAE,wBAAwB,EAAE,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAC/K,MAAM,SAAS,GAAG,6BAA6B,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9E,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC;WAC7I,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC;WACjE,UAAU,CAAC,SAAS,CAAC,KAAK,SAAS,CAAC,SAAS,CAAC,gBAAgB;WAC9D,UAAU,CAAC,WAAW,CAAC,iBAAiB,CAAC,KAAK,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,iBAAiB,CAAC;WAC/F,WAAW,CAAC,sBAAsB,KAAK,IAAI;QAAE,MAAM,OAAO,EAAE,CAAC;IAClE,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,IAAI,GAAG,6BAA6B,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,cAAc,KAAK,GAAG,IAAI,WAAW,CAAC,aAAa,KAAK,eAAe;eAChF,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,aAAa,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,aAAa,CAAC,OAAO;eACtG,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,aAAa,CAAC,OAAO;YAAE,MAAM,OAAO,EAAE,CAAC;IACzE,CAAC;SAAM,IAAI,CAAC,CAAC,UAAU,EAAE,mBAAmB,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,IAAI,WAAW,CAAC,mBAAmB,KAAK,IAAI;QAAE,MAAM,OAAO,EAAE,CAAC;IACvJ,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,EAA2C,CAAC;AACvH,CAAC;AAED,SAAS,KAAK,CAAC,KAAc,EAAE,MAAgB;IAC7C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;WACzG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAAE,MAAM,OAAO,EAAE,CAAC;IAC5E,OAAO,KAAgC,CAAC;AAC1C,CAAC;AACD,SAAS,SAAS,CAAC,KAAc,IAAqB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/H,SAAS,OAAO,KAAiB,OAAO,IAAI,UAAU,CAAC,iEAAiE,EAAE,EAAE,IAAI,EAAE,iCAAiC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}