agent-hitch 0.1.1 → 0.2.1

package/dist/src/evals/request.js

@@ -0,0 +1,109 @@
+export const DEFAULT_EVAL_TIMEOUT_MS = 15 * 60 * 1_000;
+export const DEFAULT_EVAL_SETUP_TIMEOUT_MS = 30 * 60 * 1_000;
+const HARBOR_EVAL_BYPASS_ARG = {
+    codex: "--dangerously-bypass-approvals-and-sandbox",
+    opencode: "--dangerously-skip-permissions",
+};
+export function newEvalId() {
+    return `eval_${randomUUID().replaceAll("-", "")}`;
+}
+export async function validateEvalRequest(input) {
+    if (!input || typeof input !== "object" || Array.isArray(input))
+        throw invalidInput("eval request must be a JSON object");
+    const allowed = new Set([
+        "schema_version", "backend", "dataset", "harness_ref", "model", "attempts",
+        "max_concurrent", "timeout_ms", "setup_timeout_ms", "agent_args", "pass_env",
+    ]);
+    const unexpected = Object.keys(input).find((field) => !allowed.has(field));
+    if (unexpected)
+        throw invalidInput(`unknown eval request field: ${unexpected}`);
+    if (input.schema_version !== undefined && input.schema_version !== SCHEMA_VERSION)
+        throw invalidInput(`unsupported schema_version: ${input.schema_version}`);
+    if ((input.backend || "harbor") !== "harbor")
+        throw invalidInput("only the harbor eval backend is currently supported");
+    if (typeof input.dataset !== "string" || !input.dataset.trim())
+        throw invalidInput("dataset must be a non-empty string");
+    if (typeof input.harness_ref !== "string" || !input.harness_ref.trim())
+        throw invalidInput("harness_ref must be a non-empty string");
+    const reference = parseHarnessReference(input.harness_ref);
+    if (reference.selector.type === "installed") {
+        throw invalidInput("eval requires an immutable harness ref: version:<exact> or commit:<sha>");
+    }
+    assertExactLocalGitEvalReference(reference);
+    const benchmark = await resolveBenchmarkReference(input.dataset.trim());
+    const attempts = positiveInteger(input.attempts ?? 1, "attempts");
+    const maxConcurrent = positiveInteger(input.max_concurrent ?? 4, "max_concurrent");
+    const timeout = nonNegativeNumber(input.timeout_ms ?? DEFAULT_EVAL_TIMEOUT_MS, "timeout_ms");
+    const setupTimeout = nonNegativeNumber(input.setup_timeout_ms ?? DEFAULT_EVAL_SETUP_TIMEOUT_MS, "setup_timeout_ms");
+    if (input.model !== undefined && typeof input.model !== "string")
+        throw invalidInput("model must be a string");
+    if (input.agent_args !== undefined && (!Array.isArray(input.agent_args) || input.agent_args.some((value) => typeof value !== "string"))) {
+        throw invalidInput("agent_args must be an array of strings");
+    }
+    if (input.pass_env !== undefined && (!Array.isArray(input.pass_env) || input.pass_env.some((value) => typeof value !== "string"))) {
+        throw invalidInput("pass_env must be an array of strings");
+    }
+    const agentArgs = Array.isArray(input.agent_args) ? [...input.agent_args] : [];
+    const bypassArg = HARBOR_EVAL_BYPASS_ARG[reference.harness_id];
+    if (bypassArg && !agentArgs.includes(bypassArg))
+        agentArgs.unshift(bypassArg);
+    return {
+        schema_version: SCHEMA_VERSION,
+        backend: "harbor",
+        dataset: String(input.dataset).trim(),
+        harness_ref: reference.canonical,
+        model: typeof input.model === "string" ? input.model : "",
+        attempts,
+        max_concurrent: maxConcurrent,
+        timeout_ms: timeout,
+        setup_timeout_ms: setupTimeout,
+        agent_args: agentArgs,
+        pass_env: [...new Set(Array.isArray(input.pass_env) ? input.pass_env : [])],
+        benchmark_id: benchmark.benchmark_id,
+        benchmark_revision: benchmark.benchmark_revision,
+    };
+}
+export async function resolveBenchmarkReference(dataset) {
+    const raw = dataset.trim();
+    const local = path.resolve(raw);
+    try {
+        if ((await stat(local)).isDirectory()) {
+            return {
+                benchmark_id: `local:${path.basename(local)}`,
+                benchmark_revision: await workspaceDigest(local, { excludedTopLevel: new Set([".git"]) }),
+            };
+        }
+    }
+    catch (error) {
+        if (error.code !== "ENOENT")
+            throw error;
+    }
+    const separator = raw.lastIndexOf("@");
+    if (separator <= 0 || separator === raw.length - 1) {
+        throw invalidInput("dataset must include an immutable revision (for example benchmark@1.0)");
+    }
+    const benchmarkId = raw.slice(0, separator);
+    const revision = raw.slice(separator + 1);
+    if (revision.toLowerCase() === "latest")
+        throw invalidInput("dataset revision cannot be latest");
+    return { benchmark_id: benchmarkId, benchmark_revision: revision };
+}
+function positiveInteger(value, name) {
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed <= 0)
+        throw invalidInput(`${name} must be a positive integer`);
+    return parsed;
+}
+function nonNegativeNumber(value, name) {
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed) || parsed < 0)
+        throw invalidInput(`${name} must be a non-negative number`);
+    return parsed;
+}
+import { randomUUID } from "node:crypto";
+import { stat } from "node:fs/promises";
+import path from "node:path";
+import { SCHEMA_VERSION, invalidInput } from "../foundation/index.js";
+import { assertExactLocalGitEvalReference, parseHarnessReference } from "../revisions/index.js";
+import { workspaceDigest } from "../workspaces/index.js";
+//# sourceMappingURL=request.js.map

package/dist/src/evals/request.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request.js","sourceRoot":"","sources":["../../../src/evals/request.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACvD,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAE7D,MAAM,sBAAsB,GAAqC;IAC/D,KAAK,EAAE,4CAA4C;IACnD,QAAQ,EAAE,gCAAgC;CAC3C,CAAC;AAEF,MAAM,UAAU,SAAS;IACvB,OAAO,QAAQ,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,EAAY,CAAC;AAC9D,CAAC;AAgBD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAuB;IAC/D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC;QACtB,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU;QAC1E,gBAAgB,EAAE,YAAY,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU;KAC7E,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,IAAI,UAAU;QAAE,MAAM,YAAY,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,KAAK,CAAC,cAAc,KAAK,cAAc;QAAE,MAAM,YAAY,CAAC,+BAA+B,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7J,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,qDAAqD,CAAC,CAAC;IACxH,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,oCAAoC,CAAC,CAAC;IACzH,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE;QAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACrI,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,MAAM,YAAY,CAAC,yEAAyE,CAAC,CAAC;IAChG,CAAC;IACD,gCAAgC,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;IAClE,MAAM,aAAa,GAAG,eAAe,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,IAAI,uBAAuB,EAAE,YAAY,CAAC,CAAC;IAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,gBAAgB,IAAI,6BAA6B,EAAE,kBAAkB,CAAC,CAAC;IACpH,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,wBAAwB,CAAC,CAAC;IAC/G,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxI,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QAClI,MAAM,YAAY,CAAC,sCAAsC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,CAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3F,MAAM,SAAS,GAAG,sBAAsB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAC/D,IAAI,SAAS,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9E,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;QACrC,WAAW,EAAE,SAAS,CAAC,SAAS;QAChC,KAAK,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACzD,QAAQ;QACR,cAAc,EAAE,aAAa;QAC7B,UAAU,EAAE,OAAO;QACnB,gBAAgB,EAAE,YAAY;QAC9B,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvF,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;KACjD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAAe;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACtC,OAAO;gBACL,YAAY,EAAE,SAAS,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBAC7C,kBAAkB,EAAE,MAAM,eAAe,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;aAC1F,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,YAAY,CAAC,wEAAwE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,WAAW,EAAE,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,mCAAmC,CAAC,CAAC;IACjG,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,IAAY;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACvG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,IAAY;IACrD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,IAAI,gCAAgC,CAAC,CAAC;IACxG,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,gCAAgC,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAChG,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/src/evals/service.js

@@ -0,0 +1,227 @@
+import path from "node:path";
+import { resolveHarness } from "../artifacts/index.js";
+import { HitchError, SCHEMA_VERSION, atomicWriteJSON, ensureDir, invalidInput, statePaths } from "../foundation/index.js";
+import { parseHarnessReference } from "../revisions/index.js";
+import { buildLocalGitTransport, lockedHarnessRef, runHarborBackend, verifyLocalGitTransport } from "../backends/index.js";
+import { ensureControllerRuntime, writeRuntimeReference } from "../controller-runtime/index.js";
+import { importEvalTrialRuns, validateEvalTrialReferences } from "./trial-import.js";
+import { EvalEventSink } from "./events.js";
+import { newEvalId, validateEvalRequest } from "./request.js";
+export async function runEval({ evalId = newEvalId(), request, root, env = process.env, harborExecutable, signal, onEvent }) {
+    if (!root)
+        throw invalidInput("a Hitch state root is required for eval");
+    const normalized = await validateEvalRequest(request);
+    const evalDirectory = await ensureDir(path.join(statePaths(root).evals, evalId));
+    const startedAt = new Date();
+    const sink = new EvalEventSink(evalDirectory, evalId, onEvent);
+    await sink.open();
+    await atomicWriteJSON(path.join(evalDirectory, "request.json"), normalized);
+    let result;
+    let trialRefs = [];
+    try {
+        sink.emit({ type: "eval.started", backend: normalized.backend, dataset: normalized.dataset });
+        const requestedReference = parseHarnessReference(normalized.harness_ref);
+        const resolvedRevision = await resolveHarness(normalized.harness_ref, { root, env });
+        let localTransport;
+        if (resolvedRevision.source.type === "git" && resolvedRevision.source.registered !== true) {
+            const selector = requestedReference.selector;
+            if (selector.type !== "commit" || !selector.source?.explicit)
+                throw invalidInput("Harbor local Git eval requires an explicit git+file source");
+            if (resolvedRevision.revision.commit !== selector.value) {
+                throw new HitchError("local Git exact commit resolved to a different object", { code: "local_source_integrity_mismatch", exitCode: 12 });
+            }
+            localTransport = await buildLocalGitTransport({
+                evalDirectory,
+                resolvedRevision,
+                sourceDirectory: selector.source.local_path,
+                env,
+                ...(signal ? { signal } : {}),
+            });
+            sink.emit({
+                type: "eval.local-source.prepared",
+                resolution_identity: localTransport.manifest.resolution_identity,
+                commit: localTransport.manifest.commit,
+                payload_sha256: localTransport.manifest.payload_sha256,
+                payload_bytes: localTransport.manifest.payload_bytes,
+                status: "verified",
+            });
+        }
+        // Phase 2: the shared, read-only, SHA-256-addressed controller runtime
+        // cache replaces the per-eval Hitch runtime copy (spec §4).
+        const controllerRuntime = await ensureControllerRuntime({ root });
+        const runtimeRefFile = await writeRuntimeReference(evalDirectory, controllerRuntime);
+        sink.emit({
+            type: "eval.controller-runtime",
+            runtime_id: controllerRuntime.runtime_id,
+            cache_hit: controllerRuntime.cache_hit,
+            reference: runtimeRefFile,
+        });
+        const plan = {
+            schema_version: SCHEMA_VERSION,
+            eval_id: evalId,
+            backend: "harbor",
+            candidate: {
+                id: "candidate-1",
+                requested_harness_ref: normalized.harness_ref,
+                harness_ref: lockedHarnessRef(resolvedRevision),
+                harness_id: resolvedRevision.harness_id,
+                revision_identity: resolvedRevision.identity,
+                model: normalized.model || null,
+            },
+            dataset: normalized.dataset,
+            benchmark_id: normalized.benchmark_id,
+            benchmark_revision: normalized.benchmark_revision,
+            attempts: normalized.attempts,
+            max_concurrent: normalized.max_concurrent,
+            controller_runtime: {
+                runtime_id: controllerRuntime.runtime_id,
+                manifest_digest: controllerRuntime.manifest_digest,
+            },
+            ...(localTransport ? { local_source_transport: transportSummary(localTransport) } : {}),
+            created_at: new Date().toISOString(),
+        };
+        await atomicWriteJSON(path.join(evalDirectory, "resolution.json"), resolvedRevision);
+        await atomicWriteJSON(path.join(evalDirectory, "plan.json"), plan);
+        sink.emit({
+            type: "eval.planned",
+            harness: resolvedRevision.harness_id,
+            revision_identity: resolvedRevision.identity,
+            attempts: normalized.attempts,
+        });
+        if (localTransport) {
+            await verifyLocalGitTransport(localTransport, {
+                expected: {
+                    harnessId: resolvedRevision.harness_id,
+                    resolutionIdentity: resolvedRevision.identity,
+                    commit: resolvedRevision.revision.commit,
+                },
+                env,
+                ...(signal ? { signal } : {}),
+            });
+        }
+        const backendRun = await runHarborBackend({
+            evalId,
+            evalDirectory,
+            request: normalized,
+            root,
+            resolvedRevision,
+            runtimeDirectory: controllerRuntime.directory,
+            runtimeId: controllerRuntime.runtime_id,
+            ...(localTransport ? { localTransport } : {}),
+            env,
+            ...(harborExecutable !== undefined ? { harborExecutable } : {}),
+            ...(signal ? { signal } : {}),
+            emit: (event) => sink.emit(event),
+        });
+        trialRefs = await importEvalTrialRuns({
+            root,
+            evalId,
+            evalDirectory,
+            request: normalized,
+            resolvedRevision,
+            benchmarkId: normalized.benchmark_id,
+            benchmarkRevision: normalized.benchmark_revision,
+            runtimeId: controllerRuntime.runtime_id,
+            rawResult: backendRun.rawResult,
+        });
+        await validateEvalTrialReferences(root, evalId, trialRefs, {
+            benchmarkId: normalized.benchmark_id,
+            benchmarkRevision: normalized.benchmark_revision,
+        });
+        const cancelled = signal?.aborted;
+        const localSourceFailure = localTransport ? localSourceBackendFailure(backendRun.rawResult) : false;
+        const succeeded = !cancelled && !localSourceFailure && backendRun.backend.process_exit_code === 0 && backendRun.rawResult !== null;
+        result = {
+            schema_version: SCHEMA_VERSION,
+            eval_id: evalId,
+            status: cancelled ? "cancelled" : succeeded ? "succeeded" : "failed",
+            exit_code: cancelled ? 9 : succeeded ? 0 : 13,
+            backend: backendRun.backend,
+            candidate: plan.candidate,
+            dataset: normalized.dataset,
+            benchmark_id: normalized.benchmark_id,
+            benchmark_revision: normalized.benchmark_revision,
+            trials: trialRefs,
+            summary: summarizeTrialRefs(trialRefs),
+            backend_summary: backendRun.summary,
+            ...(localTransport ? { local_source_transport: transportSummary(localTransport) } : {}),
+            ...(succeeded ? {} : {
+                error: {
+                    code: cancelled ? "cancelled" : localSourceFailure ? "local_source_materialize_failed" : "harbor_failed",
+                    message: cancelled
+                        ? "eval was cancelled"
+                        : localSourceFailure
+                            ? "Harbor rejected the transported local Git source before candidate execution"
+                            : backendRun.rawResult === null
+                                ? `Harbor exited without a result (code ${backendRun.backend.process_exit_code ?? "null"})`
+                                : `Harbor exited with code ${backendRun.backend.process_exit_code ?? "null"}`,
+                },
+            }),
+            started_at: startedAt.toISOString(),
+            completed_at: new Date().toISOString(),
+        };
+    }
+    catch (error) {
+        const typed = error instanceof HitchError;
+        result = {
+            schema_version: SCHEMA_VERSION,
+            eval_id: evalId,
+            status: signal?.aborted ? "cancelled" : "failed",
+            exit_code: signal?.aborted ? 9 : typed ? error.exitCode : 12,
+            error: {
+                code: signal?.aborted ? "cancelled" : typed ? error.code : "internal_error",
+                message: error?.message || String(error),
+            },
+            benchmark_id: normalized.benchmark_id,
+            benchmark_revision: normalized.benchmark_revision,
+            trials: trialRefs,
+            started_at: startedAt.toISOString(),
+            completed_at: new Date().toISOString(),
+        };
+    }
+    await atomicWriteJSON(path.join(evalDirectory, "result.json"), result);
+    sink.emit({ type: result.status === "succeeded" ? "eval.completed" : "eval.failed", status: result.status, exit_code: result.exit_code, error: result.error });
+    await sink.close();
+    return result;
+}
+function summarizeTrialRefs(trials) {
+    const valid = trials.filter((trial) => trial.observation_status === "valid" && typeof trial.reward === "number");
+    const rewards = valid.map((trial) => trial.reward);
+    const aggregate = rewards.length
+        ? {
+            count: rewards.length,
+            mean: rewards.reduce((sum, reward) => sum + reward, 0) / rewards.length,
+            min: Math.min(...rewards),
+            max: Math.max(...rewards),
+        }
+        : null;
+    return {
+        n_trials: trials.length,
+        n_completed: valid.length,
+        n_invalid: trials.length - valid.length,
+        primary_reward: aggregate?.mean ?? null,
+        rewards: aggregate ? { reward: aggregate } : {},
+    };
+}
+function localSourceBackendFailure(rawResult) {
+    if (!rawResult)
+        return false;
+    // The bridge uses this fixed, non-secret marker when setup cannot verify or
+    // materialize the uploaded source. Do not surface provider exception text in
+    // the durable Hitch error, because it may contain backend diagnostics.
+    return JSON.stringify(rawResult).includes("hitch-local-source-materialize:");
+}
+function transportSummary(transport) {
+    const manifest = transport.manifest;
+    return {
+        kind: manifest.kind,
+        resolution_identity: manifest.resolution_identity,
+        commit: manifest.commit,
+        tree: manifest.tree,
+        payload_sha256: manifest.payload_sha256,
+        payload_bytes: manifest.payload_bytes,
+        object_count: manifest.object_count,
+        file_count: manifest.file_count,
+    };
+}
+//# sourceMappingURL=service.js.map

package/dist/src/evals/service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"service.js","sourceRoot":"","sources":["../../../src/evals/service.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,YAAY,EAAY,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpI,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAE3H,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAGhG,OAAO,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAuB9D,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAkB;IACzI,IAAI,CAAC,IAAI;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IACjF,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/D,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IAClB,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5E,IAAI,MAAkB,CAAC;IACvB,IAAI,SAAS,GAAkD,EAAE,CAAC;IAClE,IAAI,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9F,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QACzE,MAAM,gBAAgB,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;QACrF,IAAI,cAAgD,CAAC;QACrD,IAAI,gBAAgB,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK,IAAI,gBAAgB,CAAC,MAAM,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;YAC1F,MAAM,QAAQ,GAAG,kBAAkB,CAAC,QAAQ,CAAC;YAC7C,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ;gBAAE,MAAM,YAAY,CAAC,4DAA4D,CAAC,CAAC;YAC/I,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACxD,MAAM,IAAI,UAAU,CAAC,uDAAuD,EAAE,EAAE,IAAI,EAAE,iCAAiC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3I,CAAC;YACD,cAAc,GAAG,MAAM,sBAAsB,CAAC;gBAC5C,aAAa;gBACb,gBAAgB;gBAChB,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;gBAC3C,GAAG;gBACH,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9B,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC;gBACR,IAAI,EAAE,4BAA4B;gBAClC,mBAAmB,EAAE,cAAc,CAAC,QAAQ,CAAC,mBAAmB;gBAChE,MAAM,EAAE,cAAc,CAAC,QAAQ,CAAC,MAAM;gBACtC,cAAc,EAAE,cAAc,CAAC,QAAQ,CAAC,cAAc;gBACtD,aAAa,EAAE,cAAc,CAAC,QAAQ,CAAC,aAAa;gBACpD,MAAM,EAAE,UAAU;aACnB,CAAC,CAAC;QACL,CAAC;QACD,uEAAuE;QACvE,4DAA4D;QAC5D,MAAM,iBAAiB,GAA+B,MAAM,uBAAuB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;QACrF,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,yBAAyB;YAC/B,UAAU,EAAE,iBAAiB,CAAC,UAAU;YACxC,SAAS,EAAE,iBAAiB,CAAC,SAAS;YACtC,SAAS,EAAE,cAAc;SAC1B,CAAC,CAAC;QACH,MAAM,IAAI,GAAG;YACX,cAAc,EAAE,cAAc;YAC9B,OAAO,EAAE,MAAM;YACf,OAAO,EAAE,QAAQ;YACjB,SAAS,EAAE;gBACT,EAAE,EAAE,aAAa;gBACjB,qBAAqB,EAAE,UAAU,CAAC,WAAW;gBAC7C,WAAW,EAAE,gBAAgB,CAAC,gBAAgB,CAAC;gBAC/C,UAAU,EAAE,gBAAgB,CAAC,UAAU;gBACvC,iBAAiB,EAAE,gBAAgB,CAAC,QAAQ;gBAC5C,KAAK,EAAE,UAAU,CAAC,KAAK,IAAI,IAAI;aAChC;YACD,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,YAAY,EAAE,UAAU,CAAC,YAAY;YACrC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB;YACjD,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,cAAc,EAAE,UAAU,CAAC,cAAc;YACzC,kBAAkB,EAAE;gBAClB,UAAU,EAAE,iBAAiB,CAAC,UAAU;gBACxC,eAAe,EAAE,iBAAiB,CAAC,eAAe;aACnD;YACD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvF,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACrC,CAAC;QACF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,CAAC;QACnE,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,cAAc;YACpB,OAAO,EAAE,gBAAgB,CAAC,UAAU;YACpC,iBAAiB,EAAE,gBAAgB,CAAC,QAAQ;YAC5C,QAAQ,EAAE,UAAU,CAAC,QAAQ;SAC9B,CAAC,CAAC;QACH,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,uBAAuB,CAAC,cAAc,EAAE;gBAC5C,QAAQ,EAAE;oBACR,SAAS,EAAE,gBAAgB,CAAC,UAAU;oBACtC,kBAAkB,EAAE,gBAAgB,CAAC,QAAQ;oBAC7C,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,MAAgB;iBACnD;gBACD,GAAG;gBACH,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC;YACxC,MAAM;YACN,aAAa;YACb,OAAO,EAAE,UAAU;YACnB,IAAI;YACJ,gBAAgB;YAChB,gBAAgB,EAAE,iBAAiB,CAAC,SAAS;YAC7C,SAAS,EAAE,iBAAiB,CAAC,UAAU;YACvC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG;YACH,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;SAClC,CAAC,CAAC;QACH,SAAS,GAAG,MAAM,mBAAmB,CAAC;YACpC,IAAI;YACJ,MAAM;YACN,aAAa;YACb,OAAO,EAAE,UAAU;YACnB,gBAAgB;YAChB,WAAW,EAAE,UAAU,CAAC,YAAY;YACpC,iBAAiB,EAAE,UAAU,CAAC,kBAAkB;YAChD,SAAS,EAAE,iBAAiB,CAAC,UAAU;YACvC,SAAS,EAAE,UAAU,CAAC,SAAS;SAChC,CAAC,CAAC;QACH,MAAM,2BAA2B,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE;YACzD,WAAW,EAAE,UAAU,CAAC,YAAY;YACpC,iBAAiB,EAAE,UAAU,CAAC,kBAAkB;SACjD,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,EAAE,OAAO,CAAC;QAClC,MAAM,kBAAkB,GAAG,cAAc,CAAC,CAAC,CAAC,yBAAyB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACpG,MAAM,SAAS,GAAG,CAAC,SAAS,IAAI,CAAC,kBAAkB,IAAI,UAAU,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,UAAU,CAAC,SAAS,KAAK,IAAI,CAAC;QACnI,MAAM,GAAG;YACP,cAAc,EAAE,cAAc;YAC9B,OAAO,EAAE,MAAM;YACf,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YACpE,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;YAC7C,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,YAAY,EAAE,UAAU,CAAC,YAAY;YACrC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB;YACjD,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,kBAAkB,CAAC,SAAS,CAAC;YACtC,eAAe,EAAE,UAAU,CAAC,OAAO;YACnC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvF,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACnB,KAAK,EAAE;oBACL,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,eAAe;oBACxG,OAAO,EAAE,SAAS;wBAChB,CAAC,CAAC,oBAAoB;wBACtB,CAAC,CAAC,kBAAkB;4BAClB,CAAC,CAAC,6EAA6E;4BACjF,CAAC,CAAC,UAAU,CAAC,SAAS,KAAK,IAAI;gCAC7B,CAAC,CAAC,wCAAwC,UAAU,CAAC,OAAO,CAAC,iBAAiB,IAAI,MAAM,GAAG;gCAC3F,CAAC,CAAC,2BAA2B,UAAU,CAAC,OAAO,CAAC,iBAAiB,IAAI,MAAM,EAAE;iBAClF;aACF,CAAC;YACF,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;YACnC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,KAAK,YAAY,UAAU,CAAC;QAC1C,MAAM,GAAG;YACP,cAAc,EAAE,cAAc;YAC9B,OAAO,EAAE,MAAM;YACf,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YAChD,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;YAC5D,KAAK,EAAE;gBACL,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;gBAC3E,OAAO,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;aACpD;YACD,YAAY,EAAE,UAAU,CAAC,YAAY;YACrC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB;YACjD,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;YACnC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvC,CAAC;IACJ,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;IACvE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/J,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAqD;IAC/E,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACjH,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAgB,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM;QAC9B,CAAC,CAAC;YACE,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;YACvE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;YACzB,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;SAC1B;QACH,CAAC,CAAC,IAAI,CAAC;IACT,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,MAAM;QACvB,WAAW,EAAE,KAAK,CAAC,MAAM;QACzB,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QACvC,cAAc,EAAE,SAAS,EAAE,IAAI,IAAI,IAAI;QACvC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,yBAAyB,CAAC,SAAyC;IAC1E,IAAI,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IAC7B,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,gBAAgB,CAAC,SAA+B;IACvD,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;IACpC,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;QACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,UAAU,EAAE,QAAQ,CAAC,UAAU;KAChC,CAAC;AACJ,CAAC"}