agent-hitch 0.2.0 → 0.2.1

package/dist/src/evals/service.js

@@ -0,0 +1,227 @@
+import path from "node:path";
+import { resolveHarness } from "../artifacts/index.js";
+import { HitchError, SCHEMA_VERSION, atomicWriteJSON, ensureDir, invalidInput, statePaths } from "../foundation/index.js";
+import { parseHarnessReference } from "../revisions/index.js";
+import { buildLocalGitTransport, lockedHarnessRef, runHarborBackend, verifyLocalGitTransport } from "../backends/index.js";
+import { ensureControllerRuntime, writeRuntimeReference } from "../controller-runtime/index.js";
+import { importEvalTrialRuns, validateEvalTrialReferences } from "./trial-import.js";
+import { EvalEventSink } from "./events.js";
+import { newEvalId, validateEvalRequest } from "./request.js";
+export async function runEval({ evalId = newEvalId(), request, root, env = process.env, harborExecutable, signal, onEvent }) {
+    if (!root)
+        throw invalidInput("a Hitch state root is required for eval");
+    const normalized = await validateEvalRequest(request);
+    const evalDirectory = await ensureDir(path.join(statePaths(root).evals, evalId));
+    const startedAt = new Date();
+    const sink = new EvalEventSink(evalDirectory, evalId, onEvent);
+    await sink.open();
+    await atomicWriteJSON(path.join(evalDirectory, "request.json"), normalized);
+    let result;
+    let trialRefs = [];
+    try {
+        sink.emit({ type: "eval.started", backend: normalized.backend, dataset: normalized.dataset });
+        const requestedReference = parseHarnessReference(normalized.harness_ref);
+        const resolvedRevision = await resolveHarness(normalized.harness_ref, { root, env });
+        let localTransport;
+        if (resolvedRevision.source.type === "git" && resolvedRevision.source.registered !== true) {
+            const selector = requestedReference.selector;
+            if (selector.type !== "commit" || !selector.source?.explicit)
+                throw invalidInput("Harbor local Git eval requires an explicit git+file source");
+            if (resolvedRevision.revision.commit !== selector.value) {
+                throw new HitchError("local Git exact commit resolved to a different object", { code: "local_source_integrity_mismatch", exitCode: 12 });
+            }
+            localTransport = await buildLocalGitTransport({
+                evalDirectory,
+                resolvedRevision,
+                sourceDirectory: selector.source.local_path,
+                env,
+                ...(signal ? { signal } : {}),
+            });
+            sink.emit({
+                type: "eval.local-source.prepared",
+                resolution_identity: localTransport.manifest.resolution_identity,
+                commit: localTransport.manifest.commit,
+                payload_sha256: localTransport.manifest.payload_sha256,
+                payload_bytes: localTransport.manifest.payload_bytes,
+                status: "verified",
+            });
+        }
+        // Phase 2: the shared, read-only, SHA-256-addressed controller runtime
+        // cache replaces the per-eval Hitch runtime copy (spec §4).
+        const controllerRuntime = await ensureControllerRuntime({ root });
+        const runtimeRefFile = await writeRuntimeReference(evalDirectory, controllerRuntime);
+        sink.emit({
+            type: "eval.controller-runtime",
+            runtime_id: controllerRuntime.runtime_id,
+            cache_hit: controllerRuntime.cache_hit,
+            reference: runtimeRefFile,
+        });
+        const plan = {
+            schema_version: SCHEMA_VERSION,
+            eval_id: evalId,
+            backend: "harbor",
+            candidate: {
+                id: "candidate-1",
+                requested_harness_ref: normalized.harness_ref,
+                harness_ref: lockedHarnessRef(resolvedRevision),
+                harness_id: resolvedRevision.harness_id,
+                revision_identity: resolvedRevision.identity,
+                model: normalized.model || null,
+            },
+            dataset: normalized.dataset,
+            benchmark_id: normalized.benchmark_id,
+            benchmark_revision: normalized.benchmark_revision,
+            attempts: normalized.attempts,
+            max_concurrent: normalized.max_concurrent,
+            controller_runtime: {
+                runtime_id: controllerRuntime.runtime_id,
+                manifest_digest: controllerRuntime.manifest_digest,
+            },
+            ...(localTransport ? { local_source_transport: transportSummary(localTransport) } : {}),
+            created_at: new Date().toISOString(),
+        };
+        await atomicWriteJSON(path.join(evalDirectory, "resolution.json"), resolvedRevision);
+        await atomicWriteJSON(path.join(evalDirectory, "plan.json"), plan);
+        sink.emit({
+            type: "eval.planned",
+            harness: resolvedRevision.harness_id,
+            revision_identity: resolvedRevision.identity,
+            attempts: normalized.attempts,
+        });
+        if (localTransport) {
+            await verifyLocalGitTransport(localTransport, {
+                expected: {
+                    harnessId: resolvedRevision.harness_id,
+                    resolutionIdentity: resolvedRevision.identity,
+                    commit: resolvedRevision.revision.commit,
+                },
+                env,
+                ...(signal ? { signal } : {}),
+            });
+        }
+        const backendRun = await runHarborBackend({
+            evalId,
+            evalDirectory,
+            request: normalized,
+            root,
+            resolvedRevision,
+            runtimeDirectory: controllerRuntime.directory,
+            runtimeId: controllerRuntime.runtime_id,
+            ...(localTransport ? { localTransport } : {}),
+            env,
+            ...(harborExecutable !== undefined ? { harborExecutable } : {}),
+            ...(signal ? { signal } : {}),
+            emit: (event) => sink.emit(event),
+        });
+        trialRefs = await importEvalTrialRuns({
+            root,
+            evalId,
+            evalDirectory,
+            request: normalized,
+            resolvedRevision,
+            benchmarkId: normalized.benchmark_id,
+            benchmarkRevision: normalized.benchmark_revision,
+            runtimeId: controllerRuntime.runtime_id,
+            rawResult: backendRun.rawResult,
+        });
+        await validateEvalTrialReferences(root, evalId, trialRefs, {
+            benchmarkId: normalized.benchmark_id,
+            benchmarkRevision: normalized.benchmark_revision,
+        });
+        const cancelled = signal?.aborted;
+        const localSourceFailure = localTransport ? localSourceBackendFailure(backendRun.rawResult) : false;
+        const succeeded = !cancelled && !localSourceFailure && backendRun.backend.process_exit_code === 0 && backendRun.rawResult !== null;
+        result = {
+            schema_version: SCHEMA_VERSION,
+            eval_id: evalId,
+            status: cancelled ? "cancelled" : succeeded ? "succeeded" : "failed",
+            exit_code: cancelled ? 9 : succeeded ? 0 : 13,
+            backend: backendRun.backend,
+            candidate: plan.candidate,
+            dataset: normalized.dataset,
+            benchmark_id: normalized.benchmark_id,
+            benchmark_revision: normalized.benchmark_revision,
+            trials: trialRefs,
+            summary: summarizeTrialRefs(trialRefs),
+            backend_summary: backendRun.summary,
+            ...(localTransport ? { local_source_transport: transportSummary(localTransport) } : {}),
+            ...(succeeded ? {} : {
+                error: {
+                    code: cancelled ? "cancelled" : localSourceFailure ? "local_source_materialize_failed" : "harbor_failed",
+                    message: cancelled
+                        ? "eval was cancelled"
+                        : localSourceFailure
+                            ? "Harbor rejected the transported local Git source before candidate execution"
+                            : backendRun.rawResult === null
+                                ? `Harbor exited without a result (code ${backendRun.backend.process_exit_code ?? "null"})`
+                                : `Harbor exited with code ${backendRun.backend.process_exit_code ?? "null"}`,
+                },
+            }),
+            started_at: startedAt.toISOString(),
+            completed_at: new Date().toISOString(),
+        };
+    }
+    catch (error) {
+        const typed = error instanceof HitchError;
+        result = {
+            schema_version: SCHEMA_VERSION,
+            eval_id: evalId,
+            status: signal?.aborted ? "cancelled" : "failed",
+            exit_code: signal?.aborted ? 9 : typed ? error.exitCode : 12,
+            error: {
+                code: signal?.aborted ? "cancelled" : typed ? error.code : "internal_error",
+                message: error?.message || String(error),
+            },
+            benchmark_id: normalized.benchmark_id,
+            benchmark_revision: normalized.benchmark_revision,
+            trials: trialRefs,
+            started_at: startedAt.toISOString(),
+            completed_at: new Date().toISOString(),
+        };
+    }
+    await atomicWriteJSON(path.join(evalDirectory, "result.json"), result);
+    sink.emit({ type: result.status === "succeeded" ? "eval.completed" : "eval.failed", status: result.status, exit_code: result.exit_code, error: result.error });
+    await sink.close();
+    return result;
+}
+function summarizeTrialRefs(trials) {
+    const valid = trials.filter((trial) => trial.observation_status === "valid" && typeof trial.reward === "number");
+    const rewards = valid.map((trial) => trial.reward);
+    const aggregate = rewards.length
+        ? {
+            count: rewards.length,
+            mean: rewards.reduce((sum, reward) => sum + reward, 0) / rewards.length,
+            min: Math.min(...rewards),
+            max: Math.max(...rewards),
+        }
+        : null;
+    return {
+        n_trials: trials.length,
+        n_completed: valid.length,
+        n_invalid: trials.length - valid.length,
+        primary_reward: aggregate?.mean ?? null,
+        rewards: aggregate ? { reward: aggregate } : {},
+    };
+}
+function localSourceBackendFailure(rawResult) {
+    if (!rawResult)
+        return false;
+    // The bridge uses this fixed, non-secret marker when setup cannot verify or
+    // materialize the uploaded source. Do not surface provider exception text in
+    // the durable Hitch error, because it may contain backend diagnostics.
+    return JSON.stringify(rawResult).includes("hitch-local-source-materialize:");
+}
+function transportSummary(transport) {
+    const manifest = transport.manifest;
+    return {
+        kind: manifest.kind,
+        resolution_identity: manifest.resolution_identity,
+        commit: manifest.commit,
+        tree: manifest.tree,
+        payload_sha256: manifest.payload_sha256,
+        payload_bytes: manifest.payload_bytes,
+        object_count: manifest.object_count,
+        file_count: manifest.file_count,
+    };
+}
+//# sourceMappingURL=service.js.map

package/dist/src/evals/service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"service.js","sourceRoot":"","sources":["../../../src/evals/service.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,YAAY,EAAY,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpI,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAE3H,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAGhG,OAAO,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAuB9D,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAkB;IACzI,IAAI,CAAC,IAAI;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IACjF,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/D,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IAClB,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5E,IAAI,MAAkB,CAAC;IACvB,IAAI,SAAS,GAAkD,EAAE,CAAC;IAClE,IAAI,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9F,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QACzE,MAAM,gBAAgB,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;QACrF,IAAI,cAAgD,CAAC;QACrD,IAAI,gBAAgB,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK,IAAI,gBAAgB,CAAC,MAAM,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;YAC1F,MAAM,QAAQ,GAAG,kBAAkB,CAAC,QAAQ,CAAC;YAC7C,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ;gBAAE,MAAM,YAAY,CAAC,4DAA4D,CAAC,CAAC;YAC/I,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACxD,MAAM,IAAI,UAAU,CAAC,uDAAuD,EAAE,EAAE,IAAI,EAAE,iCAAiC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3I,CAAC;YACD,cAAc,GAAG,MAAM,sBAAsB,CAAC;gBAC5C,aAAa;gBACb,gBAAgB;gBAChB,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;gBAC3C,GAAG;gBACH,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9B,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC;gBACR,IAAI,EAAE,4BAA4B;gBAClC,mBAAmB,EAAE,cAAc,CAAC,QAAQ,CAAC,mBAAmB;gBAChE,MAAM,EAAE,cAAc,CAAC,QAAQ,CAAC,MAAM;gBACtC,cAAc,EAAE,cAAc,CAAC,QAAQ,CAAC,cAAc;gBACtD,aAAa,EAAE,cAAc,CAAC,QAAQ,CAAC,aAAa;gBACpD,MAAM,EAAE,UAAU;aACnB,CAAC,CAAC;QACL,CAAC;QACD,uEAAuE;QACvE,4DAA4D;QAC5D,MAAM,iBAAiB,GAA+B,MAAM,uBAAuB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;QACrF,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,yBAAyB;YAC/B,UAAU,EAAE,iBAAiB,CAAC,UAAU;YACxC,SAAS,EAAE,iBAAiB,CAAC,SAAS;YACtC,SAAS,EAAE,cAAc;SAC1B,CAAC,CAAC;QACH,MAAM,IAAI,GAAG;YACX,cAAc,EAAE,cAAc;YAC9B,OAAO,EAAE,MAAM;YACf,OAAO,EAAE,QAAQ;YACjB,SAAS,EAAE;gBACT,EAAE,EAAE,aAAa;gBACjB,qBAAqB,EAAE,UAAU,CAAC,WAAW;gBAC7C,WAAW,EAAE,gBAAgB,CAAC,gBAAgB,CAAC;gBAC/C,UAAU,EAAE,gBAAgB,CAAC,UAAU;gBACvC,iBAAiB,EAAE,gBAAgB,CAAC,QAAQ;gBAC5C,KAAK,EAAE,UAAU,CAAC,KAAK,IAAI,IAAI;aAChC;YACD,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,YAAY,EAAE,UAAU,CAAC,YAAY;YACrC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB;YACjD,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,cAAc,EAAE,UAAU,CAAC,cAAc;YACzC,kBAAkB,EAAE;gBAClB,UAAU,EAAE,iBAAiB,CAAC,UAAU;gBACxC,eAAe,EAAE,iBAAiB,CAAC,eAAe;aACnD;YACD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvF,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACrC,CAAC;QACF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,CAAC;QACnE,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,cAAc;YACpB,OAAO,EAAE,gBAAgB,CAAC,UAAU;YACpC,iBAAiB,EAAE,gBAAgB,CAAC,QAAQ;YAC5C,QAAQ,EAAE,UAAU,CAAC,QAAQ;SAC9B,CAAC,CAAC;QACH,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,uBAAuB,CAAC,cAAc,EAAE;gBAC5C,QAAQ,EAAE;oBACR,SAAS,EAAE,gBAAgB,CAAC,UAAU;oBACtC,kBAAkB,EAAE,gBAAgB,CAAC,QAAQ;oBAC7C,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,MAAgB;iBACnD;gBACD,GAAG;gBACH,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC;YACxC,MAAM;YACN,aAAa;YACb,OAAO,EAAE,UAAU;YACnB,IAAI;YACJ,gBAAgB;YAChB,gBAAgB,EAAE,iBAAiB,CAAC,SAAS;YAC7C,SAAS,EAAE,iBAAiB,CAAC,UAAU;YACvC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG;YACH,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;SAClC,CAAC,CAAC;QACH,SAAS,GAAG,MAAM,mBAAmB,CAAC;YACpC,IAAI;YACJ,MAAM;YACN,aAAa;YACb,OAAO,EAAE,UAAU;YACnB,gBAAgB;YAChB,WAAW,EAAE,UAAU,CAAC,YAAY;YACpC,iBAAiB,EAAE,UAAU,CAAC,kBAAkB;YAChD,SAAS,EAAE,iBAAiB,CAAC,UAAU;YACvC,SAAS,EAAE,UAAU,CAAC,SAAS;SAChC,CAAC,CAAC;QACH,MAAM,2BAA2B,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE;YACzD,WAAW,EAAE,UAAU,CAAC,YAAY;YACpC,iBAAiB,EAAE,UAAU,CAAC,kBAAkB;SACjD,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,EAAE,OAAO,CAAC;QAClC,MAAM,kBAAkB,GAAG,cAAc,CAAC,CAAC,CAAC,yBAAyB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACpG,MAAM,SAAS,GAAG,CAAC,SAAS,IAAI,CAAC,kBAAkB,IAAI,UAAU,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,UAAU,CAAC,SAAS,KAAK,IAAI,CAAC;QACnI,MAAM,GAAG;YACP,cAAc,EAAE,cAAc;YAC9B,OAAO,EAAE,MAAM;YACf,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YACpE,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;YAC7C,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,YAAY,EAAE,UAAU,CAAC,YAAY;YACrC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB;YACjD,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,kBAAkB,CAAC,SAAS,CAAC;YACtC,eAAe,EAAE,UAAU,CAAC,OAAO;YACnC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvF,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACnB,KAAK,EAAE;oBACL,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,eAAe;oBACxG,OAAO,EAAE,SAAS;wBAChB,CAAC,CAAC,oBAAoB;wBACtB,CAAC,CAAC,kBAAkB;4BAClB,CAAC,CAAC,6EAA6E;4BACjF,CAAC,CAAC,UAAU,CAAC,SAAS,KAAK,IAAI;gCAC7B,CAAC,CAAC,wCAAwC,UAAU,CAAC,OAAO,CAAC,iBAAiB,IAAI,MAAM,GAAG;gCAC3F,CAAC,CAAC,2BAA2B,UAAU,CAAC,OAAO,CAAC,iBAAiB,IAAI,MAAM,EAAE;iBAClF;aACF,CAAC;YACF,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;YACnC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,KAAK,YAAY,UAAU,CAAC;QAC1C,MAAM,GAAG;YACP,cAAc,EAAE,cAAc;YAC9B,OAAO,EAAE,MAAM;YACf,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YAChD,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;YAC5D,KAAK,EAAE;gBACL,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;gBAC3E,OAAO,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;aACpD;YACD,YAAY,EAAE,UAAU,CAAC,YAAY;YACrC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB;YACjD,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;YACnC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvC,CAAC;IACJ,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;IACvE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/J,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAqD;IAC/E,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACjH,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAgB,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM;QAC9B,CAAC,CAAC;YACE,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;YACvE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;YACzB,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;SAC1B;QACH,CAAC,CAAC,IAAI,CAAC;IACT,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,MAAM;QACvB,WAAW,EAAE,KAAK,CAAC,MAAM;QACzB,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QACvC,cAAc,EAAE,SAAS,EAAE,IAAI,IAAI,IAAI;QACvC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,yBAAyB,CAAC,SAAyC;IAC1E,IAAI,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IAC7B,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,gBAAgB,CAAC,SAA+B;IACvD,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;IACpC,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;QACjD,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,UAAU,EAAE,QAAQ,CAAC,UAAU;KAChC,CAAC;AACJ,CAAC"}

package/dist/src/evals/trial-import.js

@@ -0,0 +1,394 @@
+import { cp, lstat, mkdtemp, readFile, readdir, rename, rm, stat } from "node:fs/promises";
+import path from "node:path";
+import { atomicWriteJSON, ensureDir, readJSON, statePaths, writePrivateFile } from "../foundation/index.js";
+import { validateRunContext } from "../domain/index.js";
+import { newRunId, safeAgentArgsForPersistence } from "../runs/index.js";
+import { benchmarkTaskDigest, benchmarkVerifierIdentity, defaultModelIdentity, loadRunRecord, projectRunRecord, sha256JSON, } from "../runs/index.js";
+/** Import every Harbor trial into the authoritative runs/ store. */
+export async function importEvalTrialRuns(options) {
+    const trials = Array.isArray(options.rawResult?.trial_results)
+        ? options.rawResult.trial_results
+        : [];
+    const refs = [];
+    for (const [index, trial] of trials.entries()) {
+        // Harbor's result task_name may be display-qualified (for example,
+        // terminal-bench/regex-log), while the persisted trial lock contains the
+        // canonical task id used by the in-container bridge and exported run.
+        // Keep task_name only as a compatibility fallback for Harbor runtimes
+        // that do not persist a lock.json.
+        const fallbackTaskId = nonEmpty(trial.task_name) || `trial-${index + 1}`;
+        const trialId = nonEmpty(trial.trial_name) || `${fallbackTaskId}__${index + 1}`;
+        const attempt = trialAttempt(trialId);
+        const trialDirectory = path.join(options.evalDirectory, "harbor", "job", trialId);
+        const bundle = await findRunBundle(trialDirectory);
+        let taskId = fallbackTaskId;
+        let published = false;
+        try {
+            taskId = await lockedTaskId(trialDirectory) || fallbackTaskId;
+            if (bundle) {
+                refs.push(await importRunBundle({ ...options, trial, taskId, trialId, attempt, bundle }));
+                published = true;
+            }
+            else {
+                refs.push(await createDiagnosticRun({ ...options, trial, taskId, trialId, attempt }));
+            }
+        }
+        catch (error) {
+            if (bundle) {
+                await atomicWriteJSON(path.join(path.dirname(bundle), "hitch-run-import-error.json"), {
+                    schema_version: "1",
+                    trial_id: trialId,
+                    code: "run_bundle_import_failed",
+                    message: error.message,
+                    recorded_at: new Date().toISOString(),
+                }).catch(() => { });
+            }
+            refs.push(await createDiagnosticRun({
+                ...options,
+                trial: {
+                    ...trial,
+                    exception_info: "hitch-run-bundle-import-failed",
+                    hitch_import_error: error.message,
+                },
+                taskId,
+                trialId,
+                attempt,
+            }));
+        }
+        finally {
+            // The exported bundle is staging, not a second trajectory authority.
+            // Delete it only after atomic publication. A failed import keeps the
+            // complete bundle beside its diagnostic so it can be inspected/retried.
+            if (bundle && published && isWithin(options.evalDirectory, bundle)) {
+                await rm(bundle, { recursive: true, force: true });
+            }
+        }
+    }
+    return refs;
+}
+async function lockedTaskId(trialDirectory) {
+    const lockPath = path.join(trialDirectory, "lock.json");
+    let lock;
+    try {
+        lock = await readJSON(lockPath);
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return null;
+        throw new Error(`Harbor trial lock is unreadable: ${lockPath}`, { cause: error });
+    }
+    if (!lock || typeof lock !== "object" || Array.isArray(lock)) {
+        throw new Error(`Harbor trial lock is invalid: ${lockPath}`);
+    }
+    const task = lock.task;
+    if (!task || typeof task !== "object" || Array.isArray(task)) {
+        throw new Error(`Harbor trial lock has no task.name: ${lockPath}`);
+    }
+    const taskId = nonEmpty(task.name);
+    if (!taskId)
+        throw new Error(`Harbor trial lock has no task.name: ${lockPath}`);
+    return taskId;
+}
+async function importRunBundle(input) {
+    await validateBundleTree(input.bundle);
+    const manifestValue = await readJSON(path.join(input.bundle, "manifest.json"));
+    const record = projectRunRecord(manifestValue);
+    if (!/^run_[a-f0-9]{32}$/.test(record.run_id))
+        throw new Error(`invalid exported run_id: ${record.run_id}`);
+    const context = validateRunContext(record.context);
+    if (context.kind !== "benchmark_task"
+        || context.benchmark_id !== input.benchmarkId
+        || context.benchmark_revision !== input.benchmarkRevision
+        || context.task_id !== input.taskId) {
+        throw new Error(`exported run ${record.run_id} has a mismatched benchmark context`);
+    }
+    const expectedVerifier = benchmarkVerifierIdentity(input.benchmarkId, input.benchmarkRevision);
+    if (context.verifier_identity !== expectedVerifier)
+        throw new Error(`exported run ${record.run_id} has a mismatched verifier identity`);
+    if (record.parent?.eval_id !== input.evalId
+        || record.parent.trial_id !== input.trialId
+        || record.parent.attempt !== input.attempt) {
+        throw new Error(`exported run ${record.run_id} has a mismatched eval parent`);
+    }
+    const stagingParent = await mkdtemp(path.join(await ensureDir(statePaths(input.root).temporary), "eval-run-import-"));
+    const staging = path.join(stagingParent, record.run_id);
+    try {
+        await cp(input.bundle, staging, { recursive: true, errorOnExist: true, force: false });
+        await readJSON(path.join(staging, "resolution.json"));
+        await validateJSONLines(path.join(staging, "events.jsonl"));
+        const verifier = verifierResult(input.trial);
+        const verifierRef = verifier ? "verifier/result.json" : undefined;
+        if (verifier)
+            await atomicWriteJSON(path.join(staging, verifierRef), verifier);
+        const beforeObservation = await loadRunRecord(staging, { verifyTrajectory: true });
+        const observation = trialObservation(input.trial, beforeObservation.record.status, beforeObservation.trajectory_status, beforeObservation.record_status, verifierRef);
+        const manifest = await readJSON(path.join(staging, "manifest.json"));
+        const portableManifest = withoutKeys(manifest, [
+            "workspace", "source_workspace", "execution_workspace",
+            "managed_workspace", "executable", "artifact_entrypoint",
+        ]);
+        const request = await readJSON(path.join(staging, "request.json"));
+        await atomicWriteJSON(path.join(staging, "request.json"), { ...request, cwd: "." });
+        const result = await readJSON(path.join(staging, "result.json"));
+        await atomicWriteJSON(path.join(staging, "result.json"), withoutKeys(result, ["workspace"]));
+        await atomicWriteJSON(path.join(staging, "manifest.json"), {
+            ...portableManifest,
+            observation,
+            result_ref: "result.json",
+            ...(manifest.trajectory_ref ? {} : { trajectory_ref: "trajectory.ref.json" }),
+            sealed: true,
+        });
+        const verified = await loadRunRecord(staging, { verifyTrajectory: true });
+        if (verified.record.observation?.status !== observation.status)
+            throw new Error(`failed to seal observation for ${record.run_id}`);
+        const destination = path.join(statePaths(input.root).runs, record.run_id);
+        await ensureDir(path.dirname(destination));
+        try {
+            await stat(destination);
+            throw new Error(`run destination already exists: ${record.run_id}`);
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+        }
+        await rename(staging, destination);
+        return evalTrialRef(input, record.run_id, observation);
+    }
+    finally {
+        await rm(stagingParent, { recursive: true, force: true });
+    }
+}
+async function createDiagnosticRun(input) {
+    const runId = newRunId();
+    const runDirectory = await ensureDir(path.join(statePaths(input.root).runs, runId));
+    const now = new Date().toISOString();
+    const verifier = verifierResult(input.trial);
+    const verifierRef = verifier ? "verifier/result.json" : undefined;
+    if (verifier)
+        await atomicWriteJSON(path.join(runDirectory, verifierRef), verifier);
+    const reason = input.trial.exception_info
+        ? "infrastructure_failure"
+        : verifier ? "trajectory_missing_or_corrupt" : "verifier_result_missing";
+    const observation = {
+        status: "invalid",
+        invalid_reason: reason,
+        ...(verifierRef ? { verifier_result_ref: verifierRef } : {}),
+    };
+    const context = {
+        kind: "benchmark_task",
+        benchmark_id: input.benchmarkId,
+        benchmark_revision: input.benchmarkRevision,
+        task_id: input.taskId,
+        task_digest: benchmarkTaskDigest(input.benchmarkId, input.benchmarkRevision, input.taskId),
+        verifier_identity: benchmarkVerifierIdentity(input.benchmarkId, input.benchmarkRevision),
+    };
+    const safeAgentArgs = safeAgentArgsForPersistence(input.request.agent_args);
+    const argsDigest = safeAgentArgs.length ? sha256JSON(safeAgentArgs) : undefined;
+    const artifactId = typeof input.trial.agent_result?.metadata === "object"
+        ? input.trial.agent_result.metadata?.hitch_artifact_id
+        : undefined;
+    await atomicWriteJSON(path.join(runDirectory, "request.json"), {
+        schema_version: "1",
+        context,
+        parent: { kind: "eval", eval_id: input.evalId, trial_id: input.trialId, attempt: input.attempt },
+        task_id: input.taskId,
+        prompt: null,
+        diagnostic: "trial bundle was not exported",
+    });
+    await atomicWriteJSON(path.join(runDirectory, "resolution.json"), input.resolvedRevision);
+    await writePrivateFile(path.join(runDirectory, "events.jsonl"), `${JSON.stringify({
+        schema_version: "1",
+        sequence: 1,
+        timestamp: now,
+        run_id: runId,
+        type: "eval.trial.import_failed",
+        reason,
+    })}\n`);
+    await atomicWriteJSON(path.join(runDirectory, "result.json"), {
+        schema_version: "1",
+        run_id: runId,
+        status: "failed",
+        exit_code: 12,
+        error: { code: reason, message: "Harbor trial completed without an importable Hitch run bundle" },
+        completed_at: now,
+    });
+    await atomicWriteJSON(path.join(runDirectory, "manifest.json"), {
+        schema_version: "1",
+        run_id: runId,
+        context,
+        parent: { kind: "eval", eval_id: input.evalId, trial_id: input.trialId, attempt: input.attempt },
+        status: "failed",
+        harness: {
+            harness_id: input.resolvedRevision.harness_id,
+            requested_ref: input.request.harness_ref,
+            revision_identity: input.resolvedRevision.identity,
+            ...(artifactId && /^sha256:[0-9a-f]{64}$/.test(artifactId) ? { artifact_id: artifactId } : {}),
+            ...(argsDigest ? { agent_args_sha256: argsDigest } : {}),
+        },
+        model: defaultModelIdentity(input.request.model, input.resolvedRevision.harness_id),
+        protocol: {
+            timeout_ms: input.request.timeout_ms,
+            workspace_mode: "shared",
+            ...(input.runtimeId && /^sha256:[0-9a-f]{64}$/.test(input.runtimeId)
+                ? { environment_identity: input.runtimeId }
+                : {}),
+        },
+        observation,
+        request_ref: "request.json",
+        resolution_ref: "resolution.json",
+        result_ref: "result.json",
+        created_at: now,
+        completed_at: now,
+        sealed: true,
+    });
+    return evalTrialRef(input, runId, observation);
+}
+function evalTrialRef(input, runId, observation) {
+    return {
+        trial_id: input.trialId,
+        run_id: runId,
+        task_id: input.taskId,
+        attempt: input.attempt,
+        observation_status: observation.status,
+        ...(observation.reward !== undefined ? { reward: observation.reward } : {}),
+        ...(observation.verifier_result_ref ? { verifier_result_ref: observation.verifier_result_ref } : {}),
+        ...(observation.invalid_reason ? { invalid_reason: observation.invalid_reason } : {}),
+    };
+}
+function trialObservation(trial, runStatus, trajectoryStatus, recordStatus, verifierRef) {
+    if (trial.exception_info)
+        return { status: "invalid", invalid_reason: "infrastructure_failure", ...(verifierRef ? { verifier_result_ref: verifierRef } : {}) };
+    if (runStatus === "cancelled")
+        return { status: "invalid", invalid_reason: "cancelled", ...(verifierRef ? { verifier_result_ref: verifierRef } : {}) };
+    if (runStatus !== "succeeded")
+        return { status: "invalid", invalid_reason: "infrastructure_failure", ...(verifierRef ? { verifier_result_ref: verifierRef } : {}) };
+    if (recordStatus !== "valid")
+        return { status: "invalid", invalid_reason: "infrastructure_failure", ...(verifierRef ? { verifier_result_ref: verifierRef } : {}) };
+    if (trajectoryStatus !== "valid")
+        return { status: "invalid", invalid_reason: "trajectory_missing_or_corrupt", ...(verifierRef ? { verifier_result_ref: verifierRef } : {}) };
+    const reward = primaryReward(trial);
+    if (reward === undefined || !verifierRef)
+        return { status: "invalid", invalid_reason: "verifier_result_missing" };
+    return { status: "valid", reward, verifier_result_ref: verifierRef };
+}
+function verifierResult(trial) {
+    return trial.verifier_result && typeof trial.verifier_result === "object" && !Array.isArray(trial.verifier_result)
+        ? trial.verifier_result
+        : null;
+}
+function primaryReward(trial) {
+    const rewards = (verifierResult(trial)?.rewards || {});
+    const preferred = rewards.reward;
+    if (typeof preferred === "number" && Number.isFinite(preferred))
+        return preferred;
+    return Object.values(rewards).find((value) => typeof value === "number" && Number.isFinite(value));
+}
+async function findRunBundle(root, depth = 0) {
+    if (depth > 5)
+        return null;
+    try {
+        const manifest = path.join(root, "manifest.json");
+        if ((await lstat(manifest)).isFile() && path.basename(root) === "hitch-run-bundle")
+            return root;
+    }
+    catch { /* Continue scanning. */ }
+    let entries;
+    try {
+        entries = await readdir(root, { withFileTypes: true });
+    }
+    catch {
+        return null;
+    }
+    for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
+        if (!entry.isDirectory())
+            continue;
+        const found = await findRunBundle(path.join(root, entry.name), depth + 1);
+        if (found)
+            return found;
+    }
+    return null;
+}
+async function validateBundleTree(root) {
+    let files = 0;
+    let bytes = 0;
+    const walk = async (directory) => {
+        for (const entry of await readdir(directory, { withFileTypes: true })) {
+            const target = path.join(directory, entry.name);
+            const info = await lstat(target);
+            if (info.isSymbolicLink())
+                throw new Error(`run bundle contains a symbolic link: ${path.relative(root, target)}`);
+            if (info.isDirectory()) {
+                await walk(target);
+            }
+            else if (info.isFile()) {
+                files += 1;
+                bytes += info.size;
+                if (files > 100_000 || bytes > 1024 * 1024 * 1024)
+                    throw new Error("run bundle exceeds import limits");
+            }
+            else {
+                throw new Error(`run bundle contains a special file: ${path.relative(root, target)}`);
+            }
+        }
+    };
+    await walk(root);
+}
+function trialAttempt(trialId) {
+    const match = trialId.match(/__(\d+)$/);
+    const value = Number(match?.[1]);
+    // Modern Harbor trial ids carry a random suffix. Each such id identifies a
+    // distinct trial, whose execution attempt starts at one.
+    return Number.isInteger(value) && value > 0 ? value : 1;
+}
+function nonEmpty(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+function isWithin(root, candidate) {
+    const relative = path.relative(path.resolve(root), path.resolve(candidate));
+    return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
+}
+function withoutKeys(record, keys) {
+    const result = { ...record };
+    for (const key of keys)
+        delete result[key];
+    return result;
+}
+async function validateJSONLines(file) {
+    const content = await readFile(file, "utf8");
+    for (const [index, line] of content.split(/\r?\n/).entries()) {
+        if (!line)
+            continue;
+        try {
+            JSON.parse(line);
+        }
+        catch (error) {
+            throw new Error(`invalid JSONL at ${path.basename(file)}:${index + 1}: ${error.message}`);
+        }
+    }
+}
+export async function validateEvalTrialReferences(root, evalId, trials, expected) {
+    for (const trial of trials) {
+        const loaded = await loadRunRecord(path.join(statePaths(root).runs, trial.run_id), { verifyTrajectory: false });
+        const record = loaded.record;
+        if (record.context.kind !== "benchmark_task")
+            throw new Error(`eval trial ${trial.trial_id} references a non-benchmark run`);
+        if (expected && (record.context.benchmark_id !== expected.benchmarkId
+            || record.context.benchmark_revision !== expected.benchmarkRevision
+            || record.context.verifier_identity !== benchmarkVerifierIdentity(expected.benchmarkId, expected.benchmarkRevision)))
+            throw new Error(`eval trial ${trial.trial_id} benchmark identity mismatch`);
+        if (record.parent?.eval_id !== evalId || record.parent.trial_id !== trial.trial_id || record.parent.attempt !== trial.attempt) {
+            throw new Error(`eval trial ${trial.trial_id} parent mismatch`);
+        }
+        if (record.context.task_id !== trial.task_id)
+            throw new Error(`eval trial ${trial.trial_id} task mismatch`);
+        if (record.observation?.status !== trial.observation_status)
+            throw new Error(`eval trial ${trial.trial_id} observation status mismatch`);
+        if (record.observation?.reward !== trial.reward)
+            throw new Error(`eval trial ${trial.trial_id} reward mismatch`);
+        if (record.observation?.verifier_result_ref !== trial.verifier_result_ref)
+            throw new Error(`eval trial ${trial.trial_id} verifier ref mismatch`);
+        if (record.observation?.invalid_reason !== trial.invalid_reason)
+            throw new Error(`eval trial ${trial.trial_id} invalid reason mismatch`);
+    }
+}
+//# sourceMappingURL=trial-import.js.map