agent-hitch 0.2.5 → 0.2.7

package/dist/src/evals/infrastructure-retry.js

@@ -0,0 +1,218 @@
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { runHarborBackend } from "../backends/index.js";
+import { HitchError } from "../foundation/index.js";
+import { importEvalTrialRun, importEvalTrialRuns, TrialBundlePendingError, validateEvalTrialReferences } from "./trial-import.js";
+import { replaceInvalidEvalProgressTrial, writeEvalProgress } from "./progress.js";
+const INFRASTRUCTURE_REASONS = new Set([
+    "infrastructure_failure",
+    "verifier_infrastructure_failure",
+    "verifier_result_missing",
+]);
+// Verifier failures are retried inside the original live Harbor trial by the
+// custom verifier wrapper. Starting another Harbor trial would execute the
+// candidate agent again, so only non-verifier trial infrastructure failures
+// are eligible for this outer retry path.
+const RETRYABLE_TRIAL_INFRASTRUCTURE_REASONS = new Set([
+    "infrastructure_failure",
+]);
+export async function runInfrastructureRetries(options) {
+    let progress = options.progress;
+    let retryCandidates = retryableInfrastructureTrials(options.initialRefs);
+    const runs = [];
+    for (let retry = 1; retry <= options.request.infrastructure_retries && retryCandidates.length > 0; retry += 1) {
+        if (options.signal?.aborted)
+            break;
+        const retryTriggers = [...retryCandidates];
+        const taskNames = [...new Set(retryCandidates.map((ref) => ref.task_id))].sort();
+        const backoffMs = options.request.infrastructure_retry_backoff_ms * retry;
+        options.sink.emit({
+            type: "eval.infrastructure-retry.scheduled",
+            execution_kind: "physical-infrastructure-retry",
+            candidate_executes: true,
+            attempt: options.logicalAttempt,
+            retry,
+            tasks: taskNames,
+            backoff_ms: backoffMs,
+        });
+        if (backoffMs > 0) {
+            if (options.signal)
+                await delay(backoffMs, undefined, { signal: options.signal });
+            else
+                await delay(backoffMs);
+        }
+        if (options.signal?.aborted)
+            break;
+        const backendDirectory = path.join(options.backendBaseDirectory || path.join(options.evalDirectory, "infrastructure-retries"), `retry-${String(retry).padStart(4, "0")}`, options.request.attempts === 1 ? "harbor" : attemptDirectoryName(options.logicalAttempt));
+        const harborJobDirectory = path.join(backendDirectory, "job");
+        const retryRefs = [];
+        const selectedTasks = new Set(taskNames);
+        let lifecycle;
+        const retryEnvironmentImages = () => lifecycle?.environmentImages ?? options.environmentImages;
+        const executionEvidence = async () => lifecycle?.captureExecutionEvidence
+            ? { executionEvidence: await lifecycle.captureExecutionEvidence() }
+            : {};
+        const publish = async (ref) => {
+            if (ref.attempt !== options.logicalAttempt || !selectedTasks.has(ref.task_id)) {
+                throw new HitchError(`Harbor infrastructure retry returned an unselected trial: ${ref.task_id}#${ref.attempt}`, {
+                    code: "eval_infrastructure_retry_trial_mismatch",
+                    exitCode: 12,
+                });
+            }
+            const existing = retryRefs.find((current) => current.trial_id === ref.trial_id);
+            if (existing) {
+                if (JSON.stringify(existing) !== JSON.stringify(ref))
+                    throw new Error(`infrastructure retry trial identity changed: ${ref.trial_id}`);
+                return;
+            }
+            retryRefs.push(ref);
+            await validateEvalTrialReferences(options.root, options.evalId, [ref], {
+                benchmarkId: options.request.benchmark_id,
+                benchmarkRevision: options.request.benchmark_revision,
+            });
+            if (ref.observation_status === "valid") {
+                progress = replaceInvalidEvalProgressTrial(progress, ref);
+                await writeEvalProgress(options.evalDirectory, progress);
+            }
+            options.sink.emit({
+                type: ref.observation_status === "valid"
+                    ? "eval.infrastructure-retry.repaired"
+                    : "eval.infrastructure-retry.failed",
+                attempt: options.logicalAttempt,
+                retry,
+                task_id: ref.task_id,
+                trial_id: ref.trial_id,
+                run_id: ref.run_id,
+                observation_status: ref.observation_status,
+                ...(ref.invalid_reason ? { invalid_reason: ref.invalid_reason } : {}),
+                generation: progress.generation,
+            });
+        };
+        let run;
+        try {
+            lifecycle = await options.beginRetry?.({ retry, logicalAttempt: options.logicalAttempt, taskNames, triggers: retryTriggers, backendDirectory });
+            run = await runHarborBackend({
+                evalId: options.evalId,
+                evalDirectory: options.evalDirectory,
+                backendDirectory,
+                logicalAttempt: options.logicalAttempt,
+                taskNames,
+                request: { ...options.request, attempts: 1 },
+                root: options.root,
+                resolvedRevision: options.resolvedRevision,
+                runtimeDirectory: options.controllerRuntime.directory,
+                runtimeId: options.controllerRuntime.runtime_id,
+                preparedArtifact: options.preparedArtifact,
+                ...(options.executionResources ? { executionResources: options.executionResources } : {}),
+                ...(options.resolvedImages ? { resolvedImages: options.resolvedImages } : {}),
+                ...(options.interactionCaptureExporter ? { modelProxy: options.interactionCaptureExporter.route } : {}),
+                ...lifecycle?.backend,
+                env: options.env,
+                ...(options.harborExecutable !== undefined ? { harborExecutable: options.harborExecutable } : {}),
+                ...(options.signal ? { signal: options.signal } : {}),
+                ...(options.trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs: options.trialBundleGraceMs }),
+                emit: (event) => options.sink.emit({ ...event, execution_kind: "physical-infrastructure-retry", infrastructure_retry: retry, logical_attempt: options.logicalAttempt, ...(lifecycle ? { work_id: lifecycle.workId, lease_id: lifecycle.leaseId } : {}) }),
+                onTrialSettled: async (trial, context) => {
+                    try {
+                        const environmentImages = retryEnvironmentImages();
+                        const ref = await importEvalTrialRun({
+                            root: options.root,
+                            evalId: options.evalId,
+                            evalDirectory: options.evalDirectory,
+                            harborJobDirectory,
+                            expectedAttempt: options.logicalAttempt,
+                            request: options.request,
+                            resolvedRevision: options.resolvedRevision,
+                            benchmarkId: options.request.benchmark_id,
+                            benchmarkRevision: options.request.benchmark_revision,
+                            publicationMode: "replace-invalid",
+                            runtimeId: options.controllerRuntime.runtime_id,
+                            env: options.env,
+                            ...(options.modelCapturePlan ? { modelCapturePlan: options.modelCapturePlan } : {}),
+                            ...(options.interactionCaptureExporter ? {
+                                interactionCaptureExporter: options.interactionCaptureExporter,
+                            } : {}),
+                            ...await executionEvidence(),
+                            ...(environmentImages ? { environmentImages } : {}),
+                            requireCompleteMarker: true,
+                            allowMissingBundleDiagnostic: context.bundleWaitExpired,
+                        }, trial, retryRefs.length, retryRefs);
+                        await publish(ref);
+                        return true;
+                    }
+                    catch (error) {
+                        if (error instanceof TrialBundlePendingError)
+                            return false;
+                        throw error;
+                    }
+                },
+            });
+            const environmentImages = retryEnvironmentImages();
+            const terminalRefs = await importEvalTrialRuns({
+                root: options.root,
+                evalId: options.evalId,
+                evalDirectory: options.evalDirectory,
+                harborJobDirectory,
+                expectedAttempt: options.logicalAttempt,
+                request: options.request,
+                resolvedRevision: options.resolvedRevision,
+                benchmarkId: options.request.benchmark_id,
+                benchmarkRevision: options.request.benchmark_revision,
+                publicationMode: "replace-invalid",
+                runtimeId: options.controllerRuntime.runtime_id,
+                env: options.env,
+                ...(options.modelCapturePlan ? { modelCapturePlan: options.modelCapturePlan } : {}),
+                ...(options.interactionCaptureExporter ? {
+                    interactionCaptureExporter: options.interactionCaptureExporter,
+                } : {}),
+                ...await executionEvidence(),
+                ...(environmentImages ? { environmentImages } : {}),
+                rawResult: run.rawResult,
+            }, retryRefs);
+            for (const ref of terminalRefs)
+                await publish(ref);
+            if (run.rawResult !== null)
+                assertBackendTrialSet(run.rawResult, retryRefs);
+            runs.push({ attempt: options.logicalAttempt, retry, tasks: taskNames, triggers: retryTriggers, refs: retryRefs, run, ...(lifecycle ? { leaseId: lifecycle.leaseId, workId: lifecycle.workId } : {}) });
+        }
+        finally {
+            await lifecycle?.close();
+        }
+        options.sink.emit({
+            type: "eval.infrastructure-retry.completed",
+            execution_kind: "physical-infrastructure-retry",
+            candidate_executes: true,
+            attempt: options.logicalAttempt,
+            retry,
+            tasks: taskNames,
+            repaired_tasks: retryRefs.filter((ref) => ref.observation_status === "valid").map((ref) => ref.task_id).sort(),
+            remaining_tasks: retryableInfrastructureTrials(retryRefs).map((ref) => ref.task_id).sort(),
+        });
+        if (run.backend.process_exit_code !== 0 || run.rawResult === null || options.stopAfterResult?.(run.rawResult))
+            break;
+        retryCandidates = retryableInfrastructureTrials(retryRefs);
+    }
+    return { progress, runs };
+}
+export function retryableInfrastructureTrials(trials) {
+    return trials.filter((trial) => trial.observation_status === "invalid"
+        && trial.invalid_reason !== undefined
+        && RETRYABLE_TRIAL_INFRASTRUCTURE_REASONS.has(trial.invalid_reason));
+}
+export function infrastructureFailureTrials(trials) {
+    return trials.filter((trial) => trial.observation_status === "invalid"
+        && trial.invalid_reason !== undefined
+        && INFRASTRUCTURE_REASONS.has(trial.invalid_reason));
+}
+function assertBackendTrialSet(rawResult, refs) {
+    const trials = Array.isArray(rawResult.trial_results) ? rawResult.trial_results : [];
+    const backendIds = new Set(trials.map((trial) => typeof trial.trial_name === "string" ? trial.trial_name : null).filter((value) => value !== null));
+    const refIds = new Set(refs.map((ref) => ref.trial_id));
+    if (backendIds.size !== trials.length || backendIds.size !== refIds.size || [...backendIds].some((id) => !refIds.has(id))) {
+        throw new Error("Harbor infrastructure retry trial set does not match published eval progress");
+    }
+}
+function attemptDirectoryName(attempt) {
+    return `attempt-${String(attempt).padStart(4, "0")}`;
+}
+//# sourceMappingURL=infrastructure-retry.js.map

package/dist/src/evals/infrastructure-retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"infrastructure-retry.js","sourceRoot":"","sources":["../../../src/evals/infrastructure-retry.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAIxD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAClI,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAInF,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,wBAAwB;IACxB,iCAAiC;IACjC,yBAAyB;CAC1B,CAAC,CAAC;AAEH,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,0CAA0C;AAC1C,MAAM,sCAAsC,GAAG,IAAI,GAAG,CAAC;IACrD,wBAAwB;CACzB,CAAC,CAAC;AA6DH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,OAAwC;IAExC,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,eAAe,GAAG,6BAA6B,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACzE,MAAM,IAAI,GAA6B,EAAE,CAAC;IAC1C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC9G,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACnC,MAAM,aAAa,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,+BAA+B,GAAG,KAAK,CAAC;QAC1E,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QACH,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;;gBAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QAEnC,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAChC,OAAO,CAAC,oBAAoB,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,wBAAwB,CAAC,EAC1F,SAAS,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EACzC,OAAO,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,cAAc,CAAC,CACzF,CAAC;QACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAqB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,SAAmD,CAAC;QACxD,MAAM,sBAAsB,GAAG,GAAyC,EAAE,CAAC,SAAS,EAAE,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,CAAC;QACrI,MAAM,iBAAiB,GAAG,KAAK,IAA0D,EAAE,CAAC,SAAS,EAAE,wBAAwB;YAC7H,CAAC,CAAC,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC,wBAAwB,EAAE,EAAE;YACnE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,OAAO,GAAG,KAAK,EAAE,GAAmB,EAAiB,EAAE;YAC3D,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO,CAAC,cAAc,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9E,MAAM,IAAI,UAAU,CAAC,6DAA6D,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE;oBAC9G,IAAI,EAAE,0CAA0C;oBAChD,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC;YAChF,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtI,OAAO;YACT,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,MAAM,2BAA2B,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;gBACrE,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;aACtD,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACvC,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC1D,MAAM,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,GAAG,CAAC,kBAAkB,KAAK,OAAO;oBACtC,CAAC,CAAC,oCAAoC;oBACtC,CAAC,CAAC,kCAAkC;gBACtC,OAAO,EAAE,OAAO,CAAC,cAAc;gBAC/B,KAAK;gBACL,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gBAC1C,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrE,UAAU,EAAE,QAAQ,CAAC,UAAU;aAChC,CAAC,CAAC;QACL,CAAC,CAAC;QACF,IAAI,GAAwB,CAAC;QAC7B,IAAI,CAAC;YACH,SAAS,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAChJ,GAAG,GAAG,MAAM,gBAAgB,CAAC;gBAC3B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,gBAAgB;gBAChB,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,SAAS;gBACT,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE;gBAC5C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,SAAS;gBACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;gBAC/C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzF,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,0BAA0B,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvG,GAAG,SAAS,EAAE,OAAO;gBACrB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;gBACvG,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,cAAc,EAAE,+BAA+B,EAAE,oBAAoB,EAAE,KAAK,EAAE,eAAe,EAAE,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBACzP,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAoB,EAAE;oBACzD,IAAI,CAAC;wBACH,MAAM,iBAAiB,GAAG,sBAAsB,EAAE,CAAC;wBACnD,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;4BACnC,IAAI,EAAE,OAAO,CAAC,IAAI;4BAClB,MAAM,EAAE,OAAO,CAAC,MAAM;4BACtB,aAAa,EAAE,OAAO,CAAC,aAAa;4BACpC,kBAAkB;4BAClB,eAAe,EAAE,OAAO,CAAC,cAAc;4BACvC,OAAO,EAAE,OAAO,CAAC,OAAO;4BACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;4BAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;4BACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;4BACrD,eAAe,EAAE,iBAAiB;4BAClC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;4BAC/C,GAAG,EAAE,OAAO,CAAC,GAAG;4BAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACnF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC;gCACvC,0BAA0B,EAAE,OAAO,CAAC,0BAA0B;6BAC/D,CAAC,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,MAAM,iBAAiB,EAAE;4BAC5B,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACnD,qBAAqB,EAAE,IAAI;4BAC3B,4BAA4B,EAAE,OAAO,CAAC,iBAAiB;yBACxD,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;wBACvC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;wBACnB,OAAO,IAAI,CAAC;oBACd,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,KAAK,YAAY,uBAAuB;4BAAE,OAAO,KAAK,CAAC;wBAC3D,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,MAAM,iBAAiB,GAAG,sBAAsB,EAAE,CAAC;YACnD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC;gBAC7C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,kBAAkB;gBAClB,eAAe,EAAE,OAAO,CAAC,cAAc;gBACvC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;gBACrD,eAAe,EAAE,iBAAiB;gBAClC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;gBAC/C,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC;oBACvC,0BAA0B,EAAE,OAAO,CAAC,0BAA0B;iBAC/D,CAAC,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,MAAM,iBAAiB,EAAE;gBAC5B,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnD,SAAS,EAAE,GAAG,CAAC,SAAS;aACzB,EAAE,SAAS,CAAC,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,YAAY;gBAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;gBAAE,qBAAqB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC5E,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACzM,CAAC;gBAAS,CAAC;YACT,MAAM,SAAS,EAAE,KAAK,EAAE,CAAC;QAC3B,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;YAC9G,eAAe,EAAE,6BAA6B,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;SAC3F,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,MAAM;QACrH,eAAe,GAAG,6BAA6B,CAAC,SAAS,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAiC;IAC7E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sCAAsC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAAiC;IAC3E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAkC,EAAE,IAA+B;IAChG,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC;IACrK,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxD,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1H,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe;IAC3C,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC"}

package/dist/src/evals/interaction-capture-import.js

@@ -0,0 +1,49 @@
+import { HitchError, atomicWriteJSON, ensureDir } from "../foundation/index.js";
+import { loadInteractionCapture } from "../model-access/index.js";
+export async function importTrialInteractionCapture(input, runId, staging) {
+    const plan = input.modelCapturePlan;
+    await writeTrialCapturePolicy(staging, plan);
+    const expectsProxy = plan?.effective_mode === "proxy" || plan?.effective_mode === "hybrid";
+    const existing = expectsProxy
+        ? await loadInteractionCapture(staging).catch((error) => {
+            if (error.code === "ENOENT")
+                return null;
+            throw error;
+        })
+        : null;
+    if (existing) {
+        const ref = existing.ref;
+        if (ref.run_id !== runId || ref.mode !== plan?.effective_mode || ref.required !== plan.required
+            || ref.topology !== plan.topology || plan.required && ref.completeness !== "complete") {
+            throw new HitchError("remote model interaction capture does not match its sealed plan", { code: "model_capture_incomplete", exitCode: 12 });
+        }
+        return;
+    }
+    if (!input.interactionCaptureExporter) {
+        if (expectsProxy && plan.required) {
+            throw new HitchError("required model interaction capture is unavailable", { code: "model_capture_incomplete", exitCode: 12 });
+        }
+        return;
+    }
+    const ref = await input.interactionCaptureExporter.finalizeRun(runId, staging);
+    if (ref.run_id !== runId || ref.mode !== input.interactionCaptureExporter.route.mode) {
+        throw new HitchError("model interaction capture identity does not match the run", { code: "model_capture_incomplete", exitCode: 12 });
+    }
+    if (input.interactionCaptureExporter.plan.required && ref.completeness !== "complete") {
+        throw new HitchError("required model interaction capture is incomplete", { code: "model_capture_incomplete", exitCode: 12 });
+    }
+}
+export async function writeTrialCapturePolicy(staging, plan) {
+    if (!plan)
+        return;
+    await ensureDir(`${staging}/interactions`);
+    await atomicWriteJSON(`${staging}/interactions/capture.policy.json`, {
+        schema_version: "1",
+        requested_mode: plan.requested_mode,
+        effective_mode: plan.effective_mode,
+        required: plan.required,
+        ...(plan.topology ? { topology: plan.topology } : {}),
+        ...(plan.degraded_reason ? { degraded_reason: plan.degraded_reason } : {}),
+    });
+}
+//# sourceMappingURL=interaction-capture-import.js.map

package/dist/src/evals/interaction-capture-import.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interaction-capture-import.js","sourceRoot":"","sources":["../../../src/evals/interaction-capture-import.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAChF,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAGlE,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAGnD,EAAE,KAAa,EAAE,OAAe;IAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,gBAAgB,CAAC;IACpC,MAAM,uBAAuB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,EAAE,cAAc,KAAK,OAAO,IAAI,IAAI,EAAE,cAAc,KAAK,QAAQ,CAAC;IAC3F,MAAM,QAAQ,GAAG,YAAY;QAC3B,CAAC,CAAC,MAAM,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACtD,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC;YACpE,MAAM,KAAK,CAAC;QACd,CAAC,CAAC;QACF,CAAC,CAAC,IAAI,CAAC;IACT,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;QACzB,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,cAAc,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ;eAC1F,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACxF,MAAM,IAAI,UAAU,CAAC,iEAAiE,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9I,CAAC;QACD,OAAO;IACT,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,CAAC;QACtC,IAAI,YAAY,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,IAAI,UAAU,CAAC,mDAAmD,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAChI,CAAC;QACD,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,0BAA0B,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/E,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,0BAA0B,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACrF,MAAM,IAAI,UAAU,CAAC,2DAA2D,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACxI,CAAC;IACD,IAAI,KAAK,CAAC,0BAA0B,CAAC,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;QACtF,MAAM,IAAI,UAAU,CAAC,kDAAkD,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/H,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAe,EAAE,IAAyB;IACtF,IAAI,CAAC,IAAI;QAAE,OAAO;IAClB,MAAM,SAAS,CAAC,GAAG,OAAO,eAAe,CAAC,CAAC;IAC3C,MAAM,eAAe,CAAC,GAAG,OAAO,mCAAmC,EAAE;QACnE,cAAc,EAAE,GAAG;QACnB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3E,CAAC,CAAC;AACL,CAAC"}