agent-hitch 0.2.6 → 0.2.8

package/dist/src/evals/eval-lifecycle-events.js

@@ -0,0 +1,29 @@
+export function emitEvalPlanLifecycle(sink, plan, planningStartedAt) {
+    sink.emit({
+        type: "eval.plan.created",
+        duration_ms: Date.now() - planningStartedAt,
+        work_items: plan.work_items.length,
+        membership: plan.membership,
+    });
+    for (const workItem of plan.work_items)
+        sink.emit({
+            type: "eval.work.queued",
+            work_id: workItem.work_id,
+            task_ids: workItem.task_ids,
+            reservation: workItem.reservation,
+        });
+    for (const workItem of plan.work_items)
+        if (workItem.scheduling)
+            sink.emit({
+                type: "eval.work.priority-computed",
+                work_id: workItem.work_id,
+                slot_id: workItem.slots[0],
+                task_id: workItem.task_ids[0],
+                policy: workItem.scheduling.policy,
+                estimated_duration_ms: workItem.scheduling.estimated_duration_ms,
+                remaining_path_ms: workItem.scheduling.remaining_path_ms,
+                estimate_source: workItem.scheduling.estimate_source,
+                estimate_sample_count: workItem.scheduling.estimate_sample_count,
+            });
+}
+//# sourceMappingURL=eval-lifecycle-events.js.map

package/dist/src/evals/eval-lifecycle-events.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-lifecycle-events.js","sourceRoot":"","sources":["../../../src/evals/eval-lifecycle-events.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,qBAAqB,CAAC,IAAmB,EAAE,IAAyB,EAAE,iBAAyB;IAC7G,IAAI,CAAC,IAAI,CAAC;QACR,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,iBAAiB;QAC3C,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;QAClC,UAAU,EAAE,IAAI,CAAC,UAAU;KAC5B,CAAC,CAAC;IACH,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU;QAAE,IAAI,CAAC,IAAI,CAAC;YAChD,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,WAAW,EAAE,QAAQ,CAAC,WAAW;SAClC,CAAC,CAAC;IACH,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU;QAAE,IAAI,QAAQ,CAAC,UAAU;YAAE,IAAI,CAAC,IAAI,CAAC;gBACzE,IAAI,EAAE,6BAA6B;gBACnC,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC1B,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC7B,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,MAAM;gBAClC,qBAAqB,EAAE,QAAQ,CAAC,UAAU,CAAC,qBAAqB;gBAChE,iBAAiB,EAAE,QAAQ,CAAC,UAAU,CAAC,iBAAiB;gBACxD,eAAe,EAAE,QAAQ,CAAC,UAAU,CAAC,eAAe;gBACpD,qBAAqB,EAAE,QAAQ,CAAC,UAAU,CAAC,qBAAqB;aACjE,CAAC,CAAC;AACL,CAAC"}

package/dist/src/evals/eval-logical-plan.js

@@ -0,0 +1,51 @@
+import path from "node:path";
+import { SCHEMA_VERSION, atomicWriteJSON, readJSON } from "../foundation/index.js";
+import { writeEvalProgress } from "./progress.js";
+/** Persist the reproducible, artifact-independent part of an eval before setup work begins. */
+export async function writeEvalPlanningCheckpoint(evalDirectory, plan, progress) {
+    await Promise.all([
+        atomicWriteJSON(path.join(evalDirectory, "logical-plan.json"), plan),
+        writeEvalProgress(evalDirectory, progress),
+    ]);
+}
+export function materializeEvalPlan(logical, modelCapture, artifactFields) {
+    const { kind: _kind, model_capture: _modelCapture, ...base } = logical;
+    return {
+        ...base,
+        ...(modelCapture ? { model_capture: modelCapture } : {}),
+        ...artifactFields,
+    };
+}
+/** Read only the restart inputs that cannot be reconstructed from request.json. */
+export async function readEvalLogicalPlan(evalDirectory, evalId, request) {
+    const value = await readJSON(path.join(evalDirectory, "logical-plan.json"), null);
+    if (value === null)
+        return null;
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("eval logical plan must be an object");
+    const plan = value;
+    if (plan.schema_version !== SCHEMA_VERSION || plan.kind !== "eval-logical-plan" || plan.eval_id !== evalId
+        || plan.dataset !== request.dataset || plan.benchmark_id !== request.benchmark_id
+        || plan.benchmark_revision !== request.benchmark_revision) {
+        throw new TypeError("eval logical plan identity is invalid");
+    }
+    if (plan.model_capture !== undefined)
+        parseModelCapturePlan(plan.model_capture);
+    return plan;
+}
+export function logicalPlanModelCapture(plan) {
+    return plan?.model_capture === undefined ? undefined : parseModelCapturePlan(plan.model_capture);
+}
+function parseModelCapturePlan(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("eval logical model capture plan is invalid");
+    const plan = value;
+    const modes = new Set(["off", "native", "proxy", "hybrid"]);
+    if (!modes.has(String(plan.requested_mode)) || !modes.has(String(plan.effective_mode)) || typeof plan.required !== "boolean"
+        || plan.topology !== undefined && plan.topology !== "host-side" && plan.topology !== "in-sandbox"
+        || plan.degraded_reason !== undefined && typeof plan.degraded_reason !== "string") {
+        throw new TypeError("eval logical model capture plan is invalid");
+    }
+    return plan;
+}
+//# sourceMappingURL=eval-logical-plan.js.map

package/dist/src/evals/eval-logical-plan.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-logical-plan.js","sourceRoot":"","sources":["../../../src/evals/eval-logical-plan.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACnF,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAoBlD,+FAA+F;AAC/F,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,aAAqB,EACrB,IAAuB,EACvB,QAAwB;IAExB,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC,EAAE,IAAI,CAAC;QACpE,iBAAiB,CAAC,aAAa,EAAE,QAAQ,CAAC;KAC3C,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAA0B,EAC1B,YAA4C,EAC5C,cAAuC;IAEvC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACvE,OAAO;QACL,GAAG,IAAI;QACP,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,cAAc;KAClB,CAAC;AACJ,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,aAAqB,EACrB,MAAc,EACd,OAAoB;IAEpB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC,EAAE,IAAI,CAAC,CAAC;IAClG,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC5H,MAAM,IAAI,GAAG,KAAgC,CAAC;IAC9C,IAAI,IAAI,CAAC,cAAc,KAAK,cAAc,IAAI,IAAI,CAAC,IAAI,KAAK,mBAAmB,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM;WACrG,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY;WAC9E,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC5D,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS;QAAE,qBAAqB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAChF,OAAO,IAAoC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAA8B;IACpE,OAAO,IAAI,EAAE,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AACnG,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,4CAA4C,CAAC,CAAC;IACnI,MAAM,IAAI,GAAG,KAAgC,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,SAAS;WACvH,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,QAAQ,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,KAAK,YAAY;WAC9F,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;QACpF,MAAM,IAAI,SAAS,CAAC,4CAA4C,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAqC,CAAC;AAC/C,CAAC"}

package/dist/src/evals/eval-result-builder.js

@@ -0,0 +1,81 @@
+import { SCHEMA_VERSION } from "../foundation/index.js";
+import { preparedArtifactPlanFields } from "./eval-artifact-planning.js";
+import { infrastructureFailureTrials } from "./infrastructure-retry.js";
+import { invalidTrialSlots } from "./rerun-slots.js";
+import { localSourceBackendFailure, summarizeTrialRefs, transportSummary } from "./result-helpers.js";
+export function buildCompletedEvalResult(input) {
+    const trialRefs = input.progress.trials;
+    const localSourceFailure = input.localTransport
+        ? input.backendRuns.some(({ run }) => localSourceBackendFailure(run.rawResult))
+            || input.infrastructureRetryRuns.some(({ run }) => localSourceBackendFailure(run.rawResult))
+        : false;
+    const expectedBackendRuns = input.plannedTaskExecution ? input.executionWorkItems : input.request.attempts;
+    const backendsSucceeded = input.backendRuns.every(({ run }) => run.backend.process_exit_code === 0 && run.rawResult !== null)
+        && (input.plannedTaskExecution ? trialRefs.length === input.plannedTrials : input.backendRuns.length === expectedBackendRuns);
+    const invalidTrials = input.localTaskIds === null
+        ? trialRefs.filter((trial) => trial.observation_status !== "valid").map((trial) => ({ task_id: trial.task_id, attempt: trial.attempt }))
+        : invalidTrialSlots(input.localTaskIds, input.request.attempts, input.progress);
+    const infrastructureFailures = infrastructureFailureTrials(trialRefs);
+    const infrastructureFailureSlots = infrastructureFailures.map((trial) => ({ task_id: trial.task_id, attempt: trial.attempt }));
+    const verifierRetriesExhausted = input.request.infrastructure_retries > 0
+        && infrastructureFailures.some((trial) => trial.invalid_reason === "verifier_infrastructure_failure");
+    const infrastructureErrorCode = verifierRetriesExhausted
+        || (input.request.infrastructure_retries > 0 && input.infrastructureRetryRuns.length > 0)
+        ? "eval_infrastructure_retries_exhausted"
+        : "eval_has_infrastructure_failures";
+    const succeeded = !input.cancelled && !localSourceFailure && backendsSucceeded && invalidTrials.length === 0;
+    const singleBackend = input.backendRuns.length === 1 ? input.backendRuns[0].run : undefined;
+    return {
+        schema_version: SCHEMA_VERSION,
+        eval_id: input.evalId,
+        status: input.cancelled ? "cancelled" : succeeded ? "succeeded" : "failed",
+        exit_code: input.cancelled ? 9 : succeeded ? 0 : 13,
+        ...(singleBackend ? { backend: singleBackend.backend, backend_summary: singleBackend.summary } : {}),
+        ...(input.plannedTaskExecution ? {
+            backend_work_items: input.backendRuns.map(({ attempt, workId, tasks, leaseId, run }) => ({
+                work_id: workId, lease_id: leaseId, attempt, tasks, backend: run.backend, backend_summary: run.summary,
+            })),
+        } : input.request.attempts > 1 ? {
+            backend_runs: input.backendRuns.map(({ attempt, run }) => ({ attempt, backend: run.backend, backend_summary: run.summary })),
+        } : {}),
+        infrastructure_retry_policy: {
+            max_retries: input.request.infrastructure_retries,
+            backoff_ms: input.request.infrastructure_retry_backoff_ms,
+            verifier_execution: "same_trial_verifier_only",
+            candidate_rerun_on_verifier_failure: false,
+        },
+        ...(input.infrastructureRetryRuns.length > 0 ? {
+            infrastructure_retry_runs: input.infrastructureRetryRuns.map(({ attempt, retry, tasks, triggers, refs, run, leaseId, workId }) => ({
+                execution_kind: "physical-infrastructure-retry", ...(leaseId ? { lease_id: leaseId } : {}),
+                ...(workId ? { work_id: workId } : {}), attempt, retry, tasks, trigger_trials: triggers,
+                trials: refs, backend: run.backend, backend_summary: run.summary,
+            })),
+        } : {}),
+        candidate: input.candidate,
+        dataset: input.request.dataset,
+        benchmark_id: input.request.benchmark_id,
+        benchmark_revision: input.request.benchmark_revision,
+        generation: input.progress.generation,
+        trials: trialRefs,
+        summary: summarizeTrialRefs(trialRefs),
+        ...(input.schedulerSummary ? { scheduler_summary: input.schedulerSummary } : {}),
+        ...preparedArtifactPlanFields(input.preparedAssignments),
+        ...(input.localTransport ? { local_source_transport: transportSummary(input.localTransport) } : {}),
+        ...(succeeded ? {} : {
+            error: {
+                code: input.cancelled ? "cancelled" : localSourceFailure ? "local_source_materialize_failed"
+                    : !backendsSucceeded ? "harbor_failed" : infrastructureFailures.length > 0
+                        ? infrastructureErrorCode : "eval_has_invalid_tasks",
+                message: input.cancelled ? "eval was cancelled"
+                    : localSourceFailure ? "Harbor rejected the transported local Git source before candidate execution"
+                        : !backendsSucceeded ? `Harbor work items completed ${input.backendRuns.length}/${expectedBackendRuns}`
+                            : infrastructureFailures.length > 0
+                                ? `${infrastructureErrorCode === "eval_infrastructure_retries_exhausted" ? "infrastructure retries exhausted" : "verifier infrastructure failure"}: ${infrastructureFailureSlots.map(trial => `${trial.task_id}#${trial.attempt}`).join(", ")}`
+                                : `eval has invalid or missing trials: ${invalidTrials.map(trial => `${trial.task_id}#${trial.attempt}`).join(", ")}`,
+            },
+        }),
+        started_at: input.startedAt.toISOString(),
+        completed_at: new Date().toISOString(),
+    };
+}
+//# sourceMappingURL=eval-result-builder.js.map

package/dist/src/evals/eval-result-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-result-builder.js","sourceRoot":"","sources":["../../../src/evals/eval-result-builder.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,0BAA0B,EAAuC,MAAM,6BAA6B,CAAC;AAC9G,OAAO,EAAE,2BAA2B,EAA+B,MAAM,2BAA2B,CAAC;AACrG,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,yBAAyB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAWtG,MAAM,UAAU,wBAAwB,CAAC,KAgBxC;IACC,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxC,MAAM,kBAAkB,GAAG,KAAK,CAAC,cAAc;QAC7C,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;eAC1E,KAAK,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC9F,CAAC,CAAC,KAAK,CAAC;IACV,MAAM,mBAAmB,GAAG,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC3G,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC;WACxH,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,mBAAmB,CAAC,CAAC;IAChI,MAAM,aAAa,GAAG,KAAK,CAAC,YAAY,KAAK,IAAI;QAC/C,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACxI,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAClF,MAAM,sBAAsB,GAAG,2BAA2B,CAAC,SAAS,CAAC,CAAC;IACtE,MAAM,0BAA0B,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC/H,MAAM,wBAAwB,GAAG,KAAK,CAAC,OAAO,CAAC,sBAAsB,GAAG,CAAC;WACpE,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,cAAc,KAAK,iCAAiC,CAAC,CAAC;IACxG,MAAM,uBAAuB,GAAG,wBAAwB;WACnD,CAAC,KAAK,CAAC,OAAO,CAAC,sBAAsB,GAAG,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC;QACzF,CAAC,CAAC,uCAAuC;QACzC,CAAC,CAAC,kCAAkC,CAAC;IACvC,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,kBAAkB,IAAI,iBAAiB,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC;IAC7G,MAAM,aAAa,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7F,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QAC1E,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QACnD,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpG,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAC/B,kBAAkB,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;gBACvF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,CAAC,OAAO;aACvG,CAAC,CAAC;SACJ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;YAC/B,YAAY,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;SAC7H,CAAC,CAAC,CAAC,EAAE,CAAC;QACP,2BAA2B,EAAE;YAC3B,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,sBAAsB;YACjD,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,+BAA+B;YACzD,kBAAkB,EAAE,0BAA0B;YAC9C,mCAAmC,EAAE,KAAK;SAC3C;QACD,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAC7C,yBAAyB,EAAE,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;gBACjI,cAAc,EAAE,+BAA+B,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1F,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ;gBACvF,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,CAAC,OAAO;aACjE,CAAC,CAAC;SACJ,CAAC,CAAC,CAAC,EAAE,CAAC;QACP,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO;QAC9B,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;QACxC,kBAAkB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB;QACpD,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;QACrC,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,kBAAkB,CAAC,SAAS,CAAC;QACtC,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChF,GAAG,0BAA0B,CAAC,KAAK,CAAC,mBAAmB,CAAC;QACxD,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACnB,KAAK,EAAE;gBACL,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,iCAAiC;oBAC1F,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC;wBACxE,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,wBAAwB;gBACxD,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,oBAAoB;oBAC7C,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,6EAA6E;wBAClG,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,+BAA+B,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,mBAAmB,EAAE;4BACrG,CAAC,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC;gCACjC,CAAC,CAAC,GAAG,uBAAuB,KAAK,uCAAuC,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC,iCAAiC,KAAK,0BAA0B,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gCAC/O,CAAC,CAAC,uCAAuC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aAC9H;SACF,CAAC;QACF,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;QACzC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACvC,CAAC;AACJ,CAAC"}

package/dist/src/evals/events.js

@@ -1,6 +1,8 @@
 import { createWriteStream } from "node:fs";
+import { open, stat } from "node:fs/promises";
 import path from "node:path";
 import { SCHEMA_VERSION, ensureDir } from "../foundation/index.js";
+export const MAX_EVAL_EVENT_BYTES = 64 * 1024;
 export class EvalEventSink {
     path;
     evalId;
@@ -16,23 +18,33 @@ export class EvalEventSink {
     }
     async open() {
         await ensureDir(path.dirname(this.path));
+        this.sequence = await lastCommittedSequence(this.path);
         this.stream = createWriteStream(this.path, { flags: "a", mode: 0o600 });
         this.stream.on("error", (error) => { this.streamError ||= error; });
     }
     emit(event) {
-        const framed = {
+        const identity = {
             schema_version: SCHEMA_VERSION,
             sequence: ++this.sequence,
             timestamp: new Date().toISOString(),
             eval_id: this.evalId,
-            ...event,
         };
-        this.pending = this.pending.then(() => writeChunk(this.stream, `${JSON.stringify(framed)}\n`));
+        const framed = { ...identity, ...event };
+        const encoded = JSON.stringify(framed);
+        const persisted = Buffer.byteLength(encoded) <= MAX_EVAL_EVENT_BYTES
+            ? framed
+            : {
+                ...identity,
+                type: boundedType(event.type),
+                truncated: true,
+                original_bytes: Buffer.byteLength(encoded),
+            };
+        this.pending = this.pending.then(() => writeChunk(this.stream, `${JSON.stringify(persisted)}\n`));
         try {
-            this.onEvent(framed);
+            this.onEvent(persisted);
         }
         catch { /* Observers cannot break eval persistence. */ }
-        return framed;
+        return persisted;
     }
     async close() {
         let failure;
@@ -53,6 +65,42 @@ export class EvalEventSink {
             throw failure;
     }
 }
+function boundedType(value) {
+    const type = typeof value === "string" && value ? value : "eval.event.truncated";
+    return type.slice(0, 256);
+}
+async function lastCommittedSequence(file) {
+    let size;
+    try {
+        size = (await stat(file)).size;
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return 0;
+        throw error;
+    }
+    if (size === 0)
+        return 0;
+    const handle = await open(file, "r");
+    try {
+        const length = Math.min(size, 64 * 1024);
+        const buffer = Buffer.allocUnsafe(length);
+        const { bytesRead } = await handle.read(buffer, 0, length, size - length);
+        const committed = buffer.subarray(0, bytesRead).toString("utf8").replace(/[^\n]*$/, "");
+        const lines = committed.trimEnd().split("\n");
+        const last = lines.at(-1);
+        if (!last)
+            return 0;
+        const parsed = JSON.parse(last);
+        if (!Number.isSafeInteger(parsed.sequence) || parsed.sequence < 0) {
+            throw new TypeError("eval event sequence is invalid");
+        }
+        return parsed.sequence;
+    }
+    finally {
+        await handle.close();
+    }
+}
 function writeChunk(stream, chunk) {
     return new Promise((resolve, reject) => stream.write(chunk, (error) => error ? reject(error) : resolve()));
 }

package/dist/src/evals/events.js.map

@@ -1 +1 @@
-{"version":3,"file":"events.js","sourceRoot":"","sources":["../../../src/evals/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAEnE,MAAM,OAAO,aAAa;IACf,IAAI,CAAS;IACL,MAAM,CAAS;IACf,OAAO,CAA2C;IAC3D,QAAQ,GAAG,CAAC,CAAC;IACb,OAAO,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3C,MAAM,CAA0B;IAChC,WAAW,CAAoB;IAEvC,YAAY,aAAqB,EAAE,MAAc,EAAE,UAAoD,GAAG,EAAE,GAAE,CAAC;QAC7G,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,CAAC,KAA8B;QACjC,MAAM,MAAM,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,QAAQ,EAAE,EAAE,IAAI,CAAC,QAAQ;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,IAAI,CAAC,MAAM;YACpB,GAAG,KAAK;SACT,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAqB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9G,IAAI,CAAC;YAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,8CAA8C,CAAC,CAAC;QACtF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,OAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,MAAM,IAAI,CAAC,OAAO,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,GAAG,KAAc,CAAC;QAAC,CAAC;QACvE,IAAI,CAAC;YAAC,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,KAAK,KAAc,CAAC;QAAC,CAAC;QACrF,OAAO,KAAK,IAAI,CAAC,WAAW,CAAC;QAC7B,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC;IAC7B,CAAC;CACF;AAED,SAAS,UAAU,CAAC,MAAmB,EAAE,KAAa;IACpD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,WAAW,CAAC,MAA+B;IAClD,IAAI,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"events.js","sourceRoot":"","sources":["../../../src/evals/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAEnE,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,CAAC;AAE9C,MAAM,OAAO,aAAa;IACf,IAAI,CAAS;IACL,MAAM,CAAS;IACf,OAAO,CAA2C;IAC3D,QAAQ,GAAG,CAAC,CAAC;IACb,OAAO,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3C,MAAM,CAA0B;IAChC,WAAW,CAAoB;IAEvC,YAAY,aAAqB,EAAE,MAAc,EAAE,UAAoD,GAAG,EAAE,GAAE,CAAC;QAC7G,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,CAAC,KAA8B;QACjC,MAAM,QAAQ,GAAG;YACf,cAAc,EAAE,cAAc;YAC9B,QAAQ,EAAE,EAAE,IAAI,CAAC,QAAQ;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,IAAI,CAAC,MAAM;SACrB,CAAC;QACF,MAAM,MAAM,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,oBAAoB;YAClE,CAAC,CAAC,MAAM;YACR,CAAC,CAAC;gBACE,GAAG,QAAQ;gBACX,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC;gBAC7B,SAAS,EAAE,IAAI;gBACf,cAAc,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;aAC3C,CAAC;QACN,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAqB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACjH,IAAI,CAAC;YAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,8CAA8C,CAAC,CAAC;QACzF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,OAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,MAAM,IAAI,CAAC,OAAO,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,GAAG,KAAc,CAAC;QAAC,CAAC;QACvE,IAAI,CAAC;YAAC,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,KAAK,KAAc,CAAC;QAAC,CAAC;QACrF,OAAO,KAAK,IAAI,CAAC,WAAW,CAAC;QAC7B,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC;IAC7B,CAAC;CACF;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjF,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,IAAY;IAC/C,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,CAAC,CAAC;QACjE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,GAAG,MAAM,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACxF,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA2B,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAK,MAAM,CAAC,QAAmB,GAAG,CAAC,EAAE,CAAC;YAC9E,MAAM,IAAI,SAAS,CAAC,gCAAgC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,MAAM,CAAC,QAAkB,CAAC;IACnC,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAAmB,EAAE,KAAa;IACpD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,WAAW,CAAC,MAA+B;IAClD,IAAI,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/evals/evolution-baseline.js

@@ -0,0 +1,191 @@
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { atomicWriteJSON, ensureDir, readJSON, sha256JSON, withFileLock } from "../foundation/index.js";
+import { HitchError } from "../foundation/index.js";
+const localFlights = new Map();
+/** Persisted lineage singleflight. Only the caller that creates `submitting`
+ * may invoke the materializer; later rounds validate and return `ready`. */
+export async function ensureEvolutionBaseline(options) {
+    validateInputs(options);
+    const file = baselinePath(options.root, options.evolutionId);
+    const local = localFlights.get(file);
+    if (local)
+        return local;
+    const operation = ensureInternal(options, file);
+    localFlights.set(file, operation);
+    try {
+        return await operation;
+    }
+    finally {
+        if (localFlights.get(file) === operation)
+            localFlights.delete(file);
+    }
+}
+export function evolutionBaselineFingerprint(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length === 0) {
+        throw new TypeError("evolution baseline fingerprint input is invalid");
+    }
+    return sha256JSON(value);
+}
+export async function readEvolutionBaseline(root, evolutionId) {
+    const value = await readJSON(baselinePath(root, evolutionId), null);
+    return value === null ? null : parseEvolutionBaseline(value, evolutionId);
+}
+export function parseEvolutionBaseline(value, expectedEvolutionId) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("evolution baseline must be an object");
+    const record = value;
+    const fields = ["schema_version", "evolution_id", "baseline_fingerprint", "state", "seed_eval_id", "heldout_eval_id", "seed_result_digest", "heldout_result_digest", "created_at", "updated_at"];
+    if (Object.keys(record).some((key) => !fields.includes(key)) || record.schema_version !== "1"
+        || typeof record.evolution_id !== "string" || !record.evolution_id
+        || expectedEvolutionId !== undefined && record.evolution_id !== expectedEvolutionId
+        || typeof record.baseline_fingerprint !== "string" || !/^sha256:[a-f0-9]{64}$/.test(record.baseline_fingerprint)
+        || !["submitting", "running", "ready", "failed"].includes(record.state)
+        || typeof record.created_at !== "string" || !Number.isFinite(Date.parse(record.created_at))
+        || typeof record.updated_at !== "string" || !Number.isFinite(Date.parse(record.updated_at)))
+        throw new TypeError("evolution baseline identity is invalid");
+    const optional = ["seed_eval_id", "heldout_eval_id"];
+    for (const name of optional)
+        if (record[name] !== undefined && (typeof record[name] !== "string" || !/^eval_[a-f0-9]{32}$/.test(record[name]))) {
+            throw new TypeError(`evolution baseline ${name} is invalid`);
+        }
+    for (const name of ["seed_result_digest", "heldout_result_digest"])
+        if (record[name] !== undefined
+            && (typeof record[name] !== "string" || !/^sha256:[a-f0-9]{64}$/.test(record[name])))
+            throw new TypeError(`evolution baseline ${name} is invalid`);
+    if (record.state === "running" && (!record.seed_eval_id || !record.heldout_eval_id))
+        throw new TypeError("running evolution baseline eval identities are missing");
+    if (record.state === "ready" && (!record.seed_eval_id || !record.heldout_eval_id || !record.seed_result_digest || !record.heldout_result_digest)) {
+        throw new TypeError("ready evolution baseline references are incomplete");
+    }
+    return record;
+}
+async function ensureInternal(options, file) {
+    const ownership = await claim(options, file);
+    if (!ownership.owner)
+        return waitForOwner(options, file, ownership.baseline);
+    try {
+        const materialized = await options.materialize({
+            markRunning: (seedEvalId, heldoutEvalId) => updateOwned(options, file, (current) => ({
+                ...assertRunningIdentity(current, seedEvalId, heldoutEvalId), state: "running",
+            })).then(() => undefined),
+        });
+        const ready = await updateOwned(options, file, (current) => {
+            if (current.seed_eval_id && current.seed_eval_id !== materialized.seed_eval_id
+                || current.heldout_eval_id && current.heldout_eval_id !== materialized.heldout_eval_id) {
+                throw new TypeError("evolution baseline eval identity changed during materialization");
+            }
+            return {
+                ...current, state: "ready", seed_eval_id: checkedEvalId(materialized.seed_eval_id), heldout_eval_id: checkedEvalId(materialized.heldout_eval_id),
+                seed_result_digest: checkedDigest(materialized.seed_result_digest), heldout_result_digest: checkedDigest(materialized.heldout_result_digest),
+            };
+        });
+        await options.validateReady?.(ready);
+        return ready;
+    }
+    catch (error) {
+        await updateOwned(options, file, (current) => ({ ...current, state: "failed" })).catch(() => undefined);
+        throw error;
+    }
+}
+async function claim(options, file) {
+    await ensureDir(path.dirname(file));
+    return withBaselineLock(options, async () => {
+        const existing = await readEvolutionBaseline(options.root, options.evolutionId);
+        if (existing) {
+            assertFingerprint(existing, options.fingerprint);
+            if (existing.state === "failed")
+                throw baselineError("evolution baseline materialization previously failed", "evolution_baseline_failed");
+            if (existing.state === "ready")
+                await options.validateReady?.(existing);
+            return { owner: false, baseline: existing };
+        }
+        const now = new Date().toISOString();
+        const baseline = parseEvolutionBaseline({
+            schema_version: "1", evolution_id: options.evolutionId, baseline_fingerprint: options.fingerprint,
+            state: "submitting", created_at: now, updated_at: now,
+        }, options.evolutionId);
+        await atomicWriteJSON(file, baseline);
+        return { owner: true, baseline };
+    });
+}
+async function waitForOwner(options, file, initial) {
+    if (initial.state === "ready")
+        return initial;
+    const deadline = Date.now() + (options.maxWaitMs ?? 24 * 60 * 60 * 1_000);
+    const poll = options.pollIntervalMs ?? 250;
+    for (;;) {
+        if (options.signal?.aborted)
+            throw options.signal.reason;
+        if (Date.now() >= deadline)
+            throw baselineError("evolution baseline materialization is still in progress", "evolution_baseline_in_progress");
+        await delay(Math.min(poll, Math.max(1, deadline - Date.now())), undefined, options.signal ? { signal: options.signal } : undefined);
+        const current = await readEvolutionBaseline(options.root, options.evolutionId);
+        if (!current)
+            throw baselineError("evolution baseline record disappeared", "evolution_baseline_state_conflict");
+        assertFingerprint(current, options.fingerprint);
+        if (current.state === "failed")
+            throw baselineError("evolution baseline materialization failed", "evolution_baseline_failed");
+        if (current.state === "ready") {
+            await options.validateReady?.(current);
+            return current;
+        }
+    }
+}
+async function updateOwned(options, file, update) {
+    return withBaselineLock(options, async () => {
+        const current = await readEvolutionBaseline(options.root, options.evolutionId);
+        if (!current)
+            throw baselineError("evolution baseline ownership was lost", "evolution_baseline_state_conflict");
+        assertFingerprint(current, options.fingerprint);
+        if (current.state === "ready" || current.state === "failed")
+            throw baselineError("evolution baseline is already terminal", "evolution_baseline_state_conflict");
+        const next = parseEvolutionBaseline({ ...update(current), updated_at: new Date().toISOString() }, options.evolutionId);
+        await atomicWriteJSON(file, next);
+        return next;
+    });
+}
+function withBaselineLock(options, operation) {
+    const key = sha256JSON({ evolution_id: options.evolutionId }).slice("sha256:".length);
+    return withFileLock(path.join(options.root, "locks", "evolution-baseline"), key, operation, {
+        timeoutCode: "evolution_baseline_locked", timeoutExitCode: 12,
+    });
+}
+function validateInputs(options) {
+    if (!options.root || !options.evolutionId || !/^sha256:[a-f0-9]{64}$/.test(options.fingerprint))
+        throw new TypeError("evolution baseline request is invalid");
+    if (options.pollIntervalMs !== undefined && (!Number.isSafeInteger(options.pollIntervalMs) || options.pollIntervalMs < 1))
+        throw new TypeError("evolution baseline poll interval is invalid");
+    if (options.maxWaitMs !== undefined && (!Number.isSafeInteger(options.maxWaitMs) || options.maxWaitMs < 1))
+        throw new TypeError("evolution baseline wait is invalid");
+}
+function baselinePath(root, evolutionId) {
+    const id = sha256JSON({ evolution_id: evolutionId }).slice("sha256:".length);
+    return path.join(root, "evolutions", id, "baseline.json");
+}
+function assertFingerprint(record, fingerprint) {
+    if (record.baseline_fingerprint !== fingerprint)
+        throw baselineError("evolution baseline fingerprint changed", "evolution_baseline_fingerprint_mismatch");
+}
+function checkedEvalId(value) {
+    if (!/^eval_[a-f0-9]{32}$/.test(value))
+        throw new TypeError("evolution baseline eval id is invalid");
+    return value;
+}
+function checkedDigest(value) {
+    if (!/^sha256:[a-f0-9]{64}$/.test(value))
+        throw new TypeError("evolution baseline result digest is invalid");
+    return value;
+}
+function assertRunningIdentity(current, seedEvalId, heldoutEvalId) {
+    const seed = checkedEvalId(seedEvalId);
+    const heldout = checkedEvalId(heldoutEvalId);
+    if (current.seed_eval_id && current.seed_eval_id !== seed || current.heldout_eval_id && current.heldout_eval_id !== heldout) {
+        throw new TypeError("evolution baseline eval identity changed while running");
+    }
+    return { ...current, seed_eval_id: seed, heldout_eval_id: heldout };
+}
+function baselineError(message, code) {
+    return new HitchError(message, { code, exitCode: 12 });
+}
+//# sourceMappingURL=evolution-baseline.js.map

package/dist/src/evals/evolution-baseline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evolution-baseline.js","sourceRoot":"","sources":["../../../src/evals/evolution-baseline.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACxG,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAqCpD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAwC,CAAC;AAErE;4EAC4E;AAC5E,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAuC;IACnF,cAAc,CAAC,OAAO,CAAC,CAAC;IACxB,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACxB,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAChD,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAClC,IAAI,CAAC;QAAC,OAAO,MAAM,SAAS,CAAC;IAAC,CAAC;YAAS,CAAC;QAAC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,SAAS;YAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAAC,CAAC;AAClH,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,KAA8B;IACzE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnG,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAY,EAAE,WAAmB;IAC3E,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,CAAC;IACpF,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,sBAAsB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAc,EAAE,mBAA4B;IACjF,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IAC7H,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,MAAM,GAAG,CAAC,gBAAgB,EAAE,cAAc,EAAE,sBAAsB,EAAE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IACjM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG;WACxF,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,YAAY;WAC/D,mBAAmB,KAAK,SAAS,IAAI,MAAM,CAAC,YAAY,KAAK,mBAAmB;WAChF,OAAO,MAAM,CAAC,oBAAoB,KAAK,QAAQ,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;WAC7G,CAAC,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAe,CAAC;WAC9E,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;WACxF,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,wCAAwC,CAAC,CAAC;IAC7J,MAAM,QAAQ,GAAG,CAAC,cAAc,EAAE,iBAAiB,CAAU,CAAC;IAC9D,KAAK,MAAM,IAAI,IAAI,QAAQ;QAAE,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAW,CAAC,CAAC,EAAE,CAAC;YACzJ,MAAM,IAAI,SAAS,CAAC,sBAAsB,IAAI,aAAa,CAAC,CAAC;QAC/D,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,CAAC,oBAAoB,EAAE,uBAAuB,CAAU;QAAE,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS;eACtG,CAAC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAW,CAAC,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,sBAAsB,IAAI,aAAa,CAAC,CAAC;IAC/J,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,wDAAwD,CAAC,CAAC;IACnK,IAAI,MAAM,CAAC,KAAK,KAAK,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,EAAE,CAAC;QACjJ,MAAM,IAAI,SAAS,CAAC,oDAAoD,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,MAAwC,CAAC;AAClD,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,OAAuC,EAAE,IAAY;IACjF,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,SAAS,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC7E,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC;YAC7C,WAAW,EAAE,CAAC,UAAU,EAAE,aAAa,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACnF,GAAG,qBAAqB,CAAC,OAAO,EAAE,UAAU,EAAE,aAAa,CAAC,EAAE,KAAK,EAAE,SAAS;aAC/E,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;SAC1B,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE;YACzD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,KAAK,YAAY,CAAC,YAAY;mBACzE,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe,KAAK,YAAY,CAAC,eAAe,EAAE,CAAC;gBACzF,MAAM,IAAI,SAAS,CAAC,iEAAiE,CAAC,CAAC;YACzF,CAAC;YACD,OAAO;gBACL,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,aAAa,CAAC,YAAY,CAAC,YAAY,CAAC,EAAE,eAAe,EAAE,aAAa,CAAC,YAAY,CAAC,eAAe,CAAC;gBAChJ,kBAAkB,EAAE,aAAa,CAAC,YAAY,CAAC,kBAAkB,CAAC,EAAE,qBAAqB,EAAE,aAAa,CAAC,YAAY,CAAC,qBAAqB,CAAC;aAC7I,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC;QACrC,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxG,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,OAAuC,EAAE,IAAY;IACxE,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,OAAO,gBAAgB,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;QAC1C,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAChF,IAAI,QAAQ,EAAE,CAAC;YACb,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;YACjD,IAAI,QAAQ,CAAC,KAAK,KAAK,QAAQ;gBAAE,MAAM,aAAa,CAAC,sDAAsD,EAAE,2BAA2B,CAAC,CAAC;YAC1I,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO;gBAAE,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,CAAC;YACxE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAC9C,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,sBAAsB,CAAC;YACtC,cAAc,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,WAAW,EAAE,oBAAoB,EAAE,OAAO,CAAC,WAAW;YACjG,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG;SACtD,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACxB,MAAM,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,OAAuC,EAAE,IAAY,EAAE,OAA4B;IAC7G,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,OAAO,CAAC,cAAc,IAAI,GAAG,CAAC;IAC3C,SAAS,CAAC;QACR,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QACzD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ;YAAE,MAAM,aAAa,CAAC,yDAAyD,EAAE,gCAAgC,CAAC,CAAC;QAC7I,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACpI,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC/E,IAAI,CAAC,OAAO;YAAE,MAAM,aAAa,CAAC,uCAAuC,EAAE,mCAAmC,CAAC,CAAC;QAChH,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAChD,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ;YAAE,MAAM,aAAa,CAAC,2CAA2C,EAAE,2BAA2B,CAAC,CAAC;QAC9H,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,CAAC;YACvC,OAAO,OAAO,CAAC;QACjB,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,OAAuC,EACvC,IAAY,EACZ,MAA6D;IAE7D,OAAO,gBAAgB,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;QAC1C,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC/E,IAAI,CAAC,OAAO;YAAE,MAAM,aAAa,CAAC,uCAAuC,EAAE,mCAAmC,CAAC,CAAC;QAChH,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAChD,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ;YAAE,MAAM,aAAa,CAAC,wCAAwC,EAAE,mCAAmC,CAAC,CAAC;QAChK,MAAM,IAAI,GAAG,sBAAsB,CAAC,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACvH,MAAM,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAI,OAAuC,EAAE,SAA2B;IAC/F,MAAM,GAAG,GAAG,UAAU,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACtF,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,oBAAoB,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE;QAC1F,WAAW,EAAE,2BAA2B,EAAE,eAAe,EAAE,EAAE;KAC9D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,OAAuC;IAC7D,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IAC9J,IAAI,OAAO,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IAC9L,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,oCAAoC,CAAC,CAAC;AACxK,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,WAAmB;IACrD,MAAM,EAAE,GAAG,UAAU,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7E,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,eAAe,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,iBAAiB,CAAC,MAA2B,EAAE,WAAmB;IACzE,IAAI,MAAM,CAAC,oBAAoB,KAAK,WAAW;QAAE,MAAM,aAAa,CAAC,wCAAwC,EAAE,yCAAyC,CAAC,CAAC;AAC5J,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IACrG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IAC7G,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,qBAAqB,CAAC,OAA4B,EAAE,UAAkB,EAAE,aAAqB;IACpG,MAAM,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,aAAa,CAAC,aAAa,CAAC,CAAC;IAC7C,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe,KAAK,OAAO,EAAE,CAAC;QAC5H,MAAM,IAAI,SAAS,CAAC,wDAAwD,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,GAAG,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;AACtE,CAAC;AAED,SAAS,aAAa,CAAC,OAAe,EAAE,IAAY;IAClD,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;AACzD,CAAC"}

package/dist/src/evals/execution-evidence.js

@@ -0,0 +1,116 @@
+const RESOURCE_FIELDS = ["cpu_millis", "memory_bytes", "container_slots", "build_slots"];
+export function parseExecutionEvidence(value) {
+    const record = exact(value, [
+        "schema_version", "provider", "worker_id", "collision_domain_id", "eval_id", "work_id", "lease_id", "lease_epoch",
+        "task_id", "reservation", "enforced", "observed",
+    ], "execution evidence");
+    if (record.schema_version !== "1" || !validText(record.provider)
+        || !validText(record.worker_id) || !validText(record.collision_domain_id)
+        || typeof record.eval_id !== "string" || !/^eval_[a-f0-9]{32}$/.test(record.eval_id)
+        || typeof record.work_id !== "string" || !/^work_[a-f0-9]{32}$/.test(record.work_id)
+        || typeof record.lease_id !== "string" || !/^lease_[a-f0-9]{32}$/.test(record.lease_id)
+        || !Number.isSafeInteger(record.lease_epoch) || record.lease_epoch < 1 || !validText(record.task_id)) {
+        throw new TypeError("execution evidence identity is invalid");
+    }
+    const reservation = resources(record.reservation, "execution evidence reservation");
+    const enforcedRecord = exact(record.enforced, ["main_limits", "sidecar_limits"], "execution evidence enforced limits");
+    const mainLimits = resources(enforcedRecord.main_limits, "execution evidence main limits");
+    const sidecarLimits = sidecars(enforcedRecord.sidecar_limits);
+    const observedRecord = exact(record.observed, [
+        "status", "started_at", "collected_at", "sample_count", "containers", "unavailable_fields", "issues",
+    ], "execution evidence observation");
+    if ((observedRecord.status !== "partial" && observedRecord.status !== "unavailable")
+        || !timestamp(observedRecord.started_at) || !timestamp(observedRecord.collected_at)
+        || Date.parse(observedRecord.collected_at) < Date.parse(observedRecord.started_at)
+        || !Number.isSafeInteger(observedRecord.sample_count) || observedRecord.sample_count < 0
+        || !Array.isArray(observedRecord.containers) || observedRecord.containers.length > 256
+        || !Array.isArray(observedRecord.unavailable_fields)
+        || observedRecord.unavailable_fields.some((entry) => !["cpu_time_ns", "peak_memory_bytes", "exit_status", "image_identity"].includes(entry))
+        || new Set(observedRecord.unavailable_fields).size !== observedRecord.unavailable_fields.length
+        || !Array.isArray(observedRecord.issues) || observedRecord.issues.length > 32
+        || observedRecord.issues.some((entry) => !validText(entry, 512)))
+        throw new TypeError("execution evidence observation is invalid");
+    const containers = observedRecord.containers.map((entry, index) => container(entry, index));
+    if (new Set(containers.map((entry) => entry.container_id)).size !== containers.length
+        || (observedRecord.status === "unavailable" && containers.length !== 0))
+        throw new TypeError("execution evidence containers are invalid");
+    return {
+        schema_version: "1",
+        provider: record.provider,
+        worker_id: record.worker_id,
+        collision_domain_id: record.collision_domain_id,
+        eval_id: record.eval_id,
+        work_id: record.work_id,
+        lease_id: record.lease_id,
+        lease_epoch: record.lease_epoch,
+        task_id: record.task_id,
+        reservation,
+        enforced: { main_limits: mainLimits, sidecar_limits: sidecarLimits },
+        observed: {
+            status: observedRecord.status,
+            started_at: observedRecord.started_at,
+            collected_at: observedRecord.collected_at,
+            sample_count: observedRecord.sample_count,
+            containers,
+            unavailable_fields: [...observedRecord.unavailable_fields],
+            issues: [...observedRecord.issues],
+        },
+    };
+}
+function container(value, index) {
+    const record = exact(value, ["container_id", "first_observed_at", "last_observed_at"], `execution evidence container ${index}`, [
+        "name", "image_reference", "image_config_digest", "peak_memory_bytes", "cpu_time_ns", "oom_killed", "exit_code", "exit_reason",
+    ]);
+    if (typeof record.container_id !== "string" || !/^[a-f0-9]{12,64}$/.test(record.container_id)
+        || (record.name !== undefined && !validText(record.name, 256)) || !timestamp(record.first_observed_at) || !timestamp(record.last_observed_at)
+        || (record.image_reference !== undefined && !validText(record.image_reference, 1_024))
+        || (record.image_config_digest !== undefined && (typeof record.image_config_digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(record.image_config_digest)))
+        || Date.parse(record.last_observed_at) < Date.parse(record.first_observed_at)
+        || (record.peak_memory_bytes !== undefined && (!Number.isSafeInteger(record.peak_memory_bytes) || record.peak_memory_bytes < 0))
+        || (record.cpu_time_ns !== undefined && (!Number.isSafeInteger(record.cpu_time_ns) || record.cpu_time_ns < 0))
+        || (record.oom_killed !== undefined && typeof record.oom_killed !== "boolean")
+        || (record.exit_code !== undefined && (!Number.isSafeInteger(record.exit_code) || record.exit_code < 0 || record.exit_code > 255))
+        || (record.exit_reason !== undefined && !validText(record.exit_reason, 512)))
+        throw new TypeError(`execution evidence container ${index} is invalid`);
+    return record;
+}
+function sidecars(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("execution evidence sidecar limits are invalid");
+    const result = {};
+    for (const [name, entry] of Object.entries(value).sort(([left], [right]) => left.localeCompare(right))) {
+        const limit = exact(entry, ["cpu_millis", "memory_bytes"], `execution evidence sidecar ${name}`, ["gpu_count"]);
+        if (!validText(name, 255) || !Number.isSafeInteger(limit.cpu_millis) || limit.cpu_millis < 1
+            || !Number.isSafeInteger(limit.memory_bytes) || limit.memory_bytes < 1
+            || (limit.gpu_count !== undefined && (!Number.isSafeInteger(limit.gpu_count) || limit.gpu_count < 1)))
+            throw new TypeError("execution evidence sidecar limits are invalid");
+        result[name] = { cpu_millis: limit.cpu_millis, memory_bytes: limit.memory_bytes, ...(limit.gpu_count === undefined ? {} : { gpu_count: limit.gpu_count }) };
+    }
+    return result;
+}
+function resources(value, label) {
+    const record = exact(value, RESOURCE_FIELDS, label, ["gpu_count", "ephemeral_disk_bytes"]);
+    if (RESOURCE_FIELDS.some((name) => !Number.isSafeInteger(record[name]) || record[name] < 0))
+        throw new TypeError(`${label} is invalid`);
+    if (record.gpu_count !== undefined && (!Number.isSafeInteger(record.gpu_count) || record.gpu_count < 0))
+        throw new TypeError(`${label} is invalid`);
+    if (record.ephemeral_disk_bytes !== undefined && (!Number.isSafeInteger(record.ephemeral_disk_bytes) || record.ephemeral_disk_bytes < 0))
+        throw new TypeError(`${label} is invalid`);
+    return record;
+}
+function exact(value, required, label, optional = []) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError(`${label} must be an object`);
+    const record = value;
+    const allowed = new Set([...required, ...optional]);
+    if (required.some((key) => !(key in record)) || Object.keys(record).some((key) => !allowed.has(key)))
+        throw new TypeError(`${label} fields are invalid`);
+    return record;
+}
+function timestamp(value) {
+    return typeof value === "string" && Number.isFinite(Date.parse(value));
+}
+function validText(value, maximum = 4_096) {
+    return typeof value === "string" && value.length > 0 && value.length <= maximum && !/[\0\r\n]/.test(value);
+}
+//# sourceMappingURL=execution-evidence.js.map