agent-hitch 0.2.7 → 0.2.8

package/dist/src/evals/duration-estimator.js

@@ -0,0 +1,126 @@
+import path from "node:path";
+import { atomicWriteJSON, ensureDir, readJSON, sha256JSON, withFileLock } from "../foundation/index.js";
+import { benchmarkTaskDigest } from "../runs/index.js";
+const MAX_SAMPLES = 20;
+const DEFAULT_ESTIMATED_DURATION_MS = 15 * 60 * 1_000;
+const MAX_DURATION_MS = 7 * 24 * 60 * 60 * 1_000;
+export async function planTaskSchedulingHints(input) {
+    const hints = [];
+    for (const taskId of [...input.taskIds].sort(compareBytes)) {
+        const identity = durationIdentity(input, taskId);
+        const stats = await readStats(statsPath(input.root, identity.key), identity).catch(() => null);
+        const baseline = input.evolutionBaselineDurations?.[taskId];
+        if (baseline !== undefined && (!Number.isSafeInteger(baseline) || baseline < 1 || baseline > MAX_DURATION_MS)) {
+            throw new TypeError(`evolution baseline duration is invalid: ${taskId}`);
+        }
+        const taskBudget = await readTaskBudget(input.dataset, taskId);
+        const fallback = taskBudget ?? (input.requestTimeoutMs > 0 ? input.requestTimeoutMs : DEFAULT_ESTIMATED_DURATION_MS);
+        const estimated = baseline ?? (stats && stats.samples.length > 0
+            ? percentile(stats.samples.map((sample) => sample.duration_ms), 0.75)
+            : fallback);
+        const failures = stats?.samples.filter((sample) => sample.retryable_infrastructure_failure).length ?? 0;
+        const retryRate = stats && stats.samples.length > 0 ? failures / stats.samples.length : 0;
+        const expectedRetries = Math.min(input.infrastructureRetries, 1) * retryRate;
+        hints.push([taskId, {
+                policy: "critical-path-lpt-v1",
+                estimated_duration_ms: estimated,
+                remaining_path_ms: Math.max(estimated, Math.round(estimated * (1 + expectedRetries))),
+                estimate_source: baseline !== undefined ? "evolution-baseline" : stats && stats.samples.length > 0 ? "history-p75" : taskBudget !== null || input.requestTimeoutMs > 0 ? "task-budget" : "default",
+                estimate_sample_count: baseline !== undefined ? 1 : stats?.samples.length ?? 0,
+            }]);
+    }
+    return Object.fromEntries(hints);
+}
+export async function recordTaskDuration(input) {
+    if (!Number.isSafeInteger(input.durationMs) || input.durationMs < 1 || input.durationMs > MAX_DURATION_MS) {
+        throw new TypeError("task duration sample is invalid");
+    }
+    const identity = durationIdentity(input, input.taskId);
+    const directory = await ensureDir(path.dirname(statsPath(input.root, identity.key)));
+    const locks = await ensureDir(path.join(input.root, "locks", "scheduler-duration-stats"));
+    await withFileLock(locks, identity.key, async () => {
+        const current = await readStats(path.join(directory, `${identity.key.slice("sha256:".length)}.json`), identity).catch(() => null);
+        const samples = [...(current?.samples ?? []), {
+                duration_ms: input.durationMs,
+                retryable_infrastructure_failure: input.retryableInfrastructureFailure,
+            }].slice(-MAX_SAMPLES);
+        await atomicWriteJSON(statsPath(input.root, identity.key), {
+            schema_version: "1",
+            ...identity,
+            samples,
+            updated_at: new Date().toISOString(),
+        });
+    }, { timeoutCode: "scheduler_duration_stats_locked", timeoutExitCode: 12 });
+}
+export function schedulingHintsFromPlan(plan) {
+    const hints = new Map();
+    for (const item of plan.work_items) {
+        const taskId = item.task_ids[0];
+        if (!taskId || !item.scheduling)
+            return undefined;
+        const existing = hints.get(taskId);
+        if (existing && JSON.stringify(existing) !== JSON.stringify(item.scheduling))
+            throw new TypeError(`execution plan task scheduling hint changed: ${taskId}`);
+        hints.set(taskId, item.scheduling);
+    }
+    return Object.fromEntries([...hints.entries()].sort((left, right) => compareBytes(left[0], right[0])));
+}
+function durationIdentity(input, taskId) {
+    const taskDigest = benchmarkTaskDigest(input.benchmarkId, input.benchmarkRevision, taskId);
+    const identity = {
+        benchmark_id: input.benchmarkId,
+        benchmark_revision: input.benchmarkRevision,
+        task_id: taskId,
+        task_digest: taskDigest,
+        provider: input.provider,
+        model: input.model,
+    };
+    return { key: sha256JSON(identity), ...identity };
+}
+function statsPath(root, key) {
+    return path.join(root, "indexes", "scheduler-duration-stats", `${key.slice("sha256:".length)}.json`);
+}
+async function readStats(file, identity) {
+    const value = await readJSON(file, null);
+    if (value === null)
+        return null;
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("task duration stats must be an object");
+    const record = value;
+    const allowed = ["schema_version", "key", "benchmark_id", "benchmark_revision", "task_id", "task_digest", "provider", "model", "samples", "updated_at"];
+    if (Object.keys(record).some((key) => !allowed.includes(key)) || record.schema_version !== "1"
+        || record.key !== identity.key || record.benchmark_id !== identity.benchmark_id || record.benchmark_revision !== identity.benchmark_revision
+        || record.task_id !== identity.task_id || record.task_digest !== identity.task_digest || record.provider !== identity.provider || record.model !== identity.model
+        || !Array.isArray(record.samples) || record.samples.length > MAX_SAMPLES
+        || typeof record.updated_at !== "string" || !Number.isFinite(Date.parse(record.updated_at))) {
+        throw new TypeError("task duration stats identity is invalid");
+    }
+    const samples = record.samples.map((value) => {
+        if (!value || typeof value !== "object" || Array.isArray(value))
+            throw new TypeError("task duration sample is invalid");
+        const sample = value;
+        if (Object.keys(sample).some((key) => key !== "duration_ms" && key !== "retryable_infrastructure_failure")
+            || !Number.isSafeInteger(sample.duration_ms) || sample.duration_ms < 1 || sample.duration_ms > MAX_DURATION_MS
+            || typeof sample.retryable_infrastructure_failure !== "boolean")
+            throw new TypeError("task duration sample is invalid");
+        return { duration_ms: sample.duration_ms, retryable_infrastructure_failure: sample.retryable_infrastructure_failure };
+    });
+    return { schema_version: "1", ...identity, samples, updated_at: record.updated_at };
+}
+async function readTaskBudget(dataset, taskId) {
+    for (const file of [path.join(dataset, taskId, ".hitch-benchmark.json"), path.join(dataset, ".hitch-benchmark.json")]) {
+        const descriptor = await readJSON(file, null).catch(() => null);
+        const seconds = descriptor?.agent_timeout_sec;
+        if (typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0)
+            return Math.max(1, Math.round(seconds * 1_000));
+    }
+    return null;
+}
+function percentile(values, ratio) {
+    const sorted = [...values].sort((left, right) => left - right);
+    return sorted[Math.max(0, Math.ceil(sorted.length * ratio) - 1)];
+}
+function compareBytes(left, right) {
+    return Buffer.compare(Buffer.from(left), Buffer.from(right));
+}
+//# sourceMappingURL=duration-estimator.js.map

package/dist/src/evals/duration-estimator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"duration-estimator.js","sourceRoot":"","sources":["../../../src/evals/duration-estimator.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACxG,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACtD,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAoBjD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAW7C;IACC,MAAM,KAAK,GAAsC,EAAE,CAAC;IACpD,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAC3D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC/F,MAAM,QAAQ,GAAG,KAAK,CAAC,0BAA0B,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,eAAe,CAAC,EAAE,CAAC;YAC9G,MAAM,IAAI,SAAS,CAAC,2CAA2C,MAAM,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,UAAU,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC;QACrH,MAAM,SAAS,GAAG,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;YAC9D,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,CAAC;YACrE,CAAC,CAAC,QAAQ,CAAC,CAAC;QACd,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,gCAAgC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;QACxG,MAAM,SAAS,GAAG,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1F,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC;QAC7E,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE;gBAClB,MAAM,EAAE,sBAAsB;gBAC9B,qBAAqB,EAAE,SAAS;gBAChC,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC;gBACrF,eAAe,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,KAAK,IAAI,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;gBAClM,qBAAqB,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC;aAC/E,CAAC,CAAC,CAAC;IACN,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KASxC;IACC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,eAAe,EAAE,CAAC;QAC1G,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,0BAA0B,CAAC,CAAC,CAAC;IAC1F,MAAM,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAClI,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE;gBAC5C,WAAW,EAAE,KAAK,CAAC,UAAU;gBAC7B,gCAAgC,EAAE,KAAK,CAAC,8BAA8B;aACvE,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC;QACvB,MAAM,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;YACzD,cAAc,EAAE,GAAG;YACnB,GAAG,QAAQ;YACX,OAAO;YACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACP,CAAC,CAAC;IACnC,CAAC,EAAE,EAAE,WAAW,EAAE,iCAAiC,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAyB;IAC/D,MAAM,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAClD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO,SAAS,CAAC;QAClD,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,gDAAgD,MAAM,EAAE,CAAC,CAAC;QAC5J,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzG,CAAC;AAED,SAAS,gBAAgB,CAAC,KAKzB,EAAE,MAAc;IACf,MAAM,UAAU,GAAG,mBAAmB,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IAC3F,MAAM,QAAQ,GAAG;QACf,YAAY,EAAE,KAAK,CAAC,WAAW;QAC/B,kBAAkB,EAAE,KAAK,CAAC,iBAAiB;QAC3C,OAAO,EAAE,MAAM;QACf,WAAW,EAAE,UAAU;QACvB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC;IACF,OAAO,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,GAAW;IAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,0BAA0B,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AACvG,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,QAA6C;IAClF,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,EAAE,IAAI,CAAC,CAAC;IACzD,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IAC9H,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,OAAO,GAAG,CAAC,gBAAgB,EAAE,KAAK,EAAE,cAAc,EAAE,oBAAoB,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IACxJ,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG;WACzF,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,GAAG,IAAI,MAAM,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY,IAAI,MAAM,CAAC,kBAAkB,KAAK,QAAQ,CAAC,kBAAkB;WACzI,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;WAC9J,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,WAAW;WACrE,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,yCAAyC,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;QACxH,MAAM,MAAM,GAAG,KAAgC,CAAC;QAChD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,kCAAkC,CAAC;eACrG,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,IAAK,MAAM,CAAC,WAAsB,GAAG,CAAC,IAAK,MAAM,CAAC,WAAsB,GAAG,eAAe;eACnI,OAAO,MAAM,CAAC,gCAAgC,KAAK,SAAS;YAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;QAC1H,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAqB,EAAE,gCAAgC,EAAE,MAAM,CAAC,gCAA2C,EAAE,CAAC;IAC7I,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;AACtF,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,OAAe,EAAE,MAAc;IAC3D,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,uBAAuB,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAC,EAAE,CAAC;QACtH,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAiC,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAChG,MAAM,OAAO,GAAG,UAAU,EAAE,iBAAiB,CAAC;QAC9C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC;IAC9H,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB,EAAE,KAAa;IACjD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;IAC/D,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAW,CAAC;AAC7E,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa;IAC/C,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC"}

package/dist/src/evals/eval-lifecycle-events.js

@@ -12,5 +12,18 @@ export function emitEvalPlanLifecycle(sink, plan, planningStartedAt) {
             task_ids: workItem.task_ids,
             reservation: workItem.reservation,
         });
+    for (const workItem of plan.work_items)
+        if (workItem.scheduling)
+            sink.emit({
+                type: "eval.work.priority-computed",
+                work_id: workItem.work_id,
+                slot_id: workItem.slots[0],
+                task_id: workItem.task_ids[0],
+                policy: workItem.scheduling.policy,
+                estimated_duration_ms: workItem.scheduling.estimated_duration_ms,
+                remaining_path_ms: workItem.scheduling.remaining_path_ms,
+                estimate_source: workItem.scheduling.estimate_source,
+                estimate_sample_count: workItem.scheduling.estimate_sample_count,
+            });
 }
 //# sourceMappingURL=eval-lifecycle-events.js.map

package/dist/src/evals/eval-lifecycle-events.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-lifecycle-events.js","sourceRoot":"","sources":["../../../src/evals/eval-lifecycle-events.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,qBAAqB,CAAC,IAAmB,EAAE,IAAyB,EAAE,iBAAyB;IAC7G,IAAI,CAAC,IAAI,CAAC;QACR,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,iBAAiB;QAC3C,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;QAClC,UAAU,EAAE,IAAI,CAAC,UAAU;KAC5B,CAAC,CAAC;IACH,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU;QAAE,IAAI,CAAC,IAAI,CAAC;YAChD,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,WAAW,EAAE,QAAQ,CAAC,WAAW;SAClC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"eval-lifecycle-events.js","sourceRoot":"","sources":["../../../src/evals/eval-lifecycle-events.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,qBAAqB,CAAC,IAAmB,EAAE,IAAyB,EAAE,iBAAyB;IAC7G,IAAI,CAAC,IAAI,CAAC;QACR,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,iBAAiB;QAC3C,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;QAClC,UAAU,EAAE,IAAI,CAAC,UAAU;KAC5B,CAAC,CAAC;IACH,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU;QAAE,IAAI,CAAC,IAAI,CAAC;YAChD,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,WAAW,EAAE,QAAQ,CAAC,WAAW;SAClC,CAAC,CAAC;IACH,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU;QAAE,IAAI,QAAQ,CAAC,UAAU;YAAE,IAAI,CAAC,IAAI,CAAC;gBACzE,IAAI,EAAE,6BAA6B;gBACnC,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC1B,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC7B,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,MAAM;gBAClC,qBAAqB,EAAE,QAAQ,CAAC,UAAU,CAAC,qBAAqB;gBAChE,iBAAiB,EAAE,QAAQ,CAAC,UAAU,CAAC,iBAAiB;gBACxD,eAAe,EAAE,QAAQ,CAAC,UAAU,CAAC,eAAe;gBACpD,qBAAqB,EAAE,QAAQ,CAAC,UAAU,CAAC,qBAAqB;aACjE,CAAC,CAAC;AACL,CAAC"}

package/dist/src/evals/eval-logical-plan.js

@@ -0,0 +1,51 @@
+import path from "node:path";
+import { SCHEMA_VERSION, atomicWriteJSON, readJSON } from "../foundation/index.js";
+import { writeEvalProgress } from "./progress.js";
+/** Persist the reproducible, artifact-independent part of an eval before setup work begins. */
+export async function writeEvalPlanningCheckpoint(evalDirectory, plan, progress) {
+    await Promise.all([
+        atomicWriteJSON(path.join(evalDirectory, "logical-plan.json"), plan),
+        writeEvalProgress(evalDirectory, progress),
+    ]);
+}
+export function materializeEvalPlan(logical, modelCapture, artifactFields) {
+    const { kind: _kind, model_capture: _modelCapture, ...base } = logical;
+    return {
+        ...base,
+        ...(modelCapture ? { model_capture: modelCapture } : {}),
+        ...artifactFields,
+    };
+}
+/** Read only the restart inputs that cannot be reconstructed from request.json. */
+export async function readEvalLogicalPlan(evalDirectory, evalId, request) {
+    const value = await readJSON(path.join(evalDirectory, "logical-plan.json"), null);
+    if (value === null)
+        return null;
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("eval logical plan must be an object");
+    const plan = value;
+    if (plan.schema_version !== SCHEMA_VERSION || plan.kind !== "eval-logical-plan" || plan.eval_id !== evalId
+        || plan.dataset !== request.dataset || plan.benchmark_id !== request.benchmark_id
+        || plan.benchmark_revision !== request.benchmark_revision) {
+        throw new TypeError("eval logical plan identity is invalid");
+    }
+    if (plan.model_capture !== undefined)
+        parseModelCapturePlan(plan.model_capture);
+    return plan;
+}
+export function logicalPlanModelCapture(plan) {
+    return plan?.model_capture === undefined ? undefined : parseModelCapturePlan(plan.model_capture);
+}
+function parseModelCapturePlan(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("eval logical model capture plan is invalid");
+    const plan = value;
+    const modes = new Set(["off", "native", "proxy", "hybrid"]);
+    if (!modes.has(String(plan.requested_mode)) || !modes.has(String(plan.effective_mode)) || typeof plan.required !== "boolean"
+        || plan.topology !== undefined && plan.topology !== "host-side" && plan.topology !== "in-sandbox"
+        || plan.degraded_reason !== undefined && typeof plan.degraded_reason !== "string") {
+        throw new TypeError("eval logical model capture plan is invalid");
+    }
+    return plan;
+}
+//# sourceMappingURL=eval-logical-plan.js.map

package/dist/src/evals/eval-logical-plan.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-logical-plan.js","sourceRoot":"","sources":["../../../src/evals/eval-logical-plan.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACnF,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAoBlD,+FAA+F;AAC/F,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,aAAqB,EACrB,IAAuB,EACvB,QAAwB;IAExB,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC,EAAE,IAAI,CAAC;QACpE,iBAAiB,CAAC,aAAa,EAAE,QAAQ,CAAC;KAC3C,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAA0B,EAC1B,YAA4C,EAC5C,cAAuC;IAEvC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACvE,OAAO;QACL,GAAG,IAAI;QACP,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,cAAc;KAClB,CAAC;AACJ,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,aAAqB,EACrB,MAAc,EACd,OAAoB;IAEpB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC,EAAE,IAAI,CAAC,CAAC;IAClG,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC5H,MAAM,IAAI,GAAG,KAAgC,CAAC;IAC9C,IAAI,IAAI,CAAC,cAAc,KAAK,cAAc,IAAI,IAAI,CAAC,IAAI,KAAK,mBAAmB,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM;WACrG,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY;WAC9E,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC5D,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS;QAAE,qBAAqB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAChF,OAAO,IAAoC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAA8B;IACpE,OAAO,IAAI,EAAE,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AACnG,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,4CAA4C,CAAC,CAAC;IACnI,MAAM,IAAI,GAAG,KAAgC,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,SAAS;WACvH,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,QAAQ,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,KAAK,YAAY;WAC9F,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;QACpF,MAAM,IAAI,SAAS,CAAC,4CAA4C,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAqC,CAAC;AAC/C,CAAC"}

package/dist/src/evals/eval-result-builder.js

@@ -0,0 +1,81 @@
+import { SCHEMA_VERSION } from "../foundation/index.js";
+import { preparedArtifactPlanFields } from "./eval-artifact-planning.js";
+import { infrastructureFailureTrials } from "./infrastructure-retry.js";
+import { invalidTrialSlots } from "./rerun-slots.js";
+import { localSourceBackendFailure, summarizeTrialRefs, transportSummary } from "./result-helpers.js";
+export function buildCompletedEvalResult(input) {
+    const trialRefs = input.progress.trials;
+    const localSourceFailure = input.localTransport
+        ? input.backendRuns.some(({ run }) => localSourceBackendFailure(run.rawResult))
+            || input.infrastructureRetryRuns.some(({ run }) => localSourceBackendFailure(run.rawResult))
+        : false;
+    const expectedBackendRuns = input.plannedTaskExecution ? input.executionWorkItems : input.request.attempts;
+    const backendsSucceeded = input.backendRuns.every(({ run }) => run.backend.process_exit_code === 0 && run.rawResult !== null)
+        && (input.plannedTaskExecution ? trialRefs.length === input.plannedTrials : input.backendRuns.length === expectedBackendRuns);
+    const invalidTrials = input.localTaskIds === null
+        ? trialRefs.filter((trial) => trial.observation_status !== "valid").map((trial) => ({ task_id: trial.task_id, attempt: trial.attempt }))
+        : invalidTrialSlots(input.localTaskIds, input.request.attempts, input.progress);
+    const infrastructureFailures = infrastructureFailureTrials(trialRefs);
+    const infrastructureFailureSlots = infrastructureFailures.map((trial) => ({ task_id: trial.task_id, attempt: trial.attempt }));
+    const verifierRetriesExhausted = input.request.infrastructure_retries > 0
+        && infrastructureFailures.some((trial) => trial.invalid_reason === "verifier_infrastructure_failure");
+    const infrastructureErrorCode = verifierRetriesExhausted
+        || (input.request.infrastructure_retries > 0 && input.infrastructureRetryRuns.length > 0)
+        ? "eval_infrastructure_retries_exhausted"
+        : "eval_has_infrastructure_failures";
+    const succeeded = !input.cancelled && !localSourceFailure && backendsSucceeded && invalidTrials.length === 0;
+    const singleBackend = input.backendRuns.length === 1 ? input.backendRuns[0].run : undefined;
+    return {
+        schema_version: SCHEMA_VERSION,
+        eval_id: input.evalId,
+        status: input.cancelled ? "cancelled" : succeeded ? "succeeded" : "failed",
+        exit_code: input.cancelled ? 9 : succeeded ? 0 : 13,
+        ...(singleBackend ? { backend: singleBackend.backend, backend_summary: singleBackend.summary } : {}),
+        ...(input.plannedTaskExecution ? {
+            backend_work_items: input.backendRuns.map(({ attempt, workId, tasks, leaseId, run }) => ({
+                work_id: workId, lease_id: leaseId, attempt, tasks, backend: run.backend, backend_summary: run.summary,
+            })),
+        } : input.request.attempts > 1 ? {
+            backend_runs: input.backendRuns.map(({ attempt, run }) => ({ attempt, backend: run.backend, backend_summary: run.summary })),
+        } : {}),
+        infrastructure_retry_policy: {
+            max_retries: input.request.infrastructure_retries,
+            backoff_ms: input.request.infrastructure_retry_backoff_ms,
+            verifier_execution: "same_trial_verifier_only",
+            candidate_rerun_on_verifier_failure: false,
+        },
+        ...(input.infrastructureRetryRuns.length > 0 ? {
+            infrastructure_retry_runs: input.infrastructureRetryRuns.map(({ attempt, retry, tasks, triggers, refs, run, leaseId, workId }) => ({
+                execution_kind: "physical-infrastructure-retry", ...(leaseId ? { lease_id: leaseId } : {}),
+                ...(workId ? { work_id: workId } : {}), attempt, retry, tasks, trigger_trials: triggers,
+                trials: refs, backend: run.backend, backend_summary: run.summary,
+            })),
+        } : {}),
+        candidate: input.candidate,
+        dataset: input.request.dataset,
+        benchmark_id: input.request.benchmark_id,
+        benchmark_revision: input.request.benchmark_revision,
+        generation: input.progress.generation,
+        trials: trialRefs,
+        summary: summarizeTrialRefs(trialRefs),
+        ...(input.schedulerSummary ? { scheduler_summary: input.schedulerSummary } : {}),
+        ...preparedArtifactPlanFields(input.preparedAssignments),
+        ...(input.localTransport ? { local_source_transport: transportSummary(input.localTransport) } : {}),
+        ...(succeeded ? {} : {
+            error: {
+                code: input.cancelled ? "cancelled" : localSourceFailure ? "local_source_materialize_failed"
+                    : !backendsSucceeded ? "harbor_failed" : infrastructureFailures.length > 0
+                        ? infrastructureErrorCode : "eval_has_invalid_tasks",
+                message: input.cancelled ? "eval was cancelled"
+                    : localSourceFailure ? "Harbor rejected the transported local Git source before candidate execution"
+                        : !backendsSucceeded ? `Harbor work items completed ${input.backendRuns.length}/${expectedBackendRuns}`
+                            : infrastructureFailures.length > 0
+                                ? `${infrastructureErrorCode === "eval_infrastructure_retries_exhausted" ? "infrastructure retries exhausted" : "verifier infrastructure failure"}: ${infrastructureFailureSlots.map(trial => `${trial.task_id}#${trial.attempt}`).join(", ")}`
+                                : `eval has invalid or missing trials: ${invalidTrials.map(trial => `${trial.task_id}#${trial.attempt}`).join(", ")}`,
+            },
+        }),
+        started_at: input.startedAt.toISOString(),
+        completed_at: new Date().toISOString(),
+    };
+}
+//# sourceMappingURL=eval-result-builder.js.map

package/dist/src/evals/eval-result-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-result-builder.js","sourceRoot":"","sources":["../../../src/evals/eval-result-builder.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,0BAA0B,EAAuC,MAAM,6BAA6B,CAAC;AAC9G,OAAO,EAAE,2BAA2B,EAA+B,MAAM,2BAA2B,CAAC;AACrG,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,yBAAyB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAWtG,MAAM,UAAU,wBAAwB,CAAC,KAgBxC;IACC,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxC,MAAM,kBAAkB,GAAG,KAAK,CAAC,cAAc;QAC7C,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;eAC1E,KAAK,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,yBAAyB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC9F,CAAC,CAAC,KAAK,CAAC;IACV,MAAM,mBAAmB,GAAG,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC3G,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC;WACxH,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,mBAAmB,CAAC,CAAC;IAChI,MAAM,aAAa,GAAG,KAAK,CAAC,YAAY,KAAK,IAAI;QAC/C,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACxI,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAClF,MAAM,sBAAsB,GAAG,2BAA2B,CAAC,SAAS,CAAC,CAAC;IACtE,MAAM,0BAA0B,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC/H,MAAM,wBAAwB,GAAG,KAAK,CAAC,OAAO,CAAC,sBAAsB,GAAG,CAAC;WACpE,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,cAAc,KAAK,iCAAiC,CAAC,CAAC;IACxG,MAAM,uBAAuB,GAAG,wBAAwB;WACnD,CAAC,KAAK,CAAC,OAAO,CAAC,sBAAsB,GAAG,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC;QACzF,CAAC,CAAC,uCAAuC;QACzC,CAAC,CAAC,kCAAkC,CAAC;IACvC,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,kBAAkB,IAAI,iBAAiB,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC;IAC7G,MAAM,aAAa,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7F,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QAC1E,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QACnD,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpG,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAC/B,kBAAkB,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;gBACvF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,CAAC,OAAO;aACvG,CAAC,CAAC;SACJ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;YAC/B,YAAY,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;SAC7H,CAAC,CAAC,CAAC,EAAE,CAAC;QACP,2BAA2B,EAAE;YAC3B,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,sBAAsB;YACjD,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,+BAA+B;YACzD,kBAAkB,EAAE,0BAA0B;YAC9C,mCAAmC,EAAE,KAAK;SAC3C;QACD,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAC7C,yBAAyB,EAAE,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;gBACjI,cAAc,EAAE,+BAA+B,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1F,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ;gBACvF,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,CAAC,OAAO;aACjE,CAAC,CAAC;SACJ,CAAC,CAAC,CAAC,EAAE,CAAC;QACP,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO;QAC9B,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;QACxC,kBAAkB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB;QACpD,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;QACrC,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,kBAAkB,CAAC,SAAS,CAAC;QACtC,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChF,GAAG,0BAA0B,CAAC,KAAK,CAAC,mBAAmB,CAAC;QACxD,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACnB,KAAK,EAAE;gBACL,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,iCAAiC;oBAC1F,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC;wBACxE,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,wBAAwB;gBACxD,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,oBAAoB;oBAC7C,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,6EAA6E;wBAClG,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,+BAA+B,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,mBAAmB,EAAE;4BACrG,CAAC,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC;gCACjC,CAAC,CAAC,GAAG,uBAAuB,KAAK,uCAAuC,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC,iCAAiC,KAAK,0BAA0B,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gCAC/O,CAAC,CAAC,uCAAuC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aAC9H;SACF,CAAC;QACF,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;QACzC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACvC,CAAC;AACJ,CAAC"}

package/dist/src/evals/evolution-baseline.js

@@ -0,0 +1,191 @@
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { atomicWriteJSON, ensureDir, readJSON, sha256JSON, withFileLock } from "../foundation/index.js";
+import { HitchError } from "../foundation/index.js";
+const localFlights = new Map();
+/** Persisted lineage singleflight. Only the caller that creates `submitting`
+ * may invoke the materializer; later rounds validate and return `ready`. */
+export async function ensureEvolutionBaseline(options) {
+    validateInputs(options);
+    const file = baselinePath(options.root, options.evolutionId);
+    const local = localFlights.get(file);
+    if (local)
+        return local;
+    const operation = ensureInternal(options, file);
+    localFlights.set(file, operation);
+    try {
+        return await operation;
+    }
+    finally {
+        if (localFlights.get(file) === operation)
+            localFlights.delete(file);
+    }
+}
+export function evolutionBaselineFingerprint(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length === 0) {
+        throw new TypeError("evolution baseline fingerprint input is invalid");
+    }
+    return sha256JSON(value);
+}
+export async function readEvolutionBaseline(root, evolutionId) {
+    const value = await readJSON(baselinePath(root, evolutionId), null);
+    return value === null ? null : parseEvolutionBaseline(value, evolutionId);
+}
+export function parseEvolutionBaseline(value, expectedEvolutionId) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("evolution baseline must be an object");
+    const record = value;
+    const fields = ["schema_version", "evolution_id", "baseline_fingerprint", "state", "seed_eval_id", "heldout_eval_id", "seed_result_digest", "heldout_result_digest", "created_at", "updated_at"];
+    if (Object.keys(record).some((key) => !fields.includes(key)) || record.schema_version !== "1"
+        || typeof record.evolution_id !== "string" || !record.evolution_id
+        || expectedEvolutionId !== undefined && record.evolution_id !== expectedEvolutionId
+        || typeof record.baseline_fingerprint !== "string" || !/^sha256:[a-f0-9]{64}$/.test(record.baseline_fingerprint)
+        || !["submitting", "running", "ready", "failed"].includes(record.state)
+        || typeof record.created_at !== "string" || !Number.isFinite(Date.parse(record.created_at))
+        || typeof record.updated_at !== "string" || !Number.isFinite(Date.parse(record.updated_at)))
+        throw new TypeError("evolution baseline identity is invalid");
+    const optional = ["seed_eval_id", "heldout_eval_id"];
+    for (const name of optional)
+        if (record[name] !== undefined && (typeof record[name] !== "string" || !/^eval_[a-f0-9]{32}$/.test(record[name]))) {
+            throw new TypeError(`evolution baseline ${name} is invalid`);
+        }
+    for (const name of ["seed_result_digest", "heldout_result_digest"])
+        if (record[name] !== undefined
+            && (typeof record[name] !== "string" || !/^sha256:[a-f0-9]{64}$/.test(record[name])))
+            throw new TypeError(`evolution baseline ${name} is invalid`);
+    if (record.state === "running" && (!record.seed_eval_id || !record.heldout_eval_id))
+        throw new TypeError("running evolution baseline eval identities are missing");
+    if (record.state === "ready" && (!record.seed_eval_id || !record.heldout_eval_id || !record.seed_result_digest || !record.heldout_result_digest)) {
+        throw new TypeError("ready evolution baseline references are incomplete");
+    }
+    return record;
+}
+async function ensureInternal(options, file) {
+    const ownership = await claim(options, file);
+    if (!ownership.owner)
+        return waitForOwner(options, file, ownership.baseline);
+    try {
+        const materialized = await options.materialize({
+            markRunning: (seedEvalId, heldoutEvalId) => updateOwned(options, file, (current) => ({
+                ...assertRunningIdentity(current, seedEvalId, heldoutEvalId), state: "running",
+            })).then(() => undefined),
+        });
+        const ready = await updateOwned(options, file, (current) => {
+            if (current.seed_eval_id && current.seed_eval_id !== materialized.seed_eval_id
+                || current.heldout_eval_id && current.heldout_eval_id !== materialized.heldout_eval_id) {
+                throw new TypeError("evolution baseline eval identity changed during materialization");
+            }
+            return {
+                ...current, state: "ready", seed_eval_id: checkedEvalId(materialized.seed_eval_id), heldout_eval_id: checkedEvalId(materialized.heldout_eval_id),
+                seed_result_digest: checkedDigest(materialized.seed_result_digest), heldout_result_digest: checkedDigest(materialized.heldout_result_digest),
+            };
+        });
+        await options.validateReady?.(ready);
+        return ready;
+    }
+    catch (error) {
+        await updateOwned(options, file, (current) => ({ ...current, state: "failed" })).catch(() => undefined);
+        throw error;
+    }
+}
+async function claim(options, file) {
+    await ensureDir(path.dirname(file));
+    return withBaselineLock(options, async () => {
+        const existing = await readEvolutionBaseline(options.root, options.evolutionId);
+        if (existing) {
+            assertFingerprint(existing, options.fingerprint);
+            if (existing.state === "failed")
+                throw baselineError("evolution baseline materialization previously failed", "evolution_baseline_failed");
+            if (existing.state === "ready")
+                await options.validateReady?.(existing);
+            return { owner: false, baseline: existing };
+        }
+        const now = new Date().toISOString();
+        const baseline = parseEvolutionBaseline({
+            schema_version: "1", evolution_id: options.evolutionId, baseline_fingerprint: options.fingerprint,
+            state: "submitting", created_at: now, updated_at: now,
+        }, options.evolutionId);
+        await atomicWriteJSON(file, baseline);
+        return { owner: true, baseline };
+    });
+}
+async function waitForOwner(options, file, initial) {
+    if (initial.state === "ready")
+        return initial;
+    const deadline = Date.now() + (options.maxWaitMs ?? 24 * 60 * 60 * 1_000);
+    const poll = options.pollIntervalMs ?? 250;
+    for (;;) {
+        if (options.signal?.aborted)
+            throw options.signal.reason;
+        if (Date.now() >= deadline)
+            throw baselineError("evolution baseline materialization is still in progress", "evolution_baseline_in_progress");
+        await delay(Math.min(poll, Math.max(1, deadline - Date.now())), undefined, options.signal ? { signal: options.signal } : undefined);
+        const current = await readEvolutionBaseline(options.root, options.evolutionId);
+        if (!current)
+            throw baselineError("evolution baseline record disappeared", "evolution_baseline_state_conflict");
+        assertFingerprint(current, options.fingerprint);
+        if (current.state === "failed")
+            throw baselineError("evolution baseline materialization failed", "evolution_baseline_failed");
+        if (current.state === "ready") {
+            await options.validateReady?.(current);
+            return current;
+        }
+    }
+}
+async function updateOwned(options, file, update) {
+    return withBaselineLock(options, async () => {
+        const current = await readEvolutionBaseline(options.root, options.evolutionId);
+        if (!current)
+            throw baselineError("evolution baseline ownership was lost", "evolution_baseline_state_conflict");
+        assertFingerprint(current, options.fingerprint);
+        if (current.state === "ready" || current.state === "failed")
+            throw baselineError("evolution baseline is already terminal", "evolution_baseline_state_conflict");
+        const next = parseEvolutionBaseline({ ...update(current), updated_at: new Date().toISOString() }, options.evolutionId);
+        await atomicWriteJSON(file, next);
+        return next;
+    });
+}
+function withBaselineLock(options, operation) {
+    const key = sha256JSON({ evolution_id: options.evolutionId }).slice("sha256:".length);
+    return withFileLock(path.join(options.root, "locks", "evolution-baseline"), key, operation, {
+        timeoutCode: "evolution_baseline_locked", timeoutExitCode: 12,
+    });
+}
+function validateInputs(options) {
+    if (!options.root || !options.evolutionId || !/^sha256:[a-f0-9]{64}$/.test(options.fingerprint))
+        throw new TypeError("evolution baseline request is invalid");
+    if (options.pollIntervalMs !== undefined && (!Number.isSafeInteger(options.pollIntervalMs) || options.pollIntervalMs < 1))
+        throw new TypeError("evolution baseline poll interval is invalid");
+    if (options.maxWaitMs !== undefined && (!Number.isSafeInteger(options.maxWaitMs) || options.maxWaitMs < 1))
+        throw new TypeError("evolution baseline wait is invalid");
+}
+function baselinePath(root, evolutionId) {
+    const id = sha256JSON({ evolution_id: evolutionId }).slice("sha256:".length);
+    return path.join(root, "evolutions", id, "baseline.json");
+}
+function assertFingerprint(record, fingerprint) {
+    if (record.baseline_fingerprint !== fingerprint)
+        throw baselineError("evolution baseline fingerprint changed", "evolution_baseline_fingerprint_mismatch");
+}
+function checkedEvalId(value) {
+    if (!/^eval_[a-f0-9]{32}$/.test(value))
+        throw new TypeError("evolution baseline eval id is invalid");
+    return value;
+}
+function checkedDigest(value) {
+    if (!/^sha256:[a-f0-9]{64}$/.test(value))
+        throw new TypeError("evolution baseline result digest is invalid");
+    return value;
+}
+function assertRunningIdentity(current, seedEvalId, heldoutEvalId) {
+    const seed = checkedEvalId(seedEvalId);
+    const heldout = checkedEvalId(heldoutEvalId);
+    if (current.seed_eval_id && current.seed_eval_id !== seed || current.heldout_eval_id && current.heldout_eval_id !== heldout) {
+        throw new TypeError("evolution baseline eval identity changed while running");
+    }
+    return { ...current, seed_eval_id: seed, heldout_eval_id: heldout };
+}
+function baselineError(message, code) {
+    return new HitchError(message, { code, exitCode: 12 });
+}
+//# sourceMappingURL=evolution-baseline.js.map

package/dist/src/evals/evolution-baseline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evolution-baseline.js","sourceRoot":"","sources":["../../../src/evals/evolution-baseline.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACxG,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAqCpD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAwC,CAAC;AAErE;4EAC4E;AAC5E,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAuC;IACnF,cAAc,CAAC,OAAO,CAAC,CAAC;IACxB,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACxB,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAChD,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAClC,IAAI,CAAC;QAAC,OAAO,MAAM,SAAS,CAAC;IAAC,CAAC;YAAS,CAAC;QAAC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,SAAS;YAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAAC,CAAC;AAClH,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,KAA8B;IACzE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnG,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAY,EAAE,WAAmB;IAC3E,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,CAAC;IACpF,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,sBAAsB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAc,EAAE,mBAA4B;IACjF,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IAC7H,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,MAAM,GAAG,CAAC,gBAAgB,EAAE,cAAc,EAAE,sBAAsB,EAAE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IACjM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG;WACxF,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,YAAY;WAC/D,mBAAmB,KAAK,SAAS,IAAI,MAAM,CAAC,YAAY,KAAK,mBAAmB;WAChF,OAAO,MAAM,CAAC,oBAAoB,KAAK,QAAQ,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;WAC7G,CAAC,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAe,CAAC;WAC9E,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;WACxF,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,wCAAwC,CAAC,CAAC;IAC7J,MAAM,QAAQ,GAAG,CAAC,cAAc,EAAE,iBAAiB,CAAU,CAAC;IAC9D,KAAK,MAAM,IAAI,IAAI,QAAQ;QAAE,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAW,CAAC,CAAC,EAAE,CAAC;YACzJ,MAAM,IAAI,SAAS,CAAC,sBAAsB,IAAI,aAAa,CAAC,CAAC;QAC/D,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,CAAC,oBAAoB,EAAE,uBAAuB,CAAU;QAAE,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS;eACtG,CAAC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAW,CAAC,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,sBAAsB,IAAI,aAAa,CAAC,CAAC;IAC/J,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,wDAAwD,CAAC,CAAC;IACnK,IAAI,MAAM,CAAC,KAAK,KAAK,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,EAAE,CAAC;QACjJ,MAAM,IAAI,SAAS,CAAC,oDAAoD,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,MAAwC,CAAC;AAClD,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,OAAuC,EAAE,IAAY;IACjF,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,SAAS,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC7E,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC;YAC7C,WAAW,EAAE,CAAC,UAAU,EAAE,aAAa,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACnF,GAAG,qBAAqB,CAAC,OAAO,EAAE,UAAU,EAAE,aAAa,CAAC,EAAE,KAAK,EAAE,SAAS;aAC/E,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;SAC1B,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE;YACzD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,KAAK,YAAY,CAAC,YAAY;mBACzE,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe,KAAK,YAAY,CAAC,eAAe,EAAE,CAAC;gBACzF,MAAM,IAAI,SAAS,CAAC,iEAAiE,CAAC,CAAC;YACzF,CAAC;YACD,OAAO;gBACL,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,aAAa,CAAC,YAAY,CAAC,YAAY,CAAC,EAAE,eAAe,EAAE,aAAa,CAAC,YAAY,CAAC,eAAe,CAAC;gBAChJ,kBAAkB,EAAE,aAAa,CAAC,YAAY,CAAC,kBAAkB,CAAC,EAAE,qBAAqB,EAAE,aAAa,CAAC,YAAY,CAAC,qBAAqB,CAAC;aAC7I,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC;QACrC,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxG,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,OAAuC,EAAE,IAAY;IACxE,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,OAAO,gBAAgB,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;QAC1C,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAChF,IAAI,QAAQ,EAAE,CAAC;YACb,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;YACjD,IAAI,QAAQ,CAAC,KAAK,KAAK,QAAQ;gBAAE,MAAM,aAAa,CAAC,sDAAsD,EAAE,2BAA2B,CAAC,CAAC;YAC1I,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO;gBAAE,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,CAAC;YACxE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAC9C,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,sBAAsB,CAAC;YACtC,cAAc,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,WAAW,EAAE,oBAAoB,EAAE,OAAO,CAAC,WAAW;YACjG,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG;SACtD,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACxB,MAAM,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,OAAuC,EAAE,IAAY,EAAE,OAA4B;IAC7G,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,OAAO,CAAC,cAAc,IAAI,GAAG,CAAC;IAC3C,SAAS,CAAC;QACR,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QACzD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ;YAAE,MAAM,aAAa,CAAC,yDAAyD,EAAE,gCAAgC,CAAC,CAAC;QAC7I,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACpI,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC/E,IAAI,CAAC,OAAO;YAAE,MAAM,aAAa,CAAC,uCAAuC,EAAE,mCAAmC,CAAC,CAAC;QAChH,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAChD,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ;YAAE,MAAM,aAAa,CAAC,2CAA2C,EAAE,2BAA2B,CAAC,CAAC;QAC9H,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,CAAC;YACvC,OAAO,OAAO,CAAC;QACjB,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,OAAuC,EACvC,IAAY,EACZ,MAA6D;IAE7D,OAAO,gBAAgB,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;QAC1C,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC/E,IAAI,CAAC,OAAO;YAAE,MAAM,aAAa,CAAC,uCAAuC,EAAE,mCAAmC,CAAC,CAAC;QAChH,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAChD,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ;YAAE,MAAM,aAAa,CAAC,wCAAwC,EAAE,mCAAmC,CAAC,CAAC;QAChK,MAAM,IAAI,GAAG,sBAAsB,CAAC,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACvH,MAAM,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAI,OAAuC,EAAE,SAA2B;IAC/F,MAAM,GAAG,GAAG,UAAU,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACtF,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,oBAAoB,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE;QAC1F,WAAW,EAAE,2BAA2B,EAAE,eAAe,EAAE,EAAE;KAC9D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,OAAuC;IAC7D,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IAC9J,IAAI,OAAO,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IAC9L,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,oCAAoC,CAAC,CAAC;AACxK,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,WAAmB;IACrD,MAAM,EAAE,GAAG,UAAU,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7E,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,eAAe,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,iBAAiB,CAAC,MAA2B,EAAE,WAAmB;IACzE,IAAI,MAAM,CAAC,oBAAoB,KAAK,WAAW;QAAE,MAAM,aAAa,CAAC,wCAAwC,EAAE,yCAAyC,CAAC,CAAC;AAC5J,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IACrG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IAC7G,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,qBAAqB,CAAC,OAA4B,EAAE,UAAkB,EAAE,aAAqB;IACpG,MAAM,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,aAAa,CAAC,aAAa,CAAC,CAAC;IAC7C,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe,KAAK,OAAO,EAAE,CAAC;QAC5H,MAAM,IAAI,SAAS,CAAC,wDAAwD,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,GAAG,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;AACtE,CAAC;AAED,SAAS,aAAa,CAAC,OAAe,EAAE,IAAY;IAClD,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;AACzD,CAAC"}

package/dist/src/evals/execution-plan.js

@@ -25,6 +25,10 @@ export function buildEvalExecutionPlan(options) {
         required: false,
     });
     const tasks = options.tasks === null ? null : canonicalTasks(options.tasks);
+    if (options.taskScheduling && (tasks === null || options.workItemMode !== "task-slots")) {
+        throw new TypeError("task scheduling hints require task-slot work items with known membership");
+    }
+    const taskScheduling = tasks === null ? undefined : parseTaskScheduling(options.taskScheduling, tasks);
     const artifactAssignments = parseArtifactAssignments(options.candidate.artifactAssignments, tasks, options.candidate.artifactId);
     const taskResources = tasks === null ? undefined : parseTaskResourceRequirements(options.taskResources, tasks);
     const environmentImages = tasks === null ? [] : parseEnvironmentImageUses(options.environmentImages ?? [], tasks, "execution plan environment images");
@@ -45,7 +49,7 @@ export function buildEvalExecutionPlan(options) {
     const workItems = tasks === null
         ? [opaqueWorkItem(options.evalId, options.maxParallelism, resources, provider, artifactAssignments[0])]
         : options.workItemMode === "task-slots"
-            ? buildTaskWorkItems(options.evalId, slots, resources, provider, taskResources, environmentImages, artifactAssignments)
+            ? buildTaskWorkItems(options.evalId, slots, resources, provider, taskResources, environmentImages, artifactAssignments, taskScheduling)
             : buildAttemptWorkItems(options.evalId, tasks, slots, options.request.attempts, options.maxParallelism, resources, provider, taskResources, environmentImages, artifactAssignments);
     return parseEvalExecutionPlan({
         schema_version: "1",
@@ -206,7 +210,7 @@ function buildAttemptWorkItems(evalId, tasks, slots, attempts, maxParallelism, r
     }
     return items;
 }
-function buildTaskWorkItems(evalId, slots, resources, provider, taskResources, environmentImages = [], artifactAssignments = []) {
+function buildTaskWorkItems(evalId, slots, resources, provider, taskResources, environmentImages = [], artifactAssignments = [], taskScheduling) {
     return slots.map((slot) => {
         const imageRefs = imagesForTasks(environmentImages, [slot.task_id]);
         const assignment = artifactAssignments.find((entry) => entry.taskIds.includes(slot.task_id));
@@ -224,6 +228,7 @@ function buildTaskWorkItems(evalId, slots, resources, provider, taskResources, e
             reservation: reservationForTasks([slot.task_id], 1, resources, taskResources),
             provider,
             ...(imageRefs.length > 0 ? { image_refs: imageRefs } : {}),
+            ...(taskScheduling?.[slot.task_id] ? { scheduling: taskScheduling[slot.task_id] } : {}),
             ...artifactPin,
         };
     });
@@ -275,7 +280,7 @@ function parseWorkItem(value, evalId, provider, maxParallelism, index) {
     assertOnlyKeys(value, [
         "schema_version", "work_id", "eval_id", "backend", "logical_attempt", "task_ids", "slots",
         "opaque_membership", "requested_parallelism", "reservation", "provider",
-        "image_refs", "artifact_id", "runtime_contract",
+        "image_refs", "artifact_id", "runtime_contract", "scheduling",
     ], `eval execution plan work item ${index}`);
     if (value.schema_version !== "1" || typeof value.work_id !== "string" || !/^work_[a-f0-9]{32}$/.test(value.work_id)
         || value.eval_id !== evalId || value.backend !== "harbor"
@@ -292,6 +297,7 @@ function parseWorkItem(value, evalId, provider, maxParallelism, index) {
     const imageRefs = parseEnvironmentImageUses(value.image_refs ?? [], value.task_ids, `eval execution plan work item ${index} image refs`);
     const artifactId = value.artifact_id === undefined ? undefined : isSha256(value.artifact_id) ? value.artifact_id : (() => { throw new TypeError(`eval execution plan work item ${index} artifact id is invalid`); })();
     const runtimeContract = value.runtime_contract === undefined ? undefined : parseRuntimeContract(value.runtime_contract, `eval execution plan work item ${index}`);
+    const scheduling = value.scheduling === undefined ? undefined : parseSchedulingHint(value.scheduling, `eval execution plan work item ${index}`);
     if ((artifactId === undefined) !== (runtimeContract === undefined))
         throw new TypeError(`eval execution plan work item ${index} artifact contract is incomplete`);
     return {
@@ -300,8 +306,38 @@ function parseWorkItem(value, evalId, provider, maxParallelism, index) {
         ...(imageRefs.length > 0 ? { image_refs: imageRefs } : {}),
         ...(artifactId ? { artifact_id: artifactId } : {}),
         ...(runtimeContract ? { runtime_contract: runtimeContract } : {}),
+        ...(scheduling ? { scheduling } : {}),
     };
 }
+function parseSchedulingHint(value, label) {
+    if (!isRecord(value))
+        throw new TypeError(`${label} scheduling hint is invalid`);
+    assertOnlyKeys(value, ["policy", "estimated_duration_ms", "remaining_path_ms", "estimate_source", "estimate_sample_count"], `${label} scheduling hint`);
+    const sources = new Set(["evolution-baseline", "history-p75", "task-budget", "default"]);
+    if (value.policy !== "critical-path-lpt-v1"
+        || !Number.isSafeInteger(value.estimated_duration_ms) || value.estimated_duration_ms < 1
+        || !Number.isSafeInteger(value.remaining_path_ms) || value.remaining_path_ms < value.estimated_duration_ms
+        || !sources.has(value.estimate_source)
+        || !Number.isSafeInteger(value.estimate_sample_count) || value.estimate_sample_count < 0) {
+        throw new TypeError(`${label} scheduling hint is invalid`);
+    }
+    if (value.estimate_source === "default" || value.estimate_source === "task-budget") {
+        if (value.estimate_sample_count !== 0)
+            throw new TypeError(`${label} scheduling hint sample count is invalid`);
+    }
+    else if (value.estimate_sample_count < 1)
+        throw new TypeError(`${label} scheduling hint sample count is invalid`);
+    return value;
+}
+function parseTaskScheduling(value, tasks) {
+    if (value === undefined)
+        return undefined;
+    const keys = Object.keys(value).sort(compareBytes);
+    if (keys.length !== tasks.length || keys.some((task, index) => task !== tasks[index])) {
+        throw new TypeError("task scheduling hints must cover every planned task exactly once");
+    }
+    return Object.fromEntries(keys.map((task) => [task, parseSchedulingHint(value[task], `task ${task}`)]));
+}
 function assertPlanGraph(membership, slots, workItems, resources, taskResources) {
     if (new Set(slots.map((slot) => slot.slot_id)).size !== slots.length || new Set(workItems.map((item) => item.work_id)).size !== workItems.length) {
         throw new TypeError("eval execution plan identities are duplicated");