agent-hitch 0.2.6 → 0.2.8

package/dist/src/evals/docker-resource-observer.js

@@ -0,0 +1,207 @@
+import { runCommand } from "../foundation/index.js";
+import { DOCKER_OWNERSHIP_LABELS, dockerOwnershipLabelMap, validateDockerResourceOwnership } from "./docker-ownership.js";
+import { parseExecutionEvidence } from "./execution-evidence.js";
+import { readDockerEngineContainerStats } from "./docker-engine-stats.js";
+export function startDockerResourceObserver(options) {
+    const ownership = validateDockerResourceOwnership(options.ownership);
+    const intervalMs = options.intervalMs ?? 250;
+    if (!Number.isSafeInteger(intervalMs) || intervalMs < 50 || intervalMs > 60_000)
+        throw new TypeError("Docker resource observer interval is invalid");
+    const env = options.env ?? process.env;
+    const command = options.run ?? ((args) => runCommand(env.HITCH_DOCKER_PATH || "docker", args, {
+        env,
+        ...(options.signal ? { signal: options.signal } : {}),
+        timeoutMs: 2_000,
+        failureCode: "docker_resource_observation_failed",
+    }));
+    const engineStats = options.engineStats ?? (options.run ? undefined : (containerId) => readDockerEngineContainerStats(containerId, {
+        env,
+        ...(options.signal ? { signal: options.signal } : {}),
+        timeoutMs: 2_000,
+    }));
+    const startedAt = new Date().toISOString();
+    const containers = new Map();
+    const issues = [];
+    let sampleCount = 0;
+    let tail = Promise.resolve();
+    let pending = false;
+    let unavailable = false;
+    let stopped = false;
+    const enqueue = () => {
+        if (pending || unavailable)
+            return tail;
+        pending = true;
+        tail = tail.then(async () => {
+            if (options.signal?.aborted)
+                return;
+            try {
+                await poll(ownership, command, engineStats, containers, issues);
+                sampleCount += 1;
+            }
+            catch (error) {
+                addIssue(issues, error?.message || String(error));
+                unavailable = true;
+            }
+        }).finally(() => { pending = false; });
+        return tail;
+    };
+    const timer = setInterval(() => { if (!stopped)
+        void enqueue(); }, intervalMs);
+    timer.unref();
+    void enqueue();
+    const evidence = () => parseExecutionEvidence({
+        schema_version: "1",
+        provider: ownership.provider,
+        worker_id: options.workerId,
+        collision_domain_id: options.collisionDomainId,
+        eval_id: ownership.eval_id,
+        work_id: ownership.work_id,
+        lease_id: ownership.lease_id,
+        lease_epoch: ownership.lease_epoch,
+        task_id: ownership.task_id,
+        reservation: options.reservation,
+        enforced: { main_limits: options.mainLimits, sidecar_limits: options.sidecarLimits },
+        observed: observation(startedAt, sampleCount, containers, issues),
+    });
+    return {
+        capture: async () => { await enqueue(); return evidence(); },
+        stop: async () => {
+            stopped = true;
+            clearInterval(timer);
+            await enqueue();
+            return evidence();
+        },
+    };
+}
+async function poll(ownership, command, engineStats, containers, issues) {
+    const labels = dockerOwnershipLabelMap(ownership);
+    const listed = await command([
+        "container", "ls", "--all",
+        "--filter", `label=${DOCKER_OWNERSHIP_LABELS.rootId}=${labels[DOCKER_OWNERSHIP_LABELS.rootId]}`,
+        "--filter", `label=${DOCKER_OWNERSHIP_LABELS.leaseId}=${labels[DOCKER_OWNERSHIP_LABELS.leaseId]}`,
+        "--filter", `label=${DOCKER_OWNERSHIP_LABELS.leaseEpoch}=${labels[DOCKER_OWNERSHIP_LABELS.leaseEpoch]}`,
+        "--format", "{{.ID}}",
+    ]);
+    const ids = [...new Set(listed.stdout.split(/\r?\n/).map((entry) => entry.trim()).filter(Boolean))].sort();
+    if (ids.length > 256 || ids.some((id) => !/^[a-f0-9]{12,64}$/.test(id)))
+        throw new TypeError("Docker resource observation returned invalid container IDs");
+    for (const id of ids) {
+        try {
+            await observeContainer(id, ownership, command, engineStats, containers);
+        }
+        catch (error) {
+            addIssue(issues, error?.message || String(error));
+        }
+    }
+}
+async function observeContainer(id, ownership, command, engineStats, containers) {
+    const inspected = JSON.parse((await command(["container", "inspect", id])).stdout);
+    if (!Array.isArray(inspected) || inspected.length !== 1 || !record(inspected[0]))
+        throw new TypeError("Docker resource observation inspect response is invalid");
+    const item = inspected[0];
+    if (typeof item.Id !== "string" || !/^[a-f0-9]{12,64}$/.test(item.Id) || (!item.Id.startsWith(id) && !id.startsWith(item.Id))) {
+        throw new TypeError("Docker resource observation container identity changed");
+    }
+    const config = record(item.Config) ? item.Config : {};
+    const rawLabels = record(config.Labels) ? config.Labels : {};
+    const epoch = Number(rawLabels[DOCKER_OWNERSHIP_LABELS.leaseEpoch]);
+    const actual = validateDockerResourceOwnership({
+        root_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.rootId] ?? ""),
+        provider: String(rawLabels[DOCKER_OWNERSHIP_LABELS.provider] ?? ""),
+        eval_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.evalId] ?? ""),
+        work_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.workId] ?? ""),
+        lease_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.leaseId] ?? ""),
+        lease_epoch: epoch,
+        ...(rawLabels[DOCKER_OWNERSHIP_LABELS.taskId] === undefined ? {} : { task_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.taskId]) }),
+    });
+    const expectedLabels = dockerOwnershipLabelMap(ownership);
+    const actualLabels = dockerOwnershipLabelMap(actual);
+    if (Object.keys(expectedLabels).some((key) => actualLabels[key] !== expectedLabels[key]))
+        throw new TypeError("Docker resource observation ownership changed");
+    const now = new Date().toISOString();
+    const previous = containers.get(item.Id);
+    const observed = previous ?? { container_id: item.Id, first_observed_at: now, last_observed_at: now };
+    observed.last_observed_at = now;
+    if (typeof item.Name === "string" && item.Name.replace(/^\//, ""))
+        observed.name = item.Name.replace(/^\//, "").replace(/[\0\r\n]/g, " ").slice(0, 256);
+    if (typeof config.Image === "string" && config.Image && config.Image.length <= 1_024 && !/[\0\r\n]/.test(config.Image))
+        observed.image_reference = config.Image;
+    if (typeof item.Image === "string" && /^sha256:[a-f0-9]{64}$/.test(item.Image))
+        observed.image_config_digest = item.Image;
+    const state = record(item.State) ? item.State : {};
+    if (typeof state.OOMKilled === "boolean")
+        observed.oom_killed = Boolean(observed.oom_killed || state.OOMKilled);
+    if (state.Running === false && Number.isSafeInteger(state.ExitCode) && state.ExitCode >= 0 && state.ExitCode <= 255) {
+        observed.exit_code = state.ExitCode;
+        const rawReason = typeof state.Error === "string" && state.Error.trim()
+            ? state.Error.trim()
+            : state.OOMKilled === true ? "oom-killed" : `exit-code-${state.ExitCode}`;
+        observed.exit_reason = rawReason.replace(/[\0\r\n]/g, " ").slice(0, 512);
+    }
+    if (state.Running === true) {
+        if (engineStats) {
+            try {
+                const stats = await engineStats(item.Id);
+                if (stats.cpu_time_ns !== undefined)
+                    observed.cpu_time_ns = Math.max(observed.cpu_time_ns ?? 0, stats.cpu_time_ns);
+                if (stats.memory_bytes !== undefined)
+                    observed.peak_memory_bytes = Math.max(observed.peak_memory_bytes ?? 0, stats.memory_bytes);
+            }
+            catch { /* CLI evidence remains useful when the Engine stats API is unavailable. */ }
+        }
+        if (observed.peak_memory_bytes === undefined)
+            try {
+                const stats = await command(["container", "stats", "--no-stream", "--format", "{{json .}}", item.Id]);
+                const line = stats.stdout.split(/\r?\n/).find((entry) => entry.trim());
+                const parsed = line ? JSON.parse(line) : null;
+                const memory = record(parsed) ? parseDockerMemoryBytes(parsed.MemUsage) : null;
+                if (memory !== null)
+                    observed.peak_memory_bytes = Math.max(observed.peak_memory_bytes ?? 0, memory);
+            }
+            catch { /* Inspect evidence remains useful when stats is unavailable. */ }
+    }
+    containers.set(item.Id, observed);
+}
+function observation(startedAt, sampleCount, values, issues) {
+    const containers = [...values.values()].sort((left, right) => left.container_id.localeCompare(right.container_id));
+    const unavailable = [];
+    if (!containers.some((entry) => entry.cpu_time_ns !== undefined))
+        unavailable.push("cpu_time_ns");
+    if (!containers.some((entry) => entry.peak_memory_bytes !== undefined))
+        unavailable.push("peak_memory_bytes");
+    if (!containers.some((entry) => entry.exit_code !== undefined || entry.oom_killed !== undefined))
+        unavailable.push("exit_status");
+    if (!containers.some((entry) => entry.image_reference !== undefined || entry.image_config_digest !== undefined))
+        unavailable.push("image_identity");
+    return {
+        status: containers.length > 0 ? "partial" : "unavailable",
+        started_at: startedAt,
+        collected_at: new Date().toISOString(),
+        sample_count: sampleCount,
+        containers,
+        unavailable_fields: unavailable,
+        issues: [...issues],
+    };
+}
+export function parseDockerMemoryBytes(value) {
+    if (typeof value !== "string")
+        return null;
+    const first = value.split("/", 1)[0]?.trim() ?? "";
+    const match = first.match(/^([0-9]+(?:\.[0-9]+)?)\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)$/);
+    if (!match)
+        return null;
+    const amount = Number(match[1]);
+    const unit = match[2];
+    const powers = { B: 1, kB: 1_000, KB: 1_000, KiB: 1024, MB: 1_000_000, MiB: 1024 ** 2, GB: 1_000_000_000, GiB: 1024 ** 3, TB: 1_000_000_000_000, TiB: 1024 ** 4 };
+    const bytes = amount * powers[unit];
+    return Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : null;
+}
+function addIssue(issues, message) {
+    const bounded = message.replace(/[\0\r\n]/g, " ").slice(0, 512) || "Docker resource observation failed";
+    if (!issues.includes(bounded) && issues.length < 32)
+        issues.push(bounded);
+}
+function record(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+//# sourceMappingURL=docker-resource-observer.js.map

package/dist/src/evals/docker-resource-observer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker-resource-observer.js","sourceRoot":"","sources":["../../../src/evals/docker-resource-observer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,+BAA+B,EAAE,MAAM,uBAAuB,CAAC;AAC1H,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAmC1E,MAAM,UAAU,2BAA2B,CAAC,OAAsC;IAChF,MAAM,SAAS,GAAG,+BAA+B,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,EAAE,IAAI,UAAU,GAAG,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,8CAA8C,CAAC,CAAC;IACrJ,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,IAAc,EAAE,EAAE,CAAC,UAAU,CAC5D,GAAG,CAAC,iBAAiB,IAAI,QAAQ,EACjC,IAAI,EACJ;QACE,GAAG;QACH,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,SAAS,EAAE,KAAK;QAChB,WAAW,EAAE,oCAAoC;KAClD,CACF,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAmB,EAAE,EAAE,CAAC,8BAA8B,CAAC,WAAW,EAAE;QACzI,GAAG;QACH,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,SAAS,EAAE,KAAK;KACjB,CAAC,CAAC,CAAC;IACJ,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,IAAI,GAAG,EAA4B,CAAC;IACvD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,OAAO,GAAG,GAAkB,EAAE;QAClC,IAAI,OAAO,IAAI,WAAW;YAAE,OAAO,IAAI,CAAC;QACxC,OAAO,GAAG,IAAI,CAAC;QACf,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC1B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,OAAO;YACpC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;gBAChE,WAAW,IAAI,CAAC,CAAC;YACnB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CAAC,MAAM,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7D,WAAW,GAAG,IAAI,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO;QAAE,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/E,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,KAAK,OAAO,EAAE,CAAC;IACf,MAAM,QAAQ,GAAG,GAAwB,EAAE,CAAC,sBAAsB,CAAC;QACjE,cAAc,EAAE,GAAG;QACnB,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,SAAS,EAAE,OAAO,CAAC,QAAQ;QAC3B,mBAAmB,EAAE,OAAO,CAAC,iBAAiB;QAC9C,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,WAAW,EAAE,SAAS,CAAC,WAAW;QAClC,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,QAAQ,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,aAAa,EAAE;QACpF,QAAQ,EAAE,WAAW,CAAC,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC;KAClE,CAAC,CAAC;IACH,OAAO;QACL,OAAO,EAAE,KAAK,IAAI,EAAE,GAAG,MAAM,OAAO,EAAE,CAAC,CAAC,OAAO,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC5D,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,OAAO,GAAG,IAAI,CAAC;YACf,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,OAAO,EAAE,CAAC;YAChB,OAAO,QAAQ,EAAE,CAAC;QACpB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI,CACjB,SAAoC,EACpC,OAAyE,EACzE,WAA4G,EAC5G,UAAyC,EACzC,MAAgB;IAEhB,MAAM,MAAM,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;QAC3B,WAAW,EAAE,IAAI,EAAE,OAAO;QAC1B,UAAU,EAAE,SAAS,uBAAuB,CAAC,MAAM,IAAI,MAAM,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE;QAC/F,UAAU,EAAE,SAAS,uBAAuB,CAAC,OAAO,IAAI,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,EAAE;QACjG,UAAU,EAAE,SAAS,uBAAuB,CAAC,UAAU,IAAI,MAAM,CAAC,uBAAuB,CAAC,UAAU,CAAC,EAAE;QACvG,UAAU,EAAE,SAAS;KACtB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3G,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IAC3J,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC;YAAC,MAAM,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QAAC,CAAC;QAChF,OAAO,KAAK,EAAE,CAAC;YAAC,QAAQ,CAAC,MAAM,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EAAU,EACV,SAAoC,EACpC,OAAyE,EACzE,WAA4G,EAC5G,UAAyC;IAEzC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAY,CAAC;IAC9F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,yDAAyD,CAAC,CAAC;IACjK,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC9H,MAAM,IAAI,SAAS,CAAC,wDAAwD,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACtD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxF,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,UAAU,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,+BAA+B,CAAC;QAC7C,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChE,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnE,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChE,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChE,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClE,WAAW,EAAE,KAAK;QAClB,GAAG,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;KACnI,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,cAAc,CAAC,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IAC/J,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAqB,QAAQ,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC;IACxH,QAAQ,CAAC,gBAAgB,GAAG,GAAG,CAAC;IAChC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;QAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACxJ,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,QAAQ,CAAC,eAAe,GAAG,MAAM,CAAC,KAAK,CAAC;IAChK,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,QAAQ,CAAC,mBAAmB,GAAG,IAAI,CAAC,KAAe,CAAC;IACpI,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACnD,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,SAAS;QAAE,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAChH,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAK,KAAK,CAAC,QAAmB,IAAI,CAAC,IAAK,KAAK,CAAC,QAAmB,IAAI,GAAG,EAAE,CAAC;QAC5I,QAAQ,CAAC,SAAS,GAAG,KAAK,CAAC,QAAkB,CAAC;QAC9C,MAAM,SAAS,GAAG,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;YACrE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;YACpB,CAAC,CAAC,KAAK,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC5E,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC3B,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACzC,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS;oBAAE,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;gBACnH,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS;oBAAE,QAAQ,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,iBAAiB,IAAI,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;YACnI,CAAC;YAAC,MAAM,CAAC,CAAC,2EAA2E,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,QAAQ,CAAC,iBAAiB,KAAK,SAAS;YAAE,IAAI,CAAC;gBACjD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,CAAC,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;gBACtG,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvE,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;gBACzD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC/E,IAAI,MAAM,KAAK,IAAI;oBAAE,QAAQ,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,iBAAiB,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;YACtG,CAAC;YAAC,MAAM,CAAC,CAAC,gEAAgE,CAAC,CAAC;IAC9E,CAAC;IACD,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,WAAW,CAClB,SAAiB,EACjB,WAAmB,EACnB,MAAqC,EACrC,MAAgB;IAEhB,MAAM,UAAU,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,CAAmC,CAAC;IACrJ,MAAM,WAAW,GAA0D,EAAE,CAAC;IAC9E,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAClG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,KAAK,SAAS,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC9G,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAClI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACpJ,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa;QACzD,UAAU,EAAE,SAAS;QACrB,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACtC,YAAY,EAAE,WAAW;QACzB,UAAU;QACV,kBAAkB,EAAE,WAAW;QAC/B,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC;KACpB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAc;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACnD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAC1F,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAW,CAAC;IAChC,MAAM,MAAM,GAA2B,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,EAAE,EAAE,EAAE,aAAa,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,EAAE,EAAE,EAAE,iBAAiB,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;IAC1L,MAAM,KAAK,GAAG,MAAM,GAAI,MAAM,CAAC,IAAI,CAAY,CAAC;IAChD,OAAO,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAClE,CAAC;AAED,SAAS,QAAQ,CAAC,MAAgB,EAAE,OAAe;IACjD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,oCAAoC,CAAC;IACxG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}

package/dist/src/evals/duration-estimator.js

@@ -0,0 +1,126 @@
+import path from "node:path";
+import { atomicWriteJSON, ensureDir, readJSON, sha256JSON, withFileLock } from "../foundation/index.js";
+import { benchmarkTaskDigest } from "../runs/index.js";
+const MAX_SAMPLES = 20;
+const DEFAULT_ESTIMATED_DURATION_MS = 15 * 60 * 1_000;
+const MAX_DURATION_MS = 7 * 24 * 60 * 60 * 1_000;
+export async function planTaskSchedulingHints(input) {
+    const hints = [];
+    for (const taskId of [...input.taskIds].sort(compareBytes)) {
+        const identity = durationIdentity(input, taskId);
+        const stats = await readStats(statsPath(input.root, identity.key), identity).catch(() => null);
+        const baseline = input.evolutionBaselineDurations?.[taskId];
+        if (baseline !== undefined && (!Number.isSafeInteger(baseline) || baseline < 1 || baseline > MAX_DURATION_MS)) {
+            throw new TypeError(`evolution baseline duration is invalid: ${taskId}`);
+        }
+        const taskBudget = await readTaskBudget(input.dataset, taskId);
+        const fallback = taskBudget ?? (input.requestTimeoutMs > 0 ? input.requestTimeoutMs : DEFAULT_ESTIMATED_DURATION_MS);
+        const estimated = baseline ?? (stats && stats.samples.length > 0
+            ? percentile(stats.samples.map((sample) => sample.duration_ms), 0.75)
+            : fallback);
+        const failures = stats?.samples.filter((sample) => sample.retryable_infrastructure_failure).length ?? 0;
+        const retryRate = stats && stats.samples.length > 0 ? failures / stats.samples.length : 0;
+        const expectedRetries = Math.min(input.infrastructureRetries, 1) * retryRate;
+        hints.push([taskId, {
+                policy: "critical-path-lpt-v1",
+                estimated_duration_ms: estimated,
+                remaining_path_ms: Math.max(estimated, Math.round(estimated * (1 + expectedRetries))),
+                estimate_source: baseline !== undefined ? "evolution-baseline" : stats && stats.samples.length > 0 ? "history-p75" : taskBudget !== null || input.requestTimeoutMs > 0 ? "task-budget" : "default",
+                estimate_sample_count: baseline !== undefined ? 1 : stats?.samples.length ?? 0,
+            }]);
+    }
+    return Object.fromEntries(hints);
+}
+export async function recordTaskDuration(input) {
+    if (!Number.isSafeInteger(input.durationMs) || input.durationMs < 1 || input.durationMs > MAX_DURATION_MS) {
+        throw new TypeError("task duration sample is invalid");
+    }
+    const identity = durationIdentity(input, input.taskId);
+    const directory = await ensureDir(path.dirname(statsPath(input.root, identity.key)));
+    const locks = await ensureDir(path.join(input.root, "locks", "scheduler-duration-stats"));
+    await withFileLock(locks, identity.key, async () => {
+        const current = await readStats(path.join(directory, `${identity.key.slice("sha256:".length)}.json`), identity).catch(() => null);
+        const samples = [...(current?.samples ?? []), {
+                duration_ms: input.durationMs,
+                retryable_infrastructure_failure: input.retryableInfrastructureFailure,
+            }].slice(-MAX_SAMPLES);
+        await atomicWriteJSON(statsPath(input.root, identity.key), {
+            schema_version: "1",
+            ...identity,
+            samples,
+            updated_at: new Date().toISOString(),
+        });
+    }, { timeoutCode: "scheduler_duration_stats_locked", timeoutExitCode: 12 });
+}
+export function schedulingHintsFromPlan(plan) {
+    const hints = new Map();
+    for (const item of plan.work_items) {
+        const taskId = item.task_ids[0];
+        if (!taskId || !item.scheduling)
+            return undefined;
+        const existing = hints.get(taskId);
+        if (existing && JSON.stringify(existing) !== JSON.stringify(item.scheduling))
+            throw new TypeError(`execution plan task scheduling hint changed: ${taskId}`);
+        hints.set(taskId, item.scheduling);
+    }
+    return Object.fromEntries([...hints.entries()].sort((left, right) => compareBytes(left[0], right[0])));
+}
+function durationIdentity(input, taskId) {
+    const taskDigest = benchmarkTaskDigest(input.benchmarkId, input.benchmarkRevision, taskId);
+    const identity = {
+        benchmark_id: input.benchmarkId,
+        benchmark_revision: input.benchmarkRevision,
+        task_id: taskId,
+        task_digest: taskDigest,
+        provider: input.provider,
+        model: input.model,
+    };
+    return { key: sha256JSON(identity), ...identity };
+}
+function statsPath(root, key) {
+    return path.join(root, "indexes", "scheduler-duration-stats", `${key.slice("sha256:".length)}.json`);
+}
+async function readStats(file, identity) {
+    const value = await readJSON(file, null);
+    if (value === null)
+        return null;
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("task duration stats must be an object");
+    const record = value;
+    const allowed = ["schema_version", "key", "benchmark_id", "benchmark_revision", "task_id", "task_digest", "provider", "model", "samples", "updated_at"];
+    if (Object.keys(record).some((key) => !allowed.includes(key)) || record.schema_version !== "1"
+        || record.key !== identity.key || record.benchmark_id !== identity.benchmark_id || record.benchmark_revision !== identity.benchmark_revision
+        || record.task_id !== identity.task_id || record.task_digest !== identity.task_digest || record.provider !== identity.provider || record.model !== identity.model
+        || !Array.isArray(record.samples) || record.samples.length > MAX_SAMPLES
+        || typeof record.updated_at !== "string" || !Number.isFinite(Date.parse(record.updated_at))) {
+        throw new TypeError("task duration stats identity is invalid");
+    }
+    const samples = record.samples.map((value) => {
+        if (!value || typeof value !== "object" || Array.isArray(value))
+            throw new TypeError("task duration sample is invalid");
+        const sample = value;
+        if (Object.keys(sample).some((key) => key !== "duration_ms" && key !== "retryable_infrastructure_failure")
+            || !Number.isSafeInteger(sample.duration_ms) || sample.duration_ms < 1 || sample.duration_ms > MAX_DURATION_MS
+            || typeof sample.retryable_infrastructure_failure !== "boolean")
+            throw new TypeError("task duration sample is invalid");
+        return { duration_ms: sample.duration_ms, retryable_infrastructure_failure: sample.retryable_infrastructure_failure };
+    });
+    return { schema_version: "1", ...identity, samples, updated_at: record.updated_at };
+}
+async function readTaskBudget(dataset, taskId) {
+    for (const file of [path.join(dataset, taskId, ".hitch-benchmark.json"), path.join(dataset, ".hitch-benchmark.json")]) {
+        const descriptor = await readJSON(file, null).catch(() => null);
+        const seconds = descriptor?.agent_timeout_sec;
+        if (typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0)
+            return Math.max(1, Math.round(seconds * 1_000));
+    }
+    return null;
+}
+function percentile(values, ratio) {
+    const sorted = [...values].sort((left, right) => left - right);
+    return sorted[Math.max(0, Math.ceil(sorted.length * ratio) - 1)];
+}
+function compareBytes(left, right) {
+    return Buffer.compare(Buffer.from(left), Buffer.from(right));
+}
+//# sourceMappingURL=duration-estimator.js.map

package/dist/src/evals/duration-estimator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"duration-estimator.js","sourceRoot":"","sources":["../../../src/evals/duration-estimator.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACxG,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACtD,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAoBjD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAW7C;IACC,MAAM,KAAK,GAAsC,EAAE,CAAC;IACpD,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAC3D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC/F,MAAM,QAAQ,GAAG,KAAK,CAAC,0BAA0B,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,eAAe,CAAC,EAAE,CAAC;YAC9G,MAAM,IAAI,SAAS,CAAC,2CAA2C,MAAM,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,UAAU,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC;QACrH,MAAM,SAAS,GAAG,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;YAC9D,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,CAAC;YACrE,CAAC,CAAC,QAAQ,CAAC,CAAC;QACd,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,gCAAgC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;QACxG,MAAM,SAAS,GAAG,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1F,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC;QAC7E,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE;gBAClB,MAAM,EAAE,sBAAsB;gBAC9B,qBAAqB,EAAE,SAAS;gBAChC,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC;gBACrF,eAAe,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,KAAK,IAAI,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;gBAClM,qBAAqB,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC;aAC/E,CAAC,CAAC,CAAC;IACN,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KASxC;IACC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,eAAe,EAAE,CAAC;QAC1G,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,0BAA0B,CAAC,CAAC,CAAC;IAC1F,MAAM,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAClI,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE;gBAC5C,WAAW,EAAE,KAAK,CAAC,UAAU;gBAC7B,gCAAgC,EAAE,KAAK,CAAC,8BAA8B;aACvE,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC;QACvB,MAAM,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;YACzD,cAAc,EAAE,GAAG;YACnB,GAAG,QAAQ;YACX,OAAO;YACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACP,CAAC,CAAC;IACnC,CAAC,EAAE,EAAE,WAAW,EAAE,iCAAiC,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAyB;IAC/D,MAAM,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAClD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO,SAAS,CAAC;QAClD,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,gDAAgD,MAAM,EAAE,CAAC,CAAC;QAC5J,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzG,CAAC;AAED,SAAS,gBAAgB,CAAC,KAKzB,EAAE,MAAc;IACf,MAAM,UAAU,GAAG,mBAAmB,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IAC3F,MAAM,QAAQ,GAAG;QACf,YAAY,EAAE,KAAK,CAAC,WAAW;QAC/B,kBAAkB,EAAE,KAAK,CAAC,iBAAiB;QAC3C,OAAO,EAAE,MAAM;QACf,WAAW,EAAE,UAAU;QACvB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC;IACF,OAAO,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,GAAW;IAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,0BAA0B,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AACvG,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,QAA6C;IAClF,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,EAAE,IAAI,CAAC,CAAC;IACzD,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IAC9H,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,OAAO,GAAG,CAAC,gBAAgB,EAAE,KAAK,EAAE,cAAc,EAAE,oBAAoB,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IACxJ,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG;WACzF,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,GAAG,IAAI,MAAM,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY,IAAI,MAAM,CAAC,kBAAkB,KAAK,QAAQ,CAAC,kBAAkB;WACzI,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;WAC9J,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,WAAW;WACrE,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,yCAAyC,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;QACxH,MAAM,MAAM,GAAG,KAAgC,CAAC;QAChD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,kCAAkC,CAAC;eACrG,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,IAAK,MAAM,CAAC,WAAsB,GAAG,CAAC,IAAK,MAAM,CAAC,WAAsB,GAAG,eAAe;eACnI,OAAO,MAAM,CAAC,gCAAgC,KAAK,SAAS;YAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;QAC1H,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,WAAqB,EAAE,gCAAgC,EAAE,MAAM,CAAC,gCAA2C,EAAE,CAAC;IAC7I,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;AACtF,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,OAAe,EAAE,MAAc;IAC3D,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,uBAAuB,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAC,EAAE,CAAC;QACtH,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAiC,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAChG,MAAM,OAAO,GAAG,UAAU,EAAE,iBAAiB,CAAC;QAC9C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC;IAC9H,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB,EAAE,KAAa;IACjD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;IAC/D,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAW,CAAC;AAC7E,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa;IAC/C,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC"}

package/dist/src/evals/environment-image-planning.js

@@ -0,0 +1,164 @@
+import { HitchError } from "../foundation/index.js";
+export const DEFAULT_ENVIRONMENT_IMAGE_PLATFORM = "linux/amd64";
+export async function planEnvironmentImages(input) {
+    const platform = input.platform ?? DEFAULT_ENVIRONMENT_IMAGE_PLATFORM;
+    const uses = [];
+    const fallbacks = input.tasks.flatMap((task) => task.environment_image_fallbacks.map((entry) => ({
+        task_id: task.task_id,
+        source: entry.source,
+        service: entry.service,
+        code: entry.code,
+    })));
+    const requests = canonicalRequests(input.tasks, platform);
+    const builds = canonicalBuilds(input.tasks, platform);
+    if (input.mode === "backend") {
+        fallbacks.push(...requests.flatMap((entry) => fallbacksFor(entry, "policy-backend")));
+        fallbacks.push(...builds.flatMap((entry) => fallbacksFor(entry, "policy-backend")));
+    }
+    else {
+        if (!input.resolver)
+            fallbacks.push(...requests.flatMap((entry) => fallbacksFor(entry, "resolver-unavailable")));
+        else
+            for (const entry of requests) {
+                try {
+                    const resolved = await input.resolver({
+                        benchmarkId: input.benchmarkId,
+                        benchmarkRevision: input.benchmarkRevision,
+                        taskId: entry.taskId,
+                        reference: entry.reference,
+                        platform: entry.platform,
+                        ...(input.signal ? { signal: input.signal } : {}),
+                    });
+                    if (!validResolvedImage(entry.reference, entry.platform, resolved))
+                        throw new TypeError("environment image resolver output is invalid");
+                    uses.push({
+                        task_ids: [entry.taskId],
+                        image_id: resolved.image_id,
+                        requested_reference: entry.reference,
+                        reference: resolved.reference,
+                        manifest_digest: resolved.manifest_digest,
+                        platform: resolved.platform,
+                        resolution: "registry",
+                        cache_hit: resolved.cache_hit,
+                    });
+                }
+                catch (error) {
+                    if (error.code === "cancelled")
+                        throw error;
+                    fallbacks.push(...fallbacksFor(entry, "resolution-failed"));
+                }
+            }
+        if (!input.builder)
+            fallbacks.push(...builds.flatMap((entry) => fallbacksFor(entry, "resolver-unavailable")));
+        else
+            for (const entry of builds) {
+                try {
+                    const built = await input.builder({
+                        benchmarkId: input.benchmarkId,
+                        benchmarkRevision: input.benchmarkRevision,
+                        taskId: entry.taskId,
+                        contextDirectory: entry.contextDirectory,
+                        dockerfile: entry.dockerfile,
+                        platform: entry.platform,
+                        ...(input.signal ? { signal: input.signal } : {}),
+                    });
+                    if (!validBuiltImage(entry.platform, built))
+                        throw new TypeError("environment image builder output is invalid");
+                    uses.push({
+                        task_ids: [entry.taskId],
+                        image_id: built.image_id,
+                        requested_reference: built.requested_reference,
+                        reference: built.reference,
+                        manifest_digest: built.manifest_digest,
+                        platform: built.platform,
+                        resolution: "prebuilt",
+                        cache_hit: built.cache_hit,
+                    });
+                }
+                catch (error) {
+                    if (error.code === "cancelled")
+                        throw error;
+                    fallbacks.push(...fallbacksFor(entry, "resolution-failed"));
+                }
+            }
+    }
+    const result = { uses: canonicalUses(uses), fallbacks: canonicalFallbacks(fallbacks) };
+    if (input.mode === "prebuild-required" && result.fallbacks.length > 0) {
+        throw new HitchError("required environment image prebuild is unavailable for one or more task services", {
+            code: "environment_prebuild_unavailable",
+            exitCode: 10,
+        });
+    }
+    return result;
+}
+function canonicalBuilds(tasks, defaultPlatform) {
+    return tasks.flatMap((task) => task.environment_builds.map((entry) => ({
+        taskId: task.task_id,
+        platform: task.runtime_platform ?? defaultPlatform,
+        contextDirectory: entry.context_directory,
+        dockerfile: entry.dockerfile,
+        bindings: [{ source: entry.source, service: entry.service }],
+    }))).sort((left, right) => compare(`${left.taskId}\0${left.contextDirectory}\0${left.dockerfile}`, `${right.taskId}\0${right.contextDirectory}\0${right.dockerfile}`));
+}
+export function resolvedImageMapping(images) {
+    const result = {};
+    for (const image of images) {
+        if (image.resolution === "prebuilt")
+            continue;
+        const existing = result[image.requested_reference];
+        if (existing !== undefined && existing !== image.reference)
+            throw new TypeError("environment image reference resolved to multiple immutable images");
+        result[image.requested_reference] = image.reference;
+    }
+    return Object.fromEntries(Object.entries(result).sort(([left], [right]) => compare(left, right)));
+}
+function canonicalRequests(tasks, defaultPlatform) {
+    const grouped = new Map();
+    for (const task of tasks)
+        for (const entry of task.environment_images) {
+            const key = `${task.task_id}\0${entry.reference}`;
+            const request = grouped.get(key) ?? { taskId: task.task_id, platform: task.runtime_platform ?? defaultPlatform, reference: entry.reference, bindings: [] };
+            request.bindings.push({ source: entry.source, service: entry.service });
+            grouped.set(key, request);
+        }
+    return [...grouped.values()].sort((left, right) => compare(`${left.taskId}\0${left.reference}`, `${right.taskId}\0${right.reference}`));
+}
+function fallbacksFor(request, code) {
+    return request.bindings.map((entry) => ({ task_id: request.taskId, source: entry.source, service: entry.service, code }));
+}
+function validResolvedImage(requested, platform, resolved) {
+    return /^sha256:[a-f0-9]{64}$/.test(resolved.image_id)
+        && /^sha256:[a-f0-9]{64}$/.test(resolved.manifest_digest)
+        && resolved.platform === platform
+        && resolved.reference.endsWith(`@${resolved.manifest_digest}`)
+        && repositoryOf(requested) === repositoryOf(resolved.reference)
+        && typeof resolved.cache_hit === "boolean";
+}
+function validBuiltImage(platform, built) {
+    return /^sha256:[a-f0-9]{64}$/.test(built.image_id)
+        && /^sha256:[a-f0-9]{64}$/.test(built.manifest_digest)
+        && built.platform === platform
+        && built.reference === `${built.requested_reference}@${built.manifest_digest}`
+        && validImageReference(built.requested_reference)
+        && typeof built.cache_hit === "boolean";
+}
+function validImageReference(value) {
+    return Boolean(value) && value.length <= 1_024 && !/[\s\0]/.test(value) && !value.includes("://") && !value.includes("$");
+}
+function repositoryOf(reference) {
+    const withoutDigest = reference.split("@")[0];
+    const slash = withoutDigest.lastIndexOf("/");
+    const colon = withoutDigest.lastIndexOf(":");
+    return colon > slash ? withoutDigest.slice(0, colon) : withoutDigest;
+}
+function canonicalUses(value) {
+    return [...value].sort((left, right) => compare(`${left.task_ids[0]}\0${left.requested_reference}`, `${right.task_ids[0]}\0${right.requested_reference}`));
+}
+function canonicalFallbacks(value) {
+    const unique = new Map(value.map((entry) => [`${entry.task_id}\0${entry.source}\0${entry.service}\0${entry.code}`, entry]));
+    return [...unique.values()].sort((left, right) => compare(`${left.task_id}\0${left.source}\0${left.service}\0${left.code}`, `${right.task_id}\0${right.source}\0${right.service}\0${right.code}`));
+}
+function compare(left, right) {
+    return Buffer.compare(Buffer.from(left), Buffer.from(right));
+}
+//# sourceMappingURL=environment-image-planning.js.map

package/dist/src/evals/environment-image-planning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"environment-image-planning.js","sourceRoot":"","sources":["../../../src/evals/environment-image-planning.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AASpD,MAAM,CAAC,MAAM,kCAAkC,GAAG,aAAa,CAAC;AAEhE,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAS3C;IACC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,kCAAkC,CAAC;IACtE,MAAM,IAAI,GAA4B,EAAE,CAAC;IACzC,MAAM,SAAS,GAAiC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7H,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,IAAI,EAAE,KAAK,CAAC,IAAI;KACjB,CAAC,CAAC,CAAC,CAAC;IACL,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACtD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACtF,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC;IACtF,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,KAAK,CAAC,QAAQ;YAAE,SAAS,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC,CAAC,CAAC;;YAC5G,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC;wBACpC,WAAW,EAAE,KAAK,CAAC,WAAW;wBAC9B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;wBAC1C,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAClD,CAAC,CAAC;oBACH,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;wBAAE,MAAM,IAAI,SAAS,CAAC,8CAA8C,CAAC,CAAC;oBACxI,IAAI,CAAC,IAAI,CAAC;wBACR,QAAQ,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;wBACxB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,mBAAmB,EAAE,KAAK,CAAC,SAAS;wBACpC,SAAS,EAAE,QAAQ,CAAC,SAAS;wBAC7B,eAAe,EAAE,QAAQ,CAAC,eAAe;wBACzC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,UAAU,EAAE,UAAU;wBACtB,SAAS,EAAE,QAAQ,CAAC,SAAS;qBAC9B,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAK,KAA2B,CAAC,IAAI,KAAK,WAAW;wBAAE,MAAM,KAAK,CAAC;oBACnE,SAAS,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC,CAAC,CAAC;;YACzG,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAChC,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC;wBAChC,WAAW,EAAE,KAAK,CAAC,WAAW;wBAC9B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;wBAC1C,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;wBACxC,UAAU,EAAE,KAAK,CAAC,UAAU;wBAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAClD,CAAC,CAAC;oBACH,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC;wBAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;oBAChH,IAAI,CAAC,IAAI,CAAC;wBACR,QAAQ,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;wBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;wBAC9C,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,eAAe,EAAE,KAAK,CAAC,eAAe;wBACtC,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,UAAU,EAAE,UAAU;wBACtB,SAAS,EAAE,KAAK,CAAC,SAAS;qBAC3B,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAK,KAA2B,CAAC,IAAI,KAAK,WAAW;wBAAE,MAAM,KAAK,CAAC;oBACnE,SAAS,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC;IACvF,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,UAAU,CAAC,kFAAkF,EAAE;YACvG,IAAI,EAAE,kCAAkC;YACxC,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,KAA0C,EAAE,eAAuB;IAO1F,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrE,MAAM,EAAE,IAAI,CAAC,OAAO;QACpB,QAAQ,EAAE,IAAI,CAAC,gBAAgB,IAAI,eAAe;QAClD,gBAAgB,EAAE,KAAK,CAAC,iBAAiB;QACzC,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;KAC7D,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,gBAAgB,KAAK,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,gBAAgB,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;AACzK,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAwC;IAC3E,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,UAAU,KAAK,UAAU;YAAE,SAAS;QAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACnD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,KAAK,CAAC,SAAS;YAAE,MAAM,IAAI,SAAS,CAAC,mEAAmE,CAAC,CAAC;QACrJ,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC;IACtD,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AACpG,CAAC;AAED,SAAS,iBAAiB,CAAC,KAA0C,EAAE,eAAuB;IAM5F,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkJ,CAAC;IAC1K,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACtE,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,gBAAgB,IAAI,eAAe,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;YAC3J,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACxE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;AAC1I,CAAC;AAED,SAAS,YAAY,CACnB,OAA0G,EAC1G,IAAqE;IAErE,OAAO,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC5H,CAAC;AAED,SAAS,kBAAkB,CACzB,SAAiB,EACjB,QAAgB,EAChB,QAA2D;IAE3D,OAAO,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;WACjD,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;WACtD,QAAQ,CAAC,QAAQ,KAAK,QAAQ;WAC9B,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC;WAC3D,YAAY,CAAC,SAAS,CAAC,KAAK,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;WAC5D,OAAO,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC;AAC/C,CAAC;AAED,SAAS,eAAe,CACtB,QAAgB,EAChB,KAAuD;IAEvD,OAAO,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;WAC9C,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;WACnD,KAAK,CAAC,QAAQ,KAAK,QAAQ;WAC3B,KAAK,CAAC,SAAS,KAAK,GAAG,KAAK,CAAC,mBAAmB,IAAI,KAAK,CAAC,eAAe,EAAE;WAC3E,mBAAmB,CAAC,KAAK,CAAC,mBAAmB,CAAC;WAC9C,OAAO,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;AAC5C,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACxC,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AAC5H,CAAC;AAED,SAAS,YAAY,CAAC,SAAiB;IACrC,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAW,CAAC;IACxD,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;AACvE,CAAC;AAED,SAAS,aAAa,CAAC,KAA8B;IACnD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,mBAAmB,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;AAC7J,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAmC;IAC7D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5H,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CACvD,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,IAAI,EAAE,EAChE,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,EAAE,CACrE,CAAC,CAAC;AACL,CAAC;AAED,SAAS,OAAO,CAAC,IAAY,EAAE,KAAa;IAC1C,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC"}

package/dist/src/evals/eval-artifact-planning.js

@@ -0,0 +1,56 @@
+import { harborTrialRuntimeContract } from "../backends/index.js";
+import { HitchError } from "../foundation/index.js";
+import { prepareEvalHarborArtifact, preparedHarnessEvent } from "./prepared-harness.js";
+import { preparedArtifactSummary } from "./result-helpers.js";
+export async function prepareEvalArtifactAssignments(input) {
+    const assignments = [];
+    for (const assignment of input.taskRuntimeContracts) {
+        const runtimeContract = harborTrialRuntimeContract(assignment.docker_platform);
+        const prepared = await prepareEvalHarborArtifact({
+            builder: input.builder,
+            root: input.root,
+            resolvedRevision: input.resolvedRevision,
+            requestedReference: input.requestedReference,
+            controllerRuntime: input.controllerRuntime,
+            runtimeContract,
+            ...(input.localTransport ? { localTransport: input.localTransport } : {}),
+            env: input.env,
+            ...(input.signal ? { signal: input.signal } : {}),
+        });
+        assignments.push({ taskIds: assignment.task_ids, runtimeContract, artifact: prepared.artifact });
+        input.sink.emit({
+            ...preparedHarnessEvent(prepared.artifact, prepared.cacheHit, prepared.source, {
+                ...(prepared.builderImage ? { image: prepared.builderImage } : {}),
+                ...(prepared.builderImageId ? { imageId: prepared.builderImageId } : {}),
+            }),
+            task_ids: assignment.task_ids,
+            runtime_contract: assignment,
+        });
+    }
+    const primary = assignments[0]?.artifact;
+    if (!primary)
+        throw new HitchError("planner produced no Harbor artifact assignment", { code: "harbor_artifact_assignment_missing", exitCode: 12 });
+    const artifactsById = new Map(assignments.map((entry) => [entry.artifact.artifact_id, entry.artifact]));
+    if (artifactsById.size !== assignments.length) {
+        throw new HitchError("distinct Harbor runtime contracts produced the same artifact identity", {
+            code: "harbor_artifact_assignment_conflict",
+            exitCode: 12,
+        });
+    }
+    return { assignments, primary, artifactsById };
+}
+export function preparedArtifactPlanFields(assignments) {
+    return {
+        ...(assignments.length === 1 ? { prepared_artifact: preparedArtifactSummary(assignments[0].artifact) } : {}),
+        prepared_artifacts: assignments.map((entry) => ({
+            ...preparedArtifactSummary(entry.artifact),
+            task_ids: entry.taskIds,
+            runtime_contract: {
+                docker_platform: entry.runtimeContract.dockerPlatform,
+                artifact_platform: entry.runtimeContract.artifactPlatform,
+                node_version: entry.runtimeContract.nodeVersion,
+            },
+        })),
+    };
+}
+//# sourceMappingURL=eval-artifact-planning.js.map

package/dist/src/evals/eval-artifact-planning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-artifact-planning.js","sourceRoot":"","sources":["../../../src/evals/eval-artifact-planning.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAGlE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAKpD,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AACxF,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAQ9D,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,KAWpD;IAKC,MAAM,WAAW,GAAqC,EAAE,CAAC;IACzD,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,oBAAoB,EAAE,CAAC;QACpD,MAAM,eAAe,GAAG,0BAA0B,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QAC/E,MAAM,QAAQ,GAAG,MAAM,yBAAyB,CAAC;YAC/C,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;YAC5C,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;YAC1C,eAAe;YACf,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC,CAAC;QACH,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YACd,GAAG,oBAAoB,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE;gBAC7E,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClE,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CAAC;YACF,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;IACL,CAAC;IACD,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC;IACzC,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,UAAU,CAAC,gDAAgD,EAAE,EAAE,IAAI,EAAE,oCAAoC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACnJ,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACxG,IAAI,aAAa,CAAC,IAAI,KAAK,WAAW,CAAC,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,UAAU,CAAC,uEAAuE,EAAE;YAC5F,IAAI,EAAE,qCAAqC;YAC3C,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,WAAsD;IAC/F,OAAO;QACL,GAAG,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7G,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC9C,GAAG,uBAAuB,CAAC,KAAK,CAAC,QAAQ,CAAC;YAC1C,QAAQ,EAAE,KAAK,CAAC,OAAO;YACvB,gBAAgB,EAAE;gBAChB,eAAe,EAAE,KAAK,CAAC,eAAe,CAAC,cAAc;gBACrD,iBAAiB,EAAE,KAAK,CAAC,eAAe,CAAC,gBAAgB;gBACzD,YAAY,EAAE,KAAK,CAAC,eAAe,CAAC,WAAW;aAChD;SACF,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}

package/dist/src/evals/eval-finalization.js

@@ -0,0 +1,10 @@
+import path from "node:path";
+import { atomicWriteJSON } from "../foundation/index.js";
+export async function finalizeEvalResult(directory, sink, result) {
+    sink.emit({ type: "eval.finalizing", status: result.status, settled_trials: Array.isArray(result.trials) ? result.trials.length : 0 });
+    await atomicWriteJSON(path.join(directory, "result.json"), result);
+    sink.emit({ type: result.status === "succeeded" ? "eval.completed" : "eval.failed", status: result.status, exit_code: result.exit_code, error: result.error });
+    await sink.close();
+    return result;
+}
+//# sourceMappingURL=eval-finalization.js.map

package/dist/src/evals/eval-finalization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-finalization.js","sourceRoot":"","sources":["../../../src/evals/eval-finalization.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAIzD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB,EAAE,IAAmB,EAAE,MAAkB;IACjG,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvI,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;IACnE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/J,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC"}