agent-hitch 0.2.6 → 0.2.7

package/dist/src/evals/docker-resource-observer.js

@@ -0,0 +1,207 @@
+import { runCommand } from "../foundation/index.js";
+import { DOCKER_OWNERSHIP_LABELS, dockerOwnershipLabelMap, validateDockerResourceOwnership } from "./docker-ownership.js";
+import { parseExecutionEvidence } from "./execution-evidence.js";
+import { readDockerEngineContainerStats } from "./docker-engine-stats.js";
+export function startDockerResourceObserver(options) {
+    const ownership = validateDockerResourceOwnership(options.ownership);
+    const intervalMs = options.intervalMs ?? 250;
+    if (!Number.isSafeInteger(intervalMs) || intervalMs < 50 || intervalMs > 60_000)
+        throw new TypeError("Docker resource observer interval is invalid");
+    const env = options.env ?? process.env;
+    const command = options.run ?? ((args) => runCommand(env.HITCH_DOCKER_PATH || "docker", args, {
+        env,
+        ...(options.signal ? { signal: options.signal } : {}),
+        timeoutMs: 2_000,
+        failureCode: "docker_resource_observation_failed",
+    }));
+    const engineStats = options.engineStats ?? (options.run ? undefined : (containerId) => readDockerEngineContainerStats(containerId, {
+        env,
+        ...(options.signal ? { signal: options.signal } : {}),
+        timeoutMs: 2_000,
+    }));
+    const startedAt = new Date().toISOString();
+    const containers = new Map();
+    const issues = [];
+    let sampleCount = 0;
+    let tail = Promise.resolve();
+    let pending = false;
+    let unavailable = false;
+    let stopped = false;
+    const enqueue = () => {
+        if (pending || unavailable)
+            return tail;
+        pending = true;
+        tail = tail.then(async () => {
+            if (options.signal?.aborted)
+                return;
+            try {
+                await poll(ownership, command, engineStats, containers, issues);
+                sampleCount += 1;
+            }
+            catch (error) {
+                addIssue(issues, error?.message || String(error));
+                unavailable = true;
+            }
+        }).finally(() => { pending = false; });
+        return tail;
+    };
+    const timer = setInterval(() => { if (!stopped)
+        void enqueue(); }, intervalMs);
+    timer.unref();
+    void enqueue();
+    const evidence = () => parseExecutionEvidence({
+        schema_version: "1",
+        provider: ownership.provider,
+        worker_id: options.workerId,
+        collision_domain_id: options.collisionDomainId,
+        eval_id: ownership.eval_id,
+        work_id: ownership.work_id,
+        lease_id: ownership.lease_id,
+        lease_epoch: ownership.lease_epoch,
+        task_id: ownership.task_id,
+        reservation: options.reservation,
+        enforced: { main_limits: options.mainLimits, sidecar_limits: options.sidecarLimits },
+        observed: observation(startedAt, sampleCount, containers, issues),
+    });
+    return {
+        capture: async () => { await enqueue(); return evidence(); },
+        stop: async () => {
+            stopped = true;
+            clearInterval(timer);
+            await enqueue();
+            return evidence();
+        },
+    };
+}
+async function poll(ownership, command, engineStats, containers, issues) {
+    const labels = dockerOwnershipLabelMap(ownership);
+    const listed = await command([
+        "container", "ls", "--all",
+        "--filter", `label=${DOCKER_OWNERSHIP_LABELS.rootId}=${labels[DOCKER_OWNERSHIP_LABELS.rootId]}`,
+        "--filter", `label=${DOCKER_OWNERSHIP_LABELS.leaseId}=${labels[DOCKER_OWNERSHIP_LABELS.leaseId]}`,
+        "--filter", `label=${DOCKER_OWNERSHIP_LABELS.leaseEpoch}=${labels[DOCKER_OWNERSHIP_LABELS.leaseEpoch]}`,
+        "--format", "{{.ID}}",
+    ]);
+    const ids = [...new Set(listed.stdout.split(/\r?\n/).map((entry) => entry.trim()).filter(Boolean))].sort();
+    if (ids.length > 256 || ids.some((id) => !/^[a-f0-9]{12,64}$/.test(id)))
+        throw new TypeError("Docker resource observation returned invalid container IDs");
+    for (const id of ids) {
+        try {
+            await observeContainer(id, ownership, command, engineStats, containers);
+        }
+        catch (error) {
+            addIssue(issues, error?.message || String(error));
+        }
+    }
+}
+async function observeContainer(id, ownership, command, engineStats, containers) {
+    const inspected = JSON.parse((await command(["container", "inspect", id])).stdout);
+    if (!Array.isArray(inspected) || inspected.length !== 1 || !record(inspected[0]))
+        throw new TypeError("Docker resource observation inspect response is invalid");
+    const item = inspected[0];
+    if (typeof item.Id !== "string" || !/^[a-f0-9]{12,64}$/.test(item.Id) || (!item.Id.startsWith(id) && !id.startsWith(item.Id))) {
+        throw new TypeError("Docker resource observation container identity changed");
+    }
+    const config = record(item.Config) ? item.Config : {};
+    const rawLabels = record(config.Labels) ? config.Labels : {};
+    const epoch = Number(rawLabels[DOCKER_OWNERSHIP_LABELS.leaseEpoch]);
+    const actual = validateDockerResourceOwnership({
+        root_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.rootId] ?? ""),
+        provider: String(rawLabels[DOCKER_OWNERSHIP_LABELS.provider] ?? ""),
+        eval_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.evalId] ?? ""),
+        work_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.workId] ?? ""),
+        lease_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.leaseId] ?? ""),
+        lease_epoch: epoch,
+        ...(rawLabels[DOCKER_OWNERSHIP_LABELS.taskId] === undefined ? {} : { task_id: String(rawLabels[DOCKER_OWNERSHIP_LABELS.taskId]) }),
+    });
+    const expectedLabels = dockerOwnershipLabelMap(ownership);
+    const actualLabels = dockerOwnershipLabelMap(actual);
+    if (Object.keys(expectedLabels).some((key) => actualLabels[key] !== expectedLabels[key]))
+        throw new TypeError("Docker resource observation ownership changed");
+    const now = new Date().toISOString();
+    const previous = containers.get(item.Id);
+    const observed = previous ?? { container_id: item.Id, first_observed_at: now, last_observed_at: now };
+    observed.last_observed_at = now;
+    if (typeof item.Name === "string" && item.Name.replace(/^\//, ""))
+        observed.name = item.Name.replace(/^\//, "").replace(/[\0\r\n]/g, " ").slice(0, 256);
+    if (typeof config.Image === "string" && config.Image && config.Image.length <= 1_024 && !/[\0\r\n]/.test(config.Image))
+        observed.image_reference = config.Image;
+    if (typeof item.Image === "string" && /^sha256:[a-f0-9]{64}$/.test(item.Image))
+        observed.image_config_digest = item.Image;
+    const state = record(item.State) ? item.State : {};
+    if (typeof state.OOMKilled === "boolean")
+        observed.oom_killed = Boolean(observed.oom_killed || state.OOMKilled);
+    if (state.Running === false && Number.isSafeInteger(state.ExitCode) && state.ExitCode >= 0 && state.ExitCode <= 255) {
+        observed.exit_code = state.ExitCode;
+        const rawReason = typeof state.Error === "string" && state.Error.trim()
+            ? state.Error.trim()
+            : state.OOMKilled === true ? "oom-killed" : `exit-code-${state.ExitCode}`;
+        observed.exit_reason = rawReason.replace(/[\0\r\n]/g, " ").slice(0, 512);
+    }
+    if (state.Running === true) {
+        if (engineStats) {
+            try {
+                const stats = await engineStats(item.Id);
+                if (stats.cpu_time_ns !== undefined)
+                    observed.cpu_time_ns = Math.max(observed.cpu_time_ns ?? 0, stats.cpu_time_ns);
+                if (stats.memory_bytes !== undefined)
+                    observed.peak_memory_bytes = Math.max(observed.peak_memory_bytes ?? 0, stats.memory_bytes);
+            }
+            catch { /* CLI evidence remains useful when the Engine stats API is unavailable. */ }
+        }
+        if (observed.peak_memory_bytes === undefined)
+            try {
+                const stats = await command(["container", "stats", "--no-stream", "--format", "{{json .}}", item.Id]);
+                const line = stats.stdout.split(/\r?\n/).find((entry) => entry.trim());
+                const parsed = line ? JSON.parse(line) : null;
+                const memory = record(parsed) ? parseDockerMemoryBytes(parsed.MemUsage) : null;
+                if (memory !== null)
+                    observed.peak_memory_bytes = Math.max(observed.peak_memory_bytes ?? 0, memory);
+            }
+            catch { /* Inspect evidence remains useful when stats is unavailable. */ }
+    }
+    containers.set(item.Id, observed);
+}
+function observation(startedAt, sampleCount, values, issues) {
+    const containers = [...values.values()].sort((left, right) => left.container_id.localeCompare(right.container_id));
+    const unavailable = [];
+    if (!containers.some((entry) => entry.cpu_time_ns !== undefined))
+        unavailable.push("cpu_time_ns");
+    if (!containers.some((entry) => entry.peak_memory_bytes !== undefined))
+        unavailable.push("peak_memory_bytes");
+    if (!containers.some((entry) => entry.exit_code !== undefined || entry.oom_killed !== undefined))
+        unavailable.push("exit_status");
+    if (!containers.some((entry) => entry.image_reference !== undefined || entry.image_config_digest !== undefined))
+        unavailable.push("image_identity");
+    return {
+        status: containers.length > 0 ? "partial" : "unavailable",
+        started_at: startedAt,
+        collected_at: new Date().toISOString(),
+        sample_count: sampleCount,
+        containers,
+        unavailable_fields: unavailable,
+        issues: [...issues],
+    };
+}
+export function parseDockerMemoryBytes(value) {
+    if (typeof value !== "string")
+        return null;
+    const first = value.split("/", 1)[0]?.trim() ?? "";
+    const match = first.match(/^([0-9]+(?:\.[0-9]+)?)\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)$/);
+    if (!match)
+        return null;
+    const amount = Number(match[1]);
+    const unit = match[2];
+    const powers = { B: 1, kB: 1_000, KB: 1_000, KiB: 1024, MB: 1_000_000, MiB: 1024 ** 2, GB: 1_000_000_000, GiB: 1024 ** 3, TB: 1_000_000_000_000, TiB: 1024 ** 4 };
+    const bytes = amount * powers[unit];
+    return Number.isSafeInteger(bytes) && bytes >= 0 ? bytes : null;
+}
+function addIssue(issues, message) {
+    const bounded = message.replace(/[\0\r\n]/g, " ").slice(0, 512) || "Docker resource observation failed";
+    if (!issues.includes(bounded) && issues.length < 32)
+        issues.push(bounded);
+}
+function record(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+//# sourceMappingURL=docker-resource-observer.js.map

package/dist/src/evals/docker-resource-observer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker-resource-observer.js","sourceRoot":"","sources":["../../../src/evals/docker-resource-observer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,+BAA+B,EAAE,MAAM,uBAAuB,CAAC;AAC1H,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAmC1E,MAAM,UAAU,2BAA2B,CAAC,OAAsC;IAChF,MAAM,SAAS,GAAG,+BAA+B,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,EAAE,IAAI,UAAU,GAAG,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,8CAA8C,CAAC,CAAC;IACrJ,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,IAAc,EAAE,EAAE,CAAC,UAAU,CAC5D,GAAG,CAAC,iBAAiB,IAAI,QAAQ,EACjC,IAAI,EACJ;QACE,GAAG;QACH,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,SAAS,EAAE,KAAK;QAChB,WAAW,EAAE,oCAAoC;KAClD,CACF,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAmB,EAAE,EAAE,CAAC,8BAA8B,CAAC,WAAW,EAAE;QACzI,GAAG;QACH,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,SAAS,EAAE,KAAK;KACjB,CAAC,CAAC,CAAC;IACJ,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,IAAI,GAAG,EAA4B,CAAC;IACvD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,OAAO,GAAG,GAAkB,EAAE;QAClC,IAAI,OAAO,IAAI,WAAW;YAAE,OAAO,IAAI,CAAC;QACxC,OAAO,GAAG,IAAI,CAAC;QACf,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC1B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,OAAO;YACpC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;gBAChE,WAAW,IAAI,CAAC,CAAC;YACnB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CAAC,MAAM,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7D,WAAW,GAAG,IAAI,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO;QAAE,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/E,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,KAAK,OAAO,EAAE,CAAC;IACf,MAAM,QAAQ,GAAG,GAAwB,EAAE,CAAC,sBAAsB,CAAC;QACjE,cAAc,EAAE,GAAG;QACnB,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,SAAS,EAAE,OAAO,CAAC,QAAQ;QAC3B,mBAAmB,EAAE,OAAO,CAAC,iBAAiB;QAC9C,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,WAAW,EAAE,SAAS,CAAC,WAAW;QAClC,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,QAAQ,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,aAAa,EAAE;QACpF,QAAQ,EAAE,WAAW,CAAC,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC;KAClE,CAAC,CAAC;IACH,OAAO;QACL,OAAO,EAAE,KAAK,IAAI,EAAE,GAAG,MAAM,OAAO,EAAE,CAAC,CAAC,OAAO,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC5D,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,OAAO,GAAG,IAAI,CAAC;YACf,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,OAAO,EAAE,CAAC;YAChB,OAAO,QAAQ,EAAE,CAAC;QACpB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI,CACjB,SAAoC,EACpC,OAAyE,EACzE,WAA4G,EAC5G,UAAyC,EACzC,MAAgB;IAEhB,MAAM,MAAM,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;QAC3B,WAAW,EAAE,IAAI,EAAE,OAAO;QAC1B,UAAU,EAAE,SAAS,uBAAuB,CAAC,MAAM,IAAI,MAAM,CAAC,uBAAuB,CAAC,MAAM,CAAC,EAAE;QAC/F,UAAU,EAAE,SAAS,uBAAuB,CAAC,OAAO,IAAI,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,EAAE;QACjG,UAAU,EAAE,SAAS,uBAAuB,CAAC,UAAU,IAAI,MAAM,CAAC,uBAAuB,CAAC,UAAU,CAAC,EAAE;QACvG,UAAU,EAAE,SAAS;KACtB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3G,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IAC3J,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC;YAAC,MAAM,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QAAC,CAAC;QAChF,OAAO,KAAK,EAAE,CAAC;YAAC,QAAQ,CAAC,MAAM,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EAAU,EACV,SAAoC,EACpC,OAAyE,EACzE,WAA4G,EAC5G,UAAyC;IAEzC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAY,CAAC;IAC9F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,yDAAyD,CAAC,CAAC;IACjK,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC9H,MAAM,IAAI,SAAS,CAAC,wDAAwD,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACtD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxF,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,UAAU,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,+BAA+B,CAAC;QAC7C,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChE,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnE,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChE,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAChE,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClE,WAAW,EAAE,KAAK;QAClB,GAAG,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;KACnI,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,cAAc,CAAC,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IAC/J,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAqB,QAAQ,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC;IACxH,QAAQ,CAAC,gBAAgB,GAAG,GAAG,CAAC;IAChC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;QAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACxJ,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,QAAQ,CAAC,eAAe,GAAG,MAAM,CAAC,KAAK,CAAC;IAChK,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,QAAQ,CAAC,mBAAmB,GAAG,IAAI,CAAC,KAAe,CAAC;IACpI,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACnD,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,SAAS;QAAE,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAChH,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAK,KAAK,CAAC,QAAmB,IAAI,CAAC,IAAK,KAAK,CAAC,QAAmB,IAAI,GAAG,EAAE,CAAC;QAC5I,QAAQ,CAAC,SAAS,GAAG,KAAK,CAAC,QAAkB,CAAC;QAC9C,MAAM,SAAS,GAAG,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;YACrE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;YACpB,CAAC,CAAC,KAAK,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC5E,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC3B,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACzC,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS;oBAAE,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;gBACnH,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS;oBAAE,QAAQ,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,iBAAiB,IAAI,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;YACnI,CAAC;YAAC,MAAM,CAAC,CAAC,2EAA2E,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,QAAQ,CAAC,iBAAiB,KAAK,SAAS;YAAE,IAAI,CAAC;gBACjD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,CAAC,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;gBACtG,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvE,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;gBACzD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC/E,IAAI,MAAM,KAAK,IAAI;oBAAE,QAAQ,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,iBAAiB,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;YACtG,CAAC;YAAC,MAAM,CAAC,CAAC,gEAAgE,CAAC,CAAC;IAC9E,CAAC;IACD,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,WAAW,CAClB,SAAiB,EACjB,WAAmB,EACnB,MAAqC,EACrC,MAAgB;IAEhB,MAAM,UAAU,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,CAAmC,CAAC;IACrJ,MAAM,WAAW,GAA0D,EAAE,CAAC;IAC9E,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAClG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,KAAK,SAAS,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC9G,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAClI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACpJ,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa;QACzD,UAAU,EAAE,SAAS;QACrB,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACtC,YAAY,EAAE,WAAW;QACzB,UAAU;QACV,kBAAkB,EAAE,WAAW;QAC/B,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC;KACpB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAc;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACnD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAC1F,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAW,CAAC;IAChC,MAAM,MAAM,GAA2B,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,EAAE,EAAE,EAAE,aAAa,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,EAAE,EAAE,EAAE,iBAAiB,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;IAC1L,MAAM,KAAK,GAAG,MAAM,GAAI,MAAM,CAAC,IAAI,CAAY,CAAC;IAChD,OAAO,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAClE,CAAC;AAED,SAAS,QAAQ,CAAC,MAAgB,EAAE,OAAe;IACjD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,oCAAoC,CAAC;IACxG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}

package/dist/src/evals/environment-image-planning.js

@@ -0,0 +1,164 @@
+import { HitchError } from "../foundation/index.js";
+export const DEFAULT_ENVIRONMENT_IMAGE_PLATFORM = "linux/amd64";
+export async function planEnvironmentImages(input) {
+    const platform = input.platform ?? DEFAULT_ENVIRONMENT_IMAGE_PLATFORM;
+    const uses = [];
+    const fallbacks = input.tasks.flatMap((task) => task.environment_image_fallbacks.map((entry) => ({
+        task_id: task.task_id,
+        source: entry.source,
+        service: entry.service,
+        code: entry.code,
+    })));
+    const requests = canonicalRequests(input.tasks, platform);
+    const builds = canonicalBuilds(input.tasks, platform);
+    if (input.mode === "backend") {
+        fallbacks.push(...requests.flatMap((entry) => fallbacksFor(entry, "policy-backend")));
+        fallbacks.push(...builds.flatMap((entry) => fallbacksFor(entry, "policy-backend")));
+    }
+    else {
+        if (!input.resolver)
+            fallbacks.push(...requests.flatMap((entry) => fallbacksFor(entry, "resolver-unavailable")));
+        else
+            for (const entry of requests) {
+                try {
+                    const resolved = await input.resolver({
+                        benchmarkId: input.benchmarkId,
+                        benchmarkRevision: input.benchmarkRevision,
+                        taskId: entry.taskId,
+                        reference: entry.reference,
+                        platform: entry.platform,
+                        ...(input.signal ? { signal: input.signal } : {}),
+                    });
+                    if (!validResolvedImage(entry.reference, entry.platform, resolved))
+                        throw new TypeError("environment image resolver output is invalid");
+                    uses.push({
+                        task_ids: [entry.taskId],
+                        image_id: resolved.image_id,
+                        requested_reference: entry.reference,
+                        reference: resolved.reference,
+                        manifest_digest: resolved.manifest_digest,
+                        platform: resolved.platform,
+                        resolution: "registry",
+                        cache_hit: resolved.cache_hit,
+                    });
+                }
+                catch (error) {
+                    if (error.code === "cancelled")
+                        throw error;
+                    fallbacks.push(...fallbacksFor(entry, "resolution-failed"));
+                }
+            }
+        if (!input.builder)
+            fallbacks.push(...builds.flatMap((entry) => fallbacksFor(entry, "resolver-unavailable")));
+        else
+            for (const entry of builds) {
+                try {
+                    const built = await input.builder({
+                        benchmarkId: input.benchmarkId,
+                        benchmarkRevision: input.benchmarkRevision,
+                        taskId: entry.taskId,
+                        contextDirectory: entry.contextDirectory,
+                        dockerfile: entry.dockerfile,
+                        platform: entry.platform,
+                        ...(input.signal ? { signal: input.signal } : {}),
+                    });
+                    if (!validBuiltImage(entry.platform, built))
+                        throw new TypeError("environment image builder output is invalid");
+                    uses.push({
+                        task_ids: [entry.taskId],
+                        image_id: built.image_id,
+                        requested_reference: built.requested_reference,
+                        reference: built.reference,
+                        manifest_digest: built.manifest_digest,
+                        platform: built.platform,
+                        resolution: "prebuilt",
+                        cache_hit: built.cache_hit,
+                    });
+                }
+                catch (error) {
+                    if (error.code === "cancelled")
+                        throw error;
+                    fallbacks.push(...fallbacksFor(entry, "resolution-failed"));
+                }
+            }
+    }
+    const result = { uses: canonicalUses(uses), fallbacks: canonicalFallbacks(fallbacks) };
+    if (input.mode === "prebuild-required" && result.fallbacks.length > 0) {
+        throw new HitchError("required environment image prebuild is unavailable for one or more task services", {
+            code: "environment_prebuild_unavailable",
+            exitCode: 10,
+        });
+    }
+    return result;
+}
+function canonicalBuilds(tasks, defaultPlatform) {
+    return tasks.flatMap((task) => task.environment_builds.map((entry) => ({
+        taskId: task.task_id,
+        platform: task.runtime_platform ?? defaultPlatform,
+        contextDirectory: entry.context_directory,
+        dockerfile: entry.dockerfile,
+        bindings: [{ source: entry.source, service: entry.service }],
+    }))).sort((left, right) => compare(`${left.taskId}\0${left.contextDirectory}\0${left.dockerfile}`, `${right.taskId}\0${right.contextDirectory}\0${right.dockerfile}`));
+}
+export function resolvedImageMapping(images) {
+    const result = {};
+    for (const image of images) {
+        if (image.resolution === "prebuilt")
+            continue;
+        const existing = result[image.requested_reference];
+        if (existing !== undefined && existing !== image.reference)
+            throw new TypeError("environment image reference resolved to multiple immutable images");
+        result[image.requested_reference] = image.reference;
+    }
+    return Object.fromEntries(Object.entries(result).sort(([left], [right]) => compare(left, right)));
+}
+function canonicalRequests(tasks, defaultPlatform) {
+    const grouped = new Map();
+    for (const task of tasks)
+        for (const entry of task.environment_images) {
+            const key = `${task.task_id}\0${entry.reference}`;
+            const request = grouped.get(key) ?? { taskId: task.task_id, platform: task.runtime_platform ?? defaultPlatform, reference: entry.reference, bindings: [] };
+            request.bindings.push({ source: entry.source, service: entry.service });
+            grouped.set(key, request);
+        }
+    return [...grouped.values()].sort((left, right) => compare(`${left.taskId}\0${left.reference}`, `${right.taskId}\0${right.reference}`));
+}
+function fallbacksFor(request, code) {
+    return request.bindings.map((entry) => ({ task_id: request.taskId, source: entry.source, service: entry.service, code }));
+}
+function validResolvedImage(requested, platform, resolved) {
+    return /^sha256:[a-f0-9]{64}$/.test(resolved.image_id)
+        && /^sha256:[a-f0-9]{64}$/.test(resolved.manifest_digest)
+        && resolved.platform === platform
+        && resolved.reference.endsWith(`@${resolved.manifest_digest}`)
+        && repositoryOf(requested) === repositoryOf(resolved.reference)
+        && typeof resolved.cache_hit === "boolean";
+}
+function validBuiltImage(platform, built) {
+    return /^sha256:[a-f0-9]{64}$/.test(built.image_id)
+        && /^sha256:[a-f0-9]{64}$/.test(built.manifest_digest)
+        && built.platform === platform
+        && built.reference === `${built.requested_reference}@${built.manifest_digest}`
+        && validImageReference(built.requested_reference)
+        && typeof built.cache_hit === "boolean";
+}
+function validImageReference(value) {
+    return Boolean(value) && value.length <= 1_024 && !/[\s\0]/.test(value) && !value.includes("://") && !value.includes("$");
+}
+function repositoryOf(reference) {
+    const withoutDigest = reference.split("@")[0];
+    const slash = withoutDigest.lastIndexOf("/");
+    const colon = withoutDigest.lastIndexOf(":");
+    return colon > slash ? withoutDigest.slice(0, colon) : withoutDigest;
+}
+function canonicalUses(value) {
+    return [...value].sort((left, right) => compare(`${left.task_ids[0]}\0${left.requested_reference}`, `${right.task_ids[0]}\0${right.requested_reference}`));
+}
+function canonicalFallbacks(value) {
+    const unique = new Map(value.map((entry) => [`${entry.task_id}\0${entry.source}\0${entry.service}\0${entry.code}`, entry]));
+    return [...unique.values()].sort((left, right) => compare(`${left.task_id}\0${left.source}\0${left.service}\0${left.code}`, `${right.task_id}\0${right.source}\0${right.service}\0${right.code}`));
+}
+function compare(left, right) {
+    return Buffer.compare(Buffer.from(left), Buffer.from(right));
+}
+//# sourceMappingURL=environment-image-planning.js.map

package/dist/src/evals/environment-image-planning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"environment-image-planning.js","sourceRoot":"","sources":["../../../src/evals/environment-image-planning.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AASpD,MAAM,CAAC,MAAM,kCAAkC,GAAG,aAAa,CAAC;AAEhE,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAS3C;IACC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,kCAAkC,CAAC;IACtE,MAAM,IAAI,GAA4B,EAAE,CAAC;IACzC,MAAM,SAAS,GAAiC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7H,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,IAAI,EAAE,KAAK,CAAC,IAAI;KACjB,CAAC,CAAC,CAAC,CAAC;IACL,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACtD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACtF,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC;IACtF,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,KAAK,CAAC,QAAQ;YAAE,SAAS,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC,CAAC,CAAC;;YAC5G,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC;wBACpC,WAAW,EAAE,KAAK,CAAC,WAAW;wBAC9B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;wBAC1C,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAClD,CAAC,CAAC;oBACH,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;wBAAE,MAAM,IAAI,SAAS,CAAC,8CAA8C,CAAC,CAAC;oBACxI,IAAI,CAAC,IAAI,CAAC;wBACR,QAAQ,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;wBACxB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,mBAAmB,EAAE,KAAK,CAAC,SAAS;wBACpC,SAAS,EAAE,QAAQ,CAAC,SAAS;wBAC7B,eAAe,EAAE,QAAQ,CAAC,eAAe;wBACzC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,UAAU,EAAE,UAAU;wBACtB,SAAS,EAAE,QAAQ,CAAC,SAAS;qBAC9B,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAK,KAA2B,CAAC,IAAI,KAAK,WAAW;wBAAE,MAAM,KAAK,CAAC;oBACnE,SAAS,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC,CAAC,CAAC;;YACzG,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAChC,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC;wBAChC,WAAW,EAAE,KAAK,CAAC,WAAW;wBAC9B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;wBAC1C,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;wBACxC,UAAU,EAAE,KAAK,CAAC,UAAU;wBAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAClD,CAAC,CAAC;oBACH,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC;wBAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;oBAChH,IAAI,CAAC,IAAI,CAAC;wBACR,QAAQ,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;wBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;wBAC9C,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,eAAe,EAAE,KAAK,CAAC,eAAe;wBACtC,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,UAAU,EAAE,UAAU;wBACtB,SAAS,EAAE,KAAK,CAAC,SAAS;qBAC3B,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAK,KAA2B,CAAC,IAAI,KAAK,WAAW;wBAAE,MAAM,KAAK,CAAC;oBACnE,SAAS,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC;IACvF,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,UAAU,CAAC,kFAAkF,EAAE;YACvG,IAAI,EAAE,kCAAkC;YACxC,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,KAA0C,EAAE,eAAuB;IAO1F,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrE,MAAM,EAAE,IAAI,CAAC,OAAO;QACpB,QAAQ,EAAE,IAAI,CAAC,gBAAgB,IAAI,eAAe;QAClD,gBAAgB,EAAE,KAAK,CAAC,iBAAiB;QACzC,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;KAC7D,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,gBAAgB,KAAK,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,gBAAgB,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;AACzK,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAwC;IAC3E,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,UAAU,KAAK,UAAU;YAAE,SAAS;QAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACnD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,KAAK,CAAC,SAAS;YAAE,MAAM,IAAI,SAAS,CAAC,mEAAmE,CAAC,CAAC;QACrJ,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC;IACtD,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AACpG,CAAC;AAED,SAAS,iBAAiB,CAAC,KAA0C,EAAE,eAAuB;IAM5F,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkJ,CAAC;IAC1K,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACtE,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,gBAAgB,IAAI,eAAe,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;YAC3J,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACxE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC5B,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;AAC1I,CAAC;AAED,SAAS,YAAY,CACnB,OAA0G,EAC1G,IAAqE;IAErE,OAAO,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC5H,CAAC;AAED,SAAS,kBAAkB,CACzB,SAAiB,EACjB,QAAgB,EAChB,QAA2D;IAE3D,OAAO,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;WACjD,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;WACtD,QAAQ,CAAC,QAAQ,KAAK,QAAQ;WAC9B,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC;WAC3D,YAAY,CAAC,SAAS,CAAC,KAAK,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;WAC5D,OAAO,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC;AAC/C,CAAC;AAED,SAAS,eAAe,CACtB,QAAgB,EAChB,KAAuD;IAEvD,OAAO,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;WAC9C,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;WACnD,KAAK,CAAC,QAAQ,KAAK,QAAQ;WAC3B,KAAK,CAAC,SAAS,KAAK,GAAG,KAAK,CAAC,mBAAmB,IAAI,KAAK,CAAC,eAAe,EAAE;WAC3E,mBAAmB,CAAC,KAAK,CAAC,mBAAmB,CAAC;WAC9C,OAAO,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;AAC5C,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACxC,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AAC5H,CAAC;AAED,SAAS,YAAY,CAAC,SAAiB;IACrC,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAW,CAAC;IACxD,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;AACvE,CAAC;AAED,SAAS,aAAa,CAAC,KAA8B;IACnD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,mBAAmB,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;AAC7J,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAmC;IAC7D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5H,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CACvD,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,IAAI,EAAE,EAChE,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,EAAE,CACrE,CAAC,CAAC;AACL,CAAC;AAED,SAAS,OAAO,CAAC,IAAY,EAAE,KAAa;IAC1C,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC"}

package/dist/src/evals/eval-artifact-planning.js

@@ -0,0 +1,56 @@
+import { harborTrialRuntimeContract } from "../backends/index.js";
+import { HitchError } from "../foundation/index.js";
+import { prepareEvalHarborArtifact, preparedHarnessEvent } from "./prepared-harness.js";
+import { preparedArtifactSummary } from "./result-helpers.js";
+export async function prepareEvalArtifactAssignments(input) {
+    const assignments = [];
+    for (const assignment of input.taskRuntimeContracts) {
+        const runtimeContract = harborTrialRuntimeContract(assignment.docker_platform);
+        const prepared = await prepareEvalHarborArtifact({
+            builder: input.builder,
+            root: input.root,
+            resolvedRevision: input.resolvedRevision,
+            requestedReference: input.requestedReference,
+            controllerRuntime: input.controllerRuntime,
+            runtimeContract,
+            ...(input.localTransport ? { localTransport: input.localTransport } : {}),
+            env: input.env,
+            ...(input.signal ? { signal: input.signal } : {}),
+        });
+        assignments.push({ taskIds: assignment.task_ids, runtimeContract, artifact: prepared.artifact });
+        input.sink.emit({
+            ...preparedHarnessEvent(prepared.artifact, prepared.cacheHit, prepared.source, {
+                ...(prepared.builderImage ? { image: prepared.builderImage } : {}),
+                ...(prepared.builderImageId ? { imageId: prepared.builderImageId } : {}),
+            }),
+            task_ids: assignment.task_ids,
+            runtime_contract: assignment,
+        });
+    }
+    const primary = assignments[0]?.artifact;
+    if (!primary)
+        throw new HitchError("planner produced no Harbor artifact assignment", { code: "harbor_artifact_assignment_missing", exitCode: 12 });
+    const artifactsById = new Map(assignments.map((entry) => [entry.artifact.artifact_id, entry.artifact]));
+    if (artifactsById.size !== assignments.length) {
+        throw new HitchError("distinct Harbor runtime contracts produced the same artifact identity", {
+            code: "harbor_artifact_assignment_conflict",
+            exitCode: 12,
+        });
+    }
+    return { assignments, primary, artifactsById };
+}
+export function preparedArtifactPlanFields(assignments) {
+    return {
+        ...(assignments.length === 1 ? { prepared_artifact: preparedArtifactSummary(assignments[0].artifact) } : {}),
+        prepared_artifacts: assignments.map((entry) => ({
+            ...preparedArtifactSummary(entry.artifact),
+            task_ids: entry.taskIds,
+            runtime_contract: {
+                docker_platform: entry.runtimeContract.dockerPlatform,
+                artifact_platform: entry.runtimeContract.artifactPlatform,
+                node_version: entry.runtimeContract.nodeVersion,
+            },
+        })),
+    };
+}
+//# sourceMappingURL=eval-artifact-planning.js.map

package/dist/src/evals/eval-artifact-planning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-artifact-planning.js","sourceRoot":"","sources":["../../../src/evals/eval-artifact-planning.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAGlE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAKpD,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AACxF,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAQ9D,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,KAWpD;IAKC,MAAM,WAAW,GAAqC,EAAE,CAAC;IACzD,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,oBAAoB,EAAE,CAAC;QACpD,MAAM,eAAe,GAAG,0BAA0B,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QAC/E,MAAM,QAAQ,GAAG,MAAM,yBAAyB,CAAC;YAC/C,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;YAC5C,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;YAC1C,eAAe;YACf,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC,CAAC;QACH,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YACd,GAAG,oBAAoB,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE;gBAC7E,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClE,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CAAC;YACF,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;IACL,CAAC;IACD,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC;IACzC,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,UAAU,CAAC,gDAAgD,EAAE,EAAE,IAAI,EAAE,oCAAoC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACnJ,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACxG,IAAI,aAAa,CAAC,IAAI,KAAK,WAAW,CAAC,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,UAAU,CAAC,uEAAuE,EAAE;YAC5F,IAAI,EAAE,qCAAqC;YAC3C,QAAQ,EAAE,EAAE;SACb,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,WAAsD;IAC/F,OAAO;QACL,GAAG,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7G,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC9C,GAAG,uBAAuB,CAAC,KAAK,CAAC,QAAQ,CAAC;YAC1C,QAAQ,EAAE,KAAK,CAAC,OAAO;YACvB,gBAAgB,EAAE;gBAChB,eAAe,EAAE,KAAK,CAAC,eAAe,CAAC,cAAc;gBACrD,iBAAiB,EAAE,KAAK,CAAC,eAAe,CAAC,gBAAgB;gBACzD,YAAY,EAAE,KAAK,CAAC,eAAe,CAAC,WAAW;aAChD;SACF,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}

package/dist/src/evals/eval-finalization.js

@@ -0,0 +1,10 @@
+import path from "node:path";
+import { atomicWriteJSON } from "../foundation/index.js";
+export async function finalizeEvalResult(directory, sink, result) {
+    sink.emit({ type: "eval.finalizing", status: result.status, settled_trials: Array.isArray(result.trials) ? result.trials.length : 0 });
+    await atomicWriteJSON(path.join(directory, "result.json"), result);
+    sink.emit({ type: result.status === "succeeded" ? "eval.completed" : "eval.failed", status: result.status, exit_code: result.exit_code, error: result.error });
+    await sink.close();
+    return result;
+}
+//# sourceMappingURL=eval-finalization.js.map

package/dist/src/evals/eval-finalization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-finalization.js","sourceRoot":"","sources":["../../../src/evals/eval-finalization.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAIzD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB,EAAE,IAAmB,EAAE,MAAkB;IACjG,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvI,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;IACnE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/J,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACnB,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/evals/eval-lifecycle-events.js

@@ -0,0 +1,16 @@
+export function emitEvalPlanLifecycle(sink, plan, planningStartedAt) {
+    sink.emit({
+        type: "eval.plan.created",
+        duration_ms: Date.now() - planningStartedAt,
+        work_items: plan.work_items.length,
+        membership: plan.membership,
+    });
+    for (const workItem of plan.work_items)
+        sink.emit({
+            type: "eval.work.queued",
+            work_id: workItem.work_id,
+            task_ids: workItem.task_ids,
+            reservation: workItem.reservation,
+        });
+}
+//# sourceMappingURL=eval-lifecycle-events.js.map

package/dist/src/evals/eval-lifecycle-events.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-lifecycle-events.js","sourceRoot":"","sources":["../../../src/evals/eval-lifecycle-events.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,qBAAqB,CAAC,IAAmB,EAAE,IAAyB,EAAE,iBAAyB;IAC7G,IAAI,CAAC,IAAI,CAAC;QACR,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,iBAAiB;QAC3C,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;QAClC,UAAU,EAAE,IAAI,CAAC,UAAU;KAC5B,CAAC,CAAC;IACH,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU;QAAE,IAAI,CAAC,IAAI,CAAC;YAChD,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,WAAW,EAAE,QAAQ,CAAC,WAAW;SAClC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/evals/events.js

@@ -1,6 +1,8 @@
 import { createWriteStream } from "node:fs";
+import { open, stat } from "node:fs/promises";
 import path from "node:path";
 import { SCHEMA_VERSION, ensureDir } from "../foundation/index.js";
+export const MAX_EVAL_EVENT_BYTES = 64 * 1024;
 export class EvalEventSink {
     path;
     evalId;
@@ -16,23 +18,33 @@ export class EvalEventSink {
     }
     async open() {
         await ensureDir(path.dirname(this.path));
+        this.sequence = await lastCommittedSequence(this.path);
         this.stream = createWriteStream(this.path, { flags: "a", mode: 0o600 });
         this.stream.on("error", (error) => { this.streamError ||= error; });
     }
     emit(event) {
-        const framed = {
+        const identity = {
             schema_version: SCHEMA_VERSION,
             sequence: ++this.sequence,
             timestamp: new Date().toISOString(),
             eval_id: this.evalId,
-            ...event,
         };
-        this.pending = this.pending.then(() => writeChunk(this.stream, `${JSON.stringify(framed)}\n`));
+        const framed = { ...identity, ...event };
+        const encoded = JSON.stringify(framed);
+        const persisted = Buffer.byteLength(encoded) <= MAX_EVAL_EVENT_BYTES
+            ? framed
+            : {
+                ...identity,
+                type: boundedType(event.type),
+                truncated: true,
+                original_bytes: Buffer.byteLength(encoded),
+            };
+        this.pending = this.pending.then(() => writeChunk(this.stream, `${JSON.stringify(persisted)}\n`));
         try {
-            this.onEvent(framed);
+            this.onEvent(persisted);
         }
         catch { /* Observers cannot break eval persistence. */ }
-        return framed;
+        return persisted;
     }
     async close() {
         let failure;
@@ -53,6 +65,42 @@ export class EvalEventSink {
             throw failure;
     }
 }
+function boundedType(value) {
+    const type = typeof value === "string" && value ? value : "eval.event.truncated";
+    return type.slice(0, 256);
+}
+async function lastCommittedSequence(file) {
+    let size;
+    try {
+        size = (await stat(file)).size;
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return 0;
+        throw error;
+    }
+    if (size === 0)
+        return 0;
+    const handle = await open(file, "r");
+    try {
+        const length = Math.min(size, 64 * 1024);
+        const buffer = Buffer.allocUnsafe(length);
+        const { bytesRead } = await handle.read(buffer, 0, length, size - length);
+        const committed = buffer.subarray(0, bytesRead).toString("utf8").replace(/[^\n]*$/, "");
+        const lines = committed.trimEnd().split("\n");
+        const last = lines.at(-1);
+        if (!last)
+            return 0;
+        const parsed = JSON.parse(last);
+        if (!Number.isSafeInteger(parsed.sequence) || parsed.sequence < 0) {
+            throw new TypeError("eval event sequence is invalid");
+        }
+        return parsed.sequence;
+    }
+    finally {
+        await handle.close();
+    }
+}
 function writeChunk(stream, chunk) {
     return new Promise((resolve, reject) => stream.write(chunk, (error) => error ? reject(error) : resolve()));
 }

package/dist/src/evals/events.js.map

@@ -1 +1 @@
-{"version":3,"file":"events.js","sourceRoot":"","sources":["../../../src/evals/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAEnE,MAAM,OAAO,aAAa;IACf,IAAI,CAAS;IACL,MAAM,CAAS;IACf,OAAO,CAA2C;IAC3D,QAAQ,GAAG,CAAC,CAAC;IACb,OAAO,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3C,MAAM,CAA0B;IAChC,WAAW,CAAoB;IAEvC,YAAY,aAAqB,EAAE,MAAc,EAAE,UAAoD,GAAG,EAAE,GAAE,CAAC;QAC7G,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,CAAC,KAA8B;QACjC,MAAM,MAAM,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,QAAQ,EAAE,EAAE,IAAI,CAAC,QAAQ;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,IAAI,CAAC,MAAM;YACpB,GAAG,KAAK;SACT,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAqB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9G,IAAI,CAAC;YAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,8CAA8C,CAAC,CAAC;QACtF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,OAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,MAAM,IAAI,CAAC,OAAO,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,GAAG,KAAc,CAAC;QAAC,CAAC;QACvE,IAAI,CAAC;YAAC,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,KAAK,KAAc,CAAC;QAAC,CAAC;QACrF,OAAO,KAAK,IAAI,CAAC,WAAW,CAAC;QAC7B,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC;IAC7B,CAAC;CACF;AAED,SAAS,UAAU,CAAC,MAAmB,EAAE,KAAa;IACpD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,WAAW,CAAC,MAA+B;IAClD,IAAI,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"events.js","sourceRoot":"","sources":["../../../src/evals/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAEnE,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,CAAC;AAE9C,MAAM,OAAO,aAAa;IACf,IAAI,CAAS;IACL,MAAM,CAAS;IACf,OAAO,CAA2C;IAC3D,QAAQ,GAAG,CAAC,CAAC;IACb,OAAO,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3C,MAAM,CAA0B;IAChC,WAAW,CAAoB;IAEvC,YAAY,aAAqB,EAAE,MAAc,EAAE,UAAoD,GAAG,EAAE,GAAE,CAAC;QAC7G,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,CAAC,KAA8B;QACjC,MAAM,QAAQ,GAAG;YACf,cAAc,EAAE,cAAc;YAC9B,QAAQ,EAAE,EAAE,IAAI,CAAC,QAAQ;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,IAAI,CAAC,MAAM;SACrB,CAAC;QACF,MAAM,MAAM,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,oBAAoB;YAClE,CAAC,CAAC,MAAM;YACR,CAAC,CAAC;gBACE,GAAG,QAAQ;gBACX,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC;gBAC7B,SAAS,EAAE,IAAI;gBACf,cAAc,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;aAC3C,CAAC;QACN,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAqB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACjH,IAAI,CAAC;YAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,8CAA8C,CAAC,CAAC;QACzF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,OAA0B,CAAC;QAC/B,IAAI,CAAC;YAAC,MAAM,IAAI,CAAC,OAAO,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,GAAG,KAAc,CAAC;QAAC,CAAC;QACvE,IAAI,CAAC;YAAC,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,KAAK,KAAc,CAAC;QAAC,CAAC;QACrF,OAAO,KAAK,IAAI,CAAC,WAAW,CAAC;QAC7B,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC;IAC7B,CAAC;CACF;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjF,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,IAAY;IAC/C,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,CAAC,CAAC;QACjE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,GAAG,MAAM,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACxF,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA2B,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAK,MAAM,CAAC,QAAmB,GAAG,CAAC,EAAE,CAAC;YAC9E,MAAM,IAAI,SAAS,CAAC,gCAAgC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,MAAM,CAAC,QAAkB,CAAC;IACnC,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAAmB,EAAE,KAAa;IACpD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,WAAW,CAAC,MAA+B;IAClD,IAAI,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;AACL,CAAC"}