agent-hitch 0.2.7 → 0.2.8

package/dist/src/evals/benchmark-adapter-manifest.js

@@ -0,0 +1,219 @@
+import { createHash } from "node:crypto";
+import { lstat, readFile, readdir } from "node:fs/promises";
+import path from "node:path";
+import { invalidInput } from "../foundation/index.js";
+const SHA256 = /^sha256:[0-9a-f]{64}$/;
+/** Build the canonical manifest for an already-materialized Harbor dataset. */
+export async function buildBenchmarkAdapterManifest(input) {
+    const ids = [...input.taskIds].sort();
+    if (!ids.length || new Set(ids).size !== ids.length)
+        throw invalidInput("benchmark adapter task IDs must be non-empty and unique");
+    const tasks = await Promise.all(ids.map(async (id) => {
+        const valid = taskId(id);
+        const directory = path.join(path.resolve(input.dataset), valid);
+        const info = await lstat(directory).catch((error) => {
+            if (error.code === "ENOENT")
+                throw invalidInput(`manifest task is missing: ${valid}`);
+            throw error;
+        });
+        if (!info.isDirectory() || info.isSymbolicLink())
+            throw invalidInput(`manifest task is not a directory: ${valid}`);
+        return { task_id: valid, task_digest: await taskTreeDigest(directory) };
+    }));
+    const body = {
+        schema_version: "1",
+        kind: "gear-harbor-benchmark",
+        benchmark: input.benchmark,
+        adapter: input.adapter,
+        scoring: input.scoring,
+        tasks,
+    };
+    return { ...body, dataset_digest: sha256(canonicalJson(body)) };
+}
+/** Load and integrity-check the optional Gear standardized-dataset manifest. */
+export async function loadBenchmarkAdapterManifest(dataset) {
+    const root = path.resolve(dataset);
+    let raw;
+    try {
+        const info = await lstat(path.join(root, "benchmark.adapter.json"));
+        if (!info.isFile() || info.isSymbolicLink())
+            throw invalidInput("benchmark.adapter.json must be a regular file");
+        raw = JSON.parse(await readFile(path.join(root, "benchmark.adapter.json"), "utf8"));
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return null;
+        if (error instanceof SyntaxError)
+            throw invalidInput("benchmark.adapter.json is invalid JSON");
+        throw error;
+    }
+    const manifest = parseManifest(raw);
+    const actualTasks = [];
+    for (const task of manifest.tasks) {
+        const directory = path.join(root, task.task_id);
+        const info = await lstat(directory).catch((error) => {
+            if (error.code === "ENOENT")
+                throw invalidInput(`manifest task is missing: ${task.task_id}`);
+            throw error;
+        });
+        if (!info.isDirectory() || info.isSymbolicLink())
+            throw invalidInput(`manifest task is not a directory: ${task.task_id}`);
+        const digest = await taskTreeDigest(directory);
+        if (digest !== task.task_digest)
+            throw invalidInput(`manifest task digest mismatch: ${task.task_id}`);
+        actualTasks.push({ task_id: task.task_id, task_digest: digest });
+    }
+    const taskDirectories = [];
+    for (const entry of await readdir(root, { withFileTypes: true })) {
+        if (!entry.isDirectory())
+            continue;
+        try {
+            const taskInfo = await lstat(path.join(root, entry.name, "task.toml"));
+            if (taskInfo.isFile() && !taskInfo.isSymbolicLink())
+                taskDirectories.push(entry.name);
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+        }
+    }
+    if (JSON.stringify(taskDirectories.sort()) !== JSON.stringify(actualTasks.map((task) => task.task_id))) {
+        throw invalidInput("benchmark adapter manifest task set does not match the dataset");
+    }
+    const body = {
+        schema_version: manifest.schema_version,
+        kind: manifest.kind,
+        benchmark: manifest.benchmark,
+        adapter: manifest.adapter,
+        scoring: manifest.scoring,
+        tasks: actualTasks,
+    };
+    const expected = sha256(canonicalJson(body));
+    if (manifest.dataset_digest !== expected)
+        throw invalidInput("benchmark adapter dataset digest mismatch");
+    return manifest;
+}
+export function scoreWithinRange(score, definition) {
+    return score >= definition.range[0] && score <= definition.range[1];
+}
+function parseManifest(value) {
+    const record = object(value, "benchmark adapter manifest");
+    exact(record, ["schema_version", "kind", "benchmark", "adapter", "scoring", "tasks", "dataset_digest"], "benchmark adapter manifest");
+    if (record.schema_version !== "1" || record.kind !== "gear-harbor-benchmark")
+        throw invalidInput("unsupported benchmark adapter manifest");
+    const benchmark = namedRevision(record.benchmark, "benchmark");
+    const adapterRecord = object(record.adapter, "adapter");
+    exact(adapterRecord, ["id", "revision", "output_protocol"], "adapter");
+    if (adapterRecord.output_protocol !== "gear-harbor-eval-result-v1")
+        throw invalidInput("unsupported benchmark adapter output protocol");
+    const adapter = { id: identifier(adapterRecord.id, "adapter id"), revision: immutableRevision(adapterRecord.revision, "adapter revision"), output_protocol: adapterRecord.output_protocol };
+    const scoringRecord = object(record.scoring, "scoring");
+    exact(scoringRecord, ["total_score", "process_score"], "scoring");
+    const scoring = {
+        total_score: scoreDefinition(scoringRecord.total_score, "total_score"),
+        ...(scoringRecord.process_score === undefined ? {} : { process_score: scoreDefinition(scoringRecord.process_score, "process_score") }),
+    };
+    if (!Array.isArray(record.tasks) || record.tasks.length === 0)
+        throw invalidInput("benchmark adapter tasks must be a non-empty array");
+    const tasks = record.tasks.map((raw, index) => {
+        const task = object(raw, `task ${index}`);
+        exact(task, ["task_id", "task_digest"], `task ${index}`);
+        return { task_id: taskId(task.task_id), task_digest: digest(task.task_digest, `task ${index} digest`) };
+    });
+    const ids = tasks.map((task) => task.task_id);
+    if (new Set(ids).size !== ids.length || JSON.stringify(ids) !== JSON.stringify([...ids].sort())) {
+        throw invalidInput("benchmark adapter task IDs must be unique and sorted");
+    }
+    return {
+        schema_version: "1",
+        kind: "gear-harbor-benchmark",
+        benchmark,
+        adapter,
+        scoring,
+        tasks,
+        dataset_digest: digest(record.dataset_digest, "dataset digest"),
+    };
+}
+function namedRevision(value, label) {
+    const record = object(value, label);
+    exact(record, ["id", "revision"], label);
+    return { id: identifier(record.id, `${label} id`), revision: immutableRevision(record.revision, `${label} revision`) };
+}
+function scoreDefinition(value, label) {
+    const record = object(value, label);
+    exact(record, ["source_metric", "direction", "range", "reducer"], label);
+    if (record.direction !== "maximize" && record.direction !== "minimize")
+        throw invalidInput(`${label} direction is invalid`);
+    if (record.reducer !== "task-macro-mean")
+        throw invalidInput(`${label} reducer is invalid`);
+    if (!Array.isArray(record.range) || record.range.length !== 2 || record.range.some((item) => typeof item !== "number" || !Number.isFinite(item))) {
+        throw invalidInput(`${label} range is invalid`);
+    }
+    const range = [record.range[0], record.range[1]];
+    if (range[0] > range[1])
+        throw invalidInput(`${label} range is reversed`);
+    return { source_metric: identifier(record.source_metric, `${label} source metric`), direction: record.direction, range, reducer: "task-macro-mean" };
+}
+async function taskTreeDigest(root) {
+    const rows = [];
+    const visit = async (directory) => {
+        for (const entry of (await readdir(directory, { withFileTypes: true })).sort((left, right) => left.name.localeCompare(right.name))) {
+            const absolute = path.join(directory, entry.name);
+            const relative = path.relative(root, absolute).split(path.sep).join("/");
+            const info = await lstat(absolute);
+            if (info.isSymbolicLink())
+                throw invalidInput(`benchmark task contains a symlink: ${relative}`);
+            if (info.isDirectory())
+                await visit(absolute);
+            else if (info.isFile())
+                rows.push({ path: relative, mode: info.mode & 0o111 ? "executable" : "file", sha256: createHash("sha256").update(await readFile(absolute)).digest("hex") });
+            else
+                throw invalidInput(`benchmark task contains a special file: ${relative}`);
+        }
+    };
+    await visit(root);
+    return sha256(JSON.stringify(rows));
+}
+function canonicalJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonicalJson).join(",")}]`;
+    if (value && typeof value === "object") {
+        return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(",")}}`;
+    }
+    return JSON.stringify(value);
+}
+function sha256(value) {
+    return `sha256:${createHash("sha256").update(value).digest("hex")}`;
+}
+function object(value, label) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw invalidInput(`${label} must be an object`);
+    return value;
+}
+function exact(record, fields, label) {
+    const allowed = new Set(fields);
+    const extra = Object.keys(record).find((field) => !allowed.has(field));
+    if (extra)
+        throw invalidInput(`${label} has unknown field: ${extra}`);
+}
+function identifier(value, label) {
+    if (typeof value !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/.test(value))
+        throw invalidInput(`${label} is invalid`);
+    return value;
+}
+function immutableRevision(value, label) {
+    if (typeof value !== "string" || value.length === 0 || value.length > 512 || value.toLowerCase() === "latest")
+        throw invalidInput(`${label} is invalid`);
+    return value;
+}
+function taskId(value) {
+    if (typeof value !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/.test(value) || value === "." || value === "..")
+        throw invalidInput("task id is invalid");
+    return value;
+}
+function digest(value, label) {
+    if (typeof value !== "string" || !SHA256.test(value))
+        throw invalidInput(`${label} is invalid`);
+    return value;
+}
+//# sourceMappingURL=benchmark-adapter-manifest.js.map

package/dist/src/evals/benchmark-adapter-manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark-adapter-manifest.js","sourceRoot":"","sources":["../../../src/evals/benchmark-adapter-manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD,MAAM,MAAM,GAAG,uBAAuB,CAAC;AAwBvC,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAMnD;IACC,MAAM,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IACtC,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM;QAAE,MAAM,YAAY,CAAC,yDAAyD,CAAC,CAAC;IACnI,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACnD,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,KAA4B,EAAE,EAAE;YACzE,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,YAAY,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;YACtF,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE;YAAE,MAAM,YAAY,CAAC,qCAAqC,KAAK,EAAE,CAAC,CAAC;QACnH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;IAC1E,CAAC,CAAC,CAAC,CAAC;IACJ,MAAM,IAAI,GAAG;QACX,cAAc,EAAE,GAAY;QAC5B,IAAI,EAAE,uBAAgC;QACtC,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK;KACN,CAAC;IACF,OAAO,EAAE,GAAG,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,OAAe;IAChE,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE;YAAE,MAAM,YAAY,CAAC,+CAA+C,CAAC,CAAC;QACjH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IACtF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACpE,IAAI,KAAK,YAAY,WAAW;YAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;QAC/F,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,WAAW,GAAoD,EAAE,CAAC;IACxE,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,KAA4B,EAAE,EAAE;YACzE,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,YAAY,CAAC,6BAA6B,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7F,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE;YAAE,MAAM,YAAY,CAAC,qCAAqC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1H,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;QAC/C,IAAI,MAAM,KAAK,IAAI,CAAC,WAAW;YAAE,MAAM,YAAY,CAAC,kCAAkC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACtG,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACjE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;YAAE,SAAS;QACnC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;YACvE,IAAI,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE;gBAAE,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAC;QACtE,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QACvG,MAAM,YAAY,CAAC,gEAAgE,CAAC,CAAC;IACvF,CAAC;IACD,MAAM,IAAI,GAAG;QACX,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,KAAK,EAAE,WAAW;KACnB,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,IAAI,QAAQ,CAAC,cAAc,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,2CAA2C,CAAC,CAAC;IAC1G,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,UAAsC;IACpF,OAAO,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,4BAA4B,CAAC,CAAC;IAC3D,KAAK,CAAC,MAAM,EAAE,CAAC,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,4BAA4B,CAAC,CAAC;IACtI,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,KAAK,uBAAuB;QAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IAC3I,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAC/D,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACxD,KAAK,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE,SAAS,CAAC,CAAC;IACvE,IAAI,aAAa,CAAC,eAAe,KAAK,4BAA4B;QAAE,MAAM,YAAY,CAAC,+CAA+C,CAAC,CAAC;IACxI,MAAM,OAAO,GAAG,EAAE,EAAE,EAAE,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,YAAY,CAAC,EAAE,QAAQ,EAAE,iBAAiB,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,EAAE,eAAe,EAAE,aAAa,CAAC,eAAe,EAAW,CAAC;IACrM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACxD,KAAK,CAAC,aAAa,EAAE,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,SAAS,CAAC,CAAC;IAClE,MAAM,OAAO,GAA6B;QACxC,WAAW,EAAE,eAAe,CAAC,aAAa,CAAC,WAAW,EAAE,aAAa,CAAC;QACtE,GAAG,CAAC,aAAa,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,eAAe,CAAC,aAAa,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,CAAC;KACvI,CAAC;IACF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,mDAAmD,CAAC,CAAC;IACvI,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,QAAQ,KAAK,EAAE,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,QAAQ,KAAK,EAAE,CAAC,CAAC;QACzD,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;IAC1G,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAChG,MAAM,YAAY,CAAC,sDAAsD,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,IAAI,EAAE,uBAAuB;QAC7B,SAAS;QACT,OAAO;QACP,OAAO;QACP,KAAK;QACL,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC;KAChE,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,KAAa;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACpC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,CAAC;IACzC,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,KAAK,KAAK,CAAC,EAAE,QAAQ,EAAE,iBAAiB,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,KAAK,WAAW,CAAC,EAAE,CAAC;AACzH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,KAAa;IACpD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACpC,KAAK,CAAC,MAAM,EAAE,CAAC,eAAe,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;IACzE,IAAI,MAAM,CAAC,SAAS,KAAK,UAAU,IAAI,MAAM,CAAC,SAAS,KAAK,UAAU;QAAE,MAAM,YAAY,CAAC,GAAG,KAAK,uBAAuB,CAAC,CAAC;IAC5H,IAAI,MAAM,CAAC,OAAO,KAAK,iBAAiB;QAAE,MAAM,YAAY,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAC;IAC5F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACjJ,MAAM,YAAY,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAW,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAW,CAAU,CAAC;IAC9E,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAC;IAC1E,OAAO,EAAE,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,KAAK,gBAAgB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC;AACvJ,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAY;IACxC,MAAM,IAAI,GAAyE,EAAE,CAAC;IACtF,MAAM,KAAK,GAAG,KAAK,EAAE,SAAiB,EAAiB,EAAE;QACvD,KAAK,MAAM,KAAK,IAAI,CAAC,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnI,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,IAAI,CAAC,cAAc,EAAE;gBAAE,MAAM,YAAY,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;YAChG,IAAI,IAAI,CAAC,WAAW,EAAE;gBAAE,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;iBACzC,IAAI,IAAI,CAAC,MAAM,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;;gBAC/K,MAAM,YAAY,CAAC,2CAA2C,QAAQ,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC,CAAC;IACF,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IAClB,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAC3E,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACxM,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,MAAM,CAAC,KAAa;IAC3B,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAY,CAAC;AAChF,CAAC;AAED,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAC;IAClH,OAAO,KAAgC,CAAC;AAC1C,CAAC;AAED,SAAS,KAAK,CAAC,MAA+B,EAAE,MAAyB,EAAE,KAAa;IACtF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACvE,IAAI,KAAK;QAAE,MAAM,YAAY,CAAC,GAAG,KAAK,uBAAuB,KAAK,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,KAAa;IAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,qCAAqC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IAC/H,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,KAAa;IACtD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,WAAW,EAAE,KAAK,QAAQ;QAAE,MAAM,YAAY,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IACzJ,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,oCAAoC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAChK,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IAChG,OAAO,KAAe,CAAC;AACzB,CAAC"}

package/dist/src/evals/benchmark-candidate.js

@@ -0,0 +1,36 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { invalidInput } from "../foundation/index.js";
+export function assertBenchmarkCandidate(tasks, harnessId, agentArgs) {
+    if (tasks.some(task => task.driver.kind === "model-call") && (harnessId !== "model-call" || agentArgs.length)) {
+        throw invalidInput("no-tools tasks require the trusted model-call harness without agent overrides");
+    }
+    if (tasks.some(task => task.driver.kind !== "model-call" && task.requirements.includes("native-image-input")) && harnessId !== "codex") {
+        throw invalidInput("native-image agent tasks currently require the Codex image-capable harness");
+    }
+}
+/** The verified task trees contain the Package v1 requirements after export. */
+export async function assertStandardBenchmarkCandidate(dataset, manifest, harnessId, agentArgs) {
+    const tasks = [];
+    for (const { task_id } of manifest.tasks) {
+        let descriptor;
+        try {
+            descriptor = JSON.parse(await readFile(path.join(dataset, task_id, ".hitch-benchmark.json"), "utf8"));
+        }
+        catch (error) {
+            // Direct Harbor adapters need not use the Package v1 execution bridge.
+            if (error.code === "ENOENT")
+                continue;
+            throw error;
+        }
+        const task = descriptor?.task;
+        if (descriptor?.schema_version !== "1" || descriptor.task_id !== task_id
+            || !["model-call", "terminal", "tool-server"].includes(task?.driver?.kind)
+            || !Array.isArray(task?.requirements) || task.requirements.some((item) => typeof item !== "string")) {
+            throw invalidInput(`invalid compiled candidate requirements: ${task_id}`);
+        }
+        tasks.push(task);
+    }
+    assertBenchmarkCandidate(tasks, harnessId, agentArgs);
+}
+//# sourceMappingURL=benchmark-candidate.js.map

package/dist/src/evals/benchmark-candidate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark-candidate.js","sourceRoot":"","sources":["../../../src/evals/benchmark-candidate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAQtD,MAAM,UAAU,wBAAwB,CAAC,KAAuC,EAAE,SAAiB,EAAE,SAA4B;IAC/H,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9G,MAAM,YAAY,CAAC,+EAA+E,CAAC,CAAC;IACtG,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;QACvI,MAAM,YAAY,CAAC,4EAA4E,CAAC,CAAC;IACnG,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,OAAe,EACf,QAAoC,EACpC,SAAiB,EACjB,SAA4B;IAE5B,MAAM,KAAK,GAA4B,EAAE,CAAC;IAC1C,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACzC,IAAI,UAAU,CAAC;QACf,IAAI,CAAC;YACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,uBAAuB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACxG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,uEAAuE;YACvE,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,SAAS;YACjE,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC;QAC9B,IAAI,UAAU,EAAE,cAAc,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,KAAK,OAAO;eACnE,CAAC,CAAC,YAAY,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC;eACvE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC;YAC/G,MAAM,YAAY,CAAC,4CAA4C,OAAO,EAAE,CAAC,CAAC;QAC5E,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IACD,wBAAwB,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACxD,CAAC"}

package/dist/src/evals/benchmark-run.js

@@ -0,0 +1,166 @@
+import { cp, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { stringify as stringifyTOML } from "smol-toml";
+import { benchmarkTreeDigest, loadBenchmark, loadBenchmarkLock } from "../benchmarks/index.js";
+import { atomicWriteJSON, ensureDir, invalidInput, sha256JSON, statePaths, withFileLock } from "../foundation/index.js";
+import { newEvalId, validateEvalRequest } from "./request.js";
+import { parseHarnessReference } from "../revisions/index.js";
+import { assertBenchmarkCandidate } from "./benchmark-candidate.js";
+import { buildBenchmarkAdapterManifest, loadBenchmarkAdapterManifest } from "./benchmark-adapter-manifest.js";
+import { runEval } from "./service.js";
+export const STANDARD_BENCHMARK_COMPILER = "harbor-package@6";
+export async function runBenchmarkEval(options) {
+    if (Boolean(options.benchmark) === Boolean(options.benchmarkLock))
+        throw invalidInput("provide exactly one of --benchmark or --benchmark-lock");
+    if (options.request.dataset)
+        throw invalidInput("--dataset and --benchmark are mutually exclusive");
+    if (options.resumeExisting || options.precreated || options.remoteWorkExecutor || options.normalizedRequest)
+        throw invalidInput("standard benchmark packages currently support local managed runs only");
+    const loaded = options.benchmark ? await loadBenchmark(options.benchmark) : await loadBenchmarkLock(options.benchmarkLock);
+    assertBenchmarkCandidate(loaded.tasks.map(task => task.config), parseHarnessReference(String(options.request.harness_ref)).harness_id, Array.isArray(options.request.agent_args) ? options.request.agent_args : []);
+    const evalId = options.evalId ?? newEvalId();
+    const compiled = await compileBenchmark(loaded, options.root);
+    const request = {
+        ...options.request, dataset: compiled.tasks,
+        attempts: options.request.attempts ?? loaded.profile.sampling.attempts_per_task,
+        max_concurrent: options.request.max_concurrent ?? 1,
+        // Zero disables the legacy blanket timeout; each task/bridge owns its budget.
+        timeout_ms: options.request.timeout_ms ?? 0,
+        setup_timeout_ms: options.request.setup_timeout_ms ?? loaded.profile.budget.setup_timeout_ms,
+        infrastructure_retries: options.request.infrastructure_retries ?? loaded.profile.grading.infrastructure_retries,
+    };
+    const directory = path.join(statePaths(options.root).evals, evalId, "benchmark");
+    const normalizedRequest = await validateEvalRequest(request);
+    if (normalizedRequest.benchmark_id !== loaded.manifest.id)
+        throw invalidInput("compiled benchmark manifest identity mismatch");
+    return runEval({ ...options, evalId, request, normalizedRequest, executionStrategy: "local-task-slots-v1", onControlPhase: async (phase, work) => {
+            if (phase === "planning") {
+                await ensureDir(directory);
+                await atomicWriteJSON(path.join(directory, "benchmark.lock.json"), loaded.lock);
+                await atomicWriteJSON(path.join(directory, "manifest.json"), loaded.manifest);
+                await atomicWriteJSON(path.join(directory, "effective-profile.json"), {
+                    ...loaded.profile,
+                    effective: { attempts: request.attempts, max_concurrent: request.max_concurrent, timeout_ms: request.timeout_ms, seed_applied_to_model: false },
+                });
+                await atomicWriteJSON(path.join(directory, "package.json"), { source: compiled.source, tasks: compiled.tasks, package_digest: loaded.lock.package_digest, compiled_digest: compiled.digest });
+            }
+            await options.onControlPhase?.(phase, work);
+        } });
+}
+export async function compileBenchmark(loaded, root) {
+    const digest = sha256JSON({ lock: loaded.lock, compiler: STANDARD_BENCHMARK_COMPILER });
+    const store = await ensureDir(path.join(root, "store", "benchmarks"));
+    const target = path.join(store, digest.slice(7));
+    await withFileLock(path.join(store, "locks"), digest, async () => {
+        try {
+            const existing = await loadBenchmark(path.join(target, "source"));
+            if (sha256JSON(existing.lock) !== sha256JSON(loaded.lock))
+                throw invalidInput("stored benchmark integrity mismatch");
+            const metadata = JSON.parse(await readFile(path.join(target, "compiled.json"), "utf8"));
+            if (metadata.digest !== digest)
+                throw invalidInput("stored benchmark compiler identity mismatch");
+            if (metadata.tasks_digest !== await benchmarkTreeDigest(path.join(target, "tasks")))
+                throw invalidInput("compiled benchmark tasks were modified");
+            return;
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+        }
+        const temporary = await mkdtemp(path.join(store, ".building-"));
+        try {
+            await cp(loaded.directory, path.join(temporary, "source"), { recursive: true, dereference: false });
+            const snapshot = await loadBenchmark(path.join(temporary, "source"));
+            if (sha256JSON(snapshot.lock) !== sha256JSON(loaded.lock))
+                throw invalidInput("benchmark changed while being sealed");
+            await mkdir(path.join(temporary, "tasks"));
+            for (const task of snapshot.tasks) {
+                const taskPath = path.join(temporary, "tasks", task.id);
+                await cp(path.join(snapshot.directory, task.path), taskPath, { recursive: true });
+                const nativePhases = task.config.driver.kind === "tool-server" && task.config.driver.config.native_phases;
+                let finalizationTimeoutMs = loaded.profile.budget.collection_timeout_ms + loaded.profile.budget.cleanup_grace_ms;
+                if (nativePhases) {
+                    // Hitch enforces the original candidate budget in its private phase
+                    // supervisor. Harbor's outer guard also allows final evidence export
+                    // and failure teardown without extending the model's deadline.
+                    // Cancellation/export and host inspection each have a shutdown
+                    // allowance, bounded by collection_timeout_ms; snapshot has its own.
+                    const collections = nativePhases.protocol === "hitch-native-phase-control@2" ? 4 : 3;
+                    finalizationTimeoutMs = collections * loaded.profile.budget.collection_timeout_ms + 2 * loaded.profile.budget.cleanup_grace_ms;
+                }
+                // The inner Hitch invocation retains the source budget. The outer
+                // Harbor guard must also permit process retirement and result export.
+                const execution = structuredClone(task.harbor);
+                const agent = execution.agent;
+                agent.timeout_sec = Number(agent.timeout_sec) + Math.ceil(finalizationTimeoutMs / 1000);
+                await writeFile(path.join(taskPath, "task.toml"), stringifyTOML(execution));
+                await atomicWriteJSON(path.join(taskPath, ".hitch-benchmark.json"), {
+                    schema_version: "1", task_id: task.id, task: task.config,
+                    profile_digest: loaded.lock.profile_digest, profile: loaded.profile,
+                    primary_metric: loaded.manifest.primary_metric, metrics: loaded.manifest.metrics,
+                    score_contract: { total_score: loaded.manifest.primary_metric },
+                    tools: task.tools, agent_timeout_sec: task.harbor.agent.timeout_sec,
+                    agent_finalization_timeout_ms: finalizationTimeoutMs,
+                    ...(task.config.driver.kind === "model-call" ? { candidate_input: JSON.parse(await readFile(path.join(snapshot.directory, task.config.driver.config.input), "utf8")) } : {}),
+                    package_digest: loaded.lock.package_digest, task_digest: loaded.lock.tasks.find((t) => t.task_id === task.id).task_digest,
+                });
+            }
+            const primary = loaded.manifest.metrics[loaded.manifest.primary_metric];
+            const standardManifest = await buildBenchmarkAdapterManifest({
+                dataset: path.join(temporary, "tasks"),
+                benchmark: { id: loaded.manifest.id, revision: loaded.lock.package_digest },
+                adapter: { id: "hitch-package-v1-compiler", revision: digest, output_protocol: "gear-harbor-eval-result-v1" },
+                scoring: {
+                    total_score: {
+                        source_metric: loaded.manifest.primary_metric,
+                        direction: primary.direction,
+                        range: primary.range,
+                        reducer: "task-macro-mean",
+                    },
+                },
+                taskIds: snapshot.tasks.map((task) => task.id),
+            });
+            await atomicWriteJSON(path.join(temporary, "tasks", "benchmark.adapter.json"), standardManifest);
+            await atomicWriteJSON(path.join(temporary, "source", "benchmark.lock.json"), loaded.lock);
+            await atomicWriteJSON(path.join(temporary, "compiled.json"), { digest, tasks_digest: await benchmarkTreeDigest(path.join(temporary, "tasks")) });
+            await rename(temporary, target);
+        }
+        finally {
+            await rm(temporary, { recursive: true, force: true });
+        }
+    });
+    return { source: path.join(target, "source"), tasks: path.join(target, "tasks"), digest };
+}
+/** Compile any supported Package v1 benchmark into a standalone Harbor dataset. */
+export async function exportStandardBenchmarkDataset(packageDirectory, outputDirectory) {
+    const output = path.resolve(outputDirectory);
+    await mkdir(path.dirname(output), { recursive: true });
+    await mkdir(output); // Refuse to overwrite any existing path.
+    const temporaryRoot = await mkdtemp(path.join(path.dirname(output), ".hitch-standard-compile-"));
+    try {
+        const loaded = await loadBenchmark(packageDirectory);
+        const compiled = await compileBenchmark(loaded, temporaryRoot);
+        for (const entry of await readdir(compiled.tasks)) {
+            await cp(path.join(compiled.tasks, entry), path.join(output, entry), { recursive: true, errorOnExist: true, force: false });
+        }
+        const manifest = await loadBenchmarkAdapterManifest(output);
+        if (!manifest)
+            throw invalidInput("compiled benchmark did not produce a standard manifest");
+        return {
+            dataset: output,
+            benchmark_id: manifest.benchmark.id,
+            benchmark_revision: manifest.dataset_digest,
+            source_package_digest: loaded.lock.package_digest,
+            tasks: manifest.tasks.map((task) => task.task_id),
+            scoring: manifest.scoring,
+        };
+    }
+    catch (error) {
+        await rm(output, { recursive: true, force: true });
+        throw error;
+    }
+    finally {
+        await rm(temporaryRoot, { recursive: true, force: true });
+    }
+}
+//# sourceMappingURL=benchmark-run.js.map

package/dist/src/evals/benchmark-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark-run.js","sourceRoot":"","sources":["../../../src/evals/benchmark-run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChG,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE/F,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACxH,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,EAAE,6BAA6B,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAC9G,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAGvC,MAAM,CAAC,MAAM,2BAA2B,GAAG,kBAAkB,CAAC;AAE9D,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAItC;IACC,IAAI,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC;QAAE,MAAM,YAAY,CAAC,wDAAwD,CAAC,CAAC;IAChJ,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO;QAAE,MAAM,YAAY,CAAC,kDAAkD,CAAC,CAAC;IACpG,IAAI,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,kBAAkB,IAAI,OAAO,CAAC,iBAAiB;QAAE,MAAM,YAAY,CAAC,uEAAuE,CAAC,CAAC;IACzM,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,iBAAiB,CAAC,OAAO,CAAC,aAAc,CAAC,CAAC;IAC5H,wBAAwB,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,EACnI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG;QACd,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK;QAC3C,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,iBAAiB;QAC/E,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC;QACnD,8EAA8E;QAC9E,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC;QAC3C,gBAAgB,EAAE,OAAO,CAAC,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB;QAC5F,sBAAsB,EAAE,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB;KAChH,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IACjF,MAAM,iBAAiB,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7D,IAAI,iBAAiB,CAAC,YAAY,KAAK,MAAM,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC/H,OAAO,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC/I,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;gBACzB,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;gBAC3B,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAChF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC9E,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC,EAAE;oBACpE,GAAG,MAAM,CAAC,OAAO;oBACjB,SAAS,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,qBAAqB,EAAE,KAAK,EAAE;iBAChJ,CAAC,CAAC;gBACH,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAChM,CAAC;YACD,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC9C,CAAC,EAAE,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAyB,EAAE,IAAY;IAC5E,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,2BAA2B,EAAE,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;IACtE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;QAC/D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;YAClE,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;gBAAE,MAAM,YAAY,CAAC,qCAAqC,CAAC,CAAC;YACrH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,CAAC,CAA6C,CAAC;YACpI,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM;gBAAE,MAAM,YAAY,CAAC,6CAA6C,CAAC,CAAC;YAClG,IAAI,QAAQ,CAAC,YAAY,KAAK,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;YAClJ,OAAO;QACT,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAC;QAAC,CAAC;QACxF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;QAChE,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACpG,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;YACrE,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;gBAAE,MAAM,YAAY,CAAC,sCAAsC,CAAC,CAAC;YACtH,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;YAC3C,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;gBACxD,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClF,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;gBAC1G,IAAI,qBAAqB,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,qBAAqB,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;gBACjH,IAAI,YAAY,EAAE,CAAC;oBACjB,oEAAoE;oBACpE,qEAAqE;oBACrE,+DAA+D;oBAC/D,+DAA+D;oBAC/D,qEAAqE;oBACrE,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,KAAK,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrF,qBAAqB,GAAG,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,qBAAqB,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;gBACjI,CAAC;gBACD,kEAAkE;gBAClE,sEAAsE;gBACtE,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC/C,MAAM,KAAK,GAAG,SAAS,CAAC,KAAgC,CAAC;gBACzD,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAAC;gBACxF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC5E,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,uBAAuB,CAAC,EAAE;oBAClE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM;oBACxD,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO;oBACnE,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;oBAChF,cAAc,EAAE,EAAE,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE;oBAC/D,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,iBAAiB,EAAG,IAAI,CAAC,MAAM,CAAC,KAAiC,CAAC,WAAW;oBAChG,6BAA6B,EAAE,qBAAqB;oBACpD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC5K,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,CAAE,CAAC,WAAW;iBAC3H,CAAC,CAAC;YACL,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAE,CAAC;YACzE,MAAM,gBAAgB,GAAG,MAAM,6BAA6B,CAAC;gBAC3D,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC;gBACtC,SAAS,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE;gBAC3E,OAAO,EAAE,EAAE,EAAE,EAAE,2BAA2B,EAAE,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,4BAA4B,EAAE;gBAC7G,OAAO,EAAE;oBACP,WAAW,EAAE;wBACX,aAAa,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc;wBAC7C,SAAS,EAAE,OAAO,CAAC,SAAS;wBAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,OAAO,EAAE,iBAAiB;qBAC3B;iBACF;gBACD,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;aAC/C,CAAC,CAAC;YACH,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,wBAAwB,CAAC,EAAE,gBAAgB,CAAC,CAAC;YACjG,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,qBAAqB,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1F,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;YACjJ,MAAM,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;gBAAS,CAAC;YAAC,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;AAC5F,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,gBAAwB,EAAE,eAAuB;IACpG,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7C,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,yCAAyC;IAC9D,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,0BAA0B,CAAC,CAAC,CAAC;IACjG,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC/D,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9H,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,4BAA4B,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ;YAAE,MAAM,YAAY,CAAC,wDAAwD,CAAC,CAAC;QAC5F,OAAO;YACL,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE;YACnC,kBAAkB,EAAE,QAAQ,CAAC,cAAc;YAC3C,qBAAqB,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc;YACjD,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;YACjD,OAAO,EAAE,QAAQ,CAAC,OAAO;SAC1B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC"}

package/dist/src/evals/collect-only-rerun.js

@@ -116,6 +116,7 @@ async function collectSlot(input, executionPlan, resolution, runtimeId, slot, ex
                     publicationMode: "replace-invalid",
                     runtimeId,
                     env: input.env ?? process.env,
+                    ...(input.signal ? { signal: input.signal } : {}),
                     requireCompleteMarker: true,
                 }, trial, index, imported);
                 if (!imported.some((entry) => entry.trial_id === ref.trial_id))
@@ -138,7 +139,7 @@ async function collectSlot(input, executionPlan, resolution, runtimeId, slot, ex
             ref,
             source: {
                 source_trial_id: ref.trial_id,
-                source_run_id: ref.run_id,
+                ...(ref.run_group ? { source_run_group_id: ref.run_group.run_group_id } : { source_run_id: ref.run_id }),
                 source_work_id: work.work_id,
                 source_backend_directory: relativeBackend.split(path.sep).join("/"),
             },

package/dist/src/evals/collect-only-rerun.js.map

@@ -1 +1 @@
-{"version":3,"file":"collect-only-rerun.js","sourceRoot":"","sources":["../../../src/evals/collect-only-rerun.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC/F,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAElG,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AA6B7G,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAA4B;IACrE,MAAM,aAAa,GAAG,sBAAsB,CAAC,MAAM,QAAQ,CAAU,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC;IAC7H,IAAI,aAAa,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,IAAI,aAAa,CAAC,UAAU,KAAK,OAAO;QAAE,MAAM,WAAW,CAAC,mDAAmD,CAAC,CAAC;IAC3J,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAmB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACvG,IAAI,UAAU,CAAC,QAAQ,KAAK,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,6BAA6B,CAAC;QAAE,MAAM,WAAW,CAAC,0CAA0C,CAAC,CAAC;IACjL,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;IACnG,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC9B,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACvH,MAAM,aAAa,GAAqB,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,GAAG,QAAQ,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC;QACxH,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,aAAa;QAAE,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC3F,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,CAAC,QAAQ,CAAC,UAAU;QAAE,MAAM,iBAAiB,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IAC9G,MAAM,eAAe,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC3F,MAAM,cAAc,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;IACpD,MAAM,cAAc,GAAG,SAAS,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,aAAa,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,eAAe,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAChF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,YAAY,CAAC,EAAE;QACnE,cAAc,EAAE,cAAc;QAC9B,QAAQ,EAAE,KAAK,CAAC,OAAO;QACvB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,UAAU,EAAE,cAAc;QAC1B,SAAS,EAAE,kBAAkB,CAAC,cAAc,CAAC;QAC7C,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,cAAc,CAAC;QACxC,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC;QACvC,cAAc,EAAE,aAAa;QAC7B,eAAe,EAAE,cAAc;QAC/B,uBAAuB,EAAE,cAAc;QACvC,wBAAwB,EAAE,eAAe;QACzC,OAAO;QACP,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QAC/C,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,YAAY,EAAE,WAAW;QACzB,UAAU,EAAE,WAAW;KACxB,CAAC,CAAC;IACH,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,IAAI,EAAE,YAAY;QAClB,QAAQ,EAAE,KAAK,CAAC,OAAO;QACvB,UAAU,EAAE,cAAc;QAC1B,SAAS,EAAE,kBAAkB,CAAC,cAAc,CAAC;QAC7C,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,MAAM,EAAE,WAAW;QACnB,cAAc,EAAE,WAAW,CAAC,KAAK,CAAC,cAAc,CAAC;QACjD,cAAc,EAAE,aAAa;QAC7B,uBAAuB,EAAE,cAAc;QACvC,eAAe,EAAE,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC;QAChD,eAAe,EAAE,cAAc;QAC/B,wBAAwB,EAAE,eAAe;QACzC,OAAO;QACP,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QAC/C,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,YAAY,EAAE,WAAW;KAC1B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,KAA4B,EAC5B,aAAkC,EAClC,UAA4B,EAC5B,SAAiB,EACjB,IAAmB,EACnB,YAA8B;IAE9B,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1H,MAAM,IAAI,GAAG,WAAW,IAAI,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IAChH,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7F,MAAM,WAAW,CAAC,8CAA8C,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3F,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QAAC,OAAO,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;IACxE,OAAO,KAAK,EAAE,CAAC;QACb,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,WAAW,CAAC,sCAAsC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpI,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACjG,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAC/C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC;QAC5E,MAAM,GAAG,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;QACvG,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC1C,MAAM,QAAQ,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;QACnC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;oBACnC,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,aAAa,EAAE,KAAK,CAAC,aAAa;oBAClC,kBAAkB,EAAE,YAAY;oBAChC,eAAe,EAAE,IAAI,CAAC,OAAO;oBAC7B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,gBAAgB,EAAE,UAAU;oBAC5B,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;oBACvC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB;oBACnD,eAAe,EAAE,iBAAiB;oBAClC,SAAS;oBACT,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;oBAC7B,qBAAqB,EAAE,IAAI;iBAC5B,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;gBAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC;oBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,uBAAuB;oBAAE,SAAS;gBACvD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC;QAC/I,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,2BAA2B,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;YACjE,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;YACvC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB;SACpD,CAAC,CAAC;QACH,OAAO;YACL,GAAG;YACH,MAAM,EAAE;gBACN,eAAe,EAAE,GAAG,CAAC,QAAQ;gBAC7B,aAAa,EAAE,GAAG,CAAC,MAAM;gBACzB,cAAc,EAAE,IAAI,CAAC,OAAO;gBAC5B,wBAAwB,EAAE,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;aACpE;SACF,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,UAAU,CAAC,mDAAmD,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE;QAC1F,IAAI,EAAE,sCAAsC;QAC5C,QAAQ,EAAE,EAAE;KACb,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,KAA4B,EAC5B,QAAwB,EACxB,eAAgC,EAChC,SAAkB,EAClB,WAAmB;IAEnB,MAAM,MAAM,GAA4B;QACtC,GAAG,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;QAC/B,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QAC1C,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QAC7B,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS;QAC/B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO;QAC9B,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;QACxC,kBAAkB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB;QACpD,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,OAAO,EAAE,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC5C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,uCAAuC,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC;QACjK,UAAU,EAAE,OAAO,KAAK,CAAC,cAAc,EAAE,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU;QACxH,YAAY,EAAE,WAAW;KAC1B,CAAC;IACF,IAAI,SAAS;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC;IACnC,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK;QAAE,MAAM,WAAW,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IAClF,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,+BAA+B,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AACzF,CAAC"}
+{"version":3,"file":"collect-only-rerun.js","sourceRoot":"","sources":["../../../src/evals/collect-only-rerun.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC/F,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAElG,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AA+B7G,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAA4B;IACrE,MAAM,aAAa,GAAG,sBAAsB,CAAC,MAAM,QAAQ,CAAU,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC;IAC7H,IAAI,aAAa,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,IAAI,aAAa,CAAC,UAAU,KAAK,OAAO;QAAE,MAAM,WAAW,CAAC,mDAAmD,CAAC,CAAC;IAC3J,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAmB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACvG,IAAI,UAAU,CAAC,QAAQ,KAAK,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,6BAA6B,CAAC;QAAE,MAAM,WAAW,CAAC,0CAA0C,CAAC,CAAC;IACjL,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;IACnG,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC9B,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACvH,MAAM,aAAa,GAAqB,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,GAAG,QAAQ,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC;QACxH,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,aAAa;QAAE,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC3F,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,CAAC,QAAQ,CAAC,UAAU;QAAE,MAAM,iBAAiB,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IAC9G,MAAM,eAAe,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC3F,MAAM,cAAc,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;IACpD,MAAM,cAAc,GAAG,SAAS,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,aAAa,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,eAAe,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAChF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,YAAY,CAAC,EAAE;QACnE,cAAc,EAAE,cAAc;QAC9B,QAAQ,EAAE,KAAK,CAAC,OAAO;QACvB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,UAAU,EAAE,cAAc;QAC1B,SAAS,EAAE,kBAAkB,CAAC,cAAc,CAAC;QAC7C,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,cAAc,CAAC;QACxC,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC;QACvC,cAAc,EAAE,aAAa;QAC7B,eAAe,EAAE,cAAc;QAC/B,uBAAuB,EAAE,cAAc;QACvC,wBAAwB,EAAE,eAAe;QACzC,OAAO;QACP,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QAC/C,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,YAAY,EAAE,WAAW;QACzB,UAAU,EAAE,WAAW;KACxB,CAAC,CAAC;IACH,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,IAAI,EAAE,YAAY;QAClB,QAAQ,EAAE,KAAK,CAAC,OAAO;QACvB,UAAU,EAAE,cAAc;QAC1B,SAAS,EAAE,kBAAkB,CAAC,cAAc,CAAC;QAC7C,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,MAAM,EAAE,WAAW;QACnB,cAAc,EAAE,WAAW,CAAC,KAAK,CAAC,cAAc,CAAC;QACjD,cAAc,EAAE,aAAa;QAC7B,uBAAuB,EAAE,cAAc;QACvC,eAAe,EAAE,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC;QAChD,eAAe,EAAE,cAAc;QAC/B,wBAAwB,EAAE,eAAe;QACzC,OAAO;QACP,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QAC/C,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,YAAY,EAAE,WAAW;KAC1B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,KAA4B,EAC5B,aAAkC,EAClC,UAA4B,EAC5B,SAAiB,EACjB,IAAmB,EACnB,YAA8B;IAE9B,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1H,MAAM,IAAI,GAAG,WAAW,IAAI,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IAChH,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7F,MAAM,WAAW,CAAC,8CAA8C,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3F,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QAAC,OAAO,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;IACxE,OAAO,KAAK,EAAE,CAAC;QACb,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,WAAW,CAAC,sCAAsC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpI,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACjG,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAC/C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC;QAC5E,MAAM,GAAG,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;QACvG,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC1C,MAAM,QAAQ,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;QACnC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;oBACnC,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,aAAa,EAAE,KAAK,CAAC,aAAa;oBAClC,kBAAkB,EAAE,YAAY;oBAChC,eAAe,EAAE,IAAI,CAAC,OAAO;oBAC7B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,gBAAgB,EAAE,UAAU;oBAC5B,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;oBACvC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB;oBACnD,eAAe,EAAE,iBAAiB;oBAClC,SAAS;oBACT,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;oBAC7B,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjD,qBAAqB,EAAE,IAAI;iBAC5B,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;gBAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC;oBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,uBAAuB;oBAAE,SAAS;gBACvD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC;QAC/I,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,2BAA2B,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;YACjE,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;YACvC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB;SACpD,CAAC,CAAC;QACH,OAAO;YACL,GAAG;YACH,MAAM,EAAE;gBACN,eAAe,EAAE,GAAG,CAAC,QAAQ;gBAC7B,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;gBACxG,cAAc,EAAE,IAAI,CAAC,OAAO;gBAC5B,wBAAwB,EAAE,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;aACpE;SACF,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,UAAU,CAAC,mDAAmD,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE;QAC1F,IAAI,EAAE,sCAAsC;QAC5C,QAAQ,EAAE,EAAE;KACb,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,KAA4B,EAC5B,QAAwB,EACxB,eAAgC,EAChC,SAAkB,EAClB,WAAmB;IAEnB,MAAM,MAAM,GAA4B;QACtC,GAAG,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;QAC/B,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QAC1C,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QAC7B,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS;QAC/B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO;QAC9B,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;QACxC,kBAAkB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB;QACpD,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,OAAO,EAAE,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC5C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,uCAAuC,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC;QACjK,UAAU,EAAE,OAAO,KAAK,CAAC,cAAc,EAAE,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU;QACxH,YAAY,EAAE,WAAW;KAC1B,CAAC;IACF,IAAI,SAAS;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC;IACnC,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK;QAAE,MAAM,WAAW,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IAClF,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,+BAA+B,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AACzF,CAAC"}

package/dist/src/evals/directory.js

@@ -10,7 +10,7 @@ export async function prepareEvalDirectory(input) {
         if (JSON.stringify(existingRequest) !== JSON.stringify(input.request)) {
             throw new HitchError(`precreated eval request does not match: ${input.evalId}`, { code: "eval_request_conflict", exitCode: 2 });
         }
-        if (await readJSON(path.join(directory, "result.json"), null)) {
+        if (!input.replaceTerminal && await readJSON(path.join(directory, "result.json"), null)) {
             throw new HitchError(`eval is already terminal: ${input.evalId}`, { code: "eval_id_conflict", exitCode: 2 });
         }
         return directory;

package/dist/src/evals/directory.js.map

@@ -1 +1 @@
-{"version":3,"file":"directory.js","sourceRoot":"","sources":["../../../src/evals/directory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAE/E,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAK1C;IACC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAChE,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACrB,MAAM,eAAe,GAAG,MAAM,QAAQ,CAAqB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;QACvG,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,UAAU,CAAC,uCAAuC,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC3I,IAAI,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,UAAU,CAAC,2CAA2C,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAClI,CAAC;QACD,IAAI,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,UAAU,CAAC,6BAA6B,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC/G,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,IAAI,UAAU,CAAC,2BAA2B,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7G,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3E,OAAO,SAAS,CAAC;AACnB,CAAC"}
+{"version":3,"file":"directory.js","sourceRoot":"","sources":["../../../src/evals/directory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAE/E,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAM1C;IACC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAChE,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACrB,MAAM,eAAe,GAAG,MAAM,QAAQ,CAAqB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;QACvG,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,UAAU,CAAC,uCAAuC,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC3I,IAAI,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,UAAU,CAAC,2CAA2C,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAClI,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;YACxF,MAAM,IAAI,UAAU,CAAC,6BAA6B,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC/G,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,IAAI,UAAU,CAAC,2BAA2B,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7G,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3E,OAAO,SAAS,CAAC;AACnB,CAAC"}