@assemblyline-agents/cli 0.1.0

package/dist/eval-cases.js

@@ -0,0 +1,151 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { extname, relative, resolve, sep } from "node:path";
+import { validateFixtures } from "./eval-fixtures.js";
+import { validateMocks } from "./eval-mocks.js";
+export function discoverCases(agentRoot, options = {}) {
+    const evalRoot = resolve(agentRoot, "evals");
+    let files;
+    try {
+        files = readdirSync(evalRoot, { withFileTypes: true, recursive: true })
+            .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+            .map((entry) => resolve(entry.parentPath, entry.name))
+            .filter((path) => {
+            const relativePath = relative(evalRoot, path);
+            return relativePath.split(sep)[0] !== "evaluators";
+        })
+            .sort();
+    }
+    catch (error) {
+        const code = error && typeof error === "object" && "code" in error ? String(error.code) : "";
+        if (code === "ENOENT")
+            return [];
+        throw error;
+    }
+    const discovered = files.map((path) => loadCase(evalRoot, path));
+    const filter = options.filter?.toLocaleLowerCase();
+    return discovered.filter((item) => {
+        if (filter) {
+            const haystack = `${item.name} ${item.evalCase?.description ?? ""}`.toLocaleLowerCase();
+            if (!haystack.includes(filter))
+                return false;
+        }
+        if (options.tag && !item.evalCase?.tags?.includes(options.tag))
+            return false;
+        return true;
+    });
+}
+function loadCase(evalRoot, path) {
+    const relativePath = relative(evalRoot, path).replaceAll(sep, "/");
+    const fallbackName = relativePath.slice(0, relativePath.length - extname(relativePath).length);
+    try {
+        const value = JSON.parse(readFileSync(path, "utf8"));
+        const evalCase = normalizeCase(value, fallbackName);
+        return { path, name: evalCase.name, evalCase };
+    }
+    catch (error) {
+        return { path, name: fallbackName, error: error instanceof Error ? error.message : String(error) };
+    }
+}
+function normalizeCase(value, fallbackName) {
+    if (!isRecord(value))
+        throw new Error("case must be a JSON object");
+    if (!isRecord(value.input))
+        throw new Error("input must be a JSON object");
+    validateInputShape(value.input);
+    if (!isRecord(value.expect))
+        throw new Error("expect must be a JSON object");
+    if (value.name !== undefined && (typeof value.name !== "string" || value.name.trim() === "")) {
+        throw new Error("name must be a non-empty string");
+    }
+    if (value.tags !== undefined && (!Array.isArray(value.tags) || value.tags.some((tag) => typeof tag !== "string"))) {
+        throw new Error("tags must be an array of strings");
+    }
+    if (value.repetitions !== undefined
+        && (typeof value.repetitions !== "number" || !Number.isInteger(value.repetitions) || value.repetitions < 1 || value.repetitions > 100)) {
+        throw new Error("repetitions must be an integer between 1 and 100");
+    }
+    validateTrajectory(value.expect.trajectory);
+    validateEvaluators(value.expect.evaluators);
+    validateMocks(value.mocks);
+    validateFixtures(value.fixtures);
+    return { ...value, name: typeof value.name === "string" ? value.name : fallbackName };
+}
+function validateInputShape(input) {
+    const hasMessage = typeof input.message === "string";
+    const hasTurns = input.turns !== undefined;
+    if (hasMessage === hasTurns) {
+        throw new Error("input must declare exactly one of message or turns");
+    }
+    if (!hasTurns)
+        return;
+    if (input.tool !== undefined || input.toolInput !== undefined) {
+        throw new Error("input.tool and input.toolInput belong on individual turns when input.turns is used");
+    }
+    if (!Array.isArray(input.turns) || input.turns.length === 0) {
+        throw new Error("input.turns must be a non-empty array");
+    }
+    for (const [index, turn] of input.turns.entries()) {
+        if (!isRecord(turn) || typeof turn.message !== "string") {
+            throw new Error(`input.turns[${index}].message must be a string`);
+        }
+        if (turn.approve !== undefined && typeof turn.approve !== "boolean") {
+            throw new Error(`input.turns[${index}].approve must be a boolean`);
+        }
+        if (turn.expect !== undefined) {
+            if (!isRecord(turn.expect))
+                throw new Error(`input.turns[${index}].expect must be a JSON object`);
+            for (const key of ["judge", "evaluators"]) {
+                if (turn.expect[key] !== undefined) {
+                    throw new Error(`input.turns[${index}].expect.${key} is not supported; declare ${key} on the case-level expect`);
+                }
+            }
+            validateTrajectory(turn.expect.trajectory);
+        }
+    }
+}
+function validateTrajectory(value) {
+    if (value === undefined)
+        return;
+    if (!isRecord(value) || !["contains", "ordered", "exact", "forbid"].includes(String(value.mode))) {
+        throw new Error("expect.trajectory.mode must be contains, ordered, exact, or forbid");
+    }
+    if (!Array.isArray(value.steps) || value.steps.some((step) => !isRecord(step) || typeof step.tool !== "string")) {
+        throw new Error("expect.trajectory.steps must be an array of objects with a tool string");
+    }
+    for (const step of value.steps) {
+        if (!isRecord(step))
+            continue;
+        if (step.argumentsMatch !== undefined && !["partial", "exact"].includes(String(step.argumentsMatch))) {
+            throw new Error("trajectory argumentsMatch must be partial or exact");
+        }
+        if (step.resultMatch !== undefined && !["partial", "exact"].includes(String(step.resultMatch))) {
+            throw new Error("trajectory resultMatch must be partial or exact");
+        }
+    }
+}
+function validateEvaluators(value) {
+    if (value === undefined)
+        return;
+    if (!Array.isArray(value))
+        throw new Error("expect.evaluators must be an array");
+    for (const evaluator of value) {
+        if (!isRecord(evaluator) || typeof evaluator.name !== "string" || evaluator.name.trim() === "") {
+            throw new Error("each expect.evaluators entry must have a non-empty name");
+        }
+        if (!/^[A-Za-z0-9._-]+$/u.test(evaluator.name))
+            throw new Error("expect.evaluators name contains unsupported characters");
+        if (evaluator.metric !== undefined && typeof evaluator.metric !== "string") {
+            throw new Error("expect.evaluators metric must be a string");
+        }
+        if (evaluator.requirePass !== undefined && typeof evaluator.requirePass !== "boolean") {
+            throw new Error("expect.evaluators requirePass must be a boolean");
+        }
+        if (evaluator.minScore !== undefined && (typeof evaluator.minScore !== "number" || !Number.isFinite(evaluator.minScore))) {
+            throw new Error("expect.evaluators minScore must be a finite number");
+        }
+    }
+}
+function isRecord(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+//# sourceMappingURL=eval-cases.js.map

package/dist/eval-cases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-cases.js","sourceRoot":"","sources":["../src/eval-cases.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,MAAM,UAAU,aAAa,CAC3B,SAAiB,EACjB,UAA6C,EAAE;IAE/C,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC7C,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,WAAW,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;aACpE,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aACjE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;aACrD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YACf,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC9C,OAAO,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC;QACrD,CAAC,CAAC;aACD,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,GAAG,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7F,IAAI,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC;IACnD,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QAChC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,WAAW,IAAI,EAAE,EAAE,CAAC,iBAAiB,EAAE,CAAC;YACxF,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC/C,CAAC;QACD,IAAI,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7E,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,QAAgB,EAAE,IAAY;IAC9C,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACnE,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/F,IAAI,CAAC;QACH,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACrG,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,YAAoB;IACzD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IACpE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC3E,kBAAkB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAC7E,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC7F,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QAClH,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS;WAC9B,CAAC,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,IAAI,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC,EAAE,CAAC;QACzI,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IACD,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5C,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5C,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACjC,OAAO,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,EAAyB,CAAC;AAC/G,CAAC;AAED,SAAS,kBAAkB,CAAC,KAA8B;IACxD,MAAM,UAAU,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC;IACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;IAC3C,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IACxG,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,eAAe,KAAK,4BAA4B,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,eAAe,KAAK,6BAA6B,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,eAAe,KAAK,gCAAgC,CAAC,CAAC;YAClG,KAAK,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC;gBAC1C,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;oBACnC,MAAM,IAAI,KAAK,CAAC,eAAe,KAAK,YAAY,GAAG,8BAA8B,GAAG,2BAA2B,CAAC,CAAC;gBACnH,CAAC;YACH,CAAC;YACD,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACjG,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC;QAChH,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;IAC5F,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS;QAC9B,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;YACrG,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;YAC/F,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO;IAChC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACjF,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC/F,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC1H,IAAI,SAAS,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3E,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,SAAS,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,SAAS,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACtF,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,SAAS,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,OAAO,SAAS,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACzH,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}

package/dist/eval-comparison.d.ts

@@ -0,0 +1,8 @@
+import type { EvalComparison, EvalReport } from "./eval-types.js";
+export interface EvalRegressionGates {
+    maxRegressions?: number;
+    maxCostIncreasePercent?: number;
+}
+export declare function loadBaselineReport(path: string): EvalReport;
+export declare function compareEvalReports(baseline: EvalReport, current: EvalReport, options?: EvalRegressionGates): EvalComparison;
+//# sourceMappingURL=eval-comparison.d.ts.map

package/dist/eval-comparison.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAkB,cAAc,EAAkB,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAElG,MAAM,WAAW,mBAAmB;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAO3D;AAED,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,UAAU,EACpB,OAAO,EAAE,UAAU,EACnB,OAAO,GAAE,mBAAwB,GAChC,cAAc,CAiDhB"}

package/dist/eval-comparison.js

@@ -0,0 +1,90 @@
+import { readFileSync, statSync } from "node:fs";
+import { resolve } from "node:path";
+export function loadBaselineReport(path) {
+    let target = resolve(path);
+    if (statSync(target).isDirectory())
+        target = resolve(target, "results.json");
+    else if (target.endsWith("experiment.json"))
+        target = resolve(target, "..", "results.json");
+    const value = JSON.parse(readFileSync(target, "utf8"));
+    if (!isReport(value))
+        throw new Error(`Baseline ${target} is not an Assembly Line eval results file.`);
+    return value;
+}
+export function compareEvalReports(baseline, current, options = {}) {
+    const baselineCases = aggregateCases(baseline.results);
+    const currentCases = aggregateCases(current.results);
+    const regressions = [];
+    const improvements = [];
+    for (const [caseId, baselineCase] of baselineCases) {
+        const currentCase = currentCases.get(caseId);
+        if (!currentCase)
+            continue;
+        const comparison = {
+            caseId,
+            name: currentCase.name,
+            baselinePassRate: baselineCase.passRate,
+            currentPassRate: currentCase.passRate
+        };
+        if (currentCase.passRate < baselineCase.passRate)
+            regressions.push(comparison);
+        if (currentCase.passRate > baselineCase.passRate)
+            improvements.push(comparison);
+    }
+    const maxRegressions = options.maxRegressions ?? 0;
+    const gates = [{
+            name: "max-regressions",
+            passed: regressions.length <= maxRegressions,
+            actual: regressions.length,
+            limit: maxRegressions
+        }];
+    const costDeltaUsd = current.summary.totalCostUsd - baseline.summary.totalCostUsd;
+    const costDeltaPercent = baseline.summary.totalCostUsd === 0
+        ? undefined
+        : (costDeltaUsd / baseline.summary.totalCostUsd) * 100;
+    if (options.maxCostIncreasePercent !== undefined) {
+        const actual = costDeltaPercent ?? (costDeltaUsd > 0 ? null : 0);
+        gates.push({
+            name: "max-cost-increase-percent",
+            passed: actual !== null && actual <= options.maxCostIncreasePercent,
+            actual,
+            limit: options.maxCostIncreasePercent
+        });
+    }
+    return {
+        baselineExperimentId: baseline.experiment.id,
+        regressions: regressions.sort(byCaseId),
+        improvements: improvements.sort(byCaseId),
+        addedCaseIds: [...currentCases.keys()].filter((caseId) => !baselineCases.has(caseId)).sort(),
+        removedCaseIds: [...baselineCases.keys()].filter((caseId) => !currentCases.has(caseId)).sort(),
+        passRateDelta: current.summary.passRate - baseline.summary.passRate,
+        costDeltaUsd,
+        ...(costDeltaPercent === undefined ? {} : { costDeltaPercent }),
+        gates,
+        passed: gates.every((gate) => gate.passed)
+    };
+}
+function aggregateCases(results) {
+    const counts = new Map();
+    for (const result of results) {
+        const count = counts.get(result.caseId) ?? { name: result.name, passed: 0, total: 0 };
+        count.total += 1;
+        if (result.outcome === "passed")
+            count.passed += 1;
+        counts.set(result.caseId, count);
+    }
+    return new Map([...counts].map(([caseId, count]) => [caseId, {
+            name: count.name,
+            passRate: count.total === 0 ? 0 : count.passed / count.total
+        }]));
+}
+function isReport(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const report = value;
+    return report.schemaVersion === 1 && Array.isArray(report.results) && Boolean(report.experiment) && Boolean(report.summary);
+}
+function byCaseId(left, right) {
+    return left.caseId.localeCompare(right.caseId);
+}
+//# sourceMappingURL=eval-comparison.js.map

package/dist/eval-comparison.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE;QAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;SACxE,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;IAC5F,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,YAAY,MAAM,6CAA6C,CAAC,CAAC;IACvG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,QAAoB,EACpB,OAAmB,EACnB,UAA+B,EAAE;IAEjC,MAAM,aAAa,GAAG,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACvD,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,WAAW,GAAqB,EAAE,CAAC;IACzC,MAAM,YAAY,GAAqB,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,IAAI,aAAa,EAAE,CAAC;QACnD,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,WAAW;YAAE,SAAS;QAC3B,MAAM,UAAU,GAAG;YACjB,MAAM;YACN,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,gBAAgB,EAAE,YAAY,CAAC,QAAQ;YACvC,eAAe,EAAE,WAAW,CAAC,QAAQ;SACtC,CAAC;QACF,IAAI,WAAW,CAAC,QAAQ,GAAG,YAAY,CAAC,QAAQ;YAAE,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/E,IAAI,WAAW,CAAC,QAAQ,GAAG,YAAY,CAAC,QAAQ;YAAE,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC;IACnD,MAAM,KAAK,GAA4B,CAAC;YACtC,IAAI,EAAE,iBAAiB;YACvB,MAAM,EAAE,WAAW,CAAC,MAAM,IAAI,cAAc;YAC5C,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,KAAK,EAAE,cAAc;SACtB,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC;IAClF,MAAM,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,CAAC;QAC1D,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,CAAC,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC;IACzD,IAAI,OAAO,CAAC,sBAAsB,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,gBAAgB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,2BAA2B;YACjC,MAAM,EAAE,MAAM,KAAK,IAAI,IAAI,MAAM,IAAI,OAAO,CAAC,sBAAsB;YACnE,MAAM;YACN,KAAK,EAAE,OAAO,CAAC,sBAAsB;SACtC,CAAC,CAAC;IACL,CAAC;IACD,OAAO;QACL,oBAAoB,EAAE,QAAQ,CAAC,UAAU,CAAC,EAAE;QAC5C,WAAW,EAAE,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;QACvC,YAAY,EAAE,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;QACzC,YAAY,EAAE,CAAC,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE;QAC5F,cAAc,EAAE,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9F,aAAa,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ;QACnE,YAAY;QACZ,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC;QAC/D,KAAK;QACL,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;KAC3C,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAyB;IAC/C,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2D,CAAC;IAClF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACtF,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;QACjB,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ;YAAE,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;YAC3D,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK;SAC7D,CAAC,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,MAAM,GAAG,KAA4B,CAAC;IAC5C,OAAO,MAAM,CAAC,aAAa,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC9H,CAAC;AAED,SAAS,QAAQ,CAAC,IAAoB,EAAE,KAAqB;IAC3D,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AACjD,CAAC"}

package/dist/eval-custom.d.ts

@@ -0,0 +1,15 @@
+import type { EvalCustomEvaluatorExpectation, EvalEvaluator, EvalEvaluatorContext, EvalMetric } from "./eval-types.js";
+export declare class EvalEvaluatorLoader {
+    private readonly agentRoot;
+    private readonly cacheRoot;
+    private readonly provided;
+    private readonly loaded;
+    constructor(agentRoot: string, cacheRoot: string, provided?: Record<string, EvalEvaluator>);
+    load(name: string): Promise<EvalEvaluator>;
+    private compileAndLoad;
+}
+export declare function evaluateCustomExpectations(expectations: EvalCustomEvaluatorExpectation[], loader: EvalEvaluatorLoader, context: Omit<EvalEvaluatorContext, "config">): Promise<{
+    metrics: EvalMetric[];
+    failures: string[];
+}>;
+//# sourceMappingURL=eval-custom.d.ts.map

package/dist/eval-custom.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-custom.d.ts","sourceRoot":"","sources":["../src/eval-custom.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,8BAA8B,EAC9B,aAAa,EACb,oBAAoB,EACpB,UAAU,EACX,MAAM,iBAAiB,CAAC;AAEzB,qBAAa,mBAAmB;IAI5B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAL3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6C;gBAGjD,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAM;IAGzD,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;YAYlC,cAAc;CA2B7B;AAED,wBAAsB,0BAA0B,CAC9C,YAAY,EAAE,8BAA8B,EAAE,EAC9C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,IAAI,CAAC,oBAAoB,EAAE,QAAQ,CAAC,GAC5C,OAAO,CAAC;IAAE,OAAO,EAAE,UAAU,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA6BxD"}

package/dist/eval-custom.js

@@ -0,0 +1,118 @@
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { build } from "esbuild";
+export class EvalEvaluatorLoader {
+    agentRoot;
+    cacheRoot;
+    provided;
+    loaded = new Map();
+    constructor(agentRoot, cacheRoot, provided = {}) {
+        this.agentRoot = agentRoot;
+        this.cacheRoot = cacheRoot;
+        this.provided = provided;
+    }
+    async load(name) {
+        if (!/^[A-Za-z0-9._-]+$/u.test(name))
+            throw new Error(`Invalid evaluator name: ${name}`);
+        const provided = this.provided[name];
+        if (provided)
+            return validateEvaluator(name, provided);
+        let pending = this.loaded.get(name);
+        if (!pending) {
+            pending = this.compileAndLoad(name);
+            this.loaded.set(name, pending);
+        }
+        return pending;
+    }
+    async compileAndLoad(name) {
+        const sourcePath = evaluatorSourcePath(this.agentRoot, name);
+        if (!sourcePath) {
+            throw new Error(`Custom evaluator ${name} was not found under evals/evaluators/.`);
+        }
+        const source = readFileSync(sourcePath, "utf8");
+        const hash = createHash("sha256").update(sourcePath).update(source).digest("hex");
+        mkdirSync(this.cacheRoot, { recursive: true });
+        const outputPath = resolve(this.cacheRoot, `${name}-${hash}.mjs`);
+        if (!existsSync(outputPath)) {
+            const bundled = await build({
+                entryPoints: [sourcePath],
+                bundle: true,
+                write: false,
+                platform: "node",
+                format: "esm",
+                target: "node22",
+                packages: "external",
+                logLevel: "silent"
+            });
+            const output = bundled.outputFiles[0]?.text;
+            if (!output)
+                throw new Error(`Failed to compile custom evaluator ${name}.`);
+            writeFileSync(outputPath, output, { flag: "wx" });
+        }
+        const module = await import(`${pathToFileURL(outputPath).href}?v=${hash}`);
+        return validateEvaluator(name, module.default);
+    }
+}
+export async function evaluateCustomExpectations(expectations, loader, context) {
+    const metrics = [];
+    const failures = [];
+    for (const expectation of expectations) {
+        const evaluator = await loader.load(expectation.name);
+        const result = await evaluator.evaluate({ ...context, config: expectation.config ?? {} });
+        const evaluated = (Array.isArray(result) ? result : [result]).map((metric) => normalizeMetric(expectation.name, metric));
+        if (evaluated.length === 0)
+            throw new Error(`Custom evaluator ${expectation.name} returned no metrics.`);
+        if (new Set(evaluated.map((metric) => metric.key)).size !== evaluated.length) {
+            throw new Error(`Custom evaluator ${expectation.name} returned duplicate metric keys.`);
+        }
+        metrics.push(...evaluated);
+        if (expectation.requirePass !== false) {
+            for (const metric of evaluated) {
+                if (metric.pass === false)
+                    failures.push(`${expectation.name}.${metric.key} failed${metric.comment ? `: ${metric.comment}` : ""}`);
+            }
+        }
+        if (expectation.minScore !== undefined) {
+            const scored = expectation.metric
+                ? evaluated.filter((metric) => metric.key === expectation.metric)
+                : evaluated.filter((metric) => metric.score !== undefined);
+            if (scored.length === 0) {
+                failures.push(`${expectation.name} returned no numeric ${expectation.metric ? `${expectation.metric} ` : ""}metric for minScore`);
+            }
+            else if (scored.every((metric) => metric.score < expectation.minScore)) {
+                failures.push(`${expectation.name} score did not meet ${expectation.minScore}`);
+            }
+        }
+    }
+    return { metrics, failures };
+}
+function evaluatorSourcePath(agentRoot, name) {
+    const root = resolve(agentRoot, "evals", "evaluators");
+    for (const extension of [".ts", ".mts", ".js", ".mjs"]) {
+        const path = resolve(root, `${name}${extension}`);
+        if (existsSync(path) && dirname(path) === root)
+            return path;
+    }
+    return undefined;
+}
+function validateEvaluator(name, value) {
+    if (!value || typeof value !== "object" || typeof value.evaluate !== "function") {
+        throw new Error(`Custom evaluator ${name} must default-export an object with evaluate(context).`);
+    }
+    const evaluator = value;
+    if (evaluator.name !== name)
+        throw new Error(`Custom evaluator ${name} must declare name: ${JSON.stringify(name)}.`);
+    return evaluator;
+}
+function normalizeMetric(evaluator, metric) {
+    if (!metric || typeof metric !== "object" || typeof metric.key !== "string" || metric.key.trim() === "") {
+        throw new Error(`Custom evaluator ${evaluator} returned a metric without a non-empty key.`);
+    }
+    if (metric.score !== undefined && !Number.isFinite(metric.score)) {
+        throw new Error(`Custom evaluator ${evaluator} returned a non-finite score for ${metric.key}.`);
+    }
+    return { ...metric, evaluator };
+}
+//# sourceMappingURL=eval-custom.js.map

package/dist/eval-custom.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-custom.js","sourceRoot":"","sources":["../src/eval-custom.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAQhC,MAAM,OAAO,mBAAmB;IAIX;IACA;IACA;IALF,MAAM,GAAG,IAAI,GAAG,EAAkC,CAAC;IAEpE,YACmB,SAAiB,EACjB,SAAiB,EACjB,WAA0C,EAAE;QAF5C,cAAS,GAAT,SAAS,CAAQ;QACjB,cAAS,GAAT,SAAS,CAAQ;QACjB,aAAQ,GAAR,QAAQ,CAAoC;IAC5D,CAAC;IAEJ,KAAK,CAAC,IAAI,CAAC,IAAY;QACrB,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;QACzF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,QAAQ;YAAE,OAAO,iBAAiB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACvD,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,IAAY;QACvC,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,yCAAyC,CAAC,CAAC;QACrF,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClF,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,CAAC;QAClE,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC;gBAC1B,WAAW,EAAE,CAAC,UAAU,CAAC;gBACzB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,MAAM;gBAChB,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,QAAQ;gBAChB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;YAC5C,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,GAAG,CAAC,CAAC;YAC5E,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,IAAI,MAAM,IAAI,EAAE,CAA0B,CAAC;QACpG,OAAO,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,YAA8C,EAC9C,MAA2B,EAC3B,OAA6C;IAE7C,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1F,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QACzH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,WAAW,CAAC,IAAI,uBAAuB,CAAC,CAAC;QACzG,IAAI,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC7E,MAAM,IAAI,KAAK,CAAC,oBAAoB,WAAW,CAAC,IAAI,kCAAkC,CAAC,CAAC;QAC1F,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAC3B,IAAI,WAAW,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YACtC,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;gBAC/B,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK;oBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,IAAI,MAAM,CAAC,GAAG,UAAU,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrI,CAAC;QACH,CAAC;QACD,IAAI,WAAW,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM;gBAC/B,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,KAAK,WAAW,CAAC,MAAM,CAAC;gBACjE,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;YAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,wBAAwB,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;YACpI,CAAC;iBAAM,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAE,MAAM,CAAC,KAAgB,GAAI,WAAW,CAAC,QAAmB,CAAC,EAAE,CAAC;gBACjG,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,uBAAuB,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AAC/B,CAAC;AAED,SAAS,mBAAmB,CAAC,SAAiB,EAAE,IAAY;IAC1D,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IACvD,KAAK,MAAM,SAAS,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC,CAAC;QAClD,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;IAC9D,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,KAAc;IACrD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAQ,KAAuB,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QACnG,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,wDAAwD,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,SAAS,GAAG,KAAsB,CAAC;IACzC,IAAI,SAAS,CAAC,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrH,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,SAAiB,EAAE,MAAkB;IAC5D,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACxG,MAAM,IAAI,KAAK,CAAC,oBAAoB,SAAS,6CAA6C,CAAC,CAAC;IAC9F,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,oBAAoB,SAAS,oCAAoC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,CAAC;AAClC,CAAC"}

package/dist/eval-experiment.d.ts

@@ -0,0 +1,19 @@
+import type { DiscoveredEvalCase, EvalExecutionConfig, EvalExperimentManifest, EvalReport } from "./eval-types.js";
+export interface EvalExperimentArtifacts {
+    root: string;
+    experimentPath: string;
+    resultsPath: string;
+}
+export declare function createExperimentManifest(input: {
+    startedAt: string;
+    agentRevision: string;
+    agentRoot: string;
+    cases: DiscoveredEvalCase[];
+    config: EvalExecutionConfig;
+}): EvalExperimentManifest;
+export declare function createExperimentArtifacts(artifactRoot: string, manifest: EvalExperimentManifest): EvalExperimentArtifacts;
+export declare function writeExperimentManifest(manifest: EvalExperimentManifest, artifacts: EvalExperimentArtifacts): void;
+export declare function writeExperimentReport(report: EvalReport, artifacts: EvalExperimentArtifacts): void;
+export declare function fingerprintEvalSuite(agentRoot: string, cases: DiscoveredEvalCase[]): string;
+export declare function fingerprintValue(value: unknown): string;
+//# sourceMappingURL=eval-experiment.d.ts.map

package/dist/eval-experiment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-experiment.d.ts","sourceRoot":"","sources":["../src/eval-experiment.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,kBAAkB,EAClB,mBAAmB,EACnB,sBAAsB,EACtB,UAAU,EACX,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,MAAM,EAAE,mBAAmB,CAAC;CAC7B,GAAG,sBAAsB,CAkBzB;AAED,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,sBAAsB,GAAG,uBAAuB,CAOzH;AAED,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,uBAAuB,GACjC,IAAI,CAGN;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,uBAAuB,GAAG,IAAI,CAElG;AAED,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,MAAM,CAiC3F;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEvD"}

package/dist/eval-experiment.js

@@ -0,0 +1,122 @@
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { basename, relative, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { buildSync } from "esbuild";
+export function createExperimentManifest(input) {
+    const suiteFingerprint = fingerprintEvalSuite(input.agentRoot, input.cases);
+    const configFingerprint = fingerprintValue(input.config);
+    return {
+        schemaVersion: 1,
+        id: `${input.startedAt.replace(/[:.]/gu, "-")}-${suiteFingerprint.slice(0, 10)}-${randomUUID().slice(0, 8)}`,
+        startedAt: input.startedAt,
+        frameworkVersion: frameworkVersion(),
+        agentRevision: input.agentRevision,
+        suiteFingerprint,
+        configFingerprint,
+        caseCount: input.cases.length,
+        executionCount: input.cases.reduce((count, item) => count + (item.evalCase ? item.evalCase.repetitions ?? input.config.repetitions : 1), 0),
+        config: input.config
+    };
+}
+export function createExperimentArtifacts(artifactRoot, manifest) {
+    const root = resolve(artifactRoot, "eval-results", manifest.id);
+    return {
+        root,
+        experimentPath: resolve(root, "experiment.json"),
+        resultsPath: resolve(root, "results.json")
+    };
+}
+export function writeExperimentManifest(manifest, artifacts) {
+    mkdirSync(artifacts.root, { recursive: true });
+    writeJsonExclusive(artifacts.experimentPath, manifest);
+}
+export function writeExperimentReport(report, artifacts) {
+    writeJsonExclusive(artifacts.resultsPath, report);
+}
+export function fingerprintEvalSuite(agentRoot, cases) {
+    const files = new Set(cases.map((item) => item.path));
+    for (const item of cases) {
+        for (const evaluator of item.evalCase?.expect.evaluators ?? []) {
+            const source = evaluatorSourcePath(agentRoot, evaluator.name);
+            if (source)
+                files.add(source);
+        }
+    }
+    const hash = createHash("sha256");
+    for (const path of [...files].sort()) {
+        hash.update(relative(agentRoot, path).replaceAll("\\", "/"));
+        hash.update("\0");
+        hash.update(readFileSync(path));
+        hash.update("\0");
+    }
+    for (const sourcePath of evaluatorSources(agentRoot, cases)) {
+        const bundled = buildSync({
+            entryPoints: [sourcePath],
+            bundle: true,
+            write: false,
+            platform: "node",
+            format: "esm",
+            target: "node22",
+            packages: "external",
+            legalComments: "none",
+            logLevel: "silent"
+        }).outputFiles[0]?.contents;
+        if (!bundled)
+            throw new Error(`Failed to fingerprint custom evaluator ${basename(sourcePath)}.`);
+        hash.update(`bundle:${relative(agentRoot, sourcePath).replaceAll("\\", "/")}\0`);
+        hash.update(bundled);
+        hash.update("\0");
+    }
+    return hash.digest("hex");
+}
+export function fingerprintValue(value) {
+    return createHash("sha256").update(stableJson(value)).digest("hex");
+}
+function evaluatorSourcePath(agentRoot, name) {
+    if (!/^[A-Za-z0-9._-]+$/u.test(name))
+        return undefined;
+    const root = resolve(agentRoot, "evals", "evaluators");
+    for (const extension of [".ts", ".mts", ".js", ".mjs"]) {
+        const path = resolve(root, `${name}${extension}`);
+        if (existsSync(path))
+            return path;
+    }
+    return undefined;
+}
+function evaluatorSources(agentRoot, cases) {
+    const sources = new Set();
+    for (const item of cases) {
+        for (const evaluator of item.evalCase?.expect.evaluators ?? []) {
+            const source = evaluatorSourcePath(agentRoot, evaluator.name);
+            if (source)
+                sources.add(source);
+        }
+    }
+    return [...sources].sort();
+}
+function stableJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(stableJson).join(",")}]`;
+    if (value && typeof value === "object") {
+        const entries = Object.entries(value)
+            .filter(([, item]) => item !== undefined)
+            .sort(([left], [right]) => left.localeCompare(right));
+        return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`;
+    }
+    return JSON.stringify(value);
+}
+function frameworkVersion() {
+    try {
+        const packagePath = resolve(fileURLToPath(new URL(".", import.meta.url)), "..", "package.json");
+        const parsed = JSON.parse(readFileSync(packagePath, "utf8"));
+        return typeof parsed.version === "string" ? parsed.version : "unknown";
+    }
+    catch {
+        return "unknown";
+    }
+}
+function writeJsonExclusive(path, value) {
+    writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", flag: "wx" });
+}
+//# sourceMappingURL=eval-experiment.js.map

package/dist/eval-experiment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-experiment.js","sourceRoot":"","sources":["../src/eval-experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAcpC,MAAM,UAAU,wBAAwB,CAAC,KAMxC;IACC,MAAM,gBAAgB,GAAG,oBAAoB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAC5E,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzD,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,EAAE,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAC5G,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,gBAAgB,EAAE,gBAAgB,EAAE;QACpC,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,gBAAgB;QAChB,iBAAiB;QACjB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;QAC7B,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAChC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,EACpG,CAAC,CACF;QACD,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,YAAoB,EAAE,QAAgC;IAC9F,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAChE,OAAO;QACL,IAAI;QACJ,cAAc,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC;QAChD,WAAW,EAAE,OAAO,CAAC,IAAI,EAAE,cAAc,CAAC;KAC3C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,QAAgC,EAChC,SAAkC;IAElC,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,kBAAkB,CAAC,SAAS,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAAkB,EAAE,SAAkC;IAC1F,kBAAkB,CAAC,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,SAAiB,EAAE,KAA2B;IACjF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;YAC/D,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YAC9D,IAAI,MAAM;gBAAE,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClB,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,KAAK,MAAM,UAAU,IAAI,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;QAC5D,MAAM,OAAO,GAAG,SAAS,CAAC;YACxB,WAAW,EAAE,CAAC,UAAU,CAAC;YACzB,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,UAAU;YACpB,aAAa,EAAE,MAAM;YACrB,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACjG,IAAI,CAAC,MAAM,CAAC,UAAU,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QACjF,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,mBAAmB,CAAC,SAAiB,EAAE,IAAY;IAC1D,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACvD,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IACvD,KAAK,MAAM,SAAS,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC,CAAC;QAClD,IAAI,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;IACpC,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAiB,EAAE,KAA2B;IACtE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;YAC/D,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YAC9D,IAAI,MAAM;gBAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC;aAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC;aACxC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;QACxD,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACrG,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,gBAAgB;IACvB,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QAChG,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAA0B,CAAC;QACtF,OAAO,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,KAAc;IACtD,aAAa,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC/F,CAAC"}

package/dist/eval-fixtures.d.ts

@@ -0,0 +1,17 @@
+import type { MemoryStore, SandboxAdapter } from "@assemblyline-agents/runtime";
+import type { EvalFixtures } from "./eval-types.js";
+/**
+ * Case-declared starting state: sandbox files and memory documents seeded
+ * before the first turn. Sandbox seeding requires the case's runs to share a
+ * conversation-scoped sandbox session, so the runner assigns a conversationId
+ * to any case with sandbox fixtures.
+ */
+export declare function validateFixtures(value: unknown): void;
+export declare function seedEvalFixtures(input: {
+    fixtures: EvalFixtures;
+    sandbox: SandboxAdapter;
+    memoryStore: MemoryStore;
+    agentScope: string;
+    conversationId?: string;
+}): Promise<void>;
+//# sourceMappingURL=eval-fixtures.d.ts.map

package/dist/eval-fixtures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-fixtures.d.ts","sourceRoot":"","sources":["../src/eval-fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAa,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE3F,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEpD;;;;;GAKG;AAEH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAcrD;AAED,wBAAsB,gBAAgB,CAAC,KAAK,EAAE;IAC5C,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,cAAc,CAAC;IACxB,WAAW,EAAE,WAAW,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhB"}