@assemblyline-agents/cli 0.1.0

package/dist/dev.js

@@ -0,0 +1,157 @@
+import { watch } from "node:fs";
+import { relative, resolve, sep } from "node:path";
+import { buildAgent, validateAgentFolder } from "@assemblyline-agents/compiler";
+import { listenNodeRuntime, withNodeModelHarnesses } from "@assemblyline-agents/node";
+import { FileStateAdapter, LocalBlobAdapter, LocalSandboxAdapter, OpenEveRuntime, loadRuntimeBundle } from "@assemblyline-agents/runtime";
+import { printIssues } from "./help.js";
+const DEBOUNCE_MS = 200;
+export async function runDevCommand(options) {
+    if (!options.watch) {
+        await runDevOnce(options);
+        return;
+    }
+    await runDevWatch(options);
+}
+async function runDevOnce(options) {
+    const artifact = buildAgent(devBuildOptions(options));
+    const runtimeOptions = loadRuntimeBundle(artifact.root);
+    const runtime = new OpenEveRuntime(withNodeModelHarnesses({
+        ...runtimeOptions,
+        state: new FileStateAdapter(resolve(artifact.root, "dev-state.json")),
+        blob: new LocalBlobAdapter(resolve(artifact.root, "blobs")),
+        sandbox: new LocalSandboxAdapter(resolve(artifact.root, "sandbox")),
+        devMode: true
+    }));
+    const result = await runtime.run({
+        message: options.message ?? "Assembly Line dev check",
+        toolName: options.toolName ?? "echo",
+        approve: options.approve ?? false,
+        channel: "local"
+    });
+    console.log(`Assembly Line dev runtime ready for ${options.agentRoot}`);
+    console.log(`Run ${result.run.id} ${result.run.status}`);
+    console.log(result.response ?? "");
+}
+async function runDevWatch(options) {
+    const agentRoot = options.agentRoot;
+    const validation = validateAgentFolder(agentRoot, catalogOptions(options));
+    printIssues(validation.issues);
+    if (!validation.valid)
+        process.exit(1);
+    let artifact = buildAgent(devBuildOptions(options));
+    let server = await listenDevServer(artifact.root, options.port ?? 0);
+    const port = boundPort(server.server) ?? options.port ?? 0;
+    let debounceTimer;
+    let rebuilding = false;
+    let rerunRequested = false;
+    let closed = false;
+    const watcher = watch(agentRoot, { recursive: true }, (_eventType, fileName) => {
+        if (closed)
+            return;
+        if (fileName && shouldIgnoreChange(agentRoot, artifact.root, String(fileName)))
+            return;
+        if (debounceTimer)
+            clearTimeout(debounceTimer);
+        debounceTimer = setTimeout(() => {
+            void handleChange();
+        }, DEBOUNCE_MS);
+    });
+    console.log(`Assembly Line dev server listening at ${server.url}`);
+    console.log(`Watching ${agentRoot} for changes. Press Ctrl+C to stop.`);
+    async function handleChange() {
+        if (closed)
+            return;
+        if (rebuilding) {
+            rerunRequested = true;
+            return;
+        }
+        rebuilding = true;
+        try {
+            const revalidation = validateAgentFolder(agentRoot, catalogOptions(options));
+            if (!revalidation.valid) {
+                printIssues(revalidation.issues);
+                console.log("Agent is invalid; waiting for a valid agent before restarting.");
+                return;
+            }
+            const nextArtifact = buildAgent(devBuildOptions(options));
+            if (nextArtifact.manifest.agentRevision === artifact.manifest.agentRevision) {
+                console.log("Rebuilt: no manifest change; keeping the running server.");
+                return;
+            }
+            artifact = nextArtifact;
+            await closeServer(server.server);
+            server = await listenDevServer(artifact.root, port);
+            console.log(`Restarted on ${server.url} (revision ${artifact.manifest.agentRevision}).`);
+        }
+        catch (error) {
+            console.error(`Dev rebuild failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        finally {
+            rebuilding = false;
+            if (rerunRequested) {
+                rerunRequested = false;
+                void handleChange();
+            }
+        }
+    }
+    await new Promise((resolveExit) => {
+        process.once("SIGINT", () => {
+            closed = true;
+            if (debounceTimer)
+                clearTimeout(debounceTimer);
+            watcher.close();
+            void closeServer(server.server).finally(() => {
+                console.log("Stopped dev server.");
+                resolveExit();
+            });
+        });
+    });
+    process.exit(0);
+}
+async function listenDevServer(artifactRoot, port) {
+    const runtimeOptions = loadRuntimeBundle(artifactRoot);
+    return listenNodeRuntime({
+        ...runtimeOptions,
+        state: new FileStateAdapter(resolve(artifactRoot, "dev-state.json")),
+        blob: new LocalBlobAdapter(resolve(artifactRoot, "blobs")),
+        sandbox: new LocalSandboxAdapter(resolve(artifactRoot, "sandbox")),
+        devMode: true,
+        port
+    });
+}
+function devBuildOptions(options) {
+    const buildOptions = { root: options.agentRoot };
+    if (options.outDir)
+        buildOptions.outDir = options.outDir;
+    return buildOptions;
+}
+function catalogOptions(options) {
+    return options.modelCatalog ? { modelCatalog: options.modelCatalog } : {};
+}
+function shouldIgnoreChange(agentRoot, artifactRoot, fileName) {
+    const absolute = resolve(agentRoot, fileName);
+    const insideArtifact = relative(artifactRoot, absolute);
+    if (insideArtifact === "" || (!insideArtifact.startsWith("..") && !insideArtifact.includes(`..${sep}`))) {
+        return true;
+    }
+    const segments = fileName.split(/[\\/]/u);
+    return segments.some((segment) => segment === ".openeve" || segment === "node_modules" || segment.startsWith("."));
+}
+function boundPort(server) {
+    const address = server.address();
+    if (address && typeof address === "object")
+        return address.port;
+    return undefined;
+}
+function closeServer(server) {
+    return new Promise((resolveClose) => {
+        if (!server.listening) {
+            resolveClose();
+            return;
+        }
+        server.close(() => resolveClose());
+        // Dev-only: drop keep-alive sockets so restart is immediate.
+        server.closeAllConnections?.();
+    });
+}
+//# sourceMappingURL=dev.js.map

package/dist/dev.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dev.js","sourceRoot":"","sources":["../src/dev.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAkB,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAmC,MAAM,+BAA+B,CAAC;AACjH,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACtF,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAE1I,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAaxC,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAA0B;IAC5D,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAC1B,OAAO;IACT,CAAC;IACD,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,OAA0B;IAClD,MAAM,QAAQ,GAAG,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IACtD,MAAM,cAAc,GAAG,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,sBAAsB,CAAC;QACxD,GAAG,cAAc;QACjB,KAAK,EAAE,IAAI,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QACrE,IAAI,EAAE,IAAI,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC3D,OAAO,EAAE,IAAI,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACnE,OAAO,EAAE,IAAI;KACd,CAAC,CAAC,CAAC;IACJ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC/B,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,yBAAyB;QACrD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM;QACpC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK;QACjC,OAAO,EAAE,OAAO;KACjB,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,uCAAuC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;AACrC,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,OAA0B;IACnD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,MAAM,UAAU,GAAG,mBAAmB,CAAC,SAAS,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3E,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC/B,IAAI,CAAC,UAAU,CAAC,KAAK;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEvC,IAAI,QAAQ,GAAG,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IACpD,IAAI,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IACrE,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAE3D,IAAI,aAAyC,CAAC;IAC9C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,MAAM,OAAO,GAAc,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE;QACxF,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,QAAQ,IAAI,kBAAkB,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAAE,OAAO;QACvF,IAAI,aAAa;YAAE,YAAY,CAAC,aAAa,CAAC,CAAC;QAC/C,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,KAAK,YAAY,EAAE,CAAC;QACtB,CAAC,EAAE,WAAW,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,yCAAyC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,qCAAqC,CAAC,CAAC;IAExE,KAAK,UAAU,YAAY;QACzB,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,UAAU,EAAE,CAAC;YACf,cAAc,GAAG,IAAI,CAAC;YACtB,OAAO;QACT,CAAC;QACD,UAAU,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,mBAAmB,CAAC,SAAS,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7E,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;gBACxB,WAAW,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;gBAC9E,OAAO;YACT,CAAC;YACD,MAAM,YAAY,GAAG,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1D,IAAI,YAAY,CAAC,QAAQ,CAAC,aAAa,KAAK,QAAQ,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;gBAC5E,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;gBACxE,OAAO;YACT,CAAC;YACD,QAAQ,GAAG,YAAY,CAAC;YACxB,MAAM,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,GAAG,cAAc,QAAQ,CAAC,QAAQ,CAAC,aAAa,IAAI,CAAC,CAAC;QAC3F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,uBAAuB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACjG,CAAC;gBAAS,CAAC;YACT,UAAU,GAAG,KAAK,CAAC;YACnB,IAAI,cAAc,EAAE,CAAC;gBACnB,cAAc,GAAG,KAAK,CAAC;gBACvB,KAAK,YAAY,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,WAAW,EAAE,EAAE;QACtC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,IAAI,CAAC;YACd,IAAI,aAAa;gBAAE,YAAY,CAAC,aAAa,CAAC,CAAC;YAC/C,OAAO,CAAC,KAAK,EAAE,CAAC;YAChB,KAAK,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;gBAC3C,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gBACnC,WAAW,EAAE,CAAC;YAChB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,YAAoB,EAAE,IAAY;IAC/D,MAAM,cAAc,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IACvD,OAAO,iBAAiB,CAAC;QACvB,GAAG,cAAc;QACjB,KAAK,EAAE,IAAI,gBAAgB,CAAC,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;QACpE,IAAI,EAAE,IAAI,gBAAgB,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAC1D,OAAO,EAAE,IAAI,mBAAmB,CAAC,OAAO,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAClE,OAAO,EAAE,IAAI;QACb,IAAI;KACL,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe,CAAC,OAA0B;IACjD,MAAM,YAAY,GAAsC,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;IACpF,IAAI,OAAO,CAAC,MAAM;QAAE,YAAY,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACzD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,SAAS,cAAc,CAAC,OAA0B;IAChD,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,SAAS,kBAAkB,CAAC,SAAiB,EAAE,YAAoB,EAAE,QAAgB;IACnF,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC9C,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACxD,IAAI,cAAc,KAAK,EAAE,IAAI,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;QACxG,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC1C,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,UAAU,IAAI,OAAO,KAAK,cAAc,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;AACrH,CAAC;AAED,SAAS,SAAS,CAAC,MAAc;IAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;IACjC,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC;IAChE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,MAAc;IACjC,OAAO,IAAI,OAAO,CAAC,CAAC,YAAY,EAAE,EAAE;QAClC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,YAAY,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC;QACnC,6DAA6D;QAC7D,MAAM,CAAC,mBAAmB,EAAE,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/eval-assertions.d.ts

@@ -0,0 +1,5 @@
+import type { EvalActual, EvalCase, EvalDeterministicExpectations } from "./eval-types.js";
+export declare function evaluateCase(evalCase: EvalCase, actual: EvalActual): string[];
+/** Deterministic assertion pass shared by whole-case and per-turn expectations. */
+export declare function evaluateExpectations(expect: EvalDeterministicExpectations, actual: EvalActual): string[];
+//# sourceMappingURL=eval-assertions.d.ts.map

package/dist/eval-assertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-assertions.d.ts","sourceRoot":"","sources":["../src/eval-assertions.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,6BAA6B,EAAE,MAAM,iBAAiB,CAAC;AAE3F,wBAAgB,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,GAAG,MAAM,EAAE,CAE7E;AAED,mFAAmF;AACnF,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,6BAA6B,EAAE,MAAM,EAAE,UAAU,GAAG,MAAM,EAAE,CAqCxG"}

package/dist/eval-assertions.js

@@ -0,0 +1,99 @@
+import { isDeepStrictEqual } from "node:util";
+import { validateAgainstJsonSchema } from "@assemblyline-agents/core";
+import { evaluateTrajectory } from "./eval-trajectory.js";
+export function evaluateCase(evalCase, actual) {
+    return evaluateExpectations(evalCase.expect, actual);
+}
+/** Deterministic assertion pass shared by whole-case and per-turn expectations. */
+export function evaluateExpectations(expect, actual) {
+    const failures = [];
+    if (expect.status && actual.status !== expect.status) {
+        failures.push(`expected status ${expect.status}, got ${actual.status}`);
+    }
+    const output = expect.output;
+    if (output?.contains) {
+        for (const fragment of output.contains) {
+            if (!actual.response.includes(fragment))
+                failures.push(`output did not contain ${JSON.stringify(fragment)}`);
+        }
+    }
+    if (output?.matches) {
+        try {
+            if (!new RegExp(output.matches, "u").test(actual.response))
+                failures.push(`output did not match /${output.matches}/u`);
+        }
+        catch (error) {
+            failures.push(`invalid output.matches regex: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    if (output?.equals !== undefined && actual.response !== output.equals) {
+        failures.push(`expected exact output ${JSON.stringify(output.equals)}, got ${JSON.stringify(actual.response)}`);
+    }
+    if (output?.json)
+        evaluateJson(output.json, actual, failures);
+    for (const tool of expect.toolsCalled ?? []) {
+        if (!actual.toolsCalled.includes(tool))
+            failures.push(`expected tool ${tool} to be called`);
+    }
+    for (const tool of expect.toolsNotCalled ?? []) {
+        if (actual.toolsCalled.includes(tool))
+            failures.push(`expected tool ${tool} not to be called`);
+    }
+    if (expect.trajectory) {
+        failures.push(...evaluateTrajectory(expect.trajectory, actual.toolCalls ?? []));
+    }
+    if (expect.maxCostUsd !== undefined && !actual.costComplete) {
+        failures.push("maxCostUsd could not be evaluated because one or more provider cash receipts are unavailable");
+    }
+    else if (expect.maxCostUsd !== undefined && actual.costUsd > expect.maxCostUsd) {
+        failures.push(`cost $${actual.costUsd.toFixed(6)} exceeded $${expect.maxCostUsd.toFixed(6)}`);
+    }
+    return failures;
+}
+function evaluateJson(expectation, actual, failures) {
+    let parsed = actual.output;
+    let parseError;
+    if (parsed === undefined) {
+        try {
+            parsed = JSON.parse(actual.response);
+        }
+        catch (error) {
+            parseError = error instanceof Error ? error.message : String(error);
+        }
+    }
+    if (expectation.valid === true && parseError)
+        failures.push(`output was not valid JSON: ${parseError}`);
+    if (expectation.valid === false && !parseError)
+        failures.push("expected output not to be valid JSON");
+    if (parsed === undefined) {
+        if (expectation.jsonSchema !== undefined || expectation.fields !== undefined)
+            failures.push("JSON assertions require valid JSON output");
+        return;
+    }
+    if (expectation.jsonSchema !== undefined) {
+        const validation = validateAgainstJsonSchema(parsed, expectation.jsonSchema);
+        if (!validation.valid)
+            failures.push(`output schema validation failed: ${validation.errors.join("; ")}`);
+    }
+    for (const [path, expected] of Object.entries(expectation.fields ?? {})) {
+        const found = valueAtPath(parsed, path);
+        if (!found.exists)
+            failures.push(`JSON field ${path} was missing`);
+        else if (!isDeepStrictEqual(found.value, expected)) {
+            failures.push(`JSON field ${path} expected ${JSON.stringify(expected)}, got ${JSON.stringify(found.value)}`);
+        }
+    }
+}
+function valueAtPath(value, path) {
+    const segments = path.startsWith("/")
+        ? path.slice(1).split("/").filter(Boolean).map((segment) => segment.replace(/~1/gu, "/").replace(/~0/gu, "~"))
+        : path.split(".").filter(Boolean);
+    let current = value;
+    for (const segment of segments) {
+        if (!current || typeof current !== "object" || !(segment in current))
+            return { exists: false };
+        current = current[segment];
+    }
+    return { exists: true, value: current };
+}
+//# sourceMappingURL=eval-assertions.js.map

package/dist/eval-assertions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-assertions.js","sourceRoot":"","sources":["../src/eval-assertions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,yBAAyB,EAAmC,MAAM,2BAA2B,CAAC;AACvG,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAG1D,MAAM,UAAU,YAAY,CAAC,QAAkB,EAAE,MAAkB;IACjE,OAAO,oBAAoB,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,oBAAoB,CAAC,MAAqC,EAAE,MAAkB;IAC5F,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QACrD,QAAQ,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,MAAM,SAAS,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,IAAI,MAAM,EAAE,QAAQ,EAAE,CAAC;QACrB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,QAAQ,CAAC,IAAI,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC/G,CAAC;IACH,CAAC;IACD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAAE,QAAQ,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;QACzH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IACD,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QACtE,QAAQ,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAClH,CAAC;IACD,IAAI,MAAM,EAAE,IAAI;QAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC9D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,iBAAiB,IAAI,eAAe,CAAC,CAAC;IAC9F,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,cAAc,IAAI,EAAE,EAAE,CAAC;QAC/C,IAAI,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,iBAAiB,IAAI,mBAAmB,CAAC,CAAC;IACjG,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC5D,QAAQ,CAAC,IAAI,CAAC,8FAA8F,CAAC,CAAC;IAChH,CAAC;SAAM,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QACjF,QAAQ,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,YAAY,CACnB,WAA2E,EAC3E,MAAkB,EAClB,QAAkB;IAElB,IAAI,MAAM,GAA0B,MAAM,CAAC,MAAM,CAAC;IAClD,IAAI,UAA8B,CAAC;IACnC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAc,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,UAAU,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IACD,IAAI,WAAW,CAAC,KAAK,KAAK,IAAI,IAAI,UAAU;QAAE,QAAQ,CAAC,IAAI,CAAC,8BAA8B,UAAU,EAAE,CAAC,CAAC;IACxG,IAAI,WAAW,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,UAAU;QAAE,QAAQ,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACtG,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,IAAI,WAAW,CAAC,UAAU,KAAK,SAAS,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QACzI,OAAO;IACT,CAAC;IACD,IAAI,WAAW,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,yBAAyB,CAAC,MAAM,EAAE,WAAW,CAAC,UAAwB,CAAC,CAAC;QAC3F,IAAI,CAAC,UAAU,CAAC,KAAK;YAAE,QAAQ,CAAC,IAAI,CAAC,oCAAoC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3G,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QACxE,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,QAAQ,CAAC,IAAI,CAAC,cAAc,IAAI,cAAc,CAAC,CAAC;aAC9D,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;YACnD,QAAQ,CAAC,IAAI,CAAC,cAAc,IAAI,aAAa,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/G,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAc,EAAE,IAAY;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QACnC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9G,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC/F,OAAO,GAAI,OAAmC,CAAC,OAAO,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC"}

package/dist/eval-case-runner.d.ts

@@ -0,0 +1,38 @@
+import { type RuntimeOptions } from "@assemblyline-agents/runtime";
+import { EvalEvaluatorLoader } from "./eval-custom.js";
+import { type EvalRuntimeHost } from "./eval-turns.js";
+import type { DiscoveredEvalCase, EvalCaseResult } from "./eval-types.js";
+export interface EvalCaseRuntimeDependencies {
+    createRuntime?: (input: {
+        caseRoot: string;
+        runtimeOptions: RuntimeOptions;
+    }) => EvalRuntimeHost;
+    judgeComplete?: (input: {
+        model: string;
+        systemPrompt: string;
+        prompt: string;
+    }) => Promise<{
+        text: string;
+        costUsd?: number;
+    }>;
+    sleep?: (ms: number) => Promise<void>;
+}
+export declare function runEvalCaseWithRetries(input: {
+    item: DiscoveredEvalCase & {
+        evalCase: NonNullable<DiscoveredEvalCase["evalCase"]>;
+    };
+    caseId: string;
+    repetition: number;
+    repetitionsPlanned: number;
+    caseRoot: string;
+    artifactRoot: string;
+    evaluatorLoader: EvalEvaluatorLoader;
+    approve: boolean;
+    timeoutMs: number;
+    judgeModel: string;
+    retryAttempts: number;
+    retryBackoffMs: number;
+    env: Record<string, string | undefined>;
+    dependencies: EvalCaseRuntimeDependencies;
+}): Promise<EvalCaseResult>;
+//# sourceMappingURL=eval-case-runner.d.ts.map

package/dist/eval-case-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-case-runner.d.ts","sourceRoot":"","sources":["../src/eval-case-runner.ts"],"names":[],"mappings":"AAGA,OAAO,EASL,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAA8B,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAKnF,OAAO,EAA0C,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAC/F,OAAO,KAAK,EAAE,kBAAkB,EAAE,cAAc,EAAqB,MAAM,iBAAiB,CAAC;AAE7F,MAAM,WAAW,2BAA2B;IAC1C,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,cAAc,CAAA;KAAE,KAAK,eAAe,CAAC;IACjG,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChI,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAED,wBAAsB,sBAAsB,CAAC,KAAK,EAAE;IAClD,IAAI,EAAE,kBAAkB,GAAG;QAAE,QAAQ,EAAE,WAAW,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAA;KAAE,CAAC;IACrF,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,mBAAmB,CAAC;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACxC,YAAY,EAAE,2BAA2B,CAAC;CAC3C,GAAG,OAAO,CAAC,cAAc,CAAC,CAuB1B"}

package/dist/eval-case-runner.js

@@ -0,0 +1,247 @@
+import { mkdirSync } from "node:fs";
+import { resolve } from "node:path";
+import { withNodeModelHarnesses } from "@assemblyline-agents/node";
+import { durableAgentScopeForManifest, FileStateAdapter, LocalBlobAdapter, LocalSandboxAdapter, OpenEveRuntime, StateMemoryStore, loadRuntimeBundle, noopLogger } from "@assemblyline-agents/runtime";
+import { evaluateCase } from "./eval-assertions.js";
+import { evaluateCustomExpectations } from "./eval-custom.js";
+import { seedEvalFixtures } from "./eval-fixtures.js";
+import { createDefaultJudgeCompleter, EvalJudgeResponseError, judgeCase } from "./eval-judge.js";
+import { createToolStubs } from "./eval-mocks.js";
+import { isTransientProviderError, retryTransient } from "./eval-retry.js";
+import { conversationIdForCase, runConversation } from "./eval-turns.js";
+export async function runEvalCaseWithRetries(input) {
+    const startedAt = Date.now();
+    let runCostUsd = 0;
+    let judgeCostUsd = 0;
+    let attempt = 1;
+    while (true) {
+        const attemptRoot = resolve(input.caseRoot, `attempt-${String(attempt).padStart(2, "0")}`);
+        mkdirSync(attemptRoot, { recursive: true });
+        const attempted = await runOneAttempt(input, attemptRoot, attempt);
+        runCostUsd += attempted.result.runCostUsd;
+        judgeCostUsd += attempted.result.judgeCostUsd;
+        if (!attempted.retryable || attempt > input.retryAttempts) {
+            return {
+                ...attempted.result,
+                attempts: attempt,
+                durationMs: Date.now() - startedAt,
+                runCostUsd,
+                judgeCostUsd
+            };
+        }
+        await (input.dependencies.sleep ?? defaultSleep)(input.retryBackoffMs * (2 ** (attempt - 1)));
+        attempt += 1;
+    }
+}
+async function runOneAttempt(input, caseRoot, attempt) {
+    const startedAt = Date.now();
+    const item = input.item;
+    const base = {
+        caseId: input.caseId,
+        name: item.evalCase.name,
+        path: item.path,
+        tags: item.evalCase.tags ?? [],
+        repetition: input.repetition,
+        repetitionsPlanned: input.repetitionsPlanned,
+        attempts: 1
+    };
+    let runCostUsd = 0;
+    let judgeCostUsd = 0;
+    let phase = "runtime";
+    try {
+        const bundle = loadRuntimeBundle(input.artifactRoot);
+        const state = new FileStateAdapter(resolve(caseRoot, "state.json"));
+        const sandbox = new LocalSandboxAdapter(resolve(caseRoot, "sandbox"));
+        const memoryStore = new StateMemoryStore(state);
+        const toolStubs = createToolStubs(item.evalCase.mocks);
+        const runtimeOptions = withNodeModelHarnesses({
+            ...bundle,
+            state,
+            blob: new LocalBlobAdapter(resolve(caseRoot, "blobs")),
+            sandbox,
+            memoryStore,
+            ...(toolStubs ? { toolStubs } : {}),
+            deliveryMode: "record-only",
+            logger: noopLogger(),
+            env: input.env,
+            devMode: true
+        });
+        const conversationId = conversationIdForCase(item.evalCase);
+        if (item.evalCase.fixtures) {
+            await seedEvalFixtures({
+                fixtures: item.evalCase.fixtures,
+                sandbox,
+                memoryStore,
+                agentScope: durableAgentScopeForManifest(bundle.manifest),
+                ...(conversationId ? { conversationId } : {})
+            });
+        }
+        const createdRuntime = input.dependencies.createRuntime?.({ caseRoot, runtimeOptions }) ?? new OpenEveRuntime(runtimeOptions);
+        const outcome = await runConversation({
+            runtime: createdRuntime,
+            evalCase: item.evalCase,
+            caseId: input.caseId,
+            repetition: input.repetition,
+            attempt,
+            approveDefault: input.approve,
+            timeoutMs: input.timeoutMs,
+            ...(conversationId ? { conversationId } : {})
+        });
+        if (outcome.kind === "error") {
+            runCostUsd += outcome.costUsd;
+            if (outcome.runId) {
+                try {
+                    const timeline = await createdRuntime.replay(outcome.runId);
+                    runCostUsd += timeline.usage.reduce((sum, usage) => sum + (usage.costUsd ?? 0), 0);
+                }
+                catch {
+                    // Preserve the original case error when replay is unavailable.
+                }
+            }
+            const error = outcome.cause instanceof Error ? outcome.cause.message : String(outcome.cause);
+            return {
+                result: {
+                    ...base,
+                    outcome: "errored",
+                    durationMs: Date.now() - startedAt,
+                    runCostUsd,
+                    judgeCostUsd: 0,
+                    failures: [],
+                    error,
+                    ...(outcome.runId ? { runId: outcome.runId } : {})
+                },
+                retryable: isTransientProviderError(outcome.cause)
+            };
+        }
+        if (outcome.kind === "run_failed") {
+            runCostUsd += outcome.costUsd;
+            return {
+                result: {
+                    ...base,
+                    outcome: "errored",
+                    durationMs: Date.now() - startedAt,
+                    runCostUsd,
+                    judgeCostUsd: 0,
+                    failures: [],
+                    error: outcome.error,
+                    runId: outcome.runId,
+                    status: outcome.status,
+                    response: outcome.response
+                },
+                retryable: isTransientProviderError(outcome.error)
+            };
+        }
+        const actual = outcome.actual;
+        runCostUsd += actual.costUsd;
+        phase = "assertions";
+        const failures = [...outcome.turnFailures, ...evaluateCase(item.evalCase, actual)];
+        let judge;
+        if (item.evalCase.expect.judge) {
+            phase = "judge";
+            const model = item.evalCase.expect.judge.model ?? input.judgeModel;
+            const judged = await retryTransient(() => judgeCase(item.evalCase, actual, {
+                model,
+                env: input.env,
+                ...(input.dependencies.judgeComplete ? { complete: input.dependencies.judgeComplete } : {})
+            }), {
+                retryAttempts: input.retryAttempts,
+                retryBackoffMs: input.retryBackoffMs,
+                ...(input.dependencies.sleep ? { sleep: input.dependencies.sleep } : {})
+            });
+            judge = judged.value;
+            judgeCostUsd = judge.costUsd;
+            const threshold = item.evalCase.expect.judge.threshold ?? 0.7;
+            if (!judge.pass || judge.score < threshold) {
+                failures.push(`judge score ${judge.score.toFixed(3)} did not pass threshold ${threshold.toFixed(3)}: ${judge.reasoning}`);
+            }
+            // Durable verdict for local in-process runs (the case's state adapter
+            // owns the run); remote gateway runs skip — no append path over HTTP.
+            if (createdRuntime instanceof OpenEveRuntime && outcome.runId) {
+                try {
+                    await state.appendEvent(outcome.runId, "eval.judged", {
+                        caseId: input.caseId,
+                        case: item.evalCase.name,
+                        repetition: input.repetition,
+                        attempt,
+                        model,
+                        score: judge.score,
+                        pass: judge.pass,
+                        threshold,
+                        reasoning: judge.reasoning,
+                        costUsd: judge.costUsd
+                    });
+                }
+                catch {
+                    // Grading must never fail on the verdict append.
+                }
+            }
+        }
+        phase = "custom";
+        const judgeComplete = input.dependencies.judgeComplete ?? createDefaultJudgeCompleter(input.env);
+        let evaluatorCostUsd = 0;
+        const complete = async (request) => {
+            const completion = await judgeComplete({
+                model: request.model ?? input.judgeModel,
+                systemPrompt: request.systemPrompt ?? "",
+                prompt: request.prompt
+            });
+            const costUsd = completion.costUsd ?? 0;
+            evaluatorCostUsd += costUsd;
+            return { text: completion.text, costUsd };
+        };
+        const finalTimeline = outcome.timelines[outcome.timelines.length - 1];
+        if (!finalTimeline)
+            throw new Error("conversation produced no timeline");
+        const custom = await evaluateCustomExpectations(item.evalCase.expect.evaluators ?? [], input.evaluatorLoader, {
+            evalCase: item.evalCase,
+            actual,
+            timeline: finalTimeline,
+            timelines: outcome.timelines,
+            repetition: input.repetition,
+            attempt,
+            complete
+        });
+        judgeCostUsd += evaluatorCostUsd;
+        failures.push(...custom.failures);
+        const evaluated = {
+            ...base,
+            outcome: failures.length === 0 ? "passed" : "failed",
+            durationMs: Date.now() - startedAt,
+            runCostUsd,
+            judgeCostUsd,
+            failures,
+            runId: outcome.runId,
+            status: actual.status,
+            response: actual.response
+        };
+        if (actual.output !== undefined)
+            evaluated.output = actual.output;
+        if (actual.turns)
+            evaluated.turns = actual.turns;
+        if (judge)
+            evaluated.judge = judge;
+        if (custom.metrics.length > 0)
+            evaluated.metrics = custom.metrics;
+        return { result: evaluated, retryable: false };
+    }
+    catch (error) {
+        if (error instanceof EvalJudgeResponseError)
+            judgeCostUsd += error.costUsd;
+        return {
+            result: {
+                ...base,
+                outcome: "errored",
+                durationMs: Date.now() - startedAt,
+                runCostUsd,
+                judgeCostUsd,
+                failures: [],
+                error: error instanceof Error ? error.message : String(error)
+            },
+            retryable: phase === "runtime" && isTransientProviderError(error)
+        };
+    }
+}
+function defaultSleep(ms) {
+    return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
+}
+//# sourceMappingURL=eval-case-runner.js.map

package/dist/eval-case-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-case-runner.js","sourceRoot":"","sources":["../src/eval-case-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EACL,4BAA4B,EAC5B,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,UAAU,EAEX,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,0BAA0B,EAAuB,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjG,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC3E,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAwB,MAAM,iBAAiB,CAAC;AAS/F,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,KAe5C;IACC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3F,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;QACnE,UAAU,IAAI,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC;QAC1C,YAAY,IAAI,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC;QAC9C,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,OAAO,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC;YAC1D,OAAO;gBACL,GAAG,SAAS,CAAC,MAAM;gBACnB,QAAQ,EAAE,OAAO;gBACjB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,UAAU;gBACV,YAAY;aACb,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,YAAY,CAAC,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,OAAO,IAAI,CAAC,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,KAAmD,EACnD,QAAgB,EAChB,OAAe;IAEf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,MAAM,IAAI,GAAgG;QACxG,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;QACxB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,QAAQ,EAAE,CAAC;KACZ,CAAC;IACF,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,KAAK,GAAkD,SAAS,CAAC;IACrE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QACtE,MAAM,WAAW,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvD,MAAM,cAAc,GAAmB,sBAAsB,CAAC;YAC5D,GAAG,MAAM;YACT,KAAK;YACL,IAAI,EAAE,IAAI,gBAAgB,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO;YACP,WAAW;YACX,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,UAAU,EAAE;YACpB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5D,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YAC3B,MAAM,gBAAgB,CAAC;gBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ;gBAChC,OAAO;gBACP,WAAW;gBACX,UAAU,EAAE,4BAA4B,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9C,CAAC,CAAC;QACL,CAAC;QACD,MAAM,cAAc,GAAG,KAAK,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,IAAI,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;QAC9H,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC;YACpC,OAAO,EAAE,cAAc;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO;YACP,cAAc,EAAE,KAAK,CAAC,OAAO;YAC7B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9C,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC7B,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;YAC9B,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBAC5D,UAAU,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACrF,CAAC;gBAAC,MAAM,CAAC;oBACP,+DAA+D;gBACjE,CAAC;YACH,CAAC;YACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC7F,OAAO;gBACL,MAAM,EAAE;oBACN,GAAG,IAAI;oBACP,OAAO,EAAE,SAAS;oBAClB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,UAAU;oBACV,YAAY,EAAE,CAAC;oBACf,QAAQ,EAAE,EAAE;oBACZ,KAAK;oBACL,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACnD;gBACD,SAAS,EAAE,wBAAwB,CAAC,OAAO,CAAC,KAAK,CAAC;aACnD,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAClC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;YAC9B,OAAO;gBACL,MAAM,EAAE;oBACN,GAAG,IAAI;oBACP,OAAO,EAAE,SAAS;oBAClB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,UAAU;oBACV,YAAY,EAAE,CAAC;oBACf,QAAQ,EAAE,EAAE;oBACZ,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;iBAC3B;gBACD,SAAS,EAAE,wBAAwB,CAAC,OAAO,CAAC,KAAK,CAAC;aACnD,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,UAAU,IAAI,MAAM,CAAC,OAAO,CAAC;QAC7B,KAAK,GAAG,YAAY,CAAC;QACrB,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,YAAY,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QACnF,IAAI,KAAK,CAAC;QACV,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,KAAK,GAAG,OAAO,CAAC;YAChB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YACnE,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC,GAAG,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE;gBACrC,KAAK;gBACL,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5F,CAAC,EACF;gBACE,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CACF,CAAC;YACF,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YACrB,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,IAAI,GAAG,CAAC;YAC9D,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;gBAC3C,QAAQ,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YAC5H,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,cAAc,YAAY,cAAc,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAC9D,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE;wBACpD,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;wBACxB,UAAU,EAAE,KAAK,CAAC,UAAU;wBAC5B,OAAO;wBACP,KAAK;wBACL,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,SAAS;wBACT,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;qBACvB,CAAC,CAAC;gBACL,CAAC;gBAAC,MAAM,CAAC;oBACP,iDAAiD;gBACnD,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK,GAAG,QAAQ,CAAC;QACjB,MAAM,aAAa,GAAG,KAAK,CAAC,YAAY,CAAC,aAAa,IAAI,2BAA2B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjG,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAsB,KAAK,EAAE,OAAO,EAAE,EAAE;YACpD,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC;gBACrC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU;gBACxC,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,EAAE;gBACxC,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC;YACxC,gBAAgB,IAAI,OAAO,CAAC;YAC5B,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;QAC5C,CAAC,CAAC;QACF,MAAM,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtE,IAAI,CAAC,aAAa;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAC7C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,EACrC,KAAK,CAAC,eAAe,EACrB;YACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,MAAM;YACN,QAAQ,EAAE,aAAa;YACvB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO;YACP,QAAQ;SACT,CACF,CAAC;QACF,YAAY,IAAI,gBAAgB,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAClC,MAAM,SAAS,GAAmB;YAChC,GAAG,IAAI;YACP,OAAO,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACpD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,UAAU;YACV,YAAY;YACZ,QAAQ;YACR,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;YAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAClE,IAAI,MAAM,CAAC,KAAK;YAAE,SAAS,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACjD,IAAI,KAAK;YAAE,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC;QACnC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAClE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,sBAAsB;YAAE,YAAY,IAAI,KAAK,CAAC,OAAO,CAAC;QAC3E,OAAO;YACL,MAAM,EAAE;gBACN,GAAG,IAAI;gBACP,OAAO,EAAE,SAAS;gBAClB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,UAAU;gBACV,YAAY;gBACZ,QAAQ,EAAE,EAAE;gBACZ,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D;YACD,SAAS,EAAE,KAAK,KAAK,SAAS,IAAI,wBAAwB,CAAC,KAAK,CAAC;SAClE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;AACrE,CAAC"}

package/dist/eval-cases.d.ts

@@ -0,0 +1,6 @@
+import type { DiscoveredEvalCase } from "./eval-types.js";
+export declare function discoverCases(agentRoot: string, options?: {
+    filter?: string;
+    tag?: string;
+}): DiscoveredEvalCase[];
+//# sourceMappingURL=eval-cases.d.ts.map

package/dist/eval-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../src/eval-cases.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,kBAAkB,EAAY,MAAM,iBAAiB,CAAC;AAEpE,wBAAgB,aAAa,CAC3B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAO,GAC9C,kBAAkB,EAAE,CA2BtB"}