@assemblyline-agents/cli 0.1.0

package/dist/eval-fixtures.js

@@ -0,0 +1,59 @@
+import { sandboxSessionKeyForRun } from "@assemblyline-agents/runtime";
+/**
+ * Case-declared starting state: sandbox files and memory documents seeded
+ * before the first turn. Sandbox seeding requires the case's runs to share a
+ * conversation-scoped sandbox session, so the runner assigns a conversationId
+ * to any case with sandbox fixtures.
+ */
+export function validateFixtures(value) {
+    if (value === undefined)
+        return;
+    if (!isRecord(value))
+        throw new Error("fixtures must be an object");
+    for (const key of ["sandbox", "memory"]) {
+        const entries = value[key];
+        if (entries === undefined)
+            continue;
+        if (!isRecord(entries))
+            throw new Error(`fixtures.${key} must map paths to string contents`);
+        for (const [path, content] of Object.entries(entries)) {
+            if (typeof content !== "string")
+                throw new Error(`fixtures.${key} entry ${path} must be a string`);
+            if (key === "memory" && !path.startsWith("/memory/")) {
+                throw new Error(`fixtures.memory paths must start with /memory/ (got ${path})`);
+            }
+        }
+    }
+}
+export async function seedEvalFixtures(input) {
+    const sandboxFiles = Object.entries(input.fixtures.sandbox ?? {});
+    if (sandboxFiles.length > 0) {
+        if (!input.conversationId)
+            throw new Error("sandbox fixtures require a conversation-scoped session");
+        if (!input.sandbox.create)
+            throw new Error("the configured sandbox adapter does not support pre-created sessions");
+        const sessionKey = sandboxSessionKeyForRun(input.agentScope, {
+            id: "eval-fixture-seed",
+            input: { conversationId: input.conversationId }
+        });
+        const session = await input.sandbox.create({ id: "eval-fixture-seed", input: {} }, { sessionKey });
+        for (const [path, content] of sandboxFiles) {
+            await session.writeFile(path.startsWith("/") ? path : `/workspace/${path}`, content);
+        }
+        await session.dispose?.();
+    }
+    // Point reads key on the exact scope, so seed with the same scope the
+    // case's runs will resolve (conversation-scoped when a conversation id is
+    // assigned, agent-scoped otherwise).
+    const memoryScope = {
+        agentScope: input.agentScope,
+        ...(input.conversationId ? { conversationId: input.conversationId } : {})
+    };
+    for (const [path, content] of Object.entries(input.fixtures.memory ?? {})) {
+        await input.memoryStore.write(memoryScope, { path, content, mode: "replace" });
+    }
+}
+function isRecord(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+//# sourceMappingURL=eval-fixtures.js.map

package/dist/eval-fixtures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-fixtures.js","sourceRoot":"","sources":["../src/eval-fixtures.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAGvE;;;;;GAKG;AAEH,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IACpE,KAAK,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAU,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,OAAO,KAAK,SAAS;YAAE,SAAS;QACpC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,YAAY,GAAG,oCAAoC,CAAC,CAAC;QAC7F,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACtD,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,YAAY,GAAG,UAAU,IAAI,mBAAmB,CAAC,CAAC;YACnG,IAAI,GAAG,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACrD,MAAM,IAAI,KAAK,CAAC,uDAAuD,IAAI,GAAG,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAMtC;IACC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAClE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,cAAc;YAAE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QACrG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;QACnH,MAAM,UAAU,GAAG,uBAAuB,CAAC,KAAK,CAAC,UAAU,EAAE;YAC3D,EAAE,EAAE,mBAAmB;YACvB,KAAK,EAAE,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE;SACxB,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE,EAAE,EAA0B,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAC3H,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,YAAY,EAAE,CAAC;YAC3C,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;IAC5B,CAAC;IACD,sEAAsE;IACtE,0EAA0E;IAC1E,qCAAqC;IACrC,MAAM,WAAW,GAAG;QAClB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1E,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QAC1E,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}

package/dist/eval-gateway-host.d.ts

@@ -0,0 +1,34 @@
+import type { EvalCase } from "./eval-types.js";
+import type { EvalRuntimeHost } from "./eval-turns.js";
+/**
+ * HTTP-backed {@link EvalRuntimeHost}: runs eval cases against a deployed
+ * gateway instead of an in-process runtime. Backed by the node host's
+ * `POST /runs` (requires ASSEMBLY_LINE_ENABLE_API_RUNS=true and the admin token),
+ * `POST /runs/:id/cancel`, and `GET /runs/:id/timeline`.
+ *
+ * Only cases that need no local adapters can run remotely — see
+ * {@link remoteEvalIncompatibilities}. Case metadata tagging is not
+ * transmitted; `POST /runs` does not accept a metadata field.
+ */
+export interface GatewayEvalHostOptions {
+    url: string;
+    token: string;
+    fetch?: typeof globalThis.fetch;
+    /** Poll interval while a created run is still `running`. */
+    pollIntervalMs?: number;
+}
+/**
+ * Reasons a case cannot run against a remote gateway. Fixtures are seeded
+ * into local adapters the remote runtime never sees; multi-turn and seeded
+ * history need `run()` inputs `POST /runs` does not accept. Tool mocks are
+ * compatible when the target advertises the `eval-runs` capability on
+ * /healthz (ASSEMBLY_LINE_ENABLE_EVAL_RUNS); older servers refuse them upfront so
+ * a silently-ignored stub can never cause real side effects.
+ */
+export declare function remoteEvalIncompatibilities(evalCase: EvalCase, capabilities?: string[]): string[];
+export interface GatewayEvalHost extends EvalRuntimeHost {
+    /** Capabilities advertised by the target's /healthz; fetched once and cached. */
+    capabilities(): Promise<string[]>;
+}
+export declare function createGatewayEvalHost(options: GatewayEvalHostOptions): GatewayEvalHost;
+//# sourceMappingURL=eval-gateway-host.d.ts.map

package/dist/eval-gateway-host.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-gateway-host.d.ts","sourceRoot":"","sources":["../src/eval-gateway-host.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD;;;;;;;;;GASG;AAEH,MAAM,WAAW,sBAAsB;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC,4DAA4D;IAC5D,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;GAOG;AACH,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY,GAAE,MAAM,EAAO,GAAG,MAAM,EAAE,CAcrG;AAQD,MAAM,WAAW,eAAgB,SAAQ,eAAe;IACtD,iFAAiF;IACjF,YAAY,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACnC;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,sBAAsB,GAAG,eAAe,CAwGtF"}

package/dist/eval-gateway-host.js

@@ -0,0 +1,127 @@
+/**
+ * Reasons a case cannot run against a remote gateway. Fixtures are seeded
+ * into local adapters the remote runtime never sees; multi-turn and seeded
+ * history need `run()` inputs `POST /runs` does not accept. Tool mocks are
+ * compatible when the target advertises the `eval-runs` capability on
+ * /healthz (ASSEMBLY_LINE_ENABLE_EVAL_RUNS); older servers refuse them upfront so
+ * a silently-ignored stub can never cause real side effects.
+ */
+export function remoteEvalIncompatibilities(evalCase, capabilities = []) {
+    const reasons = [];
+    if ((evalCase.input.turns?.length ?? 0) > 1)
+        reasons.push("multi-turn conversation");
+    if (evalCase.input.recentHistory?.length)
+        reasons.push("seeded recent history");
+    if (evalCase.fixtures && (Object.keys(evalCase.fixtures.sandbox ?? {}).length > 0 || Object.keys(evalCase.fixtures.memory ?? {}).length > 0)) {
+        reasons.push("fixtures");
+    }
+    if (evalCase.mocks && Object.keys(evalCase.mocks.tools ?? {}).length > 0
+        && !capabilities.includes("eval-runs")) {
+        reasons.push("tool mocks");
+    }
+    return reasons;
+}
+export function createGatewayEvalHost(options) {
+    const baseUrl = options.url.replace(/\/+$/u, "");
+    const fetcher = options.fetch ?? globalThis.fetch;
+    const pollIntervalMs = options.pollIntervalMs ?? 500;
+    async function request(path, init = {}) {
+        const response = await fetcher(`${baseUrl}${path}`, {
+            method: init.method ?? "GET",
+            headers: {
+                accept: "application/json",
+                authorization: `Bearer ${options.token}`,
+                ...(init.body !== undefined ? { "content-type": "application/json" } : {})
+            },
+            ...(init.body !== undefined ? { body: init.body } : {})
+        });
+        const text = await response.text();
+        let body = null;
+        try {
+            body = text ? JSON.parse(text) : null;
+        }
+        catch {
+            body = text;
+        }
+        if (!response.ok) {
+            const message = body && typeof body === "object" && typeof body.error === "string"
+                ? body.error
+                : `gateway returned ${response.status} for ${path}`;
+            throw new Error(message);
+        }
+        return body;
+    }
+    function summarize(body) {
+        const record = (body ?? {});
+        const id = typeof record.runId === "string" && record.runId ? record.runId : record.run?.id;
+        const status = typeof record.status === "string" && record.status ? record.status : record.run?.status;
+        if (typeof id !== "string" || !id)
+            throw new Error("The gateway response did not include a run id.");
+        return {
+            id,
+            status: (typeof status === "string" && status ? status : "unknown"),
+            ...(typeof record.response === "string" ? { response: record.response } : {})
+        };
+    }
+    let capabilitiesPromise;
+    function capabilities() {
+        capabilitiesPromise ??= (async () => {
+            const body = await request("/healthz");
+            return Array.isArray(body?.capabilities)
+                ? body.capabilities.filter((item) => typeof item === "string")
+                : [];
+        })();
+        return capabilitiesPromise;
+    }
+    return {
+        capabilities,
+        async run(runOptions) {
+            if (runOptions.turn || runOptions.conversationId || runOptions.recentHistory?.length) {
+                throw new Error("Remote eval gateways do not support multi-turn conversations or seeded history yet. Run this case without --url.");
+            }
+            let evalBlock = runOptions.eval;
+            if (evalBlock !== undefined) {
+                const caps = await capabilities().catch(() => []);
+                if (!caps.includes("eval-runs")) {
+                    if (evalBlock.toolStubs !== undefined || evalBlock.autoApproveTools) {
+                        // Never let an old server silently drop stubs or auto-approval —
+                        // that would execute real tools / really deliver.
+                        throw new Error("The target gateway lacks the eval-runs capability. Set ASSEMBLY_LINE_ENABLE_EVAL_RUNS=true on the server or run without --url.");
+                    }
+                    // Bare record-only delivery degrades to the historical behavior.
+                    evalBlock = undefined;
+                }
+            }
+            const created = summarize(await request("/runs", {
+                method: "POST",
+                body: JSON.stringify({
+                    message: runOptions.message,
+                    approve: runOptions.approve ?? false,
+                    ...(runOptions.channel ? { channel: runOptions.channel } : {}),
+                    ...(runOptions.toolName ? { toolName: runOptions.toolName } : {}),
+                    ...(runOptions.input !== undefined ? { input: runOptions.input } : {}),
+                    ...(evalBlock !== undefined ? { eval: evalBlock } : {})
+                })
+            }));
+            runOptions.onRunCreated?.({ id: created.id, status: created.status });
+            let latest = created;
+            while (latest.status === "running") {
+                await new Promise((resolveSleep) => setTimeout(resolveSleep, pollIntervalMs));
+                latest = summarize(await request(`/runs/${encodeURIComponent(created.id)}`));
+            }
+            return {
+                run: { id: latest.id, status: latest.status },
+                ...(latest.response !== undefined ? { response: latest.response } : {})
+            };
+        },
+        async cancelRun(runId) {
+            return request(`/runs/${encodeURIComponent(runId)}/cancel`, { method: "POST", body: JSON.stringify({}) });
+        },
+        async replay(runId) {
+            // The timeline endpoint serializes the runtime's RunTimeline; the eval
+            // loop only reads events/toolCalls/usage fields, which survive JSON.
+            return await request(`/runs/${encodeURIComponent(runId)}/timeline`);
+        }
+    };
+}
+//# sourceMappingURL=eval-gateway-host.js.map

package/dist/eval-gateway-host.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-gateway-host.js","sourceRoot":"","sources":["../src/eval-gateway-host.ts"],"names":[],"mappings":"AAuBA;;;;;;;GAOG;AACH,MAAM,UAAU,2BAA2B,CAAC,QAAkB,EAAE,eAAyB,EAAE;IACzF,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACrF,IAAI,QAAQ,CAAC,KAAK,CAAC,aAAa,EAAE,MAAM;QAAE,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAChF,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QAC7I,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3B,CAAC;IACD,IACE,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;WACjE,CAAC,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,EACtC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAaD,MAAM,UAAU,qBAAqB,CAAC,OAA+B;IACnE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC;IAClD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,GAAG,CAAC;IAErD,KAAK,UAAU,OAAO,CAAC,IAAY,EAAE,OAA2C,EAAE;QAChF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,OAAO,GAAG,IAAI,EAAE,EAAE;YAClD,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK;YAC5B,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,aAAa,EAAE,UAAU,OAAO,CAAC,KAAK,EAAE;gBACxC,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC3E;YACD,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,IAAI,GAAY,IAAI,CAAC;QACzB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,OAAO,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAQ,IAA4B,CAAC,KAAK,KAAK,QAAQ;gBACzG,CAAC,CAAE,IAA0B,CAAC,KAAK;gBACnC,CAAC,CAAC,oBAAoB,QAAQ,CAAC,MAAM,QAAQ,IAAI,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,SAAS,CAAC,IAAa;QAC9B,MAAM,MAAM,GAAG,CAAC,IAAI,IAAI,EAAE,CAAwG,CAAC;QACnI,MAAM,EAAE,GAAG,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC;QAC5F,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC;QACvG,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACrG,OAAO;YACL,EAAE;YACF,MAAM,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAc;YAChF,GAAG,CAAC,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9E,CAAC;IACJ,CAAC;IAED,IAAI,mBAAkD,CAAC;IACvD,SAAS,YAAY;QACnB,mBAAmB,KAAK,CAAC,KAAK,IAAI,EAAE;YAClC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,UAAU,CAA+B,CAAC;YACrE,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC;gBACtC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;gBAC9E,CAAC,CAAC,EAAE,CAAC;QACT,CAAC,CAAC,EAAE,CAAC;QACL,OAAO,mBAAmB,CAAC;IAC7B,CAAC;IAED,OAAO;QACL,YAAY;QACZ,KAAK,CAAC,GAAG,CAAC,UAA2B;YACnC,IAAI,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,cAAc,IAAI,UAAU,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;gBACrF,MAAM,IAAI,KAAK,CAAC,kHAAkH,CAAC,CAAC;YACtI,CAAC;YACD,IAAI,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC;YAChC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,MAAM,YAAY,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC;gBAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAChC,IAAI,SAAS,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;wBACpE,iEAAiE;wBACjE,kDAAkD;wBAClD,MAAM,IAAI,KAAK,CAAC,gIAAgI,CAAC,CAAC;oBACpJ,CAAC;oBACD,iEAAiE;oBACjE,SAAS,GAAG,SAAS,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,OAAO,CAAC,OAAO,EAAE;gBAC/C,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,UAAU,CAAC,OAAO;oBAC3B,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,KAAK;oBACpC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC9D,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjE,GAAG,CAAC,UAAU,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtE,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACxD,CAAC;aACH,CAAC,CAAC,CAAC;YACJ,UAAU,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAiE,CAAC,CAAC;YACrI,IAAI,MAAM,GAAG,OAAO,CAAC;YACrB,OAAO,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACnC,MAAM,IAAI,OAAO,CAAO,CAAC,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;gBACpF,MAAM,GAAG,SAAS,CAAC,MAAM,OAAO,CAAC,SAAS,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,CAAC;YACD,OAAO;gBACL,GAAG,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;gBAC7C,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,KAAa;YAC3B,OAAO,OAAO,CAAC,SAAS,kBAAkB,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5G,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,KAAa;YACxB,uEAAuE;YACvE,qEAAqE;YACrE,OAAO,MAAM,OAAO,CAAC,SAAS,kBAAkB,CAAC,KAAK,CAAC,WAAW,CAAgB,CAAC;QACrF,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/eval-judge.d.ts

@@ -0,0 +1,35 @@
+import type { EvalActual, EvalCase, EvalJudgeResult } from "./eval-types.js";
+interface JudgePayload {
+    score: number;
+    pass: boolean;
+    reasoning: string;
+}
+export type EvalJudgeCompleter = (input: {
+    model: string;
+    systemPrompt: string;
+    prompt: string;
+}) => Promise<{
+    text: string;
+    costUsd?: number;
+}>;
+export declare class EvalJudgeResponseError extends Error {
+    readonly costUsd: number;
+    constructor(message: string, costUsd: number);
+}
+/** Lazily create one provider catalog and reuse it for every judge in a suite. */
+export declare function createDefaultJudgeCompleter(env?: Record<string, string | undefined>): EvalJudgeCompleter;
+export declare function robustParseJudge(text: string): JudgePayload;
+export declare function judgeCase(evalCase: EvalCase, actual: EvalActual, options: {
+    model: string;
+    env?: Record<string, string | undefined>;
+    complete?: (input: {
+        model: string;
+        systemPrompt: string;
+        prompt: string;
+    }) => Promise<{
+        text: string;
+        costUsd?: number;
+    }>;
+}): Promise<EvalJudgeResult>;
+export {};
+//# sourceMappingURL=eval-judge.d.ts.map

package/dist/eval-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-judge.d.ts","sourceRoot":"","sources":["../src/eval-judge.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE7E,UAAU,YAAY;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,KAAK,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAElD,qBAAa,sBAAuB,SAAQ,KAAK;IAClB,QAAQ,CAAC,OAAO,EAAE,MAAM;gBAAzC,OAAO,EAAE,MAAM,EAAW,OAAO,EAAE,MAAM;CAItD;AAED,kFAAkF;AAClF,wBAAgB,2BAA2B,CACzC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,kBAAkB,CAepB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAuB3D;AAED,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE;IACP,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACzC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC5H,GACA,OAAO,CAAC,eAAe,CAAC,CA2C1B"}

package/dist/eval-judge.js

@@ -0,0 +1,105 @@
+import { assistantText, createDefaultPiModels, piUsageToRuntime, resolvePiModel } from "@assemblyline-agents/pi";
+export class EvalJudgeResponseError extends Error {
+    costUsd;
+    constructor(message, costUsd) {
+        super(message);
+        this.costUsd = costUsd;
+        this.name = "EvalJudgeResponseError";
+    }
+}
+/** Lazily create one provider catalog and reuse it for every judge in a suite. */
+export function createDefaultJudgeCompleter(env = process.env) {
+    let modelsPromise;
+    return async ({ model: modelSpec, systemPrompt, prompt }) => {
+        modelsPromise ??= createDefaultPiModels(env);
+        const models = await modelsPromise;
+        const model = resolvePiModel(models, modelSpec);
+        const response = await models.completeSimple(model, {
+            systemPrompt,
+            messages: [{ role: "user", content: [{ type: "text", text: prompt }], timestamp: Date.now() }]
+        }, { env: compactEnv(env) });
+        return {
+            text: assistantText(response),
+            costUsd: piUsageToRuntime(response.usage).costUsd ?? 0
+        };
+    };
+}
+export function robustParseJudge(text) {
+    const stripped = text.trim().replace(/^```(?:json)?\s*/iu, "").replace(/\s*```$/u, "").trim();
+    const candidates = [stripped];
+    const start = stripped.indexOf("{");
+    const end = stripped.lastIndexOf("}");
+    if (start >= 0 && end > start)
+        candidates.push(stripped.slice(start, end + 1));
+    let lastError = "response did not contain a JSON object";
+    for (const candidate of candidates) {
+        try {
+            const value = JSON.parse(candidate);
+            if (!value || typeof value !== "object" || Array.isArray(value))
+                throw new Error("judge result must be an object");
+            const record = value;
+            if (typeof record.score !== "number" || !Number.isFinite(record.score) || record.score < 0 || record.score > 1) {
+                throw new Error("judge score must be a number from 0 to 1");
+            }
+            if (typeof record.pass !== "boolean")
+                throw new Error("judge pass must be boolean");
+            if (typeof record.reasoning !== "string")
+                throw new Error("judge reasoning must be a string");
+            return { score: record.score, pass: record.pass, reasoning: record.reasoning };
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error.message : String(error);
+        }
+    }
+    throw new Error(`Could not parse judge response: ${lastError}`);
+}
+export async function judgeCase(evalCase, actual, options) {
+    const judge = evalCase.expect.judge;
+    if (!judge)
+        throw new Error(`Eval case ${evalCase.name} does not declare a judge.`);
+    const systemPrompt = [
+        "You are an impartial evaluator of an AI agent result.",
+        "Evaluate only the supplied criteria and evidence.",
+        "When the evidence includes a turns array, judge the conversation as a whole.",
+        "Return JSON only: {\"score\": number from 0 to 1, \"pass\": boolean, \"reasoning\": string}."
+    ].join("\n");
+    const prompt = JSON.stringify({
+        criteria: judge.criteria,
+        input: evalCase.input,
+        actual: {
+            status: actual.status,
+            response: actual.response,
+            output: actual.output ?? null,
+            toolsCalled: actual.toolsCalled,
+            costUsd: actual.costUsd,
+            ...(actual.turns
+                ? {
+                    turns: actual.turns.map((turn) => ({
+                        message: turn.message,
+                        status: turn.status,
+                        response: turn.response,
+                        toolsCalled: turn.toolsCalled
+                    }))
+                }
+                : {})
+        }
+    }, null, 2);
+    const complete = options.complete ?? createDefaultJudgeCompleter(options.env ?? process.env);
+    const completion = await complete({ model: options.model, systemPrompt, prompt });
+    let parsed;
+    try {
+        parsed = robustParseJudge(completion.text);
+    }
+    catch (error) {
+        throw new EvalJudgeResponseError(error instanceof Error ? error.message : String(error), completion.costUsd ?? 0);
+    }
+    return {
+        ...parsed,
+        costUsd: completion.costUsd ?? 0,
+        model: options.model
+    };
+}
+function compactEnv(env) {
+    return Object.fromEntries(Object.entries(env).filter((entry) => typeof entry[1] === "string"));
+}
+//# sourceMappingURL=eval-judge.js.map

package/dist/eval-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-judge.js","sourceRoot":"","sources":["../src/eval-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,qBAAqB,EACrB,gBAAgB,EAChB,cAAc,EACf,MAAM,yBAAyB,CAAC;AAejC,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IACT;IAAtC,YAAY,OAAe,EAAW,OAAe;QACnD,KAAK,CAAC,OAAO,CAAC,CAAC;QADqB,YAAO,GAAP,OAAO,CAAQ;QAEnD,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,UAAU,2BAA2B,CACzC,MAA0C,OAAO,CAAC,GAAG;IAErD,IAAI,aAAmE,CAAC;IACxE,OAAO,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;QAC1D,aAAa,KAAK,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;QACnC,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE;YAClD,YAAY;YACZ,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SAC/F,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7B,OAAO;YACL,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC;YAC7B,OAAO,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,IAAI,CAAC;SACvD,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9F,MAAM,UAAU,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,KAAK,IAAI,CAAC,IAAI,GAAG,GAAG,KAAK;QAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,SAAS,GAAG,wCAAwC,CAAC;IACzD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC7C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;YACnH,MAAM,MAAM,GAAG,KAAgC,CAAC;YAChD,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBAC/G,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YACpF,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YAC9F,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACjF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,mCAAmC,SAAS,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAkB,EAClB,MAAkB,EAClB,OAIC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;IACpC,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,CAAC,IAAI,4BAA4B,CAAC,CAAC;IACpF,MAAM,YAAY,GAAG;QACnB,uDAAuD;QACvD,mDAAmD;QACnD,8EAA8E;QAC9E,8FAA8F;KAC/F,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE;YACN,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,IAAI;YAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,GAAG,CAAC,MAAM,CAAC,KAAK;gBACd,CAAC,CAAC;oBACE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;wBACjC,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,WAAW,EAAE,IAAI,CAAC,WAAW;qBAC9B,CAAC,CAAC;iBACJ;gBACH,CAAC,CAAC,EAAE,CAAC;SACR;KACF,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACZ,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,2BAA2B,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7F,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC;IAClF,IAAI,MAAoB,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,gBAAgB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,sBAAsB,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;IACpH,CAAC;IACD,OAAO;QACL,GAAG,MAAM;QACT,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,GAAuC;IACzD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAA6B,EAAE,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC;AAC5H,CAAC"}

package/dist/eval-mocks.d.ts

@@ -0,0 +1,10 @@
+import type { EvalMocks } from "./eval-types.js";
+/**
+ * Case-declared tool mocks: validation of the JSON shape and conversion into
+ * the runtime's `toolStubs` seam. Each attempt gets fresh stubs so `results`
+ * sequences replay deterministically across retries and repetitions.
+ */
+export declare function validateMocks(value: unknown): void;
+/** Build runtime tool stubs for one case attempt. */
+export declare function createToolStubs(mocks: EvalMocks | undefined): Record<string, (input: unknown) => unknown> | undefined;
+//# sourceMappingURL=eval-mocks.d.ts.map

package/dist/eval-mocks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-mocks.d.ts","sourceRoot":"","sources":["../src/eval-mocks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,iBAAiB,CAAC;AAE/D;;;;GAIG;AAEH,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkBlD;AAED,qDAAqD;AACrD,wBAAgB,eAAe,CAAC,KAAK,EAAE,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,GAAG,SAAS,CAOrH"}

package/dist/eval-mocks.js

@@ -0,0 +1,57 @@
+/**
+ * Case-declared tool mocks: validation of the JSON shape and conversion into
+ * the runtime's `toolStubs` seam. Each attempt gets fresh stubs so `results`
+ * sequences replay deterministically across retries and repetitions.
+ */
+export function validateMocks(value) {
+    if (value === undefined)
+        return;
+    if (!isRecord(value) || !isRecord(value.tools)) {
+        throw new Error("mocks must be an object with a tools record");
+    }
+    for (const [name, mock] of Object.entries(value.tools)) {
+        if (!isRecord(mock))
+            throw new Error(`mocks.tools.${name} must be an object`);
+        const modes = ["result", "results", "error"].filter((key) => mock[key] !== undefined);
+        if (modes.length !== 1) {
+            throw new Error(`mocks.tools.${name} must declare exactly one of result, results, or error`);
+        }
+        if (mock.results !== undefined && (!Array.isArray(mock.results) || mock.results.length === 0)) {
+            throw new Error(`mocks.tools.${name}.results must be a non-empty array`);
+        }
+        if (mock.error !== undefined && (typeof mock.error !== "string" || mock.error.trim() === "")) {
+            throw new Error(`mocks.tools.${name}.error must be a non-empty string`);
+        }
+    }
+}
+/** Build runtime tool stubs for one case attempt. */
+export function createToolStubs(mocks) {
+    if (!mocks)
+        return undefined;
+    const stubs = {};
+    for (const [name, mock] of Object.entries(mocks.tools)) {
+        stubs[name] = createStub(mock);
+    }
+    return stubs;
+}
+function createStub(mock) {
+    if (mock.error !== undefined) {
+        return () => {
+            throw new Error(mock.error);
+        };
+    }
+    if (mock.results !== undefined) {
+        const sequence = [...mock.results];
+        let index = 0;
+        return () => {
+            const result = sequence[Math.min(index, sequence.length - 1)];
+            index += 1;
+            return result;
+        };
+    }
+    return () => mock.result;
+}
+function isRecord(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+//# sourceMappingURL=eval-mocks.js.map

package/dist/eval-mocks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-mocks.js","sourceRoot":"","sources":["../src/eval-mocks.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AAEH,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QACvD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,oBAAoB,CAAC,CAAC;QAC9E,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC;QACtF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,wDAAwD,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAC9F,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,oCAAoC,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YAC7F,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,mCAAmC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;AACH,CAAC;AAED,qDAAqD;AACrD,MAAM,UAAU,eAAe,CAAC,KAA4B;IAC1D,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,MAAM,KAAK,GAAgD,EAAE,CAAC;IAC9D,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QACvD,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,IAAkB;IACpC,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,GAAG,EAAE;YACV,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9D,KAAK,IAAI,CAAC,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;AAC3B,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}

package/dist/eval-report.d.ts

@@ -0,0 +1,7 @@
+import type { EvalCaseResult, EvalComparison, EvalExperimentManifest, EvalReport } from "./eval-types.js";
+export declare function createEvalReport(experiment: EvalExperimentManifest, durationMs: number, results: EvalCaseResult[], options?: {
+    artifacts?: EvalReport["artifacts"];
+    comparison?: EvalComparison;
+}): EvalReport;
+export declare function formatEvalReport(report: EvalReport): string;
+//# sourceMappingURL=eval-report.d.ts.map

package/dist/eval-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-report.d.ts","sourceRoot":"","sources":["../src/eval-report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,UAAU,EACX,MAAM,iBAAiB,CAAC;AAEzB,wBAAgB,gBAAgB,CAC9B,UAAU,EAAE,sBAAsB,EAClC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,cAAc,EAAE,EACzB,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC;IACpC,UAAU,CAAC,EAAE,cAAc,CAAC;CACxB,GACL,UAAU,CAsCZ;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA6B3D"}

package/dist/eval-report.js

@@ -0,0 +1,89 @@
+export function createEvalReport(experiment, durationMs, results, options = {}) {
+    const passed = results.filter((result) => result.outcome === "passed").length;
+    const failed = results.filter((result) => result.outcome === "failed").length;
+    const errored = results.filter((result) => result.outcome === "errored").length;
+    const runCostUsd = results.reduce((sum, result) => sum + result.runCostUsd, 0);
+    const judgeCostUsd = results.reduce((sum, result) => sum + result.judgeCostUsd, 0);
+    const byTag = {};
+    for (const result of results) {
+        for (const tag of result.tags) {
+            const summary = byTag[tag] ?? { total: 0, passed: 0, failed: 0, errored: 0 };
+            summary.total += 1;
+            summary[result.outcome] += 1;
+            byTag[tag] = summary;
+        }
+    }
+    return {
+        schemaVersion: 1,
+        experiment,
+        startedAt: experiment.startedAt,
+        durationMs,
+        agentRevision: experiment.agentRevision,
+        results,
+        ...(options.comparison ? { comparison: options.comparison } : {}),
+        ...(options.artifacts ? { artifacts: options.artifacts } : {}),
+        summary: {
+            total: results.length,
+            uniqueCases: new Set(results.map((result) => result.caseId)).size,
+            repetitions: experiment.config.repetitions,
+            passed,
+            failed,
+            errored,
+            passRate: results.length === 0 ? 0 : passed / results.length,
+            runCostUsd,
+            judgeCostUsd,
+            totalCostUsd: runCostUsd + judgeCostUsd,
+            byTag
+        }
+    };
+}
+export function formatEvalReport(report) {
+    const lines = [];
+    for (const result of report.results) {
+        const marker = result.outcome === "passed" ? "✓" : result.outcome === "failed" ? "✗" : "⚠";
+        const cost = result.runCostUsd + result.judgeCostUsd;
+        const planned = result.repetitionsPlanned ?? report.summary.repetitions;
+        const repetition = planned > 1 ? ` [rep ${result.repetition}/${planned}]` : "";
+        const attempts = result.attempts > 1 ? ` [${result.attempts} attempts]` : "";
+        lines.push(`${marker} ${result.name}${repetition}${attempts}  ${result.durationMs}ms  $${cost.toFixed(6)}`);
+        for (const failure of result.failures)
+            lines.push(`    ${failure}`);
+        if (result.error)
+            lines.push(`    ${result.error}`);
+        for (const metric of result.metrics ?? []) {
+            const value = metric.score ?? metric.value ?? metric.pass;
+            lines.push(`    metric ${metric.evaluator}.${metric.key}: ${JSON.stringify(value)}`);
+        }
+    }
+    const summary = report.summary;
+    lines.push("");
+    lines.push(`${summary.passed}/${summary.total} executions passed (${(summary.passRate * 100).toFixed(1)}%); ${summary.failed} failed; ${summary.errored} errored`);
+    lines.push(`${summary.uniqueCases} unique cases; ${summary.repetitions} repetition${summary.repetitions === 1 ? "" : "s"} requested`);
+    lines.push(`Cost: runs $${summary.runCostUsd.toFixed(6)} + judges $${summary.judgeCostUsd.toFixed(6)} = $${summary.totalCostUsd.toFixed(6)}`);
+    const tags = Object.entries(summary.byTag).sort(([a], [b]) => a.localeCompare(b));
+    if (tags.length > 0) {
+        lines.push("Tags:");
+        for (const [tag, value] of tags)
+            lines.push(`  ${tag.padEnd(16)} ${value.passed}/${value.total} passed, ${value.failed} failed, ${value.errored} errored`);
+    }
+    if (report.comparison)
+        formatComparison(lines, report.comparison);
+    if (report.artifacts)
+        lines.push(`Artifacts: ${report.artifacts.root}`);
+    return lines.join("\n");
+}
+function formatComparison(lines, comparison) {
+    lines.push("");
+    lines.push(`Baseline ${comparison.baselineExperimentId}: ${comparison.passed ? "gates passed" : "gates failed"}`);
+    lines.push(`Pass-rate delta: ${(comparison.passRateDelta * 100).toFixed(1)} points; cost delta: $${comparison.costDeltaUsd.toFixed(6)}`);
+    for (const regression of comparison.regressions) {
+        lines.push(`  regression ${regression.name}: ${(regression.baselinePassRate * 100).toFixed(1)}% → ${(regression.currentPassRate * 100).toFixed(1)}%`);
+    }
+    for (const gate of comparison.gates) {
+        lines.push(`  ${gate.passed ? "✓" : "✗"} ${gate.name}: ${formatGateNumber(gate.actual)} (limit ${formatGateNumber(gate.limit)})`);
+    }
+}
+function formatGateNumber(value) {
+    return value === null ? "unbounded" : String(Number(value.toFixed(3)));
+}
+//# sourceMappingURL=eval-report.js.map

package/dist/eval-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-report.js","sourceRoot":"","sources":["../src/eval-report.ts"],"names":[],"mappings":"AAOA,MAAM,UAAU,gBAAgB,CAC9B,UAAkC,EAClC,UAAkB,EAClB,OAAyB,EACzB,UAGI,EAAE;IAEN,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC9E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC9E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IAChF,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC/E,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAmC,EAAE,CAAC;IACjD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;YAC7E,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;YACnB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC7B,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC;QACvB,CAAC;IACH,CAAC;IACD,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,UAAU;QACV,SAAS,EAAE,UAAU,CAAC,SAAS;QAC/B,UAAU;QACV,aAAa,EAAE,UAAU,CAAC,aAAa;QACvC,OAAO;QACP,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,OAAO,EAAE;YACP,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,WAAW,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YACjE,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,WAAW;YAC1C,MAAM;YACN,MAAM;YACN,OAAO;YACP,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;YAC5D,UAAU;YACV,YAAY;YACZ,YAAY,EAAE,UAAU,GAAG,YAAY;YACvC,KAAK;SACN;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAkB;IACjD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3F,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC;QACrD,MAAM,OAAO,GAAG,MAAM,CAAC,kBAAkB,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACxE,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,UAAU,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,QAAQ,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7E,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,CAAC,IAAI,GAAG,UAAU,GAAG,QAAQ,KAAK,MAAM,CAAC,UAAU,QAAQ,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5G,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;QACpE,IAAI,MAAM,CAAC,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACpD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC;YAC1D,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,GAAG,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACvF,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,uBAAuB,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,MAAM,YAAY,OAAO,CAAC,OAAO,UAAU,CAAC,CAAC;IACnK,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,kBAAkB,OAAO,CAAC,WAAW,cAAc,OAAO,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC;IACtI,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9I,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAClF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,OAAO,UAAU,CAAC,CAAC;IAC7J,CAAC;IACD,IAAI,MAAM,CAAC,UAAU;QAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAClE,IAAI,MAAM,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAe,EAAE,UAA0B;IACnE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,UAAU,CAAC,oBAAoB,KAAK,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;IAClH,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB,UAAU,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzI,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,WAAW,EAAE,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,gBAAgB,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,gBAAgB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACxJ,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpI,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAoB;IAC5C,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC"}

package/dist/eval-retry.d.ts

@@ -0,0 +1,10 @@
+export declare function isTransientProviderError(error: unknown): boolean;
+export declare function retryTransient<T>(operation: (attempt: number) => Promise<T>, options: {
+    retryAttempts: number;
+    retryBackoffMs: number;
+    sleep?: (ms: number) => Promise<void>;
+}): Promise<{
+    value: T;
+    attempts: number;
+}>;
+//# sourceMappingURL=eval-retry.d.ts.map

package/dist/eval-retry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-retry.d.ts","sourceRoot":"","sources":["../src/eval-retry.ts"],"names":[],"mappings":"AAKA,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAShE;AAED,wBAAsB,cAAc,CAAC,CAAC,EACpC,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EAC1C,OAAO,EAAE;IAAE,aAAa,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,GAChG,OAAO,CAAC;IAAE,KAAK,EAAE,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAYzC"}