@b4run/evals 0.8.28

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Brian Love
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,69 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/cacheplane/b4run/main/docs/brand/b4-logo-horizontal-black-on-white.png" alt="B4.run" width="180" />
+</p>
+
+# @b4run/evals
+
+Supported evaluation definitions, datasets, scorers, reports, and release gates for B4.run application behavior.
+
+**Use this when:** You are defining repeatable evaluations, scorers, or release gates for a B4.run application.
+
+## Install
+
+```bash
+pnpm add -D @b4run/evals @b4run/testing
+```
+
+## Example
+
+```ts
+import { contains, defineEval, gate, runEval } from "@b4run/evals"
+import { createAgentHarness, script } from "@b4run/testing"
+
+const evaluation = defineEval({
+  name: "support replies",
+  route: "/support#agent",
+  dataset: [
+    {
+      input: "Where is my order?",
+      fixtures: script()
+        .user("Where is my order?")
+        .replies("Your order is in transit."),
+    },
+  ],
+  scorers: [contains("order", { threshold: 1 })],
+  gate: gate.perScorer(),
+})
+
+await using harness = await createAgentHarness({
+  appRoot: process.cwd(),
+  route: "/support#agent",
+})
+const report = await runEval(evaluation, {
+  runCase: async (testCase) => {
+    if (typeof testCase.input !== "string") throw new TypeError("Expected string input")
+    return harness.run({
+      input: testCase.input,
+      ...(testCase.fixtures !== undefined ? { fixtures: testCase.fixtures } : {}),
+    })
+  },
+})
+```
+
+## Runtime and stability
+
+`@b4run/evals` is a supported node-only testing surface because it resolves JSON and JSONL datasets from disk. Deterministic agent runs come from `@b4run/testing`; scorer code still executes in replay, record, and live modes. `llmJudge()` needs a fixture, model credentials, or an injected fetch implementation.
+
+## Related
+
+- [Evals API reference](https://b4.run/docs/api/evals) — exact scorer, runner, and gate semantics.
+- [Evals](https://b4.run/docs/evals) — the application evaluation workflow.
+- [`@b4run/testing`](https://www.npmjs.com/package/@b4run/testing) and [Fixtures and Recording](https://b4.run/docs/testing-agents/fixtures) — deterministic model calls.
+
+## Maturity and support
+
+B4.run is pre-1.0, and its public surface can change. All publishable B4.run packages release together as a fixed group; review the [`@b4run/evals` changelog](https://github.com/cacheplane/b4run/blob/main/packages/evals/CHANGELOG.md) and [upgrading guide](https://b4.run/docs/upgrading) before upgrading. For support, use [GitHub Discussions](https://github.com/cacheplane/b4run/discussions); report defects in [GitHub Issues](https://github.com/cacheplane/b4run/issues).
+
+## License
+
+MIT. See the [repository license](https://github.com/cacheplane/b4run/blob/main/LICENSE).

package/dist/define-eval.d.ts

@@ -0,0 +1,3 @@
+import type { EvalDefinition } from "./types.js";
+export declare function defineEval(def: EvalDefinition): EvalDefinition;
+//# sourceMappingURL=define-eval.d.ts.map

package/dist/define-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"define-eval.d.ts","sourceRoot":"","sources":["../src/define-eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,wBAAgB,UAAU,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,CAc9D"}

package/dist/define-eval.js

@@ -0,0 +1,15 @@
+export function defineEval(def) {
+    if (!def.name || def.name.trim() === "") {
+        throw new Error("defineEval: `name` is required");
+    }
+    if (!def.scorers || def.scorers.length === 0) {
+        throw new Error(`defineEval("${def.name}"): at least one scorer is required`);
+    }
+    if (Array.isArray(def.dataset) && def.dataset.length === 0) {
+        throw new Error(`defineEval("${def.name}"): inline dataset is empty`);
+    }
+    if (def.dataset === undefined || def.dataset === null) {
+        throw new Error(`defineEval("${def.name}"): dataset is required`);
+    }
+    return def;
+}

package/dist/gate.d.ts

@@ -0,0 +1,13 @@
+import { DEFAULT_CASE_BAR, type EvalDefinition, type GatePolicy } from "./types.js";
+export declare const gate: {
+    mean(n: number): GatePolicy;
+    passRate(n: number): GatePolicy;
+    everyCase(n: number): GatePolicy;
+    perScorer(): GatePolicy;
+    all(...policies: GatePolicy[]): GatePolicy;
+    any(...policies: GatePolicy[]): GatePolicy;
+};
+/** gate wins; else threshold → mean(threshold); else informational (always passes). */
+export declare function resolveGate(def: Pick<EvalDefinition, "gate" | "threshold">): GatePolicy;
+export { DEFAULT_CASE_BAR };
+//# sourceMappingURL=gate.d.ts.map

package/dist/gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../src/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,KAAK,cAAc,EAAE,KAAK,UAAU,EAAmB,MAAM,YAAY,CAAA;AASpG,eAAO,MAAM,IAAI;IACf,IAAI,IAAI,MAAM,GAAG,UAAU;IAG3B,QAAQ,IAAI,MAAM,GAAG,UAAU;IAO/B,SAAS,IAAI,MAAM,GAAG,UAAU;IAMhC,SAAS,IAAI,UAAU;IAQvB,GAAG,cAAc,UAAU,EAAE,GAAG,UAAU;IAS1C,GAAG,cAAc,UAAU,EAAE,GAAG,UAAU;CAW3C,CAAA;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,WAAW,CAAC,GAAG,UAAU,CAIvF;AAED,OAAO,EAAE,gBAAgB,EAAE,CAAA"}

package/dist/gate.js

@@ -0,0 +1,64 @@
+import { DEFAULT_CASE_BAR } from "./types.js";
+function pass(reason) {
+    return reason !== undefined ? { passed: true, reason } : { passed: true };
+}
+function fail(reason) {
+    return { passed: false, reason };
+}
+export const gate = {
+    mean(n) {
+        return (r) => (r.mean >= n ? pass() : fail(`mean ${r.mean.toFixed(2)} < ${n}`));
+    },
+    passRate(n) {
+        return (r) => {
+            const rate = r.cases.length === 0 ? 1 : r.cases.filter((c) => c.passed).length / r.cases.length;
+            return rate >= n ? pass() : fail(`pass-rate ${rate.toFixed(2)} < ${n}`);
+        };
+    },
+    everyCase(n) {
+        return (r) => {
+            const bad = r.cases.find((c) => c.mean < n);
+            return bad ? fail(`case "${bad.name}" mean ${bad.mean.toFixed(2)} < ${n}`) : pass();
+        };
+    },
+    perScorer() {
+        return (r) => {
+            const bad = r.byScorer.find((s) => s.threshold !== undefined && s.mean < s.threshold);
+            return bad
+                ? fail(`scorer "${bad.scorer}" mean ${bad.mean.toFixed(2)} < ${bad.threshold}`)
+                : pass();
+        };
+    },
+    all(...policies) {
+        return (r) => {
+            for (const p of policies) {
+                const res = p(r);
+                if (!res.passed)
+                    return res;
+            }
+            return pass();
+        };
+    },
+    any(...policies) {
+        return (r) => {
+            const reasons = [];
+            for (const p of policies) {
+                const res = p(r);
+                if (res.passed)
+                    return pass();
+                if (res.reason)
+                    reasons.push(res.reason);
+            }
+            return fail(`no policy passed: ${reasons.join("; ")}`);
+        };
+    },
+};
+/** gate wins; else threshold → mean(threshold); else informational (always passes). */
+export function resolveGate(def) {
+    if (def.gate)
+        return def.gate;
+    if (def.threshold !== undefined)
+        return gate.mean(def.threshold);
+    return () => pass("informational (no gate)");
+}
+export { DEFAULT_CASE_BAR };

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export { defineEval } from "./define-eval.js";
+export { gate, resolveGate } from "./gate.js";
+export { type LlmJudgeOptions, llmJudge } from "./llm-judge.js";
+export { resolveDataset } from "./resolve-dataset.js";
+export { type RunEvalOptions, runEval } from "./run-eval.js";
+export { type NormalizedScore, normalizeScore } from "./score.js";
+export { contains, custom, exactMatch, jsonEquals, memoryFresh, memoryIsolated, memoryRecalled, regex, tokensUnder, toolCalled, } from "./scorers.js";
+export type { CaseResult, CaseScore, Dataset, EvalCase, EvalDefinition, EvalReport, GatePolicy, GateResult, Score, ScoredReport, Scorer, ScorerAggregate, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,KAAK,eAAe,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,KAAK,cAAc,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAC5D,OAAO,EAAE,KAAK,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AACjE,OAAO,EACL,QAAQ,EACR,MAAM,EACN,UAAU,EACV,UAAU,EACV,WAAW,EACX,cAAc,EACd,cAAc,EACd,KAAK,EACL,WAAW,EACX,UAAU,GACX,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,cAAc,EACd,UAAU,EACV,UAAU,EACV,UAAU,EACV,KAAK,EACL,YAAY,EACZ,MAAM,EACN,eAAe,GAChB,MAAM,YAAY,CAAA"}

package/dist/index.js

@@ -0,0 +1,7 @@
+export { defineEval } from "./define-eval.js";
+export { gate, resolveGate } from "./gate.js";
+export { llmJudge } from "./llm-judge.js";
+export { resolveDataset } from "./resolve-dataset.js";
+export { runEval } from "./run-eval.js";
+export { normalizeScore } from "./score.js";
+export { contains, custom, exactMatch, jsonEquals, memoryFresh, memoryIsolated, memoryRecalled, regex, tokensUnder, toolCalled, } from "./scorers.js";

package/dist/llm-judge.d.ts

@@ -0,0 +1,16 @@
+import type { Scorer } from "./types.js";
+type FetchImpl = (input: string, init: RequestInit) => Promise<Response>;
+export interface LlmJudgeOptions {
+    /** Criteria template; supports {{input}}, {{expected}}, {{output}} interpolation. */
+    readonly criteria: string;
+    readonly model?: string;
+    readonly threshold?: number;
+    readonly name?: string;
+    /** Overrides for testing; default to env + global fetch. */
+    readonly baseUrl?: string;
+    readonly apiKey?: string;
+    readonly fetchImpl?: FetchImpl;
+}
+export declare function llmJudge(opts: LlmJudgeOptions): Scorer;
+export {};
+//# sourceMappingURL=llm-judge.d.ts.map

package/dist/llm-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../src/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,MAAM,EAAE,MAAM,YAAY,CAAA;AAElD,KAAK,SAAS,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;AAExE,MAAM,WAAW,eAAe;IAC9B,qFAAqF;IACrF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,4DAA4D;IAC5D,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;CAC/B;AAMD,wBAAgB,QAAQ,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,CAmDtD"}

package/dist/llm-judge.js

@@ -0,0 +1,57 @@
+function interpolate(template, vars) {
+    return template.replace(/\{\{(\w+)\}\}/g, (_m, key) => vars[key] ?? "");
+}
+export function llmJudge(opts) {
+    const model = opts.model ?? "gpt-5-mini";
+    return {
+        name: opts.name ?? "llmJudge",
+        ...(opts.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: async (run, testCase) => {
+            const baseUrl = opts.baseUrl ?? process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1";
+            const apiKey = opts.apiKey ?? process.env.OPENAI_API_KEY ?? "";
+            const fetchImpl = opts.fetchImpl ?? ((i, init) => fetch(i, init));
+            const criteria = interpolate(opts.criteria, {
+                input: String(testCase.input ?? ""),
+                expected: JSON.stringify(testCase.expected ?? ""),
+                output: run.finalMessage,
+            });
+            const user = [
+                `Criteria: ${criteria}`,
+                `Agent output: ${run.finalMessage}`,
+                `Respond ONLY with JSON: {"score": <0..1>, "reason": "<short>"}.`,
+            ].join("\n");
+            let content;
+            try {
+                const res = await fetchImpl(`${baseUrl}/chat/completions`, {
+                    method: "POST",
+                    headers: { "content-type": "application/json", authorization: `Bearer ${apiKey}` },
+                    body: JSON.stringify({
+                        model,
+                        messages: [
+                            {
+                                role: "system",
+                                content: "You are a strict grader. Output only the requested JSON.",
+                            },
+                            { role: "user", content: user },
+                        ],
+                    }),
+                });
+                const json = (await res.json());
+                content = json.choices?.[0]?.message?.content ?? "";
+            }
+            catch (err) {
+                return {
+                    score: 0,
+                    reason: `judge request failed: ${err instanceof Error ? err.message : String(err)}`,
+                };
+            }
+            try {
+                const parsed = JSON.parse(content);
+                return { score: parsed.score, reason: parsed.reason ?? "" };
+            }
+            catch {
+                return { score: 0, reason: `could not parse judge verdict: ${content.slice(0, 120)}` };
+            }
+        },
+    };
+}

package/dist/regex-safety.d.ts

@@ -0,0 +1,2 @@
+export declare function createSafeRegexTester(expression: RegExp): (input: string) => boolean;
+//# sourceMappingURL=regex-safety.d.ts.map

package/dist/regex-safety.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"regex-safety.d.ts","sourceRoot":"","sources":["../src/regex-safety.ts"],"names":[],"mappings":"AAmDA,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CA8BpF"}

package/dist/regex-safety.js

@@ -0,0 +1,66 @@
+import { Script } from "node:vm";
+const MAX_REGEX_SOURCE_CODE_UNITS = 4_096;
+const MAX_REGEX_INPUT_CODE_UNITS = 65_536;
+const REGEX_EXECUTION_TIMEOUT_MS = 100;
+const OVERSIZED_REGEX_SOURCE_MESSAGE = "Regular expression source exceeds 4096 UTF-16 code units";
+const OVERSIZED_REGEX_INPUT_MESSAGE = "Regular expression input exceeds 65536 UTF-16 code units";
+const REGEX_TIMEOUT_MESSAGE = "Regular expression evaluation exceeded 100ms execution limit";
+const REGEX_TEST_SCRIPT = new Script("new RegExp(source, flags).test(input)", {
+    filename: "b4-regex-evaluation.vm",
+});
+function intrinsicRegExpGetter(property) {
+    const getter = Object.getOwnPropertyDescriptor(RegExp.prototype, property)?.get;
+    if (getter === undefined) {
+        throw new Error(`RegExp.prototype.${property} is unavailable`);
+    }
+    return getter;
+}
+const REGEXP_SOURCE_GETTER = intrinsicRegExpGetter("source");
+const REGEXP_FLAG_GETTERS = [
+    [intrinsicRegExpGetter("hasIndices"), "d"],
+    [intrinsicRegExpGetter("global"), "g"],
+    [intrinsicRegExpGetter("ignoreCase"), "i"],
+    [intrinsicRegExpGetter("multiline"), "m"],
+    [intrinsicRegExpGetter("dotAll"), "s"],
+    [intrinsicRegExpGetter("unicode"), "u"],
+    [intrinsicRegExpGetter("unicodeSets"), "v"],
+    [intrinsicRegExpGetter("sticky"), "y"],
+];
+function snapshotFlags(expression) {
+    let flags = "";
+    for (const [getter, flag] of REGEXP_FLAG_GETTERS) {
+        if (Reflect.apply(getter, expression, []) === true)
+            flags += flag;
+    }
+    return flags;
+}
+function isScriptExecutionTimeout(error) {
+    return (typeof error === "object" &&
+        error !== null &&
+        "code" in error &&
+        error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT");
+}
+export function createSafeRegexTester(expression) {
+    const source = Reflect.apply(REGEXP_SOURCE_GETTER, expression, []);
+    const flags = snapshotFlags(expression);
+    if (source.length > MAX_REGEX_SOURCE_CODE_UNITS) {
+        throw new RangeError(OVERSIZED_REGEX_SOURCE_MESSAGE);
+    }
+    return (input) => {
+        if (input.length > MAX_REGEX_INPUT_CODE_UNITS) {
+            throw new RangeError(OVERSIZED_REGEX_INPUT_MESSAGE);
+        }
+        try {
+            return (REGEX_TEST_SCRIPT.runInNewContext({ flags, input, source }, {
+                contextCodeGeneration: { strings: false, wasm: false },
+                timeout: REGEX_EXECUTION_TIMEOUT_MS,
+            }) === true);
+        }
+        catch (error) {
+            if (isScriptExecutionTimeout(error)) {
+                throw new RangeError(REGEX_TIMEOUT_MESSAGE);
+            }
+            throw error;
+        }
+    };
+}

package/dist/resolve-dataset.d.ts

@@ -0,0 +1,3 @@
+import type { Dataset, EvalCase } from "./types.js";
+export declare function resolveDataset(dataset: Dataset, baseDir: string): Promise<EvalCase[]>;
+//# sourceMappingURL=resolve-dataset.d.ts.map

package/dist/resolve-dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolve-dataset.d.ts","sourceRoot":"","sources":["../src/resolve-dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAEnD,wBAAsB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAiC3F"}

package/dist/resolve-dataset.js

@@ -0,0 +1,38 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute, resolve } from "node:path";
+export async function resolveDataset(dataset, baseDir) {
+    if (Array.isArray(dataset))
+        return [...dataset];
+    if (typeof dataset === "function")
+        return [...(await dataset())];
+    if (typeof dataset === "string") {
+        const path = isAbsolute(dataset) ? dataset : resolve(baseDir, dataset);
+        let raw;
+        try {
+            raw = await readFile(path, "utf8");
+        }
+        catch (err) {
+            throw new Error(`resolveDataset: cannot read dataset file "${path}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+        if (path.endsWith(".jsonl")) {
+            return raw
+                .split("\n")
+                .map((l) => l.trim())
+                .filter(Boolean)
+                .map((line, i) => {
+                try {
+                    return JSON.parse(line);
+                }
+                catch {
+                    throw new Error(`resolveDataset: invalid JSONL at line ${i + 1} in "${path}"`);
+                }
+            });
+        }
+        const parsed = JSON.parse(raw);
+        if (!Array.isArray(parsed)) {
+            throw new Error(`resolveDataset: "${path}" must contain a JSON array of cases`);
+        }
+        return parsed;
+    }
+    throw new Error("resolveDataset: dataset must be an array, a path string, or a function");
+}

package/dist/run-eval.d.ts

@@ -0,0 +1,10 @@
+import type { AgentRunResult } from "@b4run/testing";
+import { type EvalCase, type EvalDefinition, type EvalReport } from "./types.js";
+export interface RunEvalOptions {
+    /** Executes one case and returns its run result (replay or live; injected by the CLI). */
+    readonly runCase: (testCase: EvalCase) => Promise<AgentRunResult>;
+    /** Base dir for resolving a string dataset path (the eval file's directory). */
+    readonly baseDir?: string;
+}
+export declare function runEval(def: EvalDefinition, options: RunEvalOptions): Promise<EvalReport>;
+//# sourceMappingURL=run-eval.d.ts.map

package/dist/run-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval.d.ts","sourceRoot":"","sources":["../src/run-eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAIpD,OAAO,EAIL,KAAK,QAAQ,EACb,KAAK,cAAc,EACnB,KAAK,UAAU,EAEhB,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,cAAc;IAC7B,0FAA0F;IAC1F,QAAQ,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,cAAc,CAAC,CAAA;IACjE,gFAAgF;IAChF,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAC1B;AAMD,wBAAsB,OAAO,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,CAqD/F"}

package/dist/run-eval.js

@@ -0,0 +1,56 @@
+import { resolveGate } from "./gate.js";
+import { resolveDataset } from "./resolve-dataset.js";
+import { normalizeScore } from "./score.js";
+import { DEFAULT_CASE_BAR, } from "./types.js";
+function mean(nums) {
+    return nums.length === 0 ? 0 : nums.reduce((a, b) => a + b, 0) / nums.length;
+}
+export async function runEval(def, options) {
+    const cases = await resolveDataset(def.dataset, options.baseDir ?? process.cwd());
+    const thresholdOf = new Map(def.scorers.map((s) => [s.name, s.threshold]));
+    const caseResults = [];
+    for (const [index, testCase] of cases.entries()) {
+        const run = await options.runCase(testCase);
+        const scores = [];
+        for (const scorer of def.scorers) {
+            let normalized;
+            try {
+                normalized = normalizeScore(await scorer.score(run, testCase));
+            }
+            catch (err) {
+                normalized = { score: 0, reason: err instanceof Error ? err.message : String(err) };
+            }
+            scores.push({
+                scorer: scorer.name,
+                score: normalized.score,
+                ...(normalized.label !== undefined ? { label: normalized.label } : {}),
+                ...(normalized.reason !== undefined ? { reason: normalized.reason } : {}),
+            });
+        }
+        const passed = scores.every((s) => s.score >= (thresholdOf.get(s.scorer) ?? DEFAULT_CASE_BAR));
+        caseResults.push({
+            name: testCase.name ?? `case ${index + 1}`,
+            scores,
+            mean: mean(scores.map((s) => s.score)),
+            passed,
+        });
+    }
+    const byScorer = def.scorers.map((scorer) => {
+        const scorerScores = caseResults.flatMap((c) => c.scores.filter((s) => s.scorer === scorer.name).map((s) => s.score));
+        return {
+            scorer: scorer.name,
+            mean: mean(scorerScores),
+            ...(scorer.threshold !== undefined ? { threshold: scorer.threshold } : {}),
+        };
+    });
+    const overallMean = mean(caseResults.flatMap((c) => c.scores.map((s) => s.score)));
+    const scored = { name: def.name, cases: caseResults, byScorer, mean: overallMean };
+    const gated = def.gate !== undefined || def.threshold !== undefined;
+    const result = resolveGate(def)(scored);
+    return {
+        ...scored,
+        gated,
+        passed: result.passed,
+        ...(result.reason !== undefined ? { reason: result.reason } : {}),
+    };
+}

package/dist/score.d.ts

@@ -0,0 +1,8 @@
+import type { Score } from "./types.js";
+export interface NormalizedScore {
+    readonly score: number;
+    readonly label?: string;
+    readonly reason?: string;
+}
+export declare function normalizeScore(raw: Score): NormalizedScore;
+//# sourceMappingURL=score.d.ts.map

package/dist/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAEvC,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAUD,wBAAgB,cAAc,CAAC,GAAG,EAAE,KAAK,GAAG,eAAe,CAU1D"}

package/dist/score.js

@@ -0,0 +1,24 @@
+function clamp01(n) {
+    const x = Number(n);
+    if (!Number.isFinite(x))
+        return 0;
+    if (x < 0)
+        return 0;
+    if (x > 1)
+        return 1;
+    return x;
+}
+export function normalizeScore(raw) {
+    if (typeof raw === "boolean")
+        return { score: raw ? 1 : 0 };
+    if (typeof raw === "number")
+        return { score: clamp01(raw) };
+    if (raw === null || typeof raw !== "object")
+        return { score: 0 };
+    const out = { score: clamp01(raw.score) };
+    return {
+        ...out,
+        ...(raw.label !== undefined ? { label: raw.label } : {}),
+        ...(raw.reason !== undefined ? { reason: raw.reason } : {}),
+    };
+}

package/dist/scorers.d.ts

@@ -0,0 +1,50 @@
+import type { AgentRunResult } from "@b4run/testing";
+import type { EvalCase, Score, Scorer } from "./types.js";
+/** finalMessage === case.expected (string compare). */
+export declare function exactMatch(opts?: {
+    threshold?: number;
+}): Scorer;
+export declare function contains(substring: string, opts?: {
+    threshold?: number;
+}): Scorer;
+export declare function regex(re: RegExp, opts?: {
+    threshold?: number;
+}): Scorer;
+/** Deep-equals case.expected against parsed finalMessage (default) or a selector. */
+export declare function jsonEquals(opts?: {
+    threshold?: number;
+    select?: (run: AgentRunResult) => unknown;
+}): Scorer;
+export declare function toolCalled(name: string, opts?: {
+    withArgs?: Record<string, unknown>;
+    threshold?: number;
+}): Scorer;
+export declare function tokensUnder(budget: number, opts?: {
+    threshold?: number;
+}): Scorer;
+export declare function custom(fn: (run: AgentRunResult, testCase: EvalCase) => Score | Promise<Score>, opts?: {
+    name?: string;
+    threshold?: number;
+}): Scorer;
+/**
+ * Score 1 if every id in `expectedIds` appears in at least one `recall` tool
+ * result string; score 0 with a reason listing missing ids.
+ */
+export declare function memoryRecalled(expectedIds: string[], opts?: {
+    threshold?: number;
+}): Scorer;
+/**
+ * Score 1 if `run.finalMessage` contains `expectedValue` (freshness check —
+ * the newer value surfaced in the response); else 0.
+ */
+export declare function memoryFresh(expectedValue: string, opts?: {
+    threshold?: number;
+}): Scorer;
+/**
+ * Score 1 if `forbidden` does NOT appear in any recall tool output or
+ * finalMessage (no cross-namespace leak); score 0 if it leaks.
+ */
+export declare function memoryIsolated(forbidden: string, opts?: {
+    threshold?: number;
+}): Scorer;
+//# sourceMappingURL=scorers.d.ts.map

package/dist/scorers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorers.d.ts","sourceRoot":"","sources":["../src/scorers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAEpD,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAMzD,uDAAuD;AACvD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMhE;AAED,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMjF;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAQvE;AAED,qFAAqF;AACrF,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE;IAChC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAA;CAC1C,GAAG,MAAM,CAkBT;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAChE,MAAM,CAgBR;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMjF;AAED,wBAAgB,MAAM,CACpB,EAAE,EAAE,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,EACvE,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3C,MAAM,CAMR;AAaD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAY3F;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CASxF;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAcvF"}