@dawn-ai/evals 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Brian Love
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/define-eval.d.ts

@@ -0,0 +1,3 @@
+import type { EvalDefinition } from "./types.js";
+export declare function defineEval(def: EvalDefinition): EvalDefinition;
+//# sourceMappingURL=define-eval.d.ts.map

package/dist/define-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"define-eval.d.ts","sourceRoot":"","sources":["../src/define-eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,wBAAgB,UAAU,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,CAc9D"}

package/dist/define-eval.js

@@ -0,0 +1,15 @@
+export function defineEval(def) {
+    if (!def.name || def.name.trim() === "") {
+        throw new Error("defineEval: `name` is required");
+    }
+    if (!def.scorers || def.scorers.length === 0) {
+        throw new Error(`defineEval("${def.name}"): at least one scorer is required`);
+    }
+    if (Array.isArray(def.dataset) && def.dataset.length === 0) {
+        throw new Error(`defineEval("${def.name}"): inline dataset is empty`);
+    }
+    if (def.dataset === undefined || def.dataset === null) {
+        throw new Error(`defineEval("${def.name}"): dataset is required`);
+    }
+    return def;
+}

package/dist/gate.d.ts

@@ -0,0 +1,13 @@
+import { DEFAULT_CASE_BAR, type EvalDefinition, type GatePolicy } from "./types.js";
+export declare const gate: {
+    mean(n: number): GatePolicy;
+    passRate(n: number): GatePolicy;
+    everyCase(n: number): GatePolicy;
+    perScorer(): GatePolicy;
+    all(...policies: GatePolicy[]): GatePolicy;
+    any(...policies: GatePolicy[]): GatePolicy;
+};
+/** gate wins; else threshold → mean(threshold); else informational (always passes). */
+export declare function resolveGate(def: Pick<EvalDefinition, "gate" | "threshold">): GatePolicy;
+export { DEFAULT_CASE_BAR };
+//# sourceMappingURL=gate.d.ts.map

package/dist/gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../src/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,KAAK,cAAc,EAAE,KAAK,UAAU,EAAmB,MAAM,YAAY,CAAA;AASpG,eAAO,MAAM,IAAI;YACP,MAAM,GAAG,UAAU;gBAGf,MAAM,GAAG,UAAU;iBAOlB,MAAM,GAAG,UAAU;iBAMnB,UAAU;qBAQN,UAAU,EAAE,GAAG,UAAU;qBASzB,UAAU,EAAE,GAAG,UAAU;CAW3C,CAAA;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,WAAW,CAAC,GAAG,UAAU,CAIvF;AAED,OAAO,EAAE,gBAAgB,EAAE,CAAA"}

package/dist/gate.js

@@ -0,0 +1,64 @@
+import { DEFAULT_CASE_BAR } from "./types.js";
+function pass(reason) {
+    return reason !== undefined ? { passed: true, reason } : { passed: true };
+}
+function fail(reason) {
+    return { passed: false, reason };
+}
+export const gate = {
+    mean(n) {
+        return (r) => (r.mean >= n ? pass() : fail(`mean ${r.mean.toFixed(2)} < ${n}`));
+    },
+    passRate(n) {
+        return (r) => {
+            const rate = r.cases.length === 0 ? 1 : r.cases.filter((c) => c.passed).length / r.cases.length;
+            return rate >= n ? pass() : fail(`pass-rate ${rate.toFixed(2)} < ${n}`);
+        };
+    },
+    everyCase(n) {
+        return (r) => {
+            const bad = r.cases.find((c) => c.mean < n);
+            return bad ? fail(`case "${bad.name}" mean ${bad.mean.toFixed(2)} < ${n}`) : pass();
+        };
+    },
+    perScorer() {
+        return (r) => {
+            const bad = r.byScorer.find((s) => s.threshold !== undefined && s.mean < s.threshold);
+            return bad
+                ? fail(`scorer "${bad.scorer}" mean ${bad.mean.toFixed(2)} < ${bad.threshold}`)
+                : pass();
+        };
+    },
+    all(...policies) {
+        return (r) => {
+            for (const p of policies) {
+                const res = p(r);
+                if (!res.passed)
+                    return res;
+            }
+            return pass();
+        };
+    },
+    any(...policies) {
+        return (r) => {
+            const reasons = [];
+            for (const p of policies) {
+                const res = p(r);
+                if (res.passed)
+                    return pass();
+                if (res.reason)
+                    reasons.push(res.reason);
+            }
+            return fail(`no policy passed: ${reasons.join("; ")}`);
+        };
+    },
+};
+/** gate wins; else threshold → mean(threshold); else informational (always passes). */
+export function resolveGate(def) {
+    if (def.gate)
+        return def.gate;
+    if (def.threshold !== undefined)
+        return gate.mean(def.threshold);
+    return () => pass("informational (no gate)");
+}
+export { DEFAULT_CASE_BAR };

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export { defineEval } from "./define-eval.js";
+export { gate, resolveGate } from "./gate.js";
+export { type LlmJudgeOptions, llmJudge } from "./llm-judge.js";
+export { resolveDataset } from "./resolve-dataset.js";
+export { type RunEvalOptions, runEval } from "./run-eval.js";
+export { type NormalizedScore, normalizeScore } from "./score.js";
+export { contains, custom, exactMatch, jsonEquals, regex, tokensUnder, toolCalled, } from "./scorers.js";
+export type { CaseResult, CaseScore, Dataset, EvalCase, EvalDefinition, EvalReport, GatePolicy, GateResult, Score, ScoredReport, Scorer, ScorerAggregate, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,KAAK,eAAe,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,KAAK,cAAc,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAC5D,OAAO,EAAE,KAAK,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AACjE,OAAO,EACL,QAAQ,EACR,MAAM,EACN,UAAU,EACV,UAAU,EACV,KAAK,EACL,WAAW,EACX,UAAU,GACX,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,cAAc,EACd,UAAU,EACV,UAAU,EACV,UAAU,EACV,KAAK,EACL,YAAY,EACZ,MAAM,EACN,eAAe,GAChB,MAAM,YAAY,CAAA"}

package/dist/index.js

@@ -0,0 +1,7 @@
+export { defineEval } from "./define-eval.js";
+export { gate, resolveGate } from "./gate.js";
+export { llmJudge } from "./llm-judge.js";
+export { resolveDataset } from "./resolve-dataset.js";
+export { runEval } from "./run-eval.js";
+export { normalizeScore } from "./score.js";
+export { contains, custom, exactMatch, jsonEquals, regex, tokensUnder, toolCalled, } from "./scorers.js";

package/dist/llm-judge.d.ts

@@ -0,0 +1,16 @@
+import type { Scorer } from "./types.js";
+type FetchImpl = (input: string, init: RequestInit) => Promise<Response>;
+export interface LlmJudgeOptions {
+    /** Criteria template; supports {{input}}, {{expected}}, {{output}} interpolation. */
+    readonly criteria: string;
+    readonly model?: string;
+    readonly threshold?: number;
+    readonly name?: string;
+    /** Overrides for testing; default to env + global fetch. */
+    readonly baseUrl?: string;
+    readonly apiKey?: string;
+    readonly fetchImpl?: FetchImpl;
+}
+export declare function llmJudge(opts: LlmJudgeOptions): Scorer;
+export {};
+//# sourceMappingURL=llm-judge.d.ts.map

package/dist/llm-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../src/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAY,MAAM,EAAE,MAAM,YAAY,CAAA;AAElD,KAAK,SAAS,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;AAExE,MAAM,WAAW,eAAe;IAC9B,qFAAqF;IACrF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,4DAA4D;IAC5D,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;CAC/B;AAMD,wBAAgB,QAAQ,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,CAmDtD"}

package/dist/llm-judge.js

@@ -0,0 +1,57 @@
+function interpolate(template, vars) {
+    return template.replace(/\{\{(\w+)\}\}/g, (_m, key) => vars[key] ?? "");
+}
+export function llmJudge(opts) {
+    const model = opts.model ?? "gpt-4o-mini";
+    return {
+        name: opts.name ?? "llmJudge",
+        ...(opts.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: async (run, testCase) => {
+            const baseUrl = opts.baseUrl ?? process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1";
+            const apiKey = opts.apiKey ?? process.env.OPENAI_API_KEY ?? "";
+            const fetchImpl = opts.fetchImpl ?? ((i, init) => fetch(i, init));
+            const criteria = interpolate(opts.criteria, {
+                input: String(testCase.input ?? ""),
+                expected: JSON.stringify(testCase.expected ?? ""),
+                output: run.finalMessage,
+            });
+            const user = [
+                `Criteria: ${criteria}`,
+                `Agent output: ${run.finalMessage}`,
+                `Respond ONLY with JSON: {"score": <0..1>, "reason": "<short>"}.`,
+            ].join("\n");
+            let content;
+            try {
+                const res = await fetchImpl(`${baseUrl}/chat/completions`, {
+                    method: "POST",
+                    headers: { "content-type": "application/json", authorization: `Bearer ${apiKey}` },
+                    body: JSON.stringify({
+                        model,
+                        messages: [
+                            {
+                                role: "system",
+                                content: "You are a strict grader. Output only the requested JSON.",
+                            },
+                            { role: "user", content: user },
+                        ],
+                    }),
+                });
+                const json = (await res.json());
+                content = json.choices?.[0]?.message?.content ?? "";
+            }
+            catch (err) {
+                return {
+                    score: 0,
+                    reason: `judge request failed: ${err instanceof Error ? err.message : String(err)}`,
+                };
+            }
+            try {
+                const parsed = JSON.parse(content);
+                return { score: parsed.score, reason: parsed.reason ?? "" };
+            }
+            catch {
+                return { score: 0, reason: `could not parse judge verdict: ${content.slice(0, 120)}` };
+            }
+        },
+    };
+}

package/dist/resolve-dataset.d.ts

@@ -0,0 +1,3 @@
+import type { Dataset, EvalCase } from "./types.js";
+export declare function resolveDataset(dataset: Dataset, baseDir: string): Promise<EvalCase[]>;
+//# sourceMappingURL=resolve-dataset.d.ts.map

package/dist/resolve-dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolve-dataset.d.ts","sourceRoot":"","sources":["../src/resolve-dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAEnD,wBAAsB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAiC3F"}

package/dist/resolve-dataset.js

@@ -0,0 +1,38 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute, resolve } from "node:path";
+export async function resolveDataset(dataset, baseDir) {
+    if (Array.isArray(dataset))
+        return [...dataset];
+    if (typeof dataset === "function")
+        return [...(await dataset())];
+    if (typeof dataset === "string") {
+        const path = isAbsolute(dataset) ? dataset : resolve(baseDir, dataset);
+        let raw;
+        try {
+            raw = await readFile(path, "utf8");
+        }
+        catch (err) {
+            throw new Error(`resolveDataset: cannot read dataset file "${path}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+        if (path.endsWith(".jsonl")) {
+            return raw
+                .split("\n")
+                .map((l) => l.trim())
+                .filter(Boolean)
+                .map((line, i) => {
+                try {
+                    return JSON.parse(line);
+                }
+                catch {
+                    throw new Error(`resolveDataset: invalid JSONL at line ${i + 1} in "${path}"`);
+                }
+            });
+        }
+        const parsed = JSON.parse(raw);
+        if (!Array.isArray(parsed)) {
+            throw new Error(`resolveDataset: "${path}" must contain a JSON array of cases`);
+        }
+        return parsed;
+    }
+    throw new Error("resolveDataset: dataset must be an array, a path string, or a function");
+}

package/dist/run-eval.d.ts

@@ -0,0 +1,10 @@
+import type { AgentRunResult } from "@dawn-ai/testing";
+import { type EvalCase, type EvalDefinition, type EvalReport } from "./types.js";
+export interface RunEvalOptions {
+    /** Executes one case and returns its run result (replay or live; injected by the CLI). */
+    readonly runCase: (testCase: EvalCase) => Promise<AgentRunResult>;
+    /** Base dir for resolving a string dataset path (the eval file's directory). */
+    readonly baseDir?: string;
+}
+export declare function runEval(def: EvalDefinition, options: RunEvalOptions): Promise<EvalReport>;
+//# sourceMappingURL=run-eval.d.ts.map

package/dist/run-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval.d.ts","sourceRoot":"","sources":["../src/run-eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AAItD,OAAO,EAIL,KAAK,QAAQ,EACb,KAAK,cAAc,EACnB,KAAK,UAAU,EAEhB,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,cAAc;IAC7B,0FAA0F;IAC1F,QAAQ,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,cAAc,CAAC,CAAA;IACjE,gFAAgF;IAChF,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAC1B;AAMD,wBAAsB,OAAO,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,CAqD/F"}

package/dist/run-eval.js

@@ -0,0 +1,56 @@
+import { resolveGate } from "./gate.js";
+import { resolveDataset } from "./resolve-dataset.js";
+import { normalizeScore } from "./score.js";
+import { DEFAULT_CASE_BAR, } from "./types.js";
+function mean(nums) {
+    return nums.length === 0 ? 0 : nums.reduce((a, b) => a + b, 0) / nums.length;
+}
+export async function runEval(def, options) {
+    const cases = await resolveDataset(def.dataset, options.baseDir ?? process.cwd());
+    const thresholdOf = new Map(def.scorers.map((s) => [s.name, s.threshold]));
+    const caseResults = [];
+    for (const [index, testCase] of cases.entries()) {
+        const run = await options.runCase(testCase);
+        const scores = [];
+        for (const scorer of def.scorers) {
+            let normalized;
+            try {
+                normalized = normalizeScore(await scorer.score(run, testCase));
+            }
+            catch (err) {
+                normalized = { score: 0, reason: err instanceof Error ? err.message : String(err) };
+            }
+            scores.push({
+                scorer: scorer.name,
+                score: normalized.score,
+                ...(normalized.label !== undefined ? { label: normalized.label } : {}),
+                ...(normalized.reason !== undefined ? { reason: normalized.reason } : {}),
+            });
+        }
+        const passed = scores.every((s) => s.score >= (thresholdOf.get(s.scorer) ?? DEFAULT_CASE_BAR));
+        caseResults.push({
+            name: testCase.name ?? `case ${index + 1}`,
+            scores,
+            mean: mean(scores.map((s) => s.score)),
+            passed,
+        });
+    }
+    const byScorer = def.scorers.map((scorer) => {
+        const scorerScores = caseResults.flatMap((c) => c.scores.filter((s) => s.scorer === scorer.name).map((s) => s.score));
+        return {
+            scorer: scorer.name,
+            mean: mean(scorerScores),
+            ...(scorer.threshold !== undefined ? { threshold: scorer.threshold } : {}),
+        };
+    });
+    const overallMean = mean(caseResults.flatMap((c) => c.scores.map((s) => s.score)));
+    const scored = { name: def.name, cases: caseResults, byScorer, mean: overallMean };
+    const gated = def.gate !== undefined || def.threshold !== undefined;
+    const result = resolveGate(def)(scored);
+    return {
+        ...scored,
+        gated,
+        passed: result.passed,
+        ...(result.reason !== undefined ? { reason: result.reason } : {}),
+    };
+}

package/dist/score.d.ts

@@ -0,0 +1,8 @@
+import type { Score } from "./types.js";
+export interface NormalizedScore {
+    readonly score: number;
+    readonly label?: string;
+    readonly reason?: string;
+}
+export declare function normalizeScore(raw: Score): NormalizedScore;
+//# sourceMappingURL=score.d.ts.map

package/dist/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAEvC,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAUD,wBAAgB,cAAc,CAAC,GAAG,EAAE,KAAK,GAAG,eAAe,CAU1D"}

package/dist/score.js

@@ -0,0 +1,24 @@
+function clamp01(n) {
+    const x = Number(n);
+    if (!Number.isFinite(x))
+        return 0;
+    if (x < 0)
+        return 0;
+    if (x > 1)
+        return 1;
+    return x;
+}
+export function normalizeScore(raw) {
+    if (typeof raw === "boolean")
+        return { score: raw ? 1 : 0 };
+    if (typeof raw === "number")
+        return { score: clamp01(raw) };
+    if (raw === null || typeof raw !== "object")
+        return { score: 0 };
+    const out = { score: clamp01(raw.score) };
+    return {
+        ...out,
+        ...(raw.label !== undefined ? { label: raw.label } : {}),
+        ...(raw.reason !== undefined ? { reason: raw.reason } : {}),
+    };
+}

package/dist/scorers.d.ts

@@ -0,0 +1,29 @@
+import type { AgentRunResult } from "@dawn-ai/testing";
+import type { EvalCase, Score, Scorer } from "./types.js";
+/** finalMessage === case.expected (string compare). */
+export declare function exactMatch(opts?: {
+    threshold?: number;
+}): Scorer;
+export declare function contains(substring: string, opts?: {
+    threshold?: number;
+}): Scorer;
+export declare function regex(re: RegExp, opts?: {
+    threshold?: number;
+}): Scorer;
+/** Deep-equals case.expected against parsed finalMessage (default) or a selector. */
+export declare function jsonEquals(opts?: {
+    threshold?: number;
+    select?: (run: AgentRunResult) => unknown;
+}): Scorer;
+export declare function toolCalled(name: string, opts?: {
+    withArgs?: Record<string, unknown>;
+    threshold?: number;
+}): Scorer;
+export declare function tokensUnder(budget: number, opts?: {
+    threshold?: number;
+}): Scorer;
+export declare function custom(fn: (run: AgentRunResult, testCase: EvalCase) => Score | Promise<Score>, opts?: {
+    name?: string;
+    threshold?: number;
+}): Scorer;
+//# sourceMappingURL=scorers.d.ts.map

package/dist/scorers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorers.d.ts","sourceRoot":"","sources":["../src/scorers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAMzD,uDAAuD;AACvD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMhE;AAED,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMjF;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMvE;AAED,qFAAqF;AACrF,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE;IAChC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAA;CAC1C,GAAG,MAAM,CAkBT;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAChE,MAAM,CAgBR;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAMjF;AAED,wBAAgB,MAAM,CACpB,EAAE,EAAE,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,EACvE,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3C,MAAM,CAMR"}

package/dist/scorers.js

@@ -0,0 +1,80 @@
+function deepEqual(a, b) {
+    return JSON.stringify(a) === JSON.stringify(b);
+}
+/** finalMessage === case.expected (string compare). */
+export function exactMatch(opts) {
+    return {
+        name: "exactMatch",
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: (run, c) => (run.finalMessage === String(c.expected ?? "") ? 1 : 0),
+    };
+}
+export function contains(substring, opts) {
+    return {
+        name: `contains(${substring})`,
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: (run) => (run.finalMessage.includes(substring) ? 1 : 0),
+    };
+}
+export function regex(re, opts) {
+    return {
+        name: `regex(${re.source})`,
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: (run) => (re.test(run.finalMessage) ? 1 : 0),
+    };
+}
+/** Deep-equals case.expected against parsed finalMessage (default) or a selector. */
+export function jsonEquals(opts) {
+    return {
+        name: "jsonEquals",
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: (run, c) => {
+            let actual;
+            if (opts?.select) {
+                actual = opts.select(run);
+            }
+            else {
+                try {
+                    actual = JSON.parse(run.finalMessage);
+                }
+                catch {
+                    return 0;
+                }
+            }
+            return deepEqual(actual, c.expected) ? 1 : 0;
+        },
+    };
+}
+export function toolCalled(name, opts) {
+    return {
+        name: `toolCalled(${name})`,
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: (run) => {
+            const calls = run.toolCalls.filter((t) => t.name === name);
+            if (calls.length === 0)
+                return 0;
+            if (!opts?.withArgs)
+                return 1;
+            const want = opts.withArgs;
+            const hit = calls.some((call) => {
+                const args = (call.args ?? {});
+                return Object.entries(want).every(([k, v]) => deepEqual(args[k], v));
+            });
+            return hit ? 1 : 0;
+        },
+    };
+}
+export function tokensUnder(budget, opts) {
+    return {
+        name: `tokensUnder(${budget})`,
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: (run) => (run.tokens.length < budget ? 1 : 0),
+    };
+}
+export function custom(fn, opts) {
+    return {
+        name: opts?.name ?? "custom",
+        ...(opts?.threshold !== undefined ? { threshold: opts.threshold } : {}),
+        score: fn,
+    };
+}

package/dist/types.d.ts

@@ -0,0 +1,74 @@
+import type { AgentRunResult, FixtureSet, ScriptBuilder } from "@dawn-ai/testing";
+/** One dataset row. `input` is the user message for agent routes (v1). */
+export interface EvalCase {
+    readonly name?: string;
+    readonly input: unknown;
+    readonly expected?: unknown;
+    /** Per-case aimock fixtures for replay mode; ignored under --live. */
+    readonly fixtures?: FixtureSet | ScriptBuilder;
+    readonly metadata?: Record<string, unknown>;
+}
+/** Inline cases, a path to a committed .json/.jsonl, or a (sync/async) factory. */
+export type Dataset = readonly EvalCase[] | string | (() => EvalCase[] | Promise<EvalCase[]>);
+/** A scorer may return a 0..1 number, a boolean, or a rich verdict. */
+export type Score = number | boolean | {
+    readonly score: number;
+    readonly label?: string;
+    readonly reason?: string;
+};
+export interface Scorer {
+    readonly name: string;
+    /** This scorer's own pass bar (used by gate.perScorer and case-pass). */
+    readonly threshold?: number;
+    readonly score: (run: AgentRunResult, testCase: EvalCase) => Score | Promise<Score>;
+}
+export interface EvalDefinition {
+    readonly name: string;
+    /** Route key like "/chat#agent"; defaults to the co-located route at load time. */
+    readonly route?: string;
+    readonly dataset: Dataset;
+    readonly scorers: readonly Scorer[];
+    /** Sugar for gate.mean(threshold). Ignored if `gate` is set. */
+    readonly threshold?: number;
+    readonly gate?: GatePolicy;
+}
+/** Normalized score for one (case, scorer) pair. */
+export interface CaseScore {
+    readonly scorer: string;
+    readonly score: number;
+    readonly label?: string;
+    readonly reason?: string;
+}
+export interface CaseResult {
+    readonly name: string;
+    readonly scores: readonly CaseScore[];
+    readonly mean: number;
+    /** Every scorer met its bar (scorer.threshold ?? DEFAULT_CASE_BAR). */
+    readonly passed: boolean;
+}
+export interface ScorerAggregate {
+    readonly scorer: string;
+    readonly mean: number;
+    readonly threshold?: number;
+}
+/** Pre-gate report fed to gate policies. */
+export interface ScoredReport {
+    readonly name: string;
+    readonly cases: readonly CaseResult[];
+    readonly byScorer: readonly ScorerAggregate[];
+    readonly mean: number;
+}
+export interface EvalReport extends ScoredReport {
+    /** Whether a gate/threshold was configured (informational evals are false). */
+    readonly gated: boolean;
+    readonly passed: boolean;
+    readonly reason?: string;
+}
+export type GateResult = {
+    readonly passed: boolean;
+    readonly reason?: string;
+};
+export type GatePolicy = (report: ScoredReport) => GateResult;
+/** A case "passes" when every scorer ≥ its threshold, defaulting to this bar. */
+export declare const DEFAULT_CASE_BAR = 0.5;
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAEjF,0EAA0E;AAC1E,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAC3B,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,GAAG,aAAa,CAAA;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC5C;AAED,mFAAmF;AACnF,MAAM,MAAM,OAAO,GAAG,SAAS,QAAQ,EAAE,GAAG,MAAM,GAAG,CAAC,MAAM,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;AAE7F,uEAAuE;AACvE,MAAM,MAAM,KAAK,GACb,MAAM,GACN,OAAO,GACP;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAEjF,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yEAAyE;IACzE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;CACpF;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,mFAAmF;IACnF,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IACnC,gEAAgE;IAChE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAA;CAC3B;AAED,oDAAoD;AACpD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAA;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAA;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,MAAM,UAAU,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAC/E,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,YAAY,KAAK,UAAU,CAAA;AAE7D,iFAAiF;AACjF,eAAO,MAAM,gBAAgB,MAAM,CAAA"}

package/dist/types.js

@@ -0,0 +1,2 @@
+/** A case "passes" when every scorer ≥ its threshold, defaulting to this bar. */
+export const DEFAULT_CASE_BAR = 0.5;

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@dawn-ai/evals",
+  "version": "0.1.0",
+  "private": false,
+  "type": "module",
+  "license": "MIT",
+  "homepage": "https://github.com/cacheplane/dawnai/tree/main/packages/evals#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cacheplane/dawnai.git",
+    "directory": "packages/evals"
+  },
+  "bugs": {
+    "url": "https://github.com/cacheplane/dawnai/issues"
+  },
+  "engines": {
+    "node": ">=22.12.0"
+  },
+  "files": [
+    "dist"
+  ],
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "@dawn-ai/testing": "2.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "25.6.0",
+    "@dawn-ai/config-typescript": "0.4.0",
+    "@dawn-ai/testing": "2.0.0"
+  },
+  "scripts": {
+    "build": "tsc -b tsconfig.json",
+    "lint": "biome check --config-path ../config-biome/biome.json package.json src tsconfig.json vitest.config.ts",
+    "test": "vitest --run --config vitest.config.ts --passWithNoTests",
+    "typecheck": "tsc --noEmit"
+  }
+}