@agent-native/core 0.52.0 → 0.53.0

package/dist/eval/runner.js

@@ -0,0 +1,256 @@
+/**
+ * The evals runner: discover `*.eval.ts` / `evals/*.ts` files, run each eval
+ * through its scorer pipeline against the *real* agent loop, score, and report.
+ *
+ * It is the engine behind `agent-native eval` — when used as a CI deploy gate
+ * the CLI exits non-zero if any eval scores below its threshold.
+ *
+ * Two layers:
+ *   - `scoreEval` / `runEvals` — pure orchestration over an `AgentRunner` and
+ *     a list of evals. Fully unit-testable with an injected runner (no model).
+ *   - `discoverEvalFiles` / `loadEvals` — filesystem discovery + dynamic import
+ *     of author-written eval modules.
+ *
+ * Results are also (best-effort) written to the observability eval store so a
+ * dashboard can surface CI eval history next to production run evals.
+ */
+import nodePath from "node:path";
+import { pathToFileURL } from "node:url";
+import { insertEvalResult } from "../observability/store.js";
+import { DEFAULT_EVAL_THRESHOLD } from "./define-eval.js";
+import { clamp01 } from "./scorer.js";
+import { createAgentRunner } from "./agent-runner.js";
+// ─── Scoring orchestration ────────────────────────────────────────────
+/** Run one scorer's pipeline (preprocess → analyze → score → reason). */
+async function runScorer(scorer, run, runner, threshold) {
+    try {
+        const pre = scorer.preprocess ? await scorer.preprocess(run) : run;
+        const analysis = scorer.analyze
+            ? await scorer.analyze(pre, runner.analyzeContext())
+            : pre;
+        const rawScore = await scorer.generateScore(analysis);
+        const score = clamp01(rawScore);
+        const reason = scorer.generateReason
+            ? await scorer.generateReason({
+                run,
+                analysis: analysis,
+                score,
+            })
+            : undefined;
+        return { scorer: scorer.name, score, reason, passed: score >= threshold };
+    }
+    catch (err) {
+        // A scorer that throws is a failed scorer, not a crashed run — degrade
+        // gracefully so one bad scorer can't take down the whole CI gate.
+        return {
+            scorer: scorer.name,
+            score: 0,
+            reason: `Scorer errored: ${err instanceof Error ? err.message : String(err)}`,
+            passed: false,
+        };
+    }
+}
+/** Run a single eval: invoke the agent, then score with each scorer. */
+export async function scoreEval(evalCase, runner, opts = {}) {
+    const threshold = opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;
+    let run;
+    if (evalCase.run) {
+        run = await evalCase.run({
+            input: evalCase.input,
+            runAgent: (input) => runner.runAgent(input),
+        });
+    }
+    else {
+        run = await runner.runAgent(evalCase.input);
+    }
+    const scores = [];
+    for (const scorer of evalCase.scorers) {
+        scores.push(await runScorer(scorer, run, runner, threshold));
+    }
+    const avgScore = scores.length > 0
+        ? scores.reduce((s, r) => s + r.score, 0) / scores.length
+        : 0;
+    return {
+        eval: evalCase.name,
+        threshold,
+        scores,
+        // A run that errored, or any sub-threshold scorer, fails the case.
+        passed: run.ok && scores.every((s) => s.passed),
+        avgScore,
+        durationMs: run.durationMs,
+        error: run.ok ? undefined : run.error,
+    };
+}
+/** Run a batch of evals against one runner and aggregate a report. */
+export async function runEvals(evals, runner, opts = {}) {
+    const results = [];
+    for (const evalCase of evals) {
+        const row = await scoreEval(evalCase, runner, opts);
+        results.push(row);
+        if (opts.persist)
+            await persistEvalRow(row).catch(() => { });
+    }
+    const passed = results.filter((r) => r.passed).length;
+    return {
+        total: results.length,
+        passed,
+        failed: results.length - passed,
+        results,
+    };
+}
+/**
+ * Best-effort write of one eval result to the observability eval store so a
+ * dashboard can show CI eval history alongside production run evals. We write
+ * one row per (eval × scorer), tagged `evalType: "automated"` with a synthetic
+ * `eval:` run id.
+ *
+ * TODO(live-sampling): the same scorer list should also run on a sampled
+ * fraction of *real* production runs. That hook belongs in the agent loop's
+ * (not-yet-added) post-run processor seam: when a run finishes, roll the
+ * configured sample rate and, if it hits, replay the run output through these
+ * scorers and write the rows here. Wiring it now would require the in-loop
+ * processor seam another wave is adding — so this is the single intended
+ * attachment point, intentionally left as a note.
+ */
+async function persistEvalRow(row) {
+    const runId = `eval:${row.eval}:${Date.now()}`;
+    for (const s of row.scores) {
+        const result = {
+            id: crypto.randomUUID(),
+            runId,
+            threadId: null,
+            userId: null,
+            evalType: "automated",
+            criteria: `eval:${row.eval}:${s.scorer}`,
+            score: s.score,
+            reasoning: s.reason ?? null,
+            metadata: {
+                source: "cli-eval",
+                threshold: row.threshold,
+                passed: s.passed,
+            },
+            createdAt: Date.now(),
+        };
+        await insertEvalResult(result);
+    }
+}
+// ─── Discovery + loading ──────────────────────────────────────────────
+const EVAL_FILE_RE = /\.eval\.(ts|js|mjs)$/;
+const SKIP_DIRS = new Set(["node_modules", "dist", ".git", ".output", "build"]);
+/**
+ * Walk `root` for eval files. Matches two conventions:
+ *   - any `**\/*.eval.ts` (co-located with code), and
+ *   - any `*.ts` directly inside an `evals/` directory.
+ * `pattern` further filters by substring of the relative path.
+ */
+export async function discoverEvalFiles(root, pattern) {
+    const fs = await import("node:fs");
+    const out = [];
+    function isEvalFile(full, parentName) {
+        const base = nodePath.basename(full);
+        if (EVAL_FILE_RE.test(base))
+            return true;
+        if (parentName === "evals" && /\.(ts|js|mjs)$/.test(base)) {
+            // Skip obvious support files inside evals/.
+            return !/\.(spec|test|d)\.(ts|js|mjs)$/.test(base);
+        }
+        return false;
+    }
+    function walk(dir, parentName) {
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const full = nodePath.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                if (SKIP_DIRS.has(entry.name) || entry.name.startsWith("."))
+                    continue;
+                walk(full, entry.name);
+            }
+            else if (entry.isFile() && isEvalFile(full, parentName)) {
+                out.push(full);
+            }
+        }
+    }
+    walk(root, nodePath.basename(root));
+    out.sort();
+    if (!pattern)
+        return out;
+    return out.filter((f) => nodePath.relative(root, f).includes(pattern));
+}
+/** Pull `Eval` definitions out of a dynamically-imported eval module. */
+function extractEvals(mod) {
+    const candidates = [];
+    if (mod.default !== undefined)
+        candidates.push(mod.default);
+    for (const [key, value] of Object.entries(mod)) {
+        if (key === "default")
+            continue;
+        candidates.push(value);
+    }
+    const evals = [];
+    for (const c of candidates.flat()) {
+        if (c &&
+            typeof c === "object" &&
+            typeof c.name === "string" &&
+            Array.isArray(c.scorers) &&
+            c.input) {
+            evals.push(c);
+        }
+    }
+    return evals;
+}
+/** Discover and import all eval files under `root`, returning their evals. */
+export async function loadEvals(root, pattern) {
+    const files = await discoverEvalFiles(root, pattern);
+    const evals = [];
+    for (const file of files) {
+        const mod = (await import(pathToFileURL(file).href));
+        evals.push(...extractEvals(mod));
+    }
+    return { files, evals };
+}
+/**
+ * End-to-end: load evals, build a runner, score, report. The CLI wraps this
+ * and maps `report.failed > 0` to a non-zero exit code (the CI gate).
+ */
+export async function runEvalSuite(opts = {}) {
+    const cwd = opts.cwd ?? process.cwd();
+    let files = [];
+    let evals = opts.evals;
+    if (!evals) {
+        const loaded = await loadEvals(cwd, opts.pattern);
+        files = loaded.files;
+        evals = loaded.evals;
+    }
+    const runner = opts.runner ??
+        (await createAgentRunner({
+            actions: opts.actions ?? (await discoverActions(cwd)),
+            systemPrompt: opts.systemPrompt,
+        }));
+    const report = await runEvals(evals, runner, {
+        thresholdOverride: opts.thresholdOverride,
+        persist: opts.persist ?? true,
+    });
+    return { report, files };
+}
+/**
+ * Discover the app's actions so the agent under test has the real tool
+ * surface. Lazy-imports `autoDiscoverActions` to keep server-only deps out of
+ * any browser bundle that might touch this module's types.
+ */
+async function discoverActions(cwd) {
+    try {
+        const { autoDiscoverActions } = await import("../server/action-discovery.js");
+        const actionsDir = nodePath.join(cwd, "actions");
+        return await autoDiscoverActions(pathToFileURL(actionsDir + "/").href);
+    }
+    catch {
+        return {};
+    }
+}
+//# sourceMappingURL=runner.js.map

package/dist/eval/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAG7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAStD,yEAAyE;AAEzE,yEAAyE;AACzE,KAAK,UAAU,SAAS,CACtB,MAA+B,EAC/B,GAAmB,EACnB,MAAmB,EACnB,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO;YAC7B,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,GAAY,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7D,CAAC,CAAC,GAAG,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAiB,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc;YAClC,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC;gBAC1B,GAAG;gBACH,QAAQ,EAAE,QAAiB;gBAC3B,KAAK;aACN,CAAC;YACJ,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;IAC5E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uEAAuE;QACvE,kEAAkE;QAClE,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC7E,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAc,EACd,MAAmB,EACnB,OAAuC,EAAE;IAEzC,MAAM,SAAS,GACb,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAEzE,IAAI,GAAmB,CAAC;IACxB,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;QACjB,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,GAAG,CAAC;QACf,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACzD,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS;QACT,MAAM;QACN,mEAAmE;QACnE,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,QAAQ;QACR,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;KACtC,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,MAAmB,EACnB,OAA0D,EAAE;IAE5D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;QAC/B,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,cAAc,CAAC,GAAkB;IAC9C,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAA4B;YACtC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,KAAK;YACL,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,SAAS,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;YAC3B,QAAQ,EAAE;gBACR,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB;YACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QACF,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,yEAAyE;AAEzE,MAAM,YAAY,GAAG,sBAAsB,CAAC;AAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,OAAgB;IAEhB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,SAAS,UAAU,CAAC,IAAY,EAAE,UAAkB;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,IAAI,UAAU,KAAK,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,4CAA4C;YAC5C,OAAO,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,GAAW,EAAE,UAAkB;QAC3C,IAAI,OAAmC,CAAC;QACxC,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,EAAE,CAAC;IAEX,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAC;IACzB,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,yEAAyE;AACzE,SAAS,YAAY,CAAC,GAA4B;IAChD,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;QAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5D,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,IACE,CAAC;YACD,OAAO,CAAC,KAAK,QAAQ;YACrB,OAAQ,CAAU,CAAC,IAAI,KAAK,QAAQ;YACpC,KAAK,CAAC,OAAO,CAAE,CAAU,CAAC,OAAO,CAAC;YACjC,CAAU,CAAC,KAAK,EACjB,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,CAAS,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,OAAgB;IAEhB,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAGlD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC;AAuBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAA4B,EAAE;IAE9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,MAAM,MAAM,GACV,IAAI,CAAC,MAAM;QACX,CAAC,MAAM,iBAAiB,CAAC;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;YACrD,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC,CAAC,CAAC;IAEN,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE;QAC3C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;KAC9B,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,GAAW;IAEX,IAAI,CAAC;QACH,MAAM,EAAE,mBAAmB,EAAE,GAC3B,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,MAAM,mBAAmB,CAAC,aAAa,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC","sourcesContent":["/**\n * The evals runner: discover `*.eval.ts` / `evals/*.ts` files, run each eval\n * through its scorer pipeline against the *real* agent loop, score, and report.\n *\n * It is the engine behind `agent-native eval` — when used as a CI deploy gate\n * the CLI exits non-zero if any eval scores below its threshold.\n *\n * Two layers:\n *   - `scoreEval` / `runEvals` — pure orchestration over an `AgentRunner` and\n *     a list of evals. Fully unit-testable with an injected runner (no model).\n *   - `discoverEvalFiles` / `loadEvals` — filesystem discovery + dynamic import\n *     of author-written eval modules.\n *\n * Results are also (best-effort) written to the observability eval store so a\n * dashboard can surface CI eval history next to production run evals.\n */\n\nimport nodePath from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\n\nimport type { ActionEntry } from \"../agent/production-agent.js\";\nimport { insertEvalResult } from \"../observability/store.js\";\nimport type { EvalResult as ObservabilityEvalResult } from \"../observability/types.js\";\n\nimport { DEFAULT_EVAL_THRESHOLD } from \"./define-eval.js\";\nimport { clamp01 } from \"./scorer.js\";\nimport type { AgentRunner } from \"./agent-runner.js\";\nimport { createAgentRunner } from \"./agent-runner.js\";\nimport type {\n  AgentRunOutput,\n  Eval,\n  EvalResultRow,\n  EvalRunReport,\n  ScorerResult,\n} from \"./types.js\";\n\n// ─── Scoring orchestration ────────────────────────────────────────────\n\n/** Run one scorer's pipeline (preprocess → analyze → score → reason). */\nasync function runScorer(\n  scorer: Eval[\"scorers\"][number],\n  run: AgentRunOutput,\n  runner: AgentRunner,\n  threshold: number,\n): Promise<ScorerResult> {\n  try {\n    const pre = scorer.preprocess ? await scorer.preprocess(run) : run;\n    const analysis = scorer.analyze\n      ? await scorer.analyze(pre as never, runner.analyzeContext())\n      : pre;\n    const rawScore = await scorer.generateScore(analysis as never);\n    const score = clamp01(rawScore);\n    const reason = scorer.generateReason\n      ? await scorer.generateReason({\n          run,\n          analysis: analysis as never,\n          score,\n        })\n      : undefined;\n    return { scorer: scorer.name, score, reason, passed: score >= threshold };\n  } catch (err) {\n    // A scorer that throws is a failed scorer, not a crashed run — degrade\n    // gracefully so one bad scorer can't take down the whole CI gate.\n    return {\n      scorer: scorer.name,\n      score: 0,\n      reason: `Scorer errored: ${err instanceof Error ? err.message : String(err)}`,\n      passed: false,\n    };\n  }\n}\n\n/** Run a single eval: invoke the agent, then score with each scorer. */\nexport async function scoreEval(\n  evalCase: Eval,\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number } = {},\n): Promise<EvalResultRow> {\n  const threshold =\n    opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;\n\n  let run: AgentRunOutput;\n  if (evalCase.run) {\n    run = await evalCase.run({\n      input: evalCase.input,\n      runAgent: (input) => runner.runAgent(input),\n    });\n  } else {\n    run = await runner.runAgent(evalCase.input);\n  }\n\n  const scores: ScorerResult[] = [];\n  for (const scorer of evalCase.scorers) {\n    scores.push(await runScorer(scorer, run, runner, threshold));\n  }\n\n  const avgScore =\n    scores.length > 0\n      ? scores.reduce((s, r) => s + r.score, 0) / scores.length\n      : 0;\n\n  return {\n    eval: evalCase.name,\n    threshold,\n    scores,\n    // A run that errored, or any sub-threshold scorer, fails the case.\n    passed: run.ok && scores.every((s) => s.passed),\n    avgScore,\n    durationMs: run.durationMs,\n    error: run.ok ? undefined : run.error,\n  };\n}\n\n/** Run a batch of evals against one runner and aggregate a report. */\nexport async function runEvals(\n  evals: Eval[],\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number; persist?: boolean } = {},\n): Promise<EvalRunReport> {\n  const results: EvalResultRow[] = [];\n  for (const evalCase of evals) {\n    const row = await scoreEval(evalCase, runner, opts);\n    results.push(row);\n    if (opts.persist) await persistEvalRow(row).catch(() => {});\n  }\n\n  const passed = results.filter((r) => r.passed).length;\n  return {\n    total: results.length,\n    passed,\n    failed: results.length - passed,\n    results,\n  };\n}\n\n/**\n * Best-effort write of one eval result to the observability eval store so a\n * dashboard can show CI eval history alongside production run evals. We write\n * one row per (eval × scorer), tagged `evalType: \"automated\"` with a synthetic\n * `eval:` run id.\n *\n * TODO(live-sampling): the same scorer list should also run on a sampled\n * fraction of *real* production runs. That hook belongs in the agent loop's\n * (not-yet-added) post-run processor seam: when a run finishes, roll the\n * configured sample rate and, if it hits, replay the run output through these\n * scorers and write the rows here. Wiring it now would require the in-loop\n * processor seam another wave is adding — so this is the single intended\n * attachment point, intentionally left as a note.\n */\nasync function persistEvalRow(row: EvalResultRow): Promise<void> {\n  const runId = `eval:${row.eval}:${Date.now()}`;\n  for (const s of row.scores) {\n    const result: ObservabilityEvalResult = {\n      id: crypto.randomUUID(),\n      runId,\n      threadId: null,\n      userId: null,\n      evalType: \"automated\",\n      criteria: `eval:${row.eval}:${s.scorer}`,\n      score: s.score,\n      reasoning: s.reason ?? null,\n      metadata: {\n        source: \"cli-eval\",\n        threshold: row.threshold,\n        passed: s.passed,\n      },\n      createdAt: Date.now(),\n    };\n    await insertEvalResult(result);\n  }\n}\n\n// ─── Discovery + loading ──────────────────────────────────────────────\n\nconst EVAL_FILE_RE = /\\.eval\\.(ts|js|mjs)$/;\nconst SKIP_DIRS = new Set([\"node_modules\", \"dist\", \".git\", \".output\", \"build\"]);\n\n/**\n * Walk `root` for eval files. Matches two conventions:\n *   - any `**\\/*.eval.ts` (co-located with code), and\n *   - any `*.ts` directly inside an `evals/` directory.\n * `pattern` further filters by substring of the relative path.\n */\nexport async function discoverEvalFiles(\n  root: string,\n  pattern?: string,\n): Promise<string[]> {\n  const fs = await import(\"node:fs\");\n  const out: string[] = [];\n\n  function isEvalFile(full: string, parentName: string): boolean {\n    const base = nodePath.basename(full);\n    if (EVAL_FILE_RE.test(base)) return true;\n    if (parentName === \"evals\" && /\\.(ts|js|mjs)$/.test(base)) {\n      // Skip obvious support files inside evals/.\n      return !/\\.(spec|test|d)\\.(ts|js|mjs)$/.test(base);\n    }\n    return false;\n  }\n\n  function walk(dir: string, parentName: string): void {\n    let entries: import(\"node:fs\").Dirent[];\n    try {\n      entries = fs.readdirSync(dir, { withFileTypes: true });\n    } catch {\n      return;\n    }\n    for (const entry of entries) {\n      const full = nodePath.join(dir, entry.name);\n      if (entry.isDirectory()) {\n        if (SKIP_DIRS.has(entry.name) || entry.name.startsWith(\".\")) continue;\n        walk(full, entry.name);\n      } else if (entry.isFile() && isEvalFile(full, parentName)) {\n        out.push(full);\n      }\n    }\n  }\n\n  walk(root, nodePath.basename(root));\n  out.sort();\n\n  if (!pattern) return out;\n  return out.filter((f) => nodePath.relative(root, f).includes(pattern));\n}\n\n/** Pull `Eval` definitions out of a dynamically-imported eval module. */\nfunction extractEvals(mod: Record<string, unknown>): Eval[] {\n  const candidates: unknown[] = [];\n  if (mod.default !== undefined) candidates.push(mod.default);\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === \"default\") continue;\n    candidates.push(value);\n  }\n\n  const evals: Eval[] = [];\n  for (const c of candidates.flat()) {\n    if (\n      c &&\n      typeof c === \"object\" &&\n      typeof (c as Eval).name === \"string\" &&\n      Array.isArray((c as Eval).scorers) &&\n      (c as Eval).input\n    ) {\n      evals.push(c as Eval);\n    }\n  }\n  return evals;\n}\n\n/** Discover and import all eval files under `root`, returning their evals. */\nexport async function loadEvals(\n  root: string,\n  pattern?: string,\n): Promise<{ files: string[]; evals: Eval[] }> {\n  const files = await discoverEvalFiles(root, pattern);\n  const evals: Eval[] = [];\n  for (const file of files) {\n    const mod = (await import(pathToFileURL(file).href)) as Record<\n      string,\n      unknown\n    >;\n    evals.push(...extractEvals(mod));\n  }\n  return { files, evals };\n}\n\n// ─── High-level entry used by the CLI ─────────────────────────────────\n\nexport interface RunEvalSuiteOptions {\n  /** App root to discover eval files + actions under. Defaults to cwd. */\n  cwd?: string;\n  /** Substring filter on the eval file path. */\n  pattern?: string;\n  /** Global threshold override (wins over per-eval thresholds). */\n  thresholdOverride?: number;\n  /** App actions to expose to the agent. Auto-discovered when omitted. */\n  actions?: Record<string, ActionEntry>;\n  /** System prompt for runs. */\n  systemPrompt?: string;\n  /** Write results to the observability eval store (default true). */\n  persist?: boolean;\n  /** Pre-built runner (tests inject this to avoid touching engine/loop). */\n  runner?: AgentRunner;\n  /** Pre-loaded evals (tests inject this to skip filesystem discovery). */\n  evals?: Eval[];\n}\n\n/**\n * End-to-end: load evals, build a runner, score, report. The CLI wraps this\n * and maps `report.failed > 0` to a non-zero exit code (the CI gate).\n */\nexport async function runEvalSuite(\n  opts: RunEvalSuiteOptions = {},\n): Promise<{ report: EvalRunReport; files: string[] }> {\n  const cwd = opts.cwd ?? process.cwd();\n\n  let files: string[] = [];\n  let evals = opts.evals;\n  if (!evals) {\n    const loaded = await loadEvals(cwd, opts.pattern);\n    files = loaded.files;\n    evals = loaded.evals;\n  }\n\n  const runner =\n    opts.runner ??\n    (await createAgentRunner({\n      actions: opts.actions ?? (await discoverActions(cwd)),\n      systemPrompt: opts.systemPrompt,\n    }));\n\n  const report = await runEvals(evals, runner, {\n    thresholdOverride: opts.thresholdOverride,\n    persist: opts.persist ?? true,\n  });\n  return { report, files };\n}\n\n/**\n * Discover the app's actions so the agent under test has the real tool\n * surface. Lazy-imports `autoDiscoverActions` to keep server-only deps out of\n * any browser bundle that might touch this module's types.\n */\nasync function discoverActions(\n  cwd: string,\n): Promise<Record<string, ActionEntry>> {\n  try {\n    const { autoDiscoverActions } =\n      await import(\"../server/action-discovery.js\");\n    const actionsDir = nodePath.join(cwd, \"actions\");\n    return await autoDiscoverActions(pathToFileURL(actionsDir + \"/\").href);\n  } catch {\n    return {};\n  }\n}\n"]}

package/dist/eval/scorer.d.ts

@@ -0,0 +1,83 @@
+/**
+ * `createScorer` and a batteries-included set of built-in scorers.
+ *
+ * A scorer is a 4-step pipeline (preprocess → analyze → generateScore →
+ * generateReason). `createScorer` is a thin identity-with-validation factory:
+ * it enforces the one hard contract (`generateScore` is required) and returns
+ * a fully-typed `Scorer`. Built-in scorers below show both flavors:
+ *
+ *   - `exactMatch` / `contains` — pure-JS analyze, no model.
+ *   - `llmJudge` — analyze runs an LLM judge through the resolved engine
+ *     (provider-agnostic; the model is whatever the runner resolved).
+ */
+import type { AgentRunOutput, Scorer, ScorerDefinition } from "./types.js";
+/**
+ * Create a scorer from a 4-step pipeline definition.
+ *
+ * `generateScore` is the only required step. `preprocess`/`analyze` default to
+ * identity (the scorer sees the raw `AgentRunOutput`), and `generateReason` is
+ * optional.
+ */
+export declare function createScorer<Pre = AgentRunOutput, Ana = Pre>(def: ScorerDefinition<Pre, Ana>): Scorer<Pre, Ana>;
+/** Clamp any number into [0, 1]; coerce non-finite to 0. */
+export declare function clamp01(n: number): number;
+/**
+ * `exactMatch` — 1.0 when the agent's (trimmed, case-insensitive by default)
+ * text equals `expected`, else 0.0. Pure JS, no model.
+ */
+export declare function exactMatch(expected: string, opts?: {
+    caseSensitive?: boolean;
+}): Scorer<AgentRunOutput, {
+    match: boolean;
+}>;
+/**
+ * `contains` — 1.0 when the agent's text contains every required substring
+ * (case-insensitive by default). Score is the fraction matched, so a partial
+ * hit still surfaces signal. Pure JS, no model.
+ */
+export declare function contains(needles: string | string[], opts?: {
+    caseSensitive?: boolean;
+}): Scorer<AgentRunOutput, {
+    found: string[];
+    missing: string[];
+}>;
+/**
+ * `usesTool` — 1.0 when the agent invoked the named tool/action at least once.
+ * Useful as a behavioral gate ("the agent must call send-email"). Pure JS.
+ */
+export declare function usesTool(toolName: string): Scorer<AgentRunOutput, {
+    used: boolean;
+}>;
+interface JudgeVerdict {
+    score: number;
+    reasoning: string;
+}
+export interface LlmJudgeOptions {
+    /** Scorer name (defaults to `llm_judge`). */
+    name?: string;
+    /** What is being judged, e.g. "helpfulness". */
+    criteria: string;
+    /** A rubric describing what 0.0 vs 1.0 means. */
+    rubric?: string;
+    /**
+     * The score scale the judge is told to use. Output is normalized to [0,1].
+     * Defaults to a 0..1 scale.
+     */
+    scoreRange?: {
+        min: number;
+        max: number;
+    };
+}
+/**
+ * `llmJudge` — an LLM-as-judge scorer. The analyze step asks the resolved
+ * engine to score the agent output against a natural-language rubric and emit
+ * `{ "score": <n>, "reasoning": "<why>" }`. The model is whatever the runner
+ * resolved from the engine registry — this scorer NEVER hardcodes a provider
+ * or model, so evals stay provider-agnostic.
+ */
+export declare function llmJudge(opts: LlmJudgeOptions): Scorer<AgentRunOutput, {
+    verdict: JudgeVerdict | null;
+    normalized: number;
+}>;
+export {};
+//# sourceMappingURL=scorer.d.ts.map

package/dist/eval/scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../../src/eval/scorer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE3E;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG,EAC1D,GAAG,EAAE,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,GAC9B,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAgBlB;AAED,4DAA4D;AAC5D,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAGzC;AASD;;;GAGG;AACH,wBAAgB,UAAU,CACxB,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAA;CAAO,GACrC,MAAM,CAAC,cAAc,EAAE;IAAE,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC,CAiB5C;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CACtB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,IAAI,GAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAA;CAAO,GACrC,MAAM,CAAC,cAAc,EAAE;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAyBhE;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CACtB,QAAQ,EAAE,MAAM,GACf,MAAM,CAAC,cAAc,EAAE;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,CAAC,CAe3C;AAID,UAAU,YAAY;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAsBD,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CACtB,IAAI,EAAE,eAAe,GACpB,MAAM,CACP,cAAc,EACd;IAAE,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CACrD,CAmDA"}

package/dist/eval/scorer.js

@@ -0,0 +1,195 @@
+/**
+ * `createScorer` and a batteries-included set of built-in scorers.
+ *
+ * A scorer is a 4-step pipeline (preprocess → analyze → generateScore →
+ * generateReason). `createScorer` is a thin identity-with-validation factory:
+ * it enforces the one hard contract (`generateScore` is required) and returns
+ * a fully-typed `Scorer`. Built-in scorers below show both flavors:
+ *
+ *   - `exactMatch` / `contains` — pure-JS analyze, no model.
+ *   - `llmJudge` — analyze runs an LLM judge through the resolved engine
+ *     (provider-agnostic; the model is whatever the runner resolved).
+ */
+/**
+ * Create a scorer from a 4-step pipeline definition.
+ *
+ * `generateScore` is the only required step. `preprocess`/`analyze` default to
+ * identity (the scorer sees the raw `AgentRunOutput`), and `generateReason` is
+ * optional.
+ */
+export function createScorer(def) {
+    if (!def.name || typeof def.name !== "string") {
+        throw new Error("createScorer: `name` is required");
+    }
+    if (typeof def.generateScore !== "function") {
+        throw new Error(`createScorer("${def.name}"): \`generateScore\` is required`);
+    }
+    return {
+        name: def.name,
+        preprocess: def.preprocess,
+        analyze: def.analyze,
+        generateScore: def.generateScore,
+        generateReason: def.generateReason,
+    };
+}
+/** Clamp any number into [0, 1]; coerce non-finite to 0. */
+export function clamp01(n) {
+    if (!Number.isFinite(n))
+        return 0;
+    return Math.max(0, Math.min(1, n));
+}
+// ─── Built-in JS scorers ──────────────────────────────────────────────
+/** Normalize for forgiving text comparison (case + surrounding whitespace). */
+function normalize(s) {
+    return s.trim().toLowerCase();
+}
+/**
+ * `exactMatch` — 1.0 when the agent's (trimmed, case-insensitive by default)
+ * text equals `expected`, else 0.0. Pure JS, no model.
+ */
+export function exactMatch(expected, opts = {}) {
+    return createScorer({
+        name: "exact_match",
+        analyze(run) {
+            const actual = opts.caseSensitive ? run.text.trim() : normalize(run.text);
+            const want = opts.caseSensitive ? expected.trim() : normalize(expected);
+            return { match: actual === want };
+        },
+        generateScore({ match }) {
+            return match ? 1 : 0;
+        },
+        generateReason({ analysis }) {
+            return analysis.match
+                ? `Output exactly matched expected text`
+                : `Output did not exactly match expected text`;
+        },
+    });
+}
+/**
+ * `contains` — 1.0 when the agent's text contains every required substring
+ * (case-insensitive by default). Score is the fraction matched, so a partial
+ * hit still surfaces signal. Pure JS, no model.
+ */
+export function contains(needles, opts = {}) {
+    const list = (Array.isArray(needles) ? needles : [needles]).filter(Boolean);
+    return createScorer({
+        name: "contains",
+        analyze(run) {
+            const hay = opts.caseSensitive ? run.text : run.text.toLowerCase();
+            const found = [];
+            const missing = [];
+            for (const n of list) {
+                const needle = opts.caseSensitive ? n : n.toLowerCase();
+                if (hay.includes(needle))
+                    found.push(n);
+                else
+                    missing.push(n);
+            }
+            return { found, missing };
+        },
+        generateScore({ found }) {
+            return list.length === 0 ? 1 : found.length / list.length;
+        },
+        generateReason({ analysis }) {
+            if (analysis.missing.length === 0) {
+                return `All ${list.length} required phrase(s) present`;
+            }
+            return `Missing: ${analysis.missing.join(", ")}`;
+        },
+    });
+}
+/**
+ * `usesTool` — 1.0 when the agent invoked the named tool/action at least once.
+ * Useful as a behavioral gate ("the agent must call send-email"). Pure JS.
+ */
+export function usesTool(toolName) {
+    return createScorer({
+        name: `uses_tool:${toolName}`,
+        analyze(run) {
+            return { used: run.toolCalls.includes(toolName) };
+        },
+        generateScore({ used }) {
+            return used ? 1 : 0;
+        },
+        generateReason({ analysis }) {
+            return analysis.used
+                ? `Agent called \`${toolName}\``
+                : `Agent never called \`${toolName}\``;
+        },
+    });
+}
+/**
+ * Pull the first JSON object out of model text (which may be wrapped in prose
+ * or a ```json fence) and parse it into a verdict. Returns null on garbage so
+ * the caller can degrade gracefully instead of throwing.
+ */
+function parseJudgeVerdict(text) {
+    const match = text.match(/\{[\s\S]*\}/);
+    if (!match)
+        return null;
+    try {
+        const parsed = JSON.parse(match[0]);
+        if (typeof parsed.score !== "number")
+            return null;
+        return {
+            score: parsed.score,
+            reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * `llmJudge` — an LLM-as-judge scorer. The analyze step asks the resolved
+ * engine to score the agent output against a natural-language rubric and emit
+ * `{ "score": <n>, "reasoning": "<why>" }`. The model is whatever the runner
+ * resolved from the engine registry — this scorer NEVER hardcodes a provider
+ * or model, so evals stay provider-agnostic.
+ */
+export function llmJudge(opts) {
+    const min = opts.scoreRange?.min ?? 0;
+    const max = opts.scoreRange?.max ?? 1;
+    const name = opts.name ?? "llm_judge";
+    return createScorer({
+        name,
+        async analyze(run, ctx) {
+            const prompt = `You are an expert evaluator. Score the agent output below against the criteria.
+
+## Criteria
+${opts.criteria}${opts.rubric ? `\n\n## Rubric\n${opts.rubric}` : ""}
+
+## Agent Output
+${run.text || "(no text output)"}
+
+## Tools the agent used
+${run.toolCalls.length ? run.toolCalls.join(", ") : "(none)"}
+
+## Instructions
+Respond with ONLY a JSON object (no markdown, no prose outside the JSON):
+{"score": <number between ${min} and ${max}>, "reasoning": "<brief explanation>"}`;
+            const text = await ctx.judge({
+                systemPrompt: "You are an evaluation judge. Respond only with valid JSON.",
+                prompt,
+                maxOutputTokens: 512,
+            });
+            const verdict = parseJudgeVerdict(text);
+            const normalized = verdict === null
+                ? 0
+                : max > min
+                    ? (verdict.score - min) / (max - min)
+                    : verdict.score;
+            return { verdict, normalized };
+        },
+        generateScore({ normalized }) {
+            return clamp01(normalized);
+        },
+        generateReason({ analysis }) {
+            if (analysis.verdict === null) {
+                return "Judge did not return a parseable verdict";
+            }
+            return analysis.verdict.reasoning || "(no reasoning provided)";
+        },
+    });
+}
+//# sourceMappingURL=scorer.js.map

package/dist/eval/scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer.js","sourceRoot":"","sources":["../../src/eval/scorer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAC1B,GAA+B;IAE/B,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,aAAa,KAAK,UAAU,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,iBAAiB,GAAG,CAAC,IAAI,mCAAmC,CAC7D,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,aAAa,EAAE,GAAG,CAAC,aAAa;QAChC,cAAc,EAAE,GAAG,CAAC,cAAc;KACnC,CAAC;AACJ,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,yEAAyE;AAEzE,+EAA+E;AAC/E,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CACxB,QAAgB,EAChB,OAAoC,EAAE;IAEtC,OAAO,YAAY,CAAqC;QACtD,IAAI,EAAE,aAAa;QACnB,OAAO,CAAC,GAAG;YACT,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACxE,OAAO,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;QACpC,CAAC;QACD,aAAa,CAAC,EAAE,KAAK,EAAE;YACrB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,cAAc,CAAC,EAAE,QAAQ,EAAE;YACzB,OAAO,QAAQ,CAAC,KAAK;gBACnB,CAAC,CAAC,sCAAsC;gBACxC,CAAC,CAAC,4CAA4C,CAAC;QACnD,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CACtB,OAA0B,EAC1B,OAAoC,EAAE;IAEtC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5E,OAAO,YAAY,CAAyD;QAC1E,IAAI,EAAE,UAAU;QAChB,OAAO,CAAC,GAAG;YACT,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnE,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACxD,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;oBACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACvB,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC5B,CAAC;QACD,aAAa,CAAC,EAAE,KAAK,EAAE;YACrB,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5D,CAAC;QACD,cAAc,CAAC,EAAE,QAAQ,EAAE;YACzB,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClC,OAAO,OAAO,IAAI,CAAC,MAAM,6BAA6B,CAAC;YACzD,CAAC;YACD,OAAO,YAAY,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CACtB,QAAgB;IAEhB,OAAO,YAAY,CAAoC;QACrD,IAAI,EAAE,aAAa,QAAQ,EAAE;QAC7B,OAAO,CAAC,GAAG;YACT,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,aAAa,CAAC,EAAE,IAAI,EAAE;YACpB,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;QACD,cAAc,CAAC,EAAE,QAAQ,EAAE;YACzB,OAAO,QAAQ,CAAC,IAAI;gBAClB,CAAC,CAAC,kBAAkB,QAAQ,IAAI;gBAChC,CAAC,CAAC,wBAAwB,QAAQ,IAAI,CAAC;QAC3C,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AASD;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAA0B,CAAC;QAC7D,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAClD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SACxE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAgBD;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CACtB,IAAqB;IAKrB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC;IAEtC,OAAO,YAAY,CAGjB;QACA,IAAI;QACJ,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG;YACpB,MAAM,MAAM,GAAG;;;EAGnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE;;;EAGlE,GAAG,CAAC,IAAI,IAAI,kBAAkB;;;EAG9B,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ;;;;4BAIhC,GAAG,QAAQ,GAAG,wCAAwC,CAAC;YAE7E,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC;gBAC3B,YAAY,EACV,4DAA4D;gBAC9D,MAAM;gBACN,eAAe,EAAE,GAAG;aACrB,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,UAAU,GACd,OAAO,KAAK,IAAI;gBACd,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,GAAG,GAAG,GAAG;oBACT,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;oBACrC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;YACtB,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;QACjC,CAAC;QACD,aAAa,CAAC,EAAE,UAAU,EAAE;YAC1B,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;QAC7B,CAAC;QACD,cAAc,CAAC,EAAE,QAAQ,EAAE;YACzB,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC9B,OAAO,0CAA0C,CAAC;YACpD,CAAC;YACD,OAAO,QAAQ,CAAC,OAAO,CAAC,SAAS,IAAI,yBAAyB,CAAC;QACjE,CAAC;KACF,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * `createScorer` and a batteries-included set of built-in scorers.\n *\n * A scorer is a 4-step pipeline (preprocess → analyze → generateScore →\n * generateReason). `createScorer` is a thin identity-with-validation factory:\n * it enforces the one hard contract (`generateScore` is required) and returns\n * a fully-typed `Scorer`. Built-in scorers below show both flavors:\n *\n *   - `exactMatch` / `contains` — pure-JS analyze, no model.\n *   - `llmJudge` — analyze runs an LLM judge through the resolved engine\n *     (provider-agnostic; the model is whatever the runner resolved).\n */\n\nimport type { AgentRunOutput, Scorer, ScorerDefinition } from \"./types.js\";\n\n/**\n * Create a scorer from a 4-step pipeline definition.\n *\n * `generateScore` is the only required step. `preprocess`/`analyze` default to\n * identity (the scorer sees the raw `AgentRunOutput`), and `generateReason` is\n * optional.\n */\nexport function createScorer<Pre = AgentRunOutput, Ana = Pre>(\n  def: ScorerDefinition<Pre, Ana>,\n): Scorer<Pre, Ana> {\n  if (!def.name || typeof def.name !== \"string\") {\n    throw new Error(\"createScorer: `name` is required\");\n  }\n  if (typeof def.generateScore !== \"function\") {\n    throw new Error(\n      `createScorer(\"${def.name}\"): \\`generateScore\\` is required`,\n    );\n  }\n  return {\n    name: def.name,\n    preprocess: def.preprocess,\n    analyze: def.analyze,\n    generateScore: def.generateScore,\n    generateReason: def.generateReason,\n  };\n}\n\n/** Clamp any number into [0, 1]; coerce non-finite to 0. */\nexport function clamp01(n: number): number {\n  if (!Number.isFinite(n)) return 0;\n  return Math.max(0, Math.min(1, n));\n}\n\n// ─── Built-in JS scorers ──────────────────────────────────────────────\n\n/** Normalize for forgiving text comparison (case + surrounding whitespace). */\nfunction normalize(s: string): string {\n  return s.trim().toLowerCase();\n}\n\n/**\n * `exactMatch` — 1.0 when the agent's (trimmed, case-insensitive by default)\n * text equals `expected`, else 0.0. Pure JS, no model.\n */\nexport function exactMatch(\n  expected: string,\n  opts: { caseSensitive?: boolean } = {},\n): Scorer<AgentRunOutput, { match: boolean }> {\n  return createScorer<AgentRunOutput, { match: boolean }>({\n    name: \"exact_match\",\n    analyze(run) {\n      const actual = opts.caseSensitive ? run.text.trim() : normalize(run.text);\n      const want = opts.caseSensitive ? expected.trim() : normalize(expected);\n      return { match: actual === want };\n    },\n    generateScore({ match }) {\n      return match ? 1 : 0;\n    },\n    generateReason({ analysis }) {\n      return analysis.match\n        ? `Output exactly matched expected text`\n        : `Output did not exactly match expected text`;\n    },\n  });\n}\n\n/**\n * `contains` — 1.0 when the agent's text contains every required substring\n * (case-insensitive by default). Score is the fraction matched, so a partial\n * hit still surfaces signal. Pure JS, no model.\n */\nexport function contains(\n  needles: string | string[],\n  opts: { caseSensitive?: boolean } = {},\n): Scorer<AgentRunOutput, { found: string[]; missing: string[] }> {\n  const list = (Array.isArray(needles) ? needles : [needles]).filter(Boolean);\n  return createScorer<AgentRunOutput, { found: string[]; missing: string[] }>({\n    name: \"contains\",\n    analyze(run) {\n      const hay = opts.caseSensitive ? run.text : run.text.toLowerCase();\n      const found: string[] = [];\n      const missing: string[] = [];\n      for (const n of list) {\n        const needle = opts.caseSensitive ? n : n.toLowerCase();\n        if (hay.includes(needle)) found.push(n);\n        else missing.push(n);\n      }\n      return { found, missing };\n    },\n    generateScore({ found }) {\n      return list.length === 0 ? 1 : found.length / list.length;\n    },\n    generateReason({ analysis }) {\n      if (analysis.missing.length === 0) {\n        return `All ${list.length} required phrase(s) present`;\n      }\n      return `Missing: ${analysis.missing.join(\", \")}`;\n    },\n  });\n}\n\n/**\n * `usesTool` — 1.0 when the agent invoked the named tool/action at least once.\n * Useful as a behavioral gate (\"the agent must call send-email\"). Pure JS.\n */\nexport function usesTool(\n  toolName: string,\n): Scorer<AgentRunOutput, { used: boolean }> {\n  return createScorer<AgentRunOutput, { used: boolean }>({\n    name: `uses_tool:${toolName}`,\n    analyze(run) {\n      return { used: run.toolCalls.includes(toolName) };\n    },\n    generateScore({ used }) {\n      return used ? 1 : 0;\n    },\n    generateReason({ analysis }) {\n      return analysis.used\n        ? `Agent called \\`${toolName}\\``\n        : `Agent never called \\`${toolName}\\``;\n    },\n  });\n}\n\n// ─── Built-in LLM-judge scorer ────────────────────────────────────────\n\ninterface JudgeVerdict {\n  score: number;\n  reasoning: string;\n}\n\n/**\n * Pull the first JSON object out of model text (which may be wrapped in prose\n * or a ```json fence) and parse it into a verdict. Returns null on garbage so\n * the caller can degrade gracefully instead of throwing.\n */\nfunction parseJudgeVerdict(text: string): JudgeVerdict | null {\n  const match = text.match(/\\{[\\s\\S]*\\}/);\n  if (!match) return null;\n  try {\n    const parsed = JSON.parse(match[0]) as Partial<JudgeVerdict>;\n    if (typeof parsed.score !== \"number\") return null;\n    return {\n      score: parsed.score,\n      reasoning: typeof parsed.reasoning === \"string\" ? parsed.reasoning : \"\",\n    };\n  } catch {\n    return null;\n  }\n}\n\nexport interface LlmJudgeOptions {\n  /** Scorer name (defaults to `llm_judge`). */\n  name?: string;\n  /** What is being judged, e.g. \"helpfulness\". */\n  criteria: string;\n  /** A rubric describing what 0.0 vs 1.0 means. */\n  rubric?: string;\n  /**\n   * The score scale the judge is told to use. Output is normalized to [0,1].\n   * Defaults to a 0..1 scale.\n   */\n  scoreRange?: { min: number; max: number };\n}\n\n/**\n * `llmJudge` — an LLM-as-judge scorer. The analyze step asks the resolved\n * engine to score the agent output against a natural-language rubric and emit\n * `{ \"score\": <n>, \"reasoning\": \"<why>\" }`. The model is whatever the runner\n * resolved from the engine registry — this scorer NEVER hardcodes a provider\n * or model, so evals stay provider-agnostic.\n */\nexport function llmJudge(\n  opts: LlmJudgeOptions,\n): Scorer<\n  AgentRunOutput,\n  { verdict: JudgeVerdict | null; normalized: number }\n> {\n  const min = opts.scoreRange?.min ?? 0;\n  const max = opts.scoreRange?.max ?? 1;\n  const name = opts.name ?? \"llm_judge\";\n\n  return createScorer<\n    AgentRunOutput,\n    { verdict: JudgeVerdict | null; normalized: number }\n  >({\n    name,\n    async analyze(run, ctx) {\n      const prompt = `You are an expert evaluator. Score the agent output below against the criteria.\n\n## Criteria\n${opts.criteria}${opts.rubric ? `\\n\\n## Rubric\\n${opts.rubric}` : \"\"}\n\n## Agent Output\n${run.text || \"(no text output)\"}\n\n## Tools the agent used\n${run.toolCalls.length ? run.toolCalls.join(\", \") : \"(none)\"}\n\n## Instructions\nRespond with ONLY a JSON object (no markdown, no prose outside the JSON):\n{\"score\": <number between ${min} and ${max}>, \"reasoning\": \"<brief explanation>\"}`;\n\n      const text = await ctx.judge({\n        systemPrompt:\n          \"You are an evaluation judge. Respond only with valid JSON.\",\n        prompt,\n        maxOutputTokens: 512,\n      });\n      const verdict = parseJudgeVerdict(text);\n      const normalized =\n        verdict === null\n          ? 0\n          : max > min\n            ? (verdict.score - min) / (max - min)\n            : verdict.score;\n      return { verdict, normalized };\n    },\n    generateScore({ normalized }) {\n      return clamp01(normalized);\n    },\n    generateReason({ analysis }) {\n      if (analysis.verdict === null) {\n        return \"Judge did not return a parseable verdict\";\n      }\n      return analysis.verdict.reasoning || \"(no reasoning provided)\";\n    },\n  });\n}\n"]}