@graphorin/evals 0.5.0

package/dist/reporters/markdown.js

@@ -0,0 +1,59 @@
+//#region src/reporters/markdown.ts
+/** @stable */
+function renderMarkdownReport(report) {
+	const lines = [];
+	lines.push("# graphorin/evals report");
+	lines.push("");
+	lines.push("## Summary");
+	lines.push("");
+	lines.push("| Metric | Value |");
+	lines.push("| --- | --- |");
+	lines.push(`| Total cases | ${report.summary.total} |`);
+	lines.push(`| Passed | ${report.summary.passed} |`);
+	lines.push(`| Failed | ${report.summary.failed} |`);
+	lines.push(`| Avg duration (ms) | ${report.summary.avgDurationMs.toFixed(2)} |`);
+	lines.push("");
+	lines.push("## Per-scorer");
+	lines.push("");
+	lines.push("| Scorer | Pass | Fail | Avg score |");
+	lines.push("| --- | ---: | ---: | ---: |");
+	for (const [scorer, row] of Object.entries(report.summary.byScorer)) {
+		const avg = row.avgScore === null ? "n/a" : row.avgScore.toFixed(4);
+		lines.push(`| \`${scorer}\` | ${row.passed} | ${row.failed} | ${avg} |`);
+	}
+	lines.push("");
+	lines.push("## Failures");
+	lines.push("");
+	let any = false;
+	for (const r of report.results) {
+		const failures = r.scores.filter((s) => !s.result.pass);
+		if (failures.length === 0) continue;
+		any = true;
+		lines.push(`### \`${r.caseId}\` _(\`${r.durationMs}\` ms)_`);
+		lines.push("");
+		for (const f of failures) {
+			const reason = escapeMarkdownInline(f.result.reason ?? "_no reason_");
+			lines.push(`- **${f.scorer}** — ${reason}`);
+		}
+		lines.push("");
+	}
+	if (!any) {
+		lines.push("_No failures._");
+		lines.push("");
+	}
+	return lines.join("\n");
+}
+/**
+* Escape characters that would break a Markdown table row when rendered
+* inline. Backslashes must be escaped first so that an existing trailing
+* `\` in the input cannot combine with our inserted `\|` to form a
+* literal pipe. Newlines are flattened to spaces because table cells
+* cannot contain hard line breaks.
+*/
+function escapeMarkdownInline(value) {
+	return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+}
+
+//#endregion
+export { renderMarkdownReport };
+//# sourceMappingURL=markdown.js.map

package/dist/reporters/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","names":["lines: string[]"],"sources":["../../src/reporters/markdown.ts"],"sourcesContent":["/**\n * Markdown reporter. Renders an {@link EvalReport} as a markdown\n * document suitable for embedding in pull-request descriptions or\n * documentation sites.\n *\n * @packageDocumentation\n */\n\nimport type { EvalReport } from '@graphorin/observability/eval';\n\n/** @stable */\nexport function renderMarkdownReport<I, O>(report: EvalReport<I, O>): string {\n  const lines: string[] = [];\n  lines.push('# graphorin/evals report');\n  lines.push('');\n  lines.push('## Summary');\n  lines.push('');\n  lines.push('| Metric | Value |');\n  lines.push('| --- | --- |');\n  lines.push(`| Total cases | ${report.summary.total} |`);\n  lines.push(`| Passed | ${report.summary.passed} |`);\n  lines.push(`| Failed | ${report.summary.failed} |`);\n  lines.push(`| Avg duration (ms) | ${report.summary.avgDurationMs.toFixed(2)} |`);\n  lines.push('');\n  lines.push('## Per-scorer');\n  lines.push('');\n  lines.push('| Scorer | Pass | Fail | Avg score |');\n  lines.push('| --- | ---: | ---: | ---: |');\n  for (const [scorer, row] of Object.entries(report.summary.byScorer)) {\n    const avg = row.avgScore === null ? 'n/a' : row.avgScore.toFixed(4);\n    lines.push(`| \\`${scorer}\\` | ${row.passed} | ${row.failed} | ${avg} |`);\n  }\n  lines.push('');\n  lines.push('## Failures');\n  lines.push('');\n  let any = false;\n  for (const r of report.results) {\n    const failures = r.scores.filter((s) => !s.result.pass);\n    if (failures.length === 0) continue;\n    any = true;\n    lines.push(`### \\`${r.caseId}\\` _(\\`${r.durationMs}\\` ms)_`);\n    lines.push('');\n    for (const f of failures) {\n      const reason = escapeMarkdownInline(f.result.reason ?? '_no reason_');\n      lines.push(`- **${f.scorer}** — ${reason}`);\n    }\n    lines.push('');\n  }\n  if (!any) {\n    lines.push('_No failures._');\n    lines.push('');\n  }\n  return lines.join('\\n');\n}\n\n/**\n * Escape characters that would break a Markdown table row when rendered\n * inline. Backslashes must be escaped first so that an existing trailing\n * `\\` in the input cannot combine with our inserted `\\|` to form a\n * literal pipe. Newlines are flattened to spaces because table cells\n * cannot contain hard line breaks.\n */\nfunction escapeMarkdownInline(value: string): string {\n  return value.replace(/\\\\/g, '\\\\\\\\').replace(/\\|/g, '\\\\|').replace(/\\r?\\n/g, ' ');\n}\n"],"mappings":";;AAWA,SAAgB,qBAA2B,QAAkC;CAC3E,MAAMA,QAAkB,EAAE;AAC1B,OAAM,KAAK,2BAA2B;AACtC,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,aAAa;AACxB,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,qBAAqB;AAChC,OAAM,KAAK,gBAAgB;AAC3B,OAAM,KAAK,mBAAmB,OAAO,QAAQ,MAAM,IAAI;AACvD,OAAM,KAAK,cAAc,OAAO,QAAQ,OAAO,IAAI;AACnD,OAAM,KAAK,cAAc,OAAO,QAAQ,OAAO,IAAI;AACnD,OAAM,KAAK,yBAAyB,OAAO,QAAQ,cAAc,QAAQ,EAAE,CAAC,IAAI;AAChF,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,gBAAgB;AAC3B,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,uCAAuC;AAClD,OAAM,KAAK,+BAA+B;AAC1C,MAAK,MAAM,CAAC,QAAQ,QAAQ,OAAO,QAAQ,OAAO,QAAQ,SAAS,EAAE;EACnE,MAAM,MAAM,IAAI,aAAa,OAAO,QAAQ,IAAI,SAAS,QAAQ,EAAE;AACnE,QAAM,KAAK,OAAO,OAAO,OAAO,IAAI,OAAO,KAAK,IAAI,OAAO,KAAK,IAAI,IAAI;;AAE1E,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,cAAc;AACzB,OAAM,KAAK,GAAG;CACd,IAAI,MAAM;AACV,MAAK,MAAM,KAAK,OAAO,SAAS;EAC9B,MAAM,WAAW,EAAE,OAAO,QAAQ,MAAM,CAAC,EAAE,OAAO,KAAK;AACvD,MAAI,SAAS,WAAW,EAAG;AAC3B,QAAM;AACN,QAAM,KAAK,SAAS,EAAE,OAAO,SAAS,EAAE,WAAW,SAAS;AAC5D,QAAM,KAAK,GAAG;AACd,OAAK,MAAM,KAAK,UAAU;GACxB,MAAM,SAAS,qBAAqB,EAAE,OAAO,UAAU,cAAc;AACrE,SAAM,KAAK,OAAO,EAAE,OAAO,OAAO,SAAS;;AAE7C,QAAM,KAAK,GAAG;;AAEhB,KAAI,CAAC,KAAK;AACR,QAAM,KAAK,iBAAiB;AAC5B,QAAM,KAAK,GAAG;;AAEhB,QAAO,MAAM,KAAK,KAAK;;;;;;;;;AAUzB,SAAS,qBAAqB,OAAuB;AACnD,QAAO,MAAM,QAAQ,OAAO,OAAO,CAAC,QAAQ,OAAO,MAAM,CAAC,QAAQ,UAAU,IAAI"}

package/dist/reporters/terminal.d.ts

@@ -0,0 +1,9 @@
+import { EvalReport } from "@graphorin/observability/eval";
+
+//#region src/reporters/terminal.d.ts
+
+/** @stable */
+declare function renderTerminalReport<I, O>(report: EvalReport<I, O>): string;
+//#endregion
+export { renderTerminalReport };
+//# sourceMappingURL=terminal.d.ts.map

package/dist/reporters/terminal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"terminal.d.ts","names":[],"sources":["../../src/reporters/terminal.ts"],"sourcesContent":[],"mappings":";;;;;iBAUgB,mCAAmC,WAAW,GAAG"}

package/dist/reporters/terminal.js

@@ -0,0 +1,38 @@
+//#region src/reporters/terminal.ts
+/** @stable */
+function renderTerminalReport(report) {
+	const lines = [];
+	const { summary } = report;
+	lines.push("graphorin/evals — terminal report");
+	lines.push("=================================");
+	lines.push(`total:   ${summary.total}`);
+	lines.push(`passed:  ${summary.passed}`);
+	lines.push(`failed:  ${summary.failed}`);
+	lines.push(`avg ms:  ${summary.avgDurationMs.toFixed(2)}`);
+	lines.push("");
+	lines.push("per-scorer:");
+	for (const [scorer, row] of Object.entries(summary.byScorer)) {
+		const avg = row.avgScore === null ? "n/a" : row.avgScore.toFixed(4);
+		lines.push(`  ${scorer.padEnd(24)}  pass=${row.passed.toString().padStart(4)}  fail=${row.failed.toString().padStart(4)}  avg=${avg}`);
+	}
+	lines.push("");
+	lines.push("failures (first 10):");
+	let failuresShown = 0;
+	for (const r of report.results) {
+		if (failuresShown >= 10) break;
+		const failures = r.scores.filter((s) => !s.result.pass);
+		if (failures.length === 0) continue;
+		failuresShown += 1;
+		lines.push(`  • ${r.caseId} (${r.durationMs} ms)`);
+		for (const f of failures) {
+			const reason = f.result.reason ?? "<no reason>";
+			lines.push(`      ${f.scorer}: ${reason}`);
+		}
+	}
+	if (failuresShown === 0) lines.push("  (none)");
+	return lines.join("\n");
+}
+
+//#endregion
+export { renderTerminalReport };
+//# sourceMappingURL=terminal.js.map

package/dist/reporters/terminal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"terminal.js","names":["lines: string[]"],"sources":["../../src/reporters/terminal.ts"],"sourcesContent":["/**\n * Terminal reporter. Renders an {@link EvalReport} as plain ANSI-free\n * text suitable for stdout / CI logs.\n *\n * @packageDocumentation\n */\n\nimport type { EvalReport } from '@graphorin/observability/eval';\n\n/** @stable */\nexport function renderTerminalReport<I, O>(report: EvalReport<I, O>): string {\n  const lines: string[] = [];\n  const { summary } = report;\n  lines.push('graphorin/evals — terminal report');\n  lines.push('=================================');\n  lines.push(`total:   ${summary.total}`);\n  lines.push(`passed:  ${summary.passed}`);\n  lines.push(`failed:  ${summary.failed}`);\n  lines.push(`avg ms:  ${summary.avgDurationMs.toFixed(2)}`);\n  lines.push('');\n  lines.push('per-scorer:');\n  for (const [scorer, row] of Object.entries(summary.byScorer)) {\n    const avg = row.avgScore === null ? 'n/a' : row.avgScore.toFixed(4);\n    lines.push(\n      `  ${scorer.padEnd(24)}  pass=${row.passed.toString().padStart(4)}  fail=${row.failed.toString().padStart(4)}  avg=${avg}`,\n    );\n  }\n  lines.push('');\n  lines.push('failures (first 10):');\n  let failuresShown = 0;\n  for (const r of report.results) {\n    if (failuresShown >= 10) break;\n    const failures = r.scores.filter((s) => !s.result.pass);\n    if (failures.length === 0) continue;\n    failuresShown += 1;\n    lines.push(`  • ${r.caseId} (${r.durationMs} ms)`);\n    for (const f of failures) {\n      const reason = f.result.reason ?? '<no reason>';\n      lines.push(`      ${f.scorer}: ${reason}`);\n    }\n  }\n  if (failuresShown === 0) {\n    lines.push('  (none)');\n  }\n  return lines.join('\\n');\n}\n"],"mappings":";;AAUA,SAAgB,qBAA2B,QAAkC;CAC3E,MAAMA,QAAkB,EAAE;CAC1B,MAAM,EAAE,YAAY;AACpB,OAAM,KAAK,oCAAoC;AAC/C,OAAM,KAAK,oCAAoC;AAC/C,OAAM,KAAK,YAAY,QAAQ,QAAQ;AACvC,OAAM,KAAK,YAAY,QAAQ,SAAS;AACxC,OAAM,KAAK,YAAY,QAAQ,SAAS;AACxC,OAAM,KAAK,YAAY,QAAQ,cAAc,QAAQ,EAAE,GAAG;AAC1D,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,cAAc;AACzB,MAAK,MAAM,CAAC,QAAQ,QAAQ,OAAO,QAAQ,QAAQ,SAAS,EAAE;EAC5D,MAAM,MAAM,IAAI,aAAa,OAAO,QAAQ,IAAI,SAAS,QAAQ,EAAE;AACnE,QAAM,KACJ,KAAK,OAAO,OAAO,GAAG,CAAC,SAAS,IAAI,OAAO,UAAU,CAAC,SAAS,EAAE,CAAC,SAAS,IAAI,OAAO,UAAU,CAAC,SAAS,EAAE,CAAC,QAAQ,MACtH;;AAEH,OAAM,KAAK,GAAG;AACd,OAAM,KAAK,uBAAuB;CAClC,IAAI,gBAAgB;AACpB,MAAK,MAAM,KAAK,OAAO,SAAS;AAC9B,MAAI,iBAAiB,GAAI;EACzB,MAAM,WAAW,EAAE,OAAO,QAAQ,MAAM,CAAC,EAAE,OAAO,KAAK;AACvD,MAAI,SAAS,WAAW,EAAG;AAC3B,mBAAiB;AACjB,QAAM,KAAK,OAAO,EAAE,OAAO,IAAI,EAAE,WAAW,MAAM;AAClD,OAAK,MAAM,KAAK,UAAU;GACxB,MAAM,SAAS,EAAE,OAAO,UAAU;AAClC,SAAM,KAAK,SAAS,EAAE,OAAO,IAAI,SAAS;;;AAG9C,KAAI,kBAAkB,EACpB,OAAM,KAAK,WAAW;AAExB,QAAO,MAAM,KAAK,KAAK"}

package/dist/runner.d.ts

@@ -0,0 +1,11 @@
+import { EvalReport, RunOptions } from "./types.js";
+
+//#region src/runner.d.ts
+
+/**
+ * @stable
+ */
+declare function runEvals<I, O>(opts: RunOptions<I, O>): Promise<EvalReport<I, O>>;
+//#endregion
+export { runEvals };
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","names":[],"sources":["../src/runner.ts"],"sourcesContent":[],"mappings":";;;;;;;iBAsBsB,qBAAqB,WAAW,GAAG,KAAK,QAAQ,WAAW,GAAG"}

package/dist/runner.js

@@ -0,0 +1,181 @@
+//#region src/runner.ts
+/**
+* @stable
+*/
+async function runEvals(opts) {
+	const iterations = Math.max(1, opts.iterations ?? 1);
+	const concurrency = Math.max(1, opts.concurrency ?? 1);
+	const signal = opts.signal;
+	const cases = opts.dataset.cases;
+	const total = cases.length * iterations;
+	const queue = [];
+	for (let iter = 0; iter < iterations; iter++) for (let idx = 0; idx < cases.length; idx++) {
+		const sample = cases[idx];
+		if (sample === void 0) continue;
+		queue.push({
+			idx,
+			iter,
+			sample
+		});
+	}
+	const results = new Array(total);
+	let nextWorkIndex = 0;
+	let completed = 0;
+	async function worker() {
+		while (true) {
+			if (isAborted(signal)) return;
+			const myIndex = nextWorkIndex++;
+			if (myIndex >= queue.length) return;
+			const item = queue[myIndex];
+			if (item === void 0) return;
+			const baseId = item.sample.id ?? `case-${item.idx}`;
+			const caseId = iterations === 1 ? baseId : `${baseId}-iter-${item.iter}`;
+			const startedAt = Date.now();
+			let output;
+			try {
+				output = await opts.agent.run(item.sample.input, signal !== void 0 ? { signal } : void 0);
+			} catch (err) {
+				if (isAborted(signal)) return;
+				const durationMs$1 = Date.now() - startedAt;
+				results[myIndex] = {
+					caseId,
+					input: item.sample.input,
+					output: void 0,
+					durationMs: durationMs$1,
+					scores: opts.scorers.map((s) => ({
+						scorer: s.name,
+						result: {
+							pass: false,
+							reason: `agent.run threw: ${err instanceof Error ? err.message : String(err)}`
+						}
+					}))
+				};
+				completed += 1;
+				opts.onProgress?.({
+					index: completed,
+					total,
+					caseId,
+					durationMs: durationMs$1,
+					passed: false
+				});
+				continue;
+			}
+			const durationMs = Date.now() - startedAt;
+			const scores = [];
+			let allPassed = true;
+			for (const scorer of opts.scorers) {
+				const result = await safeScore(scorer, item.sample, output, durationMs);
+				scores.push({
+					scorer: scorer.name,
+					result
+				});
+				if (!result.pass) allPassed = false;
+			}
+			results[myIndex] = {
+				caseId,
+				input: item.sample.input,
+				output,
+				durationMs,
+				scores
+			};
+			completed += 1;
+			opts.onProgress?.({
+				index: completed,
+				total,
+				caseId,
+				durationMs,
+				passed: allPassed
+			});
+		}
+	}
+	const workerCount = Math.min(concurrency, queue.length);
+	await Promise.all(Array.from({ length: workerCount }, () => worker()));
+	if (isAborted(signal)) {
+		if (opts.throwOnAbort === true) throwIfAborted(signal);
+		return {
+			...summarize(results.filter((r) => r !== void 0), opts.scorers),
+			aborted: true
+		};
+	}
+	return summarize(results, opts.scorers);
+}
+async function safeScore(scorer, sampleCase, output, durationMs) {
+	try {
+		return await scorer.score({
+			case: sampleCase,
+			output,
+			durationMs
+		});
+	} catch (err) {
+		return {
+			pass: false,
+			reason: `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`
+		};
+	}
+}
+function summarize(results, scorers) {
+	const total = results.length;
+	let passed = 0;
+	let failed = 0;
+	let durationSum = 0;
+	const byScorer = {};
+	for (const scorer of scorers) byScorer[scorer.name] = {
+		passed: 0,
+		failed: 0,
+		scoreSum: 0,
+		scoreCount: 0
+	};
+	for (const r of results) {
+		durationSum += r.durationMs;
+		let passEntire = true;
+		for (const { scorer, result } of r.scores) {
+			const bucket = byScorer[scorer] ?? {
+				passed: 0,
+				failed: 0,
+				scoreSum: 0,
+				scoreCount: 0
+			};
+			if (result.pass) bucket.passed += 1;
+			else {
+				bucket.failed += 1;
+				passEntire = false;
+			}
+			if (typeof result.score === "number" && Number.isFinite(result.score)) {
+				bucket.scoreSum += result.score;
+				bucket.scoreCount += 1;
+			}
+			byScorer[scorer] = bucket;
+		}
+		if (passEntire) passed += 1;
+		else failed += 1;
+	}
+	const summaryByScorer = {};
+	for (const [scorer, b] of Object.entries(byScorer)) summaryByScorer[scorer] = {
+		passed: b.passed,
+		failed: b.failed,
+		avgScore: b.scoreCount === 0 ? null : b.scoreSum / b.scoreCount
+	};
+	return {
+		results,
+		summary: {
+			total,
+			passed,
+			failed,
+			avgDurationMs: total === 0 ? 0 : durationSum / total,
+			byScorer: Object.freeze(summaryByScorer)
+		}
+	};
+}
+function isAborted(signal) {
+	return signal?.aborted === true;
+}
+function throwIfAborted(signal) {
+	if (signal?.aborted === true) {
+		const reason = signal.reason;
+		throw reason instanceof Error ? reason : /* @__PURE__ */ new Error("Eval run aborted");
+	}
+}
+
+//#endregion
+export { runEvals };
+//# sourceMappingURL=runner.js.map

package/dist/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","names":["queue: WorkItem[]","results: EvalCaseResult<I, O>[]","output: O","durationMs","scores: EvalCaseResult<I, O>['scores'][number][]","byScorer: Record<\n    string,\n    { passed: number; failed: number; scoreSum: number; scoreCount: number }\n  >","summaryByScorer: Record<\n    string,\n    { passed: number; failed: number; avgScore: number | null }\n  >"],"sources":["../src/runner.ts"],"sourcesContent":["/**\n * Parallel eval runner. Walks every case in the dataset (across N\n * iterations), executes them with bounded concurrency, applies every\n * scorer to each output, and returns an aggregated {@link EvalReport}.\n *\n * The runner respects:\n *\n *  - `concurrency` — bounded worker pool. Default `1` (sequential).\n *  - `signal` — propagated to `agent.run(...)` if the agent accepts a\n *    second `{ signal }` argument; the runner stops issuing new work\n *    on the next loop iteration.\n *  - `onProgress` — invoked after every case with a heartbeat so\n *    long-running terminals can paint a progress bar.\n *\n * @packageDocumentation\n */\n\nimport type { Case, EvalCaseResult, EvalReport, RunOptions, ScoreResult, Scorer } from './types.js';\n\n/**\n * @stable\n */\nexport async function runEvals<I, O>(opts: RunOptions<I, O>): Promise<EvalReport<I, O>> {\n  const iterations = Math.max(1, opts.iterations ?? 1);\n  const concurrency = Math.max(1, opts.concurrency ?? 1);\n  const signal = opts.signal;\n  const cases = opts.dataset.cases;\n  const total = cases.length * iterations;\n\n  type WorkItem = { idx: number; iter: number; sample: Case<I, O> };\n  const queue: WorkItem[] = [];\n  for (let iter = 0; iter < iterations; iter++) {\n    for (let idx = 0; idx < cases.length; idx++) {\n      const sample = cases[idx];\n      if (sample === undefined) continue;\n      queue.push({ idx, iter, sample });\n    }\n  }\n\n  const results: EvalCaseResult<I, O>[] = new Array(total);\n  let nextWorkIndex = 0;\n  let completed = 0;\n\n  async function worker(): Promise<void> {\n    while (true) {\n      // EB-14: stop dispatching new work on abort, but don't throw — whatever\n      // already completed must survive into a partial report.\n      if (isAborted(signal)) return;\n      const myIndex = nextWorkIndex++;\n      if (myIndex >= queue.length) return;\n      const item = queue[myIndex];\n      if (item === undefined) return;\n      // EB-6: a caller-provided id must still be disambiguated per iteration —\n      // otherwise iterations>1 emits multiple results under one caseId and\n      // JUnit/HTML reporters render indistinguishable testcases.\n      const baseId = item.sample.id ?? `case-${item.idx}`;\n      const caseId = iterations === 1 ? baseId : `${baseId}-iter-${item.iter}`;\n      const startedAt = Date.now();\n      let output: O;\n      try {\n        output = await opts.agent.run(\n          item.sample.input,\n          signal !== undefined ? { signal } : undefined,\n        );\n      } catch (err) {\n        // An abort-induced agent failure is not a real scorer failure — drop it\n        // so it doesn't pollute the partial report with spurious fails (EB-14).\n        if (isAborted(signal)) return;\n        const durationMs = Date.now() - startedAt;\n        const failResult: EvalCaseResult<I, O> = {\n          caseId,\n          input: item.sample.input,\n          output: undefined as unknown as O,\n          durationMs,\n          scores: opts.scorers.map((s) => ({\n            scorer: s.name,\n            result: {\n              pass: false,\n              reason: `agent.run threw: ${err instanceof Error ? err.message : String(err)}`,\n            },\n          })),\n        };\n        results[myIndex] = failResult;\n        completed += 1;\n        opts.onProgress?.({\n          index: completed,\n          total,\n          caseId,\n          durationMs,\n          passed: false,\n        });\n        continue;\n      }\n      const durationMs = Date.now() - startedAt;\n      const scores: EvalCaseResult<I, O>['scores'][number][] = [];\n      let allPassed = true;\n      for (const scorer of opts.scorers) {\n        // No mid-case abort check: once agent.run completes, finish scoring so\n        // the case lands intact in the partial report rather than half-done.\n        const result = await safeScore(scorer, item.sample, output, durationMs);\n        scores.push({ scorer: scorer.name, result });\n        if (!result.pass) allPassed = false;\n      }\n      results[myIndex] = {\n        caseId,\n        input: item.sample.input,\n        output,\n        durationMs,\n        scores,\n      };\n      completed += 1;\n      opts.onProgress?.({\n        index: completed,\n        total,\n        caseId,\n        durationMs,\n        passed: allPassed,\n      });\n    }\n  }\n\n  const workerCount = Math.min(concurrency, queue.length);\n  await Promise.all(Array.from({ length: workerCount }, () => worker()));\n\n  // EB-14: an aborted run resolves with a PARTIAL report (only the cases that\n  // finished) marked `aborted: true`, instead of discarding everything — a\n  // long judged run shouldn't lose all completed work to a Ctrl+C. Callers who\n  // prefer the old throw-on-abort opt in with `throwOnAbort`.\n  if (isAborted(signal)) {\n    if (opts.throwOnAbort === true) throwIfAborted(signal);\n    const done = results.filter((r): r is EvalCaseResult<I, O> => r !== undefined);\n    return { ...summarize(done, opts.scorers), aborted: true };\n  }\n  return summarize(results, opts.scorers);\n}\n\nasync function safeScore<I, O>(\n  scorer: Scorer<I, O>,\n  sampleCase: Case<I, O>,\n  output: O,\n  durationMs: number,\n): Promise<ScoreResult> {\n  try {\n    return await scorer.score({ case: sampleCase, output, durationMs });\n  } catch (err) {\n    return {\n      pass: false,\n      reason: `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n    };\n  }\n}\n\nfunction summarize<I, O>(\n  results: ReadonlyArray<EvalCaseResult<I, O>>,\n  scorers: ReadonlyArray<Scorer<I, O>>,\n): EvalReport<I, O> {\n  const total = results.length;\n  let passed = 0;\n  let failed = 0;\n  let durationSum = 0;\n  const byScorer: Record<\n    string,\n    { passed: number; failed: number; scoreSum: number; scoreCount: number }\n  > = {};\n  for (const scorer of scorers) {\n    byScorer[scorer.name] = { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };\n  }\n  for (const r of results) {\n    durationSum += r.durationMs;\n    let passEntire = true;\n    for (const { scorer, result } of r.scores) {\n      const bucket = byScorer[scorer] ?? { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };\n      if (result.pass) bucket.passed += 1;\n      else {\n        bucket.failed += 1;\n        passEntire = false;\n      }\n      if (typeof result.score === 'number' && Number.isFinite(result.score)) {\n        bucket.scoreSum += result.score;\n        bucket.scoreCount += 1;\n      }\n      byScorer[scorer] = bucket;\n    }\n    if (passEntire) passed += 1;\n    else failed += 1;\n  }\n  const summaryByScorer: Record<\n    string,\n    { passed: number; failed: number; avgScore: number | null }\n  > = {};\n  for (const [scorer, b] of Object.entries(byScorer)) {\n    summaryByScorer[scorer] = {\n      passed: b.passed,\n      failed: b.failed,\n      avgScore: b.scoreCount === 0 ? null : b.scoreSum / b.scoreCount,\n    };\n  }\n  return {\n    results,\n    summary: {\n      total,\n      passed,\n      failed,\n      avgDurationMs: total === 0 ? 0 : durationSum / total,\n      byScorer: Object.freeze(summaryByScorer),\n    },\n  };\n}\n\n// A plain function call so TypeScript re-reads `signal.aborted` each time —\n// the flag flips during an `await` (external mutation) and inline\n// `signal?.aborted === true` checks would otherwise be narrowed away.\nfunction isAborted(signal: AbortSignal | undefined): boolean {\n  return signal?.aborted === true;\n}\n\nfunction throwIfAborted(signal: AbortSignal | undefined): void {\n  if (signal?.aborted === true) {\n    const reason = signal.reason;\n    throw reason instanceof Error ? reason : new Error('Eval run aborted');\n  }\n}\n"],"mappings":";;;;AAsBA,eAAsB,SAAe,MAAmD;CACtF,MAAM,aAAa,KAAK,IAAI,GAAG,KAAK,cAAc,EAAE;CACpD,MAAM,cAAc,KAAK,IAAI,GAAG,KAAK,eAAe,EAAE;CACtD,MAAM,SAAS,KAAK;CACpB,MAAM,QAAQ,KAAK,QAAQ;CAC3B,MAAM,QAAQ,MAAM,SAAS;CAG7B,MAAMA,QAAoB,EAAE;AAC5B,MAAK,IAAI,OAAO,GAAG,OAAO,YAAY,OACpC,MAAK,IAAI,MAAM,GAAG,MAAM,MAAM,QAAQ,OAAO;EAC3C,MAAM,SAAS,MAAM;AACrB,MAAI,WAAW,OAAW;AAC1B,QAAM,KAAK;GAAE;GAAK;GAAM;GAAQ,CAAC;;CAIrC,MAAMC,UAAkC,IAAI,MAAM,MAAM;CACxD,IAAI,gBAAgB;CACpB,IAAI,YAAY;CAEhB,eAAe,SAAwB;AACrC,SAAO,MAAM;AAGX,OAAI,UAAU,OAAO,CAAE;GACvB,MAAM,UAAU;AAChB,OAAI,WAAW,MAAM,OAAQ;GAC7B,MAAM,OAAO,MAAM;AACnB,OAAI,SAAS,OAAW;GAIxB,MAAM,SAAS,KAAK,OAAO,MAAM,QAAQ,KAAK;GAC9C,MAAM,SAAS,eAAe,IAAI,SAAS,GAAG,OAAO,QAAQ,KAAK;GAClE,MAAM,YAAY,KAAK,KAAK;GAC5B,IAAIC;AACJ,OAAI;AACF,aAAS,MAAM,KAAK,MAAM,IACxB,KAAK,OAAO,OACZ,WAAW,SAAY,EAAE,QAAQ,GAAG,OACrC;YACM,KAAK;AAGZ,QAAI,UAAU,OAAO,CAAE;IACvB,MAAMC,eAAa,KAAK,KAAK,GAAG;AAchC,YAAQ,WAbiC;KACvC;KACA,OAAO,KAAK,OAAO;KACnB,QAAQ;KACR;KACA,QAAQ,KAAK,QAAQ,KAAK,OAAO;MAC/B,QAAQ,EAAE;MACV,QAAQ;OACN,MAAM;OACN,QAAQ,oBAAoB,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;OAC7E;MACF,EAAE;KACJ;AAED,iBAAa;AACb,SAAK,aAAa;KAChB,OAAO;KACP;KACA;KACA;KACA,QAAQ;KACT,CAAC;AACF;;GAEF,MAAM,aAAa,KAAK,KAAK,GAAG;GAChC,MAAMC,SAAmD,EAAE;GAC3D,IAAI,YAAY;AAChB,QAAK,MAAM,UAAU,KAAK,SAAS;IAGjC,MAAM,SAAS,MAAM,UAAU,QAAQ,KAAK,QAAQ,QAAQ,WAAW;AACvE,WAAO,KAAK;KAAE,QAAQ,OAAO;KAAM;KAAQ,CAAC;AAC5C,QAAI,CAAC,OAAO,KAAM,aAAY;;AAEhC,WAAQ,WAAW;IACjB;IACA,OAAO,KAAK,OAAO;IACnB;IACA;IACA;IACD;AACD,gBAAa;AACb,QAAK,aAAa;IAChB,OAAO;IACP;IACA;IACA;IACA,QAAQ;IACT,CAAC;;;CAIN,MAAM,cAAc,KAAK,IAAI,aAAa,MAAM,OAAO;AACvD,OAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,aAAa,QAAQ,QAAQ,CAAC,CAAC;AAMtE,KAAI,UAAU,OAAO,EAAE;AACrB,MAAI,KAAK,iBAAiB,KAAM,gBAAe,OAAO;AAEtD,SAAO;GAAE,GAAG,UADC,QAAQ,QAAQ,MAAiC,MAAM,OAAU,EAClD,KAAK,QAAQ;GAAE,SAAS;GAAM;;AAE5D,QAAO,UAAU,SAAS,KAAK,QAAQ;;AAGzC,eAAe,UACb,QACA,YACA,QACA,YACsB;AACtB,KAAI;AACF,SAAO,MAAM,OAAO,MAAM;GAAE,MAAM;GAAY;GAAQ;GAAY,CAAC;UAC5D,KAAK;AACZ,SAAO;GACL,MAAM;GACN,QAAQ,WAAW,OAAO,KAAK,WAAW,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GAC3F;;;AAIL,SAAS,UACP,SACA,SACkB;CAClB,MAAM,QAAQ,QAAQ;CACtB,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,cAAc;CAClB,MAAMC,WAGF,EAAE;AACN,MAAK,MAAM,UAAU,QACnB,UAAS,OAAO,QAAQ;EAAE,QAAQ;EAAG,QAAQ;EAAG,UAAU;EAAG,YAAY;EAAG;AAE9E,MAAK,MAAM,KAAK,SAAS;AACvB,iBAAe,EAAE;EACjB,IAAI,aAAa;AACjB,OAAK,MAAM,EAAE,QAAQ,YAAY,EAAE,QAAQ;GACzC,MAAM,SAAS,SAAS,WAAW;IAAE,QAAQ;IAAG,QAAQ;IAAG,UAAU;IAAG,YAAY;IAAG;AACvF,OAAI,OAAO,KAAM,QAAO,UAAU;QAC7B;AACH,WAAO,UAAU;AACjB,iBAAa;;AAEf,OAAI,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,OAAO,MAAM,EAAE;AACrE,WAAO,YAAY,OAAO;AAC1B,WAAO,cAAc;;AAEvB,YAAS,UAAU;;AAErB,MAAI,WAAY,WAAU;MACrB,WAAU;;CAEjB,MAAMC,kBAGF,EAAE;AACN,MAAK,MAAM,CAAC,QAAQ,MAAM,OAAO,QAAQ,SAAS,CAChD,iBAAgB,UAAU;EACxB,QAAQ,EAAE;EACV,QAAQ,EAAE;EACV,UAAU,EAAE,eAAe,IAAI,OAAO,EAAE,WAAW,EAAE;EACtD;AAEH,QAAO;EACL;EACA,SAAS;GACP;GACA;GACA;GACA,eAAe,UAAU,IAAI,IAAI,cAAc;GAC/C,UAAU,OAAO,OAAO,gBAAgB;GACzC;EACF;;AAMH,SAAS,UAAU,QAA0C;AAC3D,QAAO,QAAQ,YAAY;;AAG7B,SAAS,eAAe,QAAuC;AAC7D,KAAI,QAAQ,YAAY,MAAM;EAC5B,MAAM,SAAS,OAAO;AACtB,QAAM,kBAAkB,QAAQ,yBAAS,IAAI,MAAM,mBAAmB"}

package/dist/scorers/code/exact-match.d.ts

@@ -0,0 +1,22 @@
+import { Scorer } from "@graphorin/observability/eval";
+
+//#region src/scorers/code/exact-match.d.ts
+
+/** @stable */
+interface ExactMatchOptions {
+  /** Optional name override. Default `'exact-match'`. */
+  readonly name?: string;
+  /** When `true`, treat strings case-insensitively. Default `false`. */
+  readonly caseInsensitive?: boolean;
+  /** When `true`, trim whitespace before comparing strings. Default `false`. */
+  readonly trim?: boolean;
+}
+/**
+ * Build an exact-match scorer.
+ *
+ * @stable
+ */
+declare function exactMatch<I = unknown, O = unknown>(options?: ExactMatchOptions): Scorer<I, O>;
+//#endregion
+export { ExactMatchOptions, exactMatch };
+//# sourceMappingURL=exact-match.d.ts.map

package/dist/scorers/code/exact-match.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exact-match.d.ts","names":[],"sources":["../../../src/scorers/code/exact-match.ts"],"sourcesContent":[],"mappings":";;;;;AA2BS,UAhBQ,iBAAA,CAgBR;;;;;;;;;;;;;iBAFO,+CACL,oBACR,OAAO,GAAG"}

package/dist/scorers/code/exact-match.js

@@ -0,0 +1,67 @@
+//#region src/scorers/code/exact-match.ts
+/**
+* Build an exact-match scorer.
+*
+* @stable
+*/
+function exactMatch(options = {}) {
+	return {
+		name: options.name ?? "exact-match",
+		async score({ case: c, output }) {
+			const expected = c.expected;
+			if (expected === void 0) return {
+				pass: false,
+				reason: "exactMatch requires a `case.expected` value."
+			};
+			const a = normalize(output, options);
+			const b = normalize(expected, options);
+			const pass = deepEqual(a, b);
+			if (pass) return {
+				pass,
+				score: 1
+			};
+			return {
+				pass,
+				score: 0,
+				reason: `expected ${truncate(JSON.stringify(b))}, received ${truncate(JSON.stringify(a))}`
+			};
+		}
+	};
+}
+function normalize(value, opts) {
+	if (typeof value === "string") {
+		let v = value;
+		if (opts.trim === true) v = v.trim();
+		if (opts.caseInsensitive === true) v = v.toLowerCase();
+		return v;
+	}
+	return value;
+}
+function deepEqual(a, b) {
+	if (a === b) return true;
+	if (typeof a !== typeof b) return false;
+	if (a === null || b === null) return a === b;
+	if (Array.isArray(a) && Array.isArray(b)) {
+		if (a.length !== b.length) return false;
+		for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false;
+		return true;
+	}
+	if (typeof a === "object" && typeof b === "object") {
+		const aKeys = Object.keys(a).sort();
+		const bKeys = Object.keys(b).sort();
+		if (aKeys.length !== bKeys.length) return false;
+		for (let i = 0; i < aKeys.length; i++) {
+			if (aKeys[i] !== bKeys[i]) return false;
+			if (!deepEqual(a[aKeys[i]], b[bKeys[i]])) return false;
+		}
+		return true;
+	}
+	return false;
+}
+function truncate(s, max = 80) {
+	return s.length > max ? `${s.slice(0, max)}…` : s;
+}
+
+//#endregion
+export { exactMatch };
+//# sourceMappingURL=exact-match.js.map

package/dist/scorers/code/exact-match.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"exact-match.js","names":[],"sources":["../../../src/scorers/code/exact-match.ts"],"sourcesContent":["/**\n * `exactMatch` — passes when `output` deeply equals `case.expected`.\n * Useful for deterministic outputs (numeric / parsed JSON / classifier\n * label).\n *\n * @packageDocumentation\n */\n\nimport type { Scorer } from '@graphorin/observability/eval';\n\n/** @stable */\nexport interface ExactMatchOptions {\n  /** Optional name override. Default `'exact-match'`. */\n  readonly name?: string;\n  /** When `true`, treat strings case-insensitively. Default `false`. */\n  readonly caseInsensitive?: boolean;\n  /** When `true`, trim whitespace before comparing strings. Default `false`. */\n  readonly trim?: boolean;\n}\n\n/**\n * Build an exact-match scorer.\n *\n * @stable\n */\nexport function exactMatch<I = unknown, O = unknown>(\n  options: ExactMatchOptions = {},\n): Scorer<I, O> {\n  const name = options.name ?? 'exact-match';\n  return {\n    name,\n    async score({ case: c, output }) {\n      const expected = c.expected;\n      if (expected === undefined) {\n        return {\n          pass: false,\n          reason: 'exactMatch requires a `case.expected` value.',\n        };\n      }\n      const a = normalize(output, options);\n      const b = normalize(expected, options);\n      const pass = deepEqual(a, b);\n      if (pass) return { pass, score: 1 };\n      return {\n        pass,\n        score: 0,\n        reason: `expected ${truncate(JSON.stringify(b))}, received ${truncate(JSON.stringify(a))}`,\n      };\n    },\n  };\n}\n\nfunction normalize(value: unknown, opts: ExactMatchOptions): unknown {\n  if (typeof value === 'string') {\n    let v = value;\n    if (opts.trim === true) v = v.trim();\n    if (opts.caseInsensitive === true) v = v.toLowerCase();\n    return v;\n  }\n  return value;\n}\n\nfunction deepEqual(a: unknown, b: unknown): boolean {\n  if (a === b) return true;\n  if (typeof a !== typeof b) return false;\n  if (a === null || b === null) return a === b;\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false;\n    for (let i = 0; i < a.length; i++) {\n      if (!deepEqual(a[i], b[i])) return false;\n    }\n    return true;\n  }\n  if (typeof a === 'object' && typeof b === 'object') {\n    const aKeys = Object.keys(a as Record<string, unknown>).sort();\n    const bKeys = Object.keys(b as Record<string, unknown>).sort();\n    if (aKeys.length !== bKeys.length) return false;\n    for (let i = 0; i < aKeys.length; i++) {\n      if (aKeys[i] !== bKeys[i]) return false;\n      if (\n        !deepEqual(\n          (a as Record<string, unknown>)[aKeys[i] as string],\n          (b as Record<string, unknown>)[bKeys[i] as string],\n        )\n      )\n        return false;\n    }\n    return true;\n  }\n  return false;\n}\n\nfunction truncate(s: string, max: number = 80): string {\n  return s.length > max ? `${s.slice(0, max)}…` : s;\n}\n"],"mappings":";;;;;;AAyBA,SAAgB,WACd,UAA6B,EAAE,EACjB;AAEd,QAAO;EACL,MAFW,QAAQ,QAAQ;EAG3B,MAAM,MAAM,EAAE,MAAM,GAAG,UAAU;GAC/B,MAAM,WAAW,EAAE;AACnB,OAAI,aAAa,OACf,QAAO;IACL,MAAM;IACN,QAAQ;IACT;GAEH,MAAM,IAAI,UAAU,QAAQ,QAAQ;GACpC,MAAM,IAAI,UAAU,UAAU,QAAQ;GACtC,MAAM,OAAO,UAAU,GAAG,EAAE;AAC5B,OAAI,KAAM,QAAO;IAAE;IAAM,OAAO;IAAG;AACnC,UAAO;IACL;IACA,OAAO;IACP,QAAQ,YAAY,SAAS,KAAK,UAAU,EAAE,CAAC,CAAC,aAAa,SAAS,KAAK,UAAU,EAAE,CAAC;IACzF;;EAEJ;;AAGH,SAAS,UAAU,OAAgB,MAAkC;AACnE,KAAI,OAAO,UAAU,UAAU;EAC7B,IAAI,IAAI;AACR,MAAI,KAAK,SAAS,KAAM,KAAI,EAAE,MAAM;AACpC,MAAI,KAAK,oBAAoB,KAAM,KAAI,EAAE,aAAa;AACtD,SAAO;;AAET,QAAO;;AAGT,SAAS,UAAU,GAAY,GAAqB;AAClD,KAAI,MAAM,EAAG,QAAO;AACpB,KAAI,OAAO,MAAM,OAAO,EAAG,QAAO;AAClC,KAAI,MAAM,QAAQ,MAAM,KAAM,QAAO,MAAM;AAC3C,KAAI,MAAM,QAAQ,EAAE,IAAI,MAAM,QAAQ,EAAE,EAAE;AACxC,MAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,OAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,IAC5B,KAAI,CAAC,UAAU,EAAE,IAAI,EAAE,GAAG,CAAE,QAAO;AAErC,SAAO;;AAET,KAAI,OAAO,MAAM,YAAY,OAAO,MAAM,UAAU;EAClD,MAAM,QAAQ,OAAO,KAAK,EAA6B,CAAC,MAAM;EAC9D,MAAM,QAAQ,OAAO,KAAK,EAA6B,CAAC,MAAM;AAC9D,MAAI,MAAM,WAAW,MAAM,OAAQ,QAAO;AAC1C,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,OAAI,MAAM,OAAO,MAAM,GAAI,QAAO;AAClC,OACE,CAAC,UACE,EAA8B,MAAM,KACpC,EAA8B,MAAM,IACtC,CAED,QAAO;;AAEX,SAAO;;AAET,QAAO;;AAGT,SAAS,SAAS,GAAW,MAAc,IAAY;AACrD,QAAO,EAAE,SAAS,MAAM,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC,KAAK"}

package/dist/scorers/code/json-path.d.ts

@@ -0,0 +1,18 @@
+import { Scorer } from "@graphorin/observability/eval";
+
+//#region src/scorers/code/json-path.d.ts
+
+/** @stable */
+interface JsonPathOptions {
+  /** Dot-separated path (`'data.user.id'`). */
+  readonly path: string;
+  /** Expected value. Compared by deep-equality. */
+  readonly equals: unknown;
+  /** Optional name override. */
+  readonly name?: string;
+}
+/** @stable */
+declare function jsonPath<I = unknown>(options: JsonPathOptions): Scorer<I, unknown>;
+//#endregion
+export { JsonPathOptions, jsonPath };
+//# sourceMappingURL=json-path.d.ts.map

package/dist/scorers/code/json-path.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-path.d.ts","names":[],"sources":["../../../src/scorers/code/json-path.ts"],"sourcesContent":[],"mappings":";;;;;UAWiB,eAAA;;;;;;;;;iBAUD,+BAA+B,kBAAkB,OAAO"}

package/dist/scorers/code/json-path.js

@@ -0,0 +1,71 @@
+//#region src/scorers/code/json-path.ts
+/** @stable */
+function jsonPath(options) {
+	return {
+		name: options.name ?? `json-path:${options.path}`,
+		async score({ output }) {
+			const actual = readPath(output, options.path);
+			const pass = deepEqual(actual, options.equals);
+			if (pass) return {
+				pass,
+				score: 1
+			};
+			return {
+				pass,
+				score: 0,
+				reason: `at '${options.path}' expected ${truncate(stringifySafe(options.equals))}, received ${truncate(stringifySafe(actual))}.`
+			};
+		}
+	};
+}
+function readPath(root, path) {
+	if (path.length === 0) return root;
+	const segments = path.split(".");
+	let cursor = root;
+	for (const segment of segments) {
+		if (cursor === null || cursor === void 0) return void 0;
+		if (Array.isArray(cursor)) {
+			const idx = Number.parseInt(segment, 10);
+			if (!Number.isFinite(idx)) return void 0;
+			cursor = cursor[idx];
+		} else if (typeof cursor === "object") cursor = cursor[segment];
+		else return;
+	}
+	return cursor;
+}
+function deepEqual(a, b) {
+	if (a === b) return true;
+	if (typeof a !== typeof b) return false;
+	if (a === null || b === null) return a === b;
+	if (Array.isArray(a) && Array.isArray(b)) {
+		if (a.length !== b.length) return false;
+		for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false;
+		return true;
+	}
+	if (typeof a === "object" && typeof b === "object") {
+		const aKeys = Object.keys(a).sort();
+		const bKeys = Object.keys(b).sort();
+		if (aKeys.length !== bKeys.length) return false;
+		for (let i = 0; i < aKeys.length; i++) {
+			if (aKeys[i] !== bKeys[i]) return false;
+			if (!deepEqual(a[aKeys[i]], b[bKeys[i]])) return false;
+		}
+		return true;
+	}
+	return false;
+}
+function truncate(s, max = 80) {
+	return s.length > max ? `${s.slice(0, max)}…` : s;
+}
+function stringifySafe(value) {
+	if (value === void 0) return "undefined";
+	try {
+		return JSON.stringify(value) ?? String(value);
+	} catch {
+		return String(value);
+	}
+}
+
+//#endregion
+export { jsonPath };
+//# sourceMappingURL=json-path.js.map

package/dist/scorers/code/json-path.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-path.js","names":["cursor: unknown"],"sources":["../../../src/scorers/code/json-path.ts"],"sourcesContent":["/**\n * `jsonPath` — passes when the value at a JSON-pointer-shaped path\n * deep-equals the caller-supplied target. Supports the dot-notation\n * subset (`a.b.c`) and array indices (`a.0.name`).\n *\n * @packageDocumentation\n */\n\nimport type { Scorer } from '@graphorin/observability/eval';\n\n/** @stable */\nexport interface JsonPathOptions {\n  /** Dot-separated path (`'data.user.id'`). */\n  readonly path: string;\n  /** Expected value. Compared by deep-equality. */\n  readonly equals: unknown;\n  /** Optional name override. */\n  readonly name?: string;\n}\n\n/** @stable */\nexport function jsonPath<I = unknown>(options: JsonPathOptions): Scorer<I, unknown> {\n  const name = options.name ?? `json-path:${options.path}`;\n  return {\n    name,\n    async score({ output }) {\n      const actual = readPath(output, options.path);\n      const pass = deepEqual(actual, options.equals);\n      if (pass) return { pass, score: 1 };\n      return {\n        pass,\n        score: 0,\n        reason:\n          `at '${options.path}' expected ${truncate(stringifySafe(options.equals))}, ` +\n          `received ${truncate(stringifySafe(actual))}.`,\n      };\n    },\n  };\n}\n\nfunction readPath(root: unknown, path: string): unknown {\n  if (path.length === 0) return root;\n  const segments = path.split('.');\n  let cursor: unknown = root;\n  for (const segment of segments) {\n    if (cursor === null || cursor === undefined) return undefined;\n    if (Array.isArray(cursor)) {\n      const idx = Number.parseInt(segment, 10);\n      if (!Number.isFinite(idx)) return undefined;\n      cursor = cursor[idx];\n    } else if (typeof cursor === 'object') {\n      cursor = (cursor as Record<string, unknown>)[segment];\n    } else {\n      return undefined;\n    }\n  }\n  return cursor;\n}\n\nfunction deepEqual(a: unknown, b: unknown): boolean {\n  if (a === b) return true;\n  if (typeof a !== typeof b) return false;\n  if (a === null || b === null) return a === b;\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false;\n    for (let i = 0; i < a.length; i++) {\n      if (!deepEqual(a[i], b[i])) return false;\n    }\n    return true;\n  }\n  if (typeof a === 'object' && typeof b === 'object') {\n    const aKeys = Object.keys(a as Record<string, unknown>).sort();\n    const bKeys = Object.keys(b as Record<string, unknown>).sort();\n    if (aKeys.length !== bKeys.length) return false;\n    for (let i = 0; i < aKeys.length; i++) {\n      if (aKeys[i] !== bKeys[i]) return false;\n      if (\n        !deepEqual(\n          (a as Record<string, unknown>)[aKeys[i] as string],\n          (b as Record<string, unknown>)[bKeys[i] as string],\n        )\n      )\n        return false;\n    }\n    return true;\n  }\n  return false;\n}\n\nfunction truncate(s: string, max: number = 80): string {\n  return s.length > max ? `${s.slice(0, max)}…` : s;\n}\n\nfunction stringifySafe(value: unknown): string {\n  if (value === undefined) return 'undefined';\n  try {\n    return JSON.stringify(value) ?? String(value);\n  } catch {\n    return String(value);\n  }\n}\n"],"mappings":";;AAqBA,SAAgB,SAAsB,SAA8C;AAElF,QAAO;EACL,MAFW,QAAQ,QAAQ,aAAa,QAAQ;EAGhD,MAAM,MAAM,EAAE,UAAU;GACtB,MAAM,SAAS,SAAS,QAAQ,QAAQ,KAAK;GAC7C,MAAM,OAAO,UAAU,QAAQ,QAAQ,OAAO;AAC9C,OAAI,KAAM,QAAO;IAAE;IAAM,OAAO;IAAG;AACnC,UAAO;IACL;IACA,OAAO;IACP,QACE,OAAO,QAAQ,KAAK,aAAa,SAAS,cAAc,QAAQ,OAAO,CAAC,CAAC,aAC7D,SAAS,cAAc,OAAO,CAAC,CAAC;IAC/C;;EAEJ;;AAGH,SAAS,SAAS,MAAe,MAAuB;AACtD,KAAI,KAAK,WAAW,EAAG,QAAO;CAC9B,MAAM,WAAW,KAAK,MAAM,IAAI;CAChC,IAAIA,SAAkB;AACtB,MAAK,MAAM,WAAW,UAAU;AAC9B,MAAI,WAAW,QAAQ,WAAW,OAAW,QAAO;AACpD,MAAI,MAAM,QAAQ,OAAO,EAAE;GACzB,MAAM,MAAM,OAAO,SAAS,SAAS,GAAG;AACxC,OAAI,CAAC,OAAO,SAAS,IAAI,CAAE,QAAO;AAClC,YAAS,OAAO;aACP,OAAO,WAAW,SAC3B,UAAU,OAAmC;MAE7C;;AAGJ,QAAO;;AAGT,SAAS,UAAU,GAAY,GAAqB;AAClD,KAAI,MAAM,EAAG,QAAO;AACpB,KAAI,OAAO,MAAM,OAAO,EAAG,QAAO;AAClC,KAAI,MAAM,QAAQ,MAAM,KAAM,QAAO,MAAM;AAC3C,KAAI,MAAM,QAAQ,EAAE,IAAI,MAAM,QAAQ,EAAE,EAAE;AACxC,MAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,OAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,IAC5B,KAAI,CAAC,UAAU,EAAE,IAAI,EAAE,GAAG,CAAE,QAAO;AAErC,SAAO;;AAET,KAAI,OAAO,MAAM,YAAY,OAAO,MAAM,UAAU;EAClD,MAAM,QAAQ,OAAO,KAAK,EAA6B,CAAC,MAAM;EAC9D,MAAM,QAAQ,OAAO,KAAK,EAA6B,CAAC,MAAM;AAC9D,MAAI,MAAM,WAAW,MAAM,OAAQ,QAAO;AAC1C,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,OAAI,MAAM,OAAO,MAAM,GAAI,QAAO;AAClC,OACE,CAAC,UACE,EAA8B,MAAM,KACpC,EAA8B,MAAM,IACtC,CAED,QAAO;;AAEX,SAAO;;AAET,QAAO;;AAGT,SAAS,SAAS,GAAW,MAAc,IAAY;AACrD,QAAO,EAAE,SAAS,MAAM,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC,KAAK;;AAGlD,SAAS,cAAc,OAAwB;AAC7C,KAAI,UAAU,OAAW,QAAO;AAChC,KAAI;AACF,SAAO,KAAK,UAAU,MAAM,IAAI,OAAO,MAAM;SACvC;AACN,SAAO,OAAO,MAAM"}

package/dist/scorers/code/predicate.d.ts

@@ -0,0 +1,18 @@
+import { Case, ScoreResult, Scorer } from "@graphorin/observability/eval";
+
+//#region src/scorers/code/predicate.d.ts
+
+/** @stable */
+interface PredicateOptions<I, O> {
+  readonly name: string;
+  readonly check: (args: {
+    readonly case: Case<I, O>;
+    readonly output: O;
+    readonly durationMs: number;
+  }) => Promise<boolean | ScoreResult> | boolean | ScoreResult;
+}
+/** @stable */
+declare function predicate<I = unknown, O = unknown>(options: PredicateOptions<I, O>): Scorer<I, O>;
+//#endregion
+export { PredicateOptions, predicate };
+//# sourceMappingURL=predicate.d.ts.map

package/dist/scorers/code/predicate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"predicate.d.ts","names":[],"sources":["../../../src/scorers/code/predicate.ts"],"sourcesContent":[],"mappings":";;;;;AAgB0B,UANT,gBAMS,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA;EAAlB,SAAA,IAAA,EAAA,MAAA;EAA2C,SAAA,KAAA,EAAA,CAAA,IAAA,EAAA;IAAW,SAAA,IAAA,EAH3C,IAG2C,CAHtC,CAGsC,EAHnC,CAGmC,CAAA;IAI9C,SAAS,MAAA,EANJ,CAMI;IAAqD,SAAA,UAAA,EAAA,MAAA;EAAG,CAAA,EAAA,GAJzE,OAIyE,CAAA,OAAA,GAJvD,WAIuD,CAAA,GAAA,OAAA,GAJ9B,WAI8B;;;AAAe,iBAAhF,SAAgF,CAAA,IAAA,OAAA,EAAA,IAAA,OAAA,CAAA,CAAA,OAAA,EAAnC,gBAAmC,CAAlB,CAAkB,EAAf,CAAe,CAAA,CAAA,EAAV,MAAU,CAAH,CAAG,EAAA,CAAA,CAAA"}

package/dist/scorers/code/predicate.js

@@ -0,0 +1,19 @@
+//#region src/scorers/code/predicate.ts
+/** @stable */
+function predicate(options) {
+	return {
+		name: options.name,
+		async score(args) {
+			const result = await options.check(args);
+			if (typeof result === "boolean") return {
+				pass: result,
+				score: result ? 1 : 0
+			};
+			return result;
+		}
+	};
+}
+
+//#endregion
+export { predicate };
+//# sourceMappingURL=predicate.js.map

package/dist/scorers/code/predicate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"predicate.js","names":[],"sources":["../../../src/scorers/code/predicate.ts"],"sourcesContent":["/**\n * `predicate` — passes when the caller-supplied predicate returns\n * truthy. The escape hatch for ad-hoc, project-specific scoring rules.\n *\n * @packageDocumentation\n */\n\nimport type { Case, ScoreResult, Scorer } from '@graphorin/observability/eval';\n\n/** @stable */\nexport interface PredicateOptions<I, O> {\n  readonly name: string;\n  readonly check: (args: {\n    readonly case: Case<I, O>;\n    readonly output: O;\n    readonly durationMs: number;\n  }) => Promise<boolean | ScoreResult> | boolean | ScoreResult;\n}\n\n/** @stable */\nexport function predicate<I = unknown, O = unknown>(options: PredicateOptions<I, O>): Scorer<I, O> {\n  return {\n    name: options.name,\n    async score(args) {\n      const result = await options.check(args);\n      if (typeof result === 'boolean') {\n        return { pass: result, score: result ? 1 : 0 };\n      }\n      return result;\n    },\n  };\n}\n"],"mappings":";;AAoBA,SAAgB,UAAoC,SAA+C;AACjG,QAAO;EACL,MAAM,QAAQ;EACd,MAAM,MAAM,MAAM;GAChB,MAAM,SAAS,MAAM,QAAQ,MAAM,KAAK;AACxC,OAAI,OAAO,WAAW,UACpB,QAAO;IAAE,MAAM;IAAQ,OAAO,SAAS,IAAI;IAAG;AAEhD,UAAO;;EAEV"}

package/dist/scorers/code/regex.d.ts

@@ -0,0 +1,15 @@
+import { Scorer } from "@graphorin/observability/eval";
+
+//#region src/scorers/code/regex.d.ts
+
+/** @stable */
+interface RegexMatchOptions {
+  readonly pattern: RegExp;
+  /** Optional name override. Default `'regex-match'`. */
+  readonly name?: string;
+}
+/** @stable */
+declare function regexMatch<I = unknown>(options: RegexMatchOptions): Scorer<I, unknown>;
+//#endregion
+export { RegexMatchOptions, regexMatch };
+//# sourceMappingURL=regex.d.ts.map

package/dist/scorers/code/regex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"regex.d.ts","names":[],"sources":["../../../src/scorers/code/regex.ts"],"sourcesContent":[],"mappings":";;;;;AAiB2E,UAP1D,iBAAA,CAO0D;oBANvD;;;;;iBAMJ,iCAAiC,oBAAoB,OAAO"}