@velajs/testing 0.5.1 → 0.6.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.6.0
+
+### Minor Changes
+
+- 5097ee4: Add `@velajs/testing/eval`: a small, model-agnostic harness for scoring the output of any string-producing function. Ships heuristic scorers (`exactMatch`, `contains`, `keyword`, `regex`), an `llmScorer` LLM-as-judge whose `judge` callback is injected (no AI SDK dependency, fails soft on unparseable replies), and `evaluate(dataset, run, scorers)` which returns per-case reports plus a per-scorer and overall aggregate.
+
 ## 0.5.1
 
 ### Patch Changes

package/README.md

@@ -95,6 +95,37 @@ expect(await res.json()).toEqual([{ id: 1, name: 'Item 1' }]);
 | `TestingModule` | Result of `compile()`: `get(token)`, `createApplication()`, `close()`. |
 | `OverrideBy` | The fluent intermediate from `overrideX()` — exposed for type-narrowing. |
 
+## Output scoring / evals
+
+`@velajs/testing/eval` is a small, model-agnostic harness for grading the output of any string-producing function — an LLM turn, an agent loop, a formatter — against heuristics or an injected LLM judge. It pulls in no AI SDK: the only model touchpoint is `llmScorer`, whose `judge` is a plain callback you supply.
+
+```ts
+import { evaluate, keyword, llmScorer } from '@velajs/testing/eval';
+
+const report = await evaluate(
+  [{ input: 'where is my order?', expected: 'shipped' }],
+  async (input) => askSupportAgent(input),
+  {
+    coverage: keyword(['shipped']),
+    helpful: llmScorer({ criteria: 'answers the question', judge: myModel }),
+  },
+);
+
+expect(report.aggregate.overall).toBeGreaterThan(0.5);
+```
+
+Each scorer returns a `[0, 1]` score (auto-clamped) with an optional reason. `evaluate` runs every case through the producer, grades each output with every scorer, and returns per-case reports plus an aggregate (`perScorer` means and an `overall` mean).
+
+| Export | Purpose |
+|---|---|
+| `evaluate(dataset, run, scorers)` | Grade a dataset; returns `{ cases, aggregate }`. |
+| `exactMatch(options?)` | 1 when the output equals `expected` (trimmed by default). |
+| `contains(needles, options?)` | Substring match, `all`-of (default) or `any`-of. |
+| `keyword(keywords, options?)` | Fractional coverage — the share of keywords present. |
+| `regex(pattern)` | 1 when the pattern matches (global/sticky flags stripped for reuse). |
+| `llmScorer({ criteria, judge })` | LLM-as-judge with an injected `judge` callback; fails soft to 0. |
+| `Scorer` | `(input) => number \| ScoreResult` (sync or async) — write your own. |
+
 ## How it's wired
 
 `@velajs/testing` consumes vela's framework primitives via `@velajs/vela/internal` (`MetadataRegistry`, `Container`, `RouteManager`, `ModuleLoader`, `ComponentManager`, `VelaApplication`, `bindAppProviders`). The same `bindAppProviders` that `VelaFactory.create` uses, so test-mode and run-mode app construction stay in lockstep automatically.

package/dist/eval/index.d.ts

@@ -0,0 +1,154 @@
+//#region src/eval/types.d.ts
+/**
+ * Shared types for the output-evaluation harness.
+ *
+ * The harness is model-agnostic: a {@link Scorer} is any function that turns one
+ * sample into a `[0, 1]` score, and {@link evaluate} drives a dataset through a
+ * producer and grades every output. Nothing here reaches for an AI SDK — the
+ * only "AI" surface is `llmScorer`, whose judge callback is injected by the
+ * caller.
+ */
+/** A value that may be produced synchronously or via a promise. */
+type Awaitable<T> = T | Promise<T>;
+/** The single sample handed to a {@link Scorer}. */
+interface ScorerInput {
+  /** The prompt/input that produced {@link ScorerInput.output}, for judge context. */
+  input?: string;
+  /** The output under test — the only field every scorer is guaranteed to see. */
+  output: string;
+  /** The reference answer, when a scorer grades against a gold value. */
+  expected?: string;
+  /** Free-form per-sample metadata carried through untouched. */
+  metadata?: Record<string, unknown>;
+}
+/** A scorer's verdict: a `[0, 1]` score plus an optional human-readable reason. */
+interface ScoreResult {
+  /** The grade, always clamped into `[0, 1]` by the harness. */
+  score: number;
+  /** Why the scorer landed on this grade. */
+  reason?: string;
+}
+/**
+ * A scorer maps one sample to a grade. Returning a bare number is shorthand for
+ * `{ score }`; return a {@link ScoreResult} to attach a reason. Sync or async.
+ */
+type Scorer = (input: ScorerInput) => Awaitable<number | ScoreResult>;
+/** One dataset row: an input plus an optional gold answer and metadata. */
+interface EvalCase {
+  input: string;
+  expected?: string;
+  metadata?: Record<string, unknown>;
+}
+/** The result of grading one case: the produced output and each scorer's verdict. */
+interface CaseReport {
+  input: string;
+  output: string;
+  /** Each scorer's verdict, keyed by the name it was registered under. */
+  scores: Record<string, ScoreResult>;
+  /** The mean of this case's scores across all scorers. */
+  average: number;
+}
+/** The roll-up across every case. */
+interface EvalAggregate {
+  /** Mean score of each scorer across all cases, keyed by scorer name. */
+  perScorer: Record<string, number>;
+  /** Mean of every case's average — the single headline number. */
+  overall: number;
+}
+/** The full run: one report per case plus the aggregate roll-up. */
+interface EvalReport {
+  cases: CaseReport[];
+  aggregate: EvalAggregate;
+}
+//#endregion
+//#region src/eval/scorers.d.ts
+/** Options for {@link exactMatch}. */
+interface ExactMatchOptions {
+  /** Trim leading/trailing whitespace before comparing (default `true`). */
+  trim?: boolean;
+  /** Compare with case sensitivity (default `true`). */
+  caseSensitive?: boolean;
+}
+/**
+ * Score 1 when the output equals the case's `expected`, else 0. Whitespace is
+ * trimmed by default; a case with no `expected` scores 0 (there is nothing to
+ * match against, so it fails closed).
+ */
+declare function exactMatch(options?: ExactMatchOptions): Scorer;
+/** Options for {@link contains}. */
+interface ContainsOptions {
+  /** `all` (default) requires every needle; `any` requires at least one. */
+  mode?: 'all' | 'any';
+  /** Match with case sensitivity (default: case-insensitive). */
+  caseSensitive?: boolean;
+}
+/**
+ * Binary substring scorer. Pass a single needle or a list; with a list, `mode`
+ * chooses whether every needle must be present (`all`, the default) or just one
+ * (`any`). Scores 1 when the condition holds, else 0.
+ */
+declare function contains(needles: string | readonly string[], options?: ContainsOptions): Scorer;
+/** Options for {@link keyword}. */
+interface KeywordOptions {
+  /** Match with case sensitivity (default: case-insensitive). */
+  caseSensitive?: boolean;
+}
+/**
+ * Fractional coverage scorer: the share of `keywords` present in the output. A
+ * rubric with three keywords, two of which appear, scores `2/3`. An empty
+ * keyword list would silently score everything 1 — the opposite of a useful
+ * eval — so it is rejected at construction time.
+ */
+declare function keyword(keywords: readonly string[], options?: KeywordOptions): Scorer;
+/**
+ * Score 1 when `pattern` matches the output, else 0. The global/sticky flags are
+ * stripped from a copy of the pattern so a scorer reused across many samples
+ * never carries `lastIndex` state between calls.
+ */
+declare function regex(pattern: RegExp): Scorer;
+/** Options for {@link llmScorer}. */
+interface LlmScorerOptions {
+  /** The rubric the judge grades against, e.g. "answers the question accurately". */
+  criteria: string;
+  /**
+   * The injected judge. Wire it to your own model call (`generateText`, a raw
+   * fetch, whatever). Keeping it a plain callback is what lets this package
+   * grade LLM output without depending on any AI SDK, and makes it trivial to
+   * fake in tests.
+   */
+  judge: (prompt: string) => Promise<string>;
+}
+/** Render the grading prompt for one sample under a rubric. */
+declare function renderJudgePrompt(criteria: string, sample: ScorerInput): string;
+/**
+ * Read a judge reply into a verdict. The score is the leading number, clamped,
+ * and the full reply is kept as the reason. Requiring the number at the very
+ * start is deliberate: a reply with no leading number fails closed to 0, and a
+ * digit buried later in the justification can never be misread as the grade.
+ */
+declare function parseJudgeReply(reply: string): ScoreResult;
+/**
+ * LLM-as-judge scorer. It builds a rubric prompt, hands it to the injected
+ * `judge`, and parses the reply into a `[0, 1]` grade. Unparseable replies fail
+ * soft (score 0 with the raw reply as the reason) rather than throwing.
+ */
+declare function llmScorer(options: LlmScorerOptions): Scorer;
+//#endregion
+//#region src/eval/evaluate.d.ts
+/**
+ * Run a whole dataset through `run` and grade each output with the named
+ * `scorers`. Every case is graded by every scorer; cases execute concurrently,
+ * so keep `run` stateless per input (or key any shared state on `input`).
+ *
+ * The returned report carries one {@link CaseReport} per case plus an aggregate:
+ * the mean of each scorer across all cases (`perScorer`) and the mean of the
+ * per-case averages (`overall`).
+ *
+ * @param dataset The cases to grade.
+ * @param run     Produces the output string for a given input.
+ * @param scorers A name → scorer map; the names key the per-scorer aggregate.
+ */
+declare function evaluate(dataset: readonly EvalCase[], run: (input: string) => Awaitable<string>, scorers: Record<string, Scorer>): Promise<EvalReport>;
+//#endregion
+export { type Awaitable, type CaseReport, type ContainsOptions, type EvalAggregate, type EvalCase, type EvalReport, type ExactMatchOptions, type KeywordOptions, type LlmScorerOptions, type ScoreResult, type Scorer, type ScorerInput, contains, evaluate, exactMatch, keyword, llmScorer, parseJudgeReply, regex, renderJudgePrompt };
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.js

@@ -0,0 +1,191 @@
+//#region src/eval/scorers.ts
+/**
+* Force a raw number into the `[0, 1]` unit range. Anything non-finite (NaN,
+* Infinity) collapses to 0 so a broken scorer fails closed rather than skewing
+* an aggregate.
+*/
+function clampUnit(value) {
+	if (!Number.isFinite(value)) return 0;
+	if (value < 0) return 0;
+	if (value > 1) return 1;
+	return value;
+}
+/** Coerce a scorer's return (a bare number or a full result) into a clamped {@link ScoreResult}. */
+function normalizeVerdict(value) {
+	if (typeof value === "number") return { score: clampUnit(value) };
+	if (value.reason === void 0) return { score: clampUnit(value.score) };
+	return {
+		score: clampUnit(value.score),
+		reason: value.reason
+	};
+}
+/** Arithmetic mean of a list; an empty list averages to 0. */
+function arithmeticMean(values) {
+	if (values.length === 0) return 0;
+	let total = 0;
+	for (const value of values) total += value;
+	return total / values.length;
+}
+/** Case-fold a string unless the caller opted into case-sensitive matching. */
+function fold(value, caseSensitive) {
+	return caseSensitive ? value : value.toLowerCase();
+}
+/**
+* Score 1 when the output equals the case's `expected`, else 0. Whitespace is
+* trimmed by default; a case with no `expected` scores 0 (there is nothing to
+* match against, so it fails closed).
+*/
+function exactMatch(options = {}) {
+	const trim = options.trim ?? true;
+	const caseSensitive = options.caseSensitive ?? true;
+	const shape = (value) => {
+		const trimmed = trim ? value.trim() : value;
+		return caseSensitive ? trimmed : trimmed.toLowerCase();
+	};
+	return ({ output, expected }) => {
+		if (expected === void 0) return {
+			score: 0,
+			reason: "no expected value to compare against"
+		};
+		return shape(output) === shape(expected) ? 1 : 0;
+	};
+}
+/**
+* Binary substring scorer. Pass a single needle or a list; with a list, `mode`
+* chooses whether every needle must be present (`all`, the default) or just one
+* (`any`). Scores 1 when the condition holds, else 0.
+*/
+function contains(needles, options = {}) {
+	const list = typeof needles === "string" ? [needles] : [...needles];
+	if (list.length === 0) throw new Error("@velajs/testing: contains() needs at least one needle");
+	const mode = options.mode ?? "all";
+	const targets = list.map((needle) => fold(needle, options.caseSensitive));
+	return ({ output }) => {
+		const haystack = fold(output, options.caseSensitive);
+		const found = targets.map((target) => haystack.includes(target));
+		return (mode === "any" ? found.some(Boolean) : found.every(Boolean)) ? 1 : 0;
+	};
+}
+/**
+* Fractional coverage scorer: the share of `keywords` present in the output. A
+* rubric with three keywords, two of which appear, scores `2/3`. An empty
+* keyword list would silently score everything 1 — the opposite of a useful
+* eval — so it is rejected at construction time.
+*/
+function keyword(keywords, options = {}) {
+	if (keywords.length === 0) throw new Error("@velajs/testing: keyword() needs at least one keyword");
+	const targets = keywords.map((word) => fold(word, options.caseSensitive));
+	return ({ output }) => {
+		const haystack = fold(output, options.caseSensitive);
+		const hits = targets.filter((target) => haystack.includes(target)).length;
+		return {
+			score: hits / targets.length,
+			reason: `${hits}/${targets.length} keywords present`
+		};
+	};
+}
+/**
+* Score 1 when `pattern` matches the output, else 0. The global/sticky flags are
+* stripped from a copy of the pattern so a scorer reused across many samples
+* never carries `lastIndex` state between calls.
+*/
+function regex(pattern) {
+	const stateless = new RegExp(pattern.source, pattern.flags.replace(/[gy]/gu, ""));
+	return ({ output }) => stateless.test(output) ? 1 : 0;
+}
+/** The regex that reads the leading number the judge is instructed to emit. */
+const LEADING_NUMBER = /^\s*([+-]?\d+(?:\.\d+)?)/u;
+/** Render the grading prompt for one sample under a rubric. */
+function renderJudgePrompt(criteria, sample) {
+	const lines = [
+		"You are grading an assistant output.",
+		`Criterion: ${criteria}`,
+		"Begin your reply with a single decimal from 0 (does not meet) to 1 (fully meets), then optionally a short justification."
+	];
+	if (sample.input !== void 0) lines.push("", `Prompt: ${sample.input}`);
+	if (sample.expected !== void 0) lines.push("", `Expected answer: ${sample.expected}`);
+	lines.push("", `Output to grade: ${sample.output}`);
+	return lines.join("\n");
+}
+/**
+* Read a judge reply into a verdict. The score is the leading number, clamped,
+* and the full reply is kept as the reason. Requiring the number at the very
+* start is deliberate: a reply with no leading number fails closed to 0, and a
+* digit buried later in the justification can never be misread as the grade.
+*/
+function parseJudgeReply(reply) {
+	const match = LEADING_NUMBER.exec(reply);
+	const reason = reply.trim();
+	if (match === null) return {
+		score: 0,
+		reason
+	};
+	return {
+		score: clampUnit(Number(match[1])),
+		reason
+	};
+}
+/**
+* LLM-as-judge scorer. It builds a rubric prompt, hands it to the injected
+* `judge`, and parses the reply into a `[0, 1]` grade. Unparseable replies fail
+* soft (score 0 with the raw reply as the reason) rather than throwing.
+*/
+function llmScorer(options) {
+	return async (sample) => parseJudgeReply(await options.judge(renderJudgePrompt(options.criteria, sample)));
+}
+//#endregion
+//#region src/eval/evaluate.ts
+/** Run every scorer over one sample and collect the verdicts keyed by scorer name. */
+async function gradeSample(sample, scorers) {
+	const graded = await Promise.all(Object.entries(scorers).map(async ([name, scorer]) => {
+		return [name, normalizeVerdict(await scorer(sample))];
+	}));
+	const scores = {};
+	for (const [name, verdict] of graded) scores[name] = verdict;
+	return scores;
+}
+/**
+* Run a whole dataset through `run` and grade each output with the named
+* `scorers`. Every case is graded by every scorer; cases execute concurrently,
+* so keep `run` stateless per input (or key any shared state on `input`).
+*
+* The returned report carries one {@link CaseReport} per case plus an aggregate:
+* the mean of each scorer across all cases (`perScorer`) and the mean of the
+* per-case averages (`overall`).
+*
+* @param dataset The cases to grade.
+* @param run     Produces the output string for a given input.
+* @param scorers A name → scorer map; the names key the per-scorer aggregate.
+*/
+async function evaluate(dataset, run, scorers) {
+	const cases = await Promise.all(dataset.map(async (testCase) => {
+		const output = await run(testCase.input);
+		const sample = {
+			input: testCase.input,
+			output
+		};
+		if (testCase.expected !== void 0) sample.expected = testCase.expected;
+		if (testCase.metadata !== void 0) sample.metadata = testCase.metadata;
+		const scores = await gradeSample(sample, scorers);
+		const average = arithmeticMean(Object.values(scores).map((verdict) => verdict.score));
+		return {
+			input: testCase.input,
+			output,
+			scores,
+			average
+		};
+	}));
+	const perScorer = {};
+	for (const name of Object.keys(scorers)) perScorer[name] = arithmeticMean(cases.map((report) => report.scores[name]?.score ?? 0));
+	return {
+		cases,
+		aggregate: {
+			perScorer,
+			overall: arithmeticMean(cases.map((report) => report.average))
+		}
+	};
+}
+//#endregion
+export { contains, evaluate, exactMatch, keyword, llmScorer, parseJudgeReply, regex, renderJudgePrompt };
+
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":[],"sources":["../../src/eval/scorers.ts","../../src/eval/evaluate.ts"],"sourcesContent":["import type { Scorer, ScorerInput, ScoreResult } from './types.js';\n\n/**\n * Force a raw number into the `[0, 1]` unit range. Anything non-finite (NaN,\n * Infinity) collapses to 0 so a broken scorer fails closed rather than skewing\n * an aggregate.\n */\nexport function clampUnit(value: number): number {\n  if (!Number.isFinite(value)) {\n    return 0;\n  }\n  if (value < 0) {\n    return 0;\n  }\n  if (value > 1) {\n    return 1;\n  }\n\n  return value;\n}\n\n/** Coerce a scorer's return (a bare number or a full result) into a clamped {@link ScoreResult}. */\nexport function normalizeVerdict(value: number | ScoreResult): ScoreResult {\n  if (typeof value === 'number') {\n    return { score: clampUnit(value) };\n  }\n  if (value.reason === undefined) {\n    return { score: clampUnit(value.score) };\n  }\n\n  return { score: clampUnit(value.score), reason: value.reason };\n}\n\n/** Arithmetic mean of a list; an empty list averages to 0. */\nexport function arithmeticMean(values: readonly number[]): number {\n  if (values.length === 0) {\n    return 0;\n  }\n\n  let total = 0;\n\n  for (const value of values) {\n    total += value;\n  }\n\n  return total / values.length;\n}\n\n/** Case-fold a string unless the caller opted into case-sensitive matching. */\nfunction fold(value: string, caseSensitive: boolean | undefined): string {\n  return caseSensitive ? value : value.toLowerCase();\n}\n\n/** Options for {@link exactMatch}. */\nexport interface ExactMatchOptions {\n  /** Trim leading/trailing whitespace before comparing (default `true`). */\n  trim?: boolean;\n  /** Compare with case sensitivity (default `true`). */\n  caseSensitive?: boolean;\n}\n\n/**\n * Score 1 when the output equals the case's `expected`, else 0. Whitespace is\n * trimmed by default; a case with no `expected` scores 0 (there is nothing to\n * match against, so it fails closed).\n */\nexport function exactMatch(options: ExactMatchOptions = {}): Scorer {\n  const trim = options.trim ?? true;\n  const caseSensitive = options.caseSensitive ?? true;\n  const shape = (value: string): string => {\n    const trimmed = trim ? value.trim() : value;\n\n    return caseSensitive ? trimmed : trimmed.toLowerCase();\n  };\n\n  return ({ output, expected }): number | ScoreResult => {\n    if (expected === undefined) {\n      return { score: 0, reason: 'no expected value to compare against' };\n    }\n\n    return shape(output) === shape(expected) ? 1 : 0;\n  };\n}\n\n/** Options for {@link contains}. */\nexport interface ContainsOptions {\n  /** `all` (default) requires every needle; `any` requires at least one. */\n  mode?: 'all' | 'any';\n  /** Match with case sensitivity (default: case-insensitive). */\n  caseSensitive?: boolean;\n}\n\n/**\n * Binary substring scorer. Pass a single needle or a list; with a list, `mode`\n * chooses whether every needle must be present (`all`, the default) or just one\n * (`any`). Scores 1 when the condition holds, else 0.\n */\nexport function contains(\n  needles: string | readonly string[],\n  options: ContainsOptions = {},\n): Scorer {\n  const list = typeof needles === 'string' ? [needles] : [...needles];\n\n  if (list.length === 0) {\n    throw new Error('@velajs/testing: contains() needs at least one needle');\n  }\n\n  const mode = options.mode ?? 'all';\n  const targets = list.map((needle) => fold(needle, options.caseSensitive));\n\n  return ({ output }): number => {\n    const haystack = fold(output, options.caseSensitive);\n    const found = targets.map((target) => haystack.includes(target));\n    const passed = mode === 'any' ? found.some(Boolean) : found.every(Boolean);\n\n    return passed ? 1 : 0;\n  };\n}\n\n/** Options for {@link keyword}. */\nexport interface KeywordOptions {\n  /** Match with case sensitivity (default: case-insensitive). */\n  caseSensitive?: boolean;\n}\n\n/**\n * Fractional coverage scorer: the share of `keywords` present in the output. A\n * rubric with three keywords, two of which appear, scores `2/3`. An empty\n * keyword list would silently score everything 1 — the opposite of a useful\n * eval — so it is rejected at construction time.\n */\nexport function keyword(keywords: readonly string[], options: KeywordOptions = {}): Scorer {\n  if (keywords.length === 0) {\n    throw new Error('@velajs/testing: keyword() needs at least one keyword');\n  }\n\n  const targets = keywords.map((word) => fold(word, options.caseSensitive));\n\n  return ({ output }): ScoreResult => {\n    const haystack = fold(output, options.caseSensitive);\n    const hits = targets.filter((target) => haystack.includes(target)).length;\n\n    return {\n      score: hits / targets.length,\n      reason: `${hits}/${targets.length} keywords present`,\n    };\n  };\n}\n\n/**\n * Score 1 when `pattern` matches the output, else 0. The global/sticky flags are\n * stripped from a copy of the pattern so a scorer reused across many samples\n * never carries `lastIndex` state between calls.\n */\nexport function regex(pattern: RegExp): Scorer {\n  const stateless = new RegExp(pattern.source, pattern.flags.replace(/[gy]/gu, ''));\n\n  return ({ output }): number => (stateless.test(output) ? 1 : 0);\n}\n\n/** Options for {@link llmScorer}. */\nexport interface LlmScorerOptions {\n  /** The rubric the judge grades against, e.g. \"answers the question accurately\". */\n  criteria: string;\n  /**\n   * The injected judge. Wire it to your own model call (`generateText`, a raw\n   * fetch, whatever). Keeping it a plain callback is what lets this package\n   * grade LLM output without depending on any AI SDK, and makes it trivial to\n   * fake in tests.\n   */\n  judge: (prompt: string) => Promise<string>;\n}\n\n/** The regex that reads the leading number the judge is instructed to emit. */\nconst LEADING_NUMBER = /^\\s*([+-]?\\d+(?:\\.\\d+)?)/u;\n\n/** Render the grading prompt for one sample under a rubric. */\nexport function renderJudgePrompt(criteria: string, sample: ScorerInput): string {\n  const lines = [\n    'You are grading an assistant output.',\n    `Criterion: ${criteria}`,\n    'Begin your reply with a single decimal from 0 (does not meet) to 1 (fully meets), then optionally a short justification.',\n  ];\n\n  if (sample.input !== undefined) {\n    lines.push('', `Prompt: ${sample.input}`);\n  }\n  if (sample.expected !== undefined) {\n    lines.push('', `Expected answer: ${sample.expected}`);\n  }\n\n  lines.push('', `Output to grade: ${sample.output}`);\n\n  return lines.join('\\n');\n}\n\n/**\n * Read a judge reply into a verdict. The score is the leading number, clamped,\n * and the full reply is kept as the reason. Requiring the number at the very\n * start is deliberate: a reply with no leading number fails closed to 0, and a\n * digit buried later in the justification can never be misread as the grade.\n */\nexport function parseJudgeReply(reply: string): ScoreResult {\n  const match = LEADING_NUMBER.exec(reply);\n  const reason = reply.trim();\n\n  if (match === null) {\n    return { score: 0, reason };\n  }\n\n  return { score: clampUnit(Number(match[1])), reason };\n}\n\n/**\n * LLM-as-judge scorer. It builds a rubric prompt, hands it to the injected\n * `judge`, and parses the reply into a `[0, 1]` grade. Unparseable replies fail\n * soft (score 0 with the raw reply as the reason) rather than throwing.\n */\nexport function llmScorer(options: LlmScorerOptions): Scorer {\n  return async (sample): Promise<ScoreResult> =>\n    parseJudgeReply(await options.judge(renderJudgePrompt(options.criteria, sample)));\n}\n","import { arithmeticMean, normalizeVerdict } from './scorers.js';\nimport type {\n  Awaitable,\n  CaseReport,\n  EvalCase,\n  EvalReport,\n  Scorer,\n  ScorerInput,\n  ScoreResult,\n} from './types.js';\n\n/** Run every scorer over one sample and collect the verdicts keyed by scorer name. */\nasync function gradeSample(\n  sample: ScorerInput,\n  scorers: Record<string, Scorer>,\n): Promise<Record<string, ScoreResult>> {\n  const graded = await Promise.all(\n    Object.entries(scorers).map(async ([name, scorer]): Promise<[string, ScoreResult]> => {\n      return [name, normalizeVerdict(await scorer(sample))];\n    }),\n  );\n\n  const scores: Record<string, ScoreResult> = {};\n\n  for (const [name, verdict] of graded) {\n    scores[name] = verdict;\n  }\n\n  return scores;\n}\n\n/**\n * Run a whole dataset through `run` and grade each output with the named\n * `scorers`. Every case is graded by every scorer; cases execute concurrently,\n * so keep `run` stateless per input (or key any shared state on `input`).\n *\n * The returned report carries one {@link CaseReport} per case plus an aggregate:\n * the mean of each scorer across all cases (`perScorer`) and the mean of the\n * per-case averages (`overall`).\n *\n * @param dataset The cases to grade.\n * @param run     Produces the output string for a given input.\n * @param scorers A name → scorer map; the names key the per-scorer aggregate.\n */\nexport async function evaluate(\n  dataset: readonly EvalCase[],\n  run: (input: string) => Awaitable<string>,\n  scorers: Record<string, Scorer>,\n): Promise<EvalReport> {\n  const cases = await Promise.all(\n    dataset.map(async (testCase): Promise<CaseReport> => {\n      const output = await run(testCase.input);\n      const sample: ScorerInput = { input: testCase.input, output };\n\n      if (testCase.expected !== undefined) {\n        sample.expected = testCase.expected;\n      }\n      if (testCase.metadata !== undefined) {\n        sample.metadata = testCase.metadata;\n      }\n\n      const scores = await gradeSample(sample, scorers);\n      const average = arithmeticMean(Object.values(scores).map((verdict) => verdict.score));\n\n      return { input: testCase.input, output, scores, average };\n    }),\n  );\n\n  const perScorer: Record<string, number> = {};\n\n  for (const name of Object.keys(scorers)) {\n    perScorer[name] = arithmeticMean(cases.map((report) => report.scores[name]?.score ?? 0));\n  }\n\n  const overall = arithmeticMean(cases.map((report) => report.average));\n\n  return { cases, aggregate: { perScorer, overall } };\n}\n"],"mappings":";;;;;;AAOA,SAAgB,UAAU,OAAuB;CAC/C,IAAI,CAAC,OAAO,SAAS,KAAK,GACxB,OAAO;CAET,IAAI,QAAQ,GACV,OAAO;CAET,IAAI,QAAQ,GACV,OAAO;CAGT,OAAO;AACT;;AAGA,SAAgB,iBAAiB,OAA0C;CACzE,IAAI,OAAO,UAAU,UACnB,OAAO,EAAE,OAAO,UAAU,KAAK,EAAE;CAEnC,IAAI,MAAM,WAAW,KAAA,GACnB,OAAO,EAAE,OAAO,UAAU,MAAM,KAAK,EAAE;CAGzC,OAAO;EAAE,OAAO,UAAU,MAAM,KAAK;EAAG,QAAQ,MAAM;CAAO;AAC/D;;AAGA,SAAgB,eAAe,QAAmC;CAChE,IAAI,OAAO,WAAW,GACpB,OAAO;CAGT,IAAI,QAAQ;CAEZ,KAAK,MAAM,SAAS,QAClB,SAAS;CAGX,OAAO,QAAQ,OAAO;AACxB;;AAGA,SAAS,KAAK,OAAe,eAA4C;CACvE,OAAO,gBAAgB,QAAQ,MAAM,YAAY;AACnD;;;;;;AAeA,SAAgB,WAAW,UAA6B,CAAC,GAAW;CAClE,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,UAA0B;EACvC,MAAM,UAAU,OAAO,MAAM,KAAK,IAAI;EAEtC,OAAO,gBAAgB,UAAU,QAAQ,YAAY;CACvD;CAEA,QAAQ,EAAE,QAAQ,eAAqC;EACrD,IAAI,aAAa,KAAA,GACf,OAAO;GAAE,OAAO;GAAG,QAAQ;EAAuC;EAGpE,OAAO,MAAM,MAAM,MAAM,MAAM,QAAQ,IAAI,IAAI;CACjD;AACF;;;;;;AAeA,SAAgB,SACd,SACA,UAA2B,CAAC,GACpB;CACR,MAAM,OAAO,OAAO,YAAY,WAAW,CAAC,OAAO,IAAI,CAAC,GAAG,OAAO;CAElE,IAAI,KAAK,WAAW,GAClB,MAAM,IAAI,MAAM,uDAAuD;CAGzE,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,UAAU,KAAK,KAAK,WAAW,KAAK,QAAQ,QAAQ,aAAa,CAAC;CAExE,QAAQ,EAAE,aAAqB;EAC7B,MAAM,WAAW,KAAK,QAAQ,QAAQ,aAAa;EACnD,MAAM,QAAQ,QAAQ,KAAK,WAAW,SAAS,SAAS,MAAM,CAAC;EAG/D,QAFe,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,MAAM,MAAM,OAAO,KAEzD,IAAI;CACtB;AACF;;;;;;;AAcA,SAAgB,QAAQ,UAA6B,UAA0B,CAAC,GAAW;CACzF,IAAI,SAAS,WAAW,GACtB,MAAM,IAAI,MAAM,uDAAuD;CAGzE,MAAM,UAAU,SAAS,KAAK,SAAS,KAAK,MAAM,QAAQ,aAAa,CAAC;CAExE,QAAQ,EAAE,aAA0B;EAClC,MAAM,WAAW,KAAK,QAAQ,QAAQ,aAAa;EACnD,MAAM,OAAO,QAAQ,QAAQ,WAAW,SAAS,SAAS,MAAM,CAAC,CAAC,CAAC;EAEnE,OAAO;GACL,OAAO,OAAO,QAAQ;GACtB,QAAQ,GAAG,KAAK,GAAG,QAAQ,OAAO;EACpC;CACF;AACF;;;;;;AAOA,SAAgB,MAAM,SAAyB;CAC7C,MAAM,YAAY,IAAI,OAAO,QAAQ,QAAQ,QAAQ,MAAM,QAAQ,UAAU,EAAE,CAAC;CAEhF,QAAQ,EAAE,aAAsB,UAAU,KAAK,MAAM,IAAI,IAAI;AAC/D;;AAgBA,MAAM,iBAAiB;;AAGvB,SAAgB,kBAAkB,UAAkB,QAA6B;CAC/E,MAAM,QAAQ;EACZ;EACA,cAAc;EACd;CACF;CAEA,IAAI,OAAO,UAAU,KAAA,GACnB,MAAM,KAAK,IAAI,WAAW,OAAO,OAAO;CAE1C,IAAI,OAAO,aAAa,KAAA,GACtB,MAAM,KAAK,IAAI,oBAAoB,OAAO,UAAU;CAGtD,MAAM,KAAK,IAAI,oBAAoB,OAAO,QAAQ;CAElD,OAAO,MAAM,KAAK,IAAI;AACxB;;;;;;;AAQA,SAAgB,gBAAgB,OAA4B;CAC1D,MAAM,QAAQ,eAAe,KAAK,KAAK;CACvC,MAAM,SAAS,MAAM,KAAK;CAE1B,IAAI,UAAU,MACZ,OAAO;EAAE,OAAO;EAAG;CAAO;CAG5B,OAAO;EAAE,OAAO,UAAU,OAAO,MAAM,EAAE,CAAC;EAAG;CAAO;AACtD;;;;;;AAOA,SAAgB,UAAU,SAAmC;CAC3D,OAAO,OAAO,WACZ,gBAAgB,MAAM,QAAQ,MAAM,kBAAkB,QAAQ,UAAU,MAAM,CAAC,CAAC;AACpF;;;;ACjNA,eAAe,YACb,QACA,SACsC;CACtC,MAAM,SAAS,MAAM,QAAQ,IAC3B,OAAO,QAAQ,OAAO,CAAC,CAAC,IAAI,OAAO,CAAC,MAAM,YAA4C;EACpF,OAAO,CAAC,MAAM,iBAAiB,MAAM,OAAO,MAAM,CAAC,CAAC;CACtD,CAAC,CACH;CAEA,MAAM,SAAsC,CAAC;CAE7C,KAAK,MAAM,CAAC,MAAM,YAAY,QAC5B,OAAO,QAAQ;CAGjB,OAAO;AACT;;;;;;;;;;;;;;AAeA,eAAsB,SACpB,SACA,KACA,SACqB;CACrB,MAAM,QAAQ,MAAM,QAAQ,IAC1B,QAAQ,IAAI,OAAO,aAAkC;EACnD,MAAM,SAAS,MAAM,IAAI,SAAS,KAAK;EACvC,MAAM,SAAsB;GAAE,OAAO,SAAS;GAAO;EAAO;EAE5D,IAAI,SAAS,aAAa,KAAA,GACxB,OAAO,WAAW,SAAS;EAE7B,IAAI,SAAS,aAAa,KAAA,GACxB,OAAO,WAAW,SAAS;EAG7B,MAAM,SAAS,MAAM,YAAY,QAAQ,OAAO;EAChD,MAAM,UAAU,eAAe,OAAO,OAAO,MAAM,CAAC,CAAC,KAAK,YAAY,QAAQ,KAAK,CAAC;EAEpF,OAAO;GAAE,OAAO,SAAS;GAAO;GAAQ;GAAQ;EAAQ;CAC1D,CAAC,CACH;CAEA,MAAM,YAAoC,CAAC;CAE3C,KAAK,MAAM,QAAQ,OAAO,KAAK,OAAO,GACpC,UAAU,QAAQ,eAAe,MAAM,KAAK,WAAW,OAAO,OAAO,KAAK,EAAE,SAAS,CAAC,CAAC;CAKzF,OAAO;EAAE;EAAO,WAAW;GAAE;GAAW,SAFxB,eAAe,MAAM,KAAK,WAAW,OAAO,OAAO,CAErB;EAAE;CAAE;AACpD"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@velajs/testing",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Testing utilities for Vela framework",
   "keywords": [
     "edge",
@@ -41,6 +41,10 @@
     "./websocket-node": {
       "types": "./dist/websocket-node/index.d.ts",
       "import": "./dist/websocket-node/index.js"
+    },
+    "./eval": {
+      "types": "./dist/eval/index.d.ts",
+      "import": "./dist/eval/index.js"
     }
   },
   "devDependencies": {
@@ -80,6 +84,7 @@
     "build": "tsdown",
     "test": "vitest run",
     "typecheck": "tsc --noEmit",
+    "typecheck:tests": "tsc --noEmit -p tsconfig.test.json",
     "lint": "oxlint .",
     "format": "oxfmt .",
     "format:check": "oxfmt --check .",
@@ -88,6 +93,6 @@
     "changeset": "changeset",
     "version-packages": "changeset version",
     "release": "pnpm build && changeset publish",
-    "verify": "pnpm lint && pnpm format:check && pnpm build && pnpm typecheck && pnpm test && pnpm publint && pnpm attw"
+    "verify": "pnpm lint && pnpm format:check && pnpm build && pnpm typecheck && pnpm typecheck:tests && pnpm test && pnpm publint && pnpm attw"
   }
 }