@mastra/evals 1.9.0 → 1.10.0-alpha.1

package/dist/vitest/index.js

@@ -0,0 +1,262 @@
+import { n as registerEvalMatchers, t as evalMatchers } from "../matchers-D3c6f71B.js";
+import { runEvals } from "@mastra/core/evals";
+//#region src/vitest/meta.ts
+/**
+* Flattens the `scores` record of a `RunEvalsResult` into `Record<string, number>`.
+* Categorized results (`agent`, `workflow`, `steps`, `trajectory`) are flattened
+* into dot-separated keys, e.g. `agent.my-scorer` or `steps.step-1.my-scorer`.
+*/
+function flattenScores(scores, prefix = "", out = {}) {
+	for (const [id, value] of Object.entries(scores)) {
+		const key = prefix ? `${prefix}.${id}` : id;
+		if (typeof value === "number") out[key] = value;
+		else if (value && typeof value === "object") flattenScores(value, key, out);
+	}
+	return out;
+}
+/**
+* Projects a `RunEvalsResult` into the serializable `MastraEvalMeta` shape
+* stored in `task.meta.mastraEval`.
+*/
+function toEvalMeta(result) {
+	const meta = {
+		scores: flattenScores(result.scores),
+		totalItems: result.summary.totalItems
+	};
+	if (result.verdict !== void 0) meta.verdict = result.verdict;
+	if (result.gateResults) meta.gateResults = result.gateResults.map((g) => ({ ...g }));
+	if (result.thresholdResults) meta.thresholdResults = result.thresholdResults.map((t) => ({ ...t }));
+	if (result.turnResults) meta.turnResults = result.turnResults;
+	return meta;
+}
+//#endregion
+//#region src/vitest/expect-evals.ts
+/** Thrown by `expectEval`/`expectEvals` `.toPass()` when the pass rate is not met. */
+var EvalPassRateError = class extends Error {
+	result;
+	constructor(message, result) {
+		super(message);
+		this.name = "EvalPassRateError";
+		this.result = result;
+	}
+};
+async function attachMetaToCurrentTest(result) {
+	try {
+		const task = (await import("vitest")).TestRunner?.getCurrentTest?.();
+		if (task?.meta) {
+			task.meta.mastraEval = toEvalMeta(result);
+			return;
+		}
+		const legacyTask = (await import("vitest/suite")).getCurrentTest?.();
+		if (legacyTask?.meta) legacyTask.meta.mastraEval = toEvalMeta(result);
+	} catch {}
+}
+function assertPassRate(result, minPassRate) {
+	if (!Number.isFinite(minPassRate) || minPassRate < 0 || minPassRate > 1) throw new Error(`toPass(minPassRate): minPassRate must be within [0, 1], got ${minPassRate}`);
+	const lines = [];
+	for (const gate of result.gateResults ?? []) if (gate.score < minPassRate) lines.push(`  ✗ gate ${gate.id}: pass rate ${formatRate(gate.score)} < required ${formatRate(minPassRate)}`);
+	for (const t of result.thresholdResults ?? []) if (!t.passed) lines.push(`  ✗ threshold ${t.id}: average score ${t.averageScore} (threshold: ${JSON.stringify(t.threshold)})`);
+	for (const turn of result.turnResults ?? []) {
+		for (const gate of turn.gateResults ?? []) if (gate.score < minPassRate) lines.push(`  ✗ turn ${turn.index} gate ${gate.id}: pass rate ${formatRate(gate.score)} < required ${formatRate(minPassRate)}`);
+		for (const t of turn.thresholdResults ?? []) if (!t.passed) lines.push(`  ✗ turn ${turn.index} threshold ${t.id}: average score ${t.averageScore} (threshold: ${JSON.stringify(t.threshold)})`);
+	}
+	if (lines.length > 0) throw new EvalPassRateError([`Eval did not pass (required pass rate: ${formatRate(minPassRate)}).`, ...lines].join("\n"), result);
+}
+function formatRate(rate) {
+	return `${Math.round(rate * 1e3) / 10}%`;
+}
+function makeAssertion(options) {
+	return { async toPass(minPassRate = 1) {
+		const result = await runEvals(options);
+		await attachMetaToCurrentTest(result);
+		assertPassRate(result, minPassRate);
+		return result;
+	} };
+}
+/**
+* Fluent eval assertion over a dataset, for use inside a regular `test()`.
+*
+* Runs `runEvals` with the given options when awaited and asserts the outcome.
+* The run's scores are attached to the current test's meta so
+* `MastraEvalsReporter` displays them in the runner output.
+*
+* Must be awaited — otherwise the test finishes before the eval runs.
+*
+* @example
+* test('capitals agent answers with the expected city', async () => {
+*   await expectEvals({
+*     target: capitalsAgent,
+*     data: [{ input: 'What is the capital of France?', groundTruth: 'Paris' }],
+*     gates: [containsGroundTruth],
+*     scorers: [{ scorer: createKeywordCoverageScorer(), threshold: 0.4 }],
+*   }).toPass(0.8);
+* });
+*/
+function expectEvals(options) {
+	return makeAssertion(options);
+}
+/**
+* Single-item variant of `expectEvals`: `data` is one item instead of an array.
+*
+* Use it with `test.each`/`it.each` for matrix testing when you want one test
+* (and one reporter entry) per data item instead of one per dataset run.
+*
+* @example
+* test.each([
+*   { input: 'What is the capital of France?', groundTruth: 'Paris' },
+*   { input: 'What is the capital of Japan?', groundTruth: 'Tokyo' },
+* ])('capitals agent: $input', async item => {
+*   await expectEval({
+*     target: capitalsAgent,
+*     data: item,
+*     gates: [containsGroundTruth],
+*   }).toPass();
+* });
+*/
+function expectEval(options) {
+	return makeAssertion({
+		...options,
+		data: [options.data]
+	});
+}
+//#endregion
+//#region src/vitest/reporter.ts
+const GREEN = "\x1B[32m";
+const RED = "\x1B[31m";
+const YELLOW = "\x1B[33m";
+const BOLD = "\x1B[1m";
+const DIM = "\x1B[2m";
+const RESET = "\x1B[0m";
+function supportsColor() {
+	return process.stdout.isTTY === true && process.env.NO_COLOR === void 0;
+}
+/**
+* Vitest reporter that prints a per-test score table for every eval run
+* using `expectEval`/`expectEvals` (or any test that populates `task.meta.mastraEval`).
+*
+* Usage in `vitest.config.ts`:
+*   test: { reporters: ['default', new MastraEvalsReporter()] }
+*/
+var MastraEvalsReporter = class {
+	color;
+	constructor(options = {}) {
+		this.color = options.color ?? supportsColor();
+	}
+	onTestRunEnd(testModules) {
+		const entries = [];
+		for (const module of testModules) for (const test of module.children.allTests()) {
+			const meta = test.meta().mastraEval;
+			if (meta) entries.push({
+				fullName: test.fullName,
+				meta
+			});
+		}
+		if (entries.length === 0) return;
+		this.print(this.render(entries));
+	}
+	/** Renders the score report as plain lines. Exposed for testing. */
+	render(entries) {
+		const lines = [];
+		lines.push("");
+		lines.push(this.style(BOLD, " Mastra Evals "));
+		for (const { fullName, meta } of entries) {
+			lines.push("");
+			lines.push(`${this.verdictBadge(meta.verdict)} ${this.style(BOLD, fullName)} ${this.style(DIM, `(${meta.totalItems} item${meta.totalItems === 1 ? "" : "s"})`)}`);
+			const rows = [];
+			for (const gate of meta.gateResults ?? []) rows.push([
+				`${gate.id} ${this.style(DIM, "(gate)")}`,
+				formatScore(gate.score),
+				this.passMark(gate.passed)
+			]);
+			for (const t of meta.thresholdResults ?? []) rows.push([
+				`${t.id} ${this.style(DIM, `(threshold: ${formatThreshold(t.threshold)})`)}`,
+				formatScore(t.averageScore),
+				this.passMark(t.passed)
+			]);
+			const covered = /* @__PURE__ */ new Set([...(meta.gateResults ?? []).map((g) => g.id), ...(meta.thresholdResults ?? []).map((t) => t.id)]);
+			for (const [id, score] of Object.entries(meta.scores)) {
+				if (covered.has(id)) continue;
+				rows.push([
+					id,
+					formatScore(score),
+					""
+				]);
+			}
+			lines.push(...this.table(rows));
+			for (const turn of meta.turnResults ?? []) {
+				const turnRows = [];
+				for (const gate of turn.gateResults ?? []) turnRows.push([
+					`${gate.id} ${this.style(DIM, "(gate)")}`,
+					formatScore(gate.score),
+					this.passMark(gate.passed)
+				]);
+				for (const t of turn.thresholdResults ?? []) turnRows.push([
+					`${t.id} ${this.style(DIM, `(threshold: ${formatThreshold(t.threshold)})`)}`,
+					formatScore(t.averageScore),
+					this.passMark(t.passed)
+				]);
+				const turnCovered = /* @__PURE__ */ new Set([...(turn.gateResults ?? []).map((g) => g.id), ...(turn.thresholdResults ?? []).map((t) => t.id)]);
+				for (const [id, score] of Object.entries(turn.scores ?? {})) {
+					if (turnCovered.has(id)) continue;
+					turnRows.push([
+						id,
+						formatScore(score),
+						""
+					]);
+				}
+				if (turnRows.length === 0) continue;
+				lines.push(`  ${this.style(DIM, `turn ${turn.index}`)}`);
+				lines.push(...this.table(turnRows, "  "));
+			}
+		}
+		const withVerdict = entries.filter((e) => e.meta.verdict !== void 0);
+		const passed = withVerdict.filter((e) => e.meta.verdict === "passed").length;
+		const failed = withVerdict.filter((e) => e.meta.verdict === "failed").length;
+		const scored = entries.length - passed - failed;
+		lines.push("");
+		const parts = [];
+		if (passed > 0) parts.push(this.style(GREEN, `${passed} passed`));
+		if (failed > 0) parts.push(this.style(RED, `${failed} failed`));
+		if (scored > 0) parts.push(this.style(YELLOW, `${scored} scored`));
+		lines.push(` Eval runs: ${entries.length} ${this.style(DIM, `(${parts.join(", ")})`)}`);
+		lines.push("");
+		return lines.join("\n");
+	}
+	table(rows, indent = "") {
+		if (rows.length === 0) return [];
+		const nameWidth = Math.max(...rows.map(([name]) => stripAnsi(name).length));
+		const scoreWidth = Math.max(5, ...rows.map(([, score]) => score.length));
+		return rows.map(([name, score, mark]) => {
+			return `${indent}   ${name}${" ".repeat(nameWidth - stripAnsi(name).length)}  ${score.padStart(scoreWidth)}  ${mark}`.trimEnd();
+		});
+	}
+	verdictBadge(verdict) {
+		if (verdict === "passed") return this.style(GREEN, "✓");
+		if (verdict === "failed") return this.style(RED, "✗");
+		return this.style(YELLOW, "•");
+	}
+	passMark(passed) {
+		return passed ? this.style(GREEN, "✓") : this.style(RED, "✗");
+	}
+	style(code, text) {
+		return this.color ? `${code}${text}${RESET}` : text;
+	}
+	print(output) {
+		process.stdout.write(`${output}\n`);
+	}
+};
+function formatScore(score) {
+	return Number.isInteger(score) ? score.toFixed(1) : score.toFixed(3).replace(/0+$/, "").replace(/\.$/, ".0");
+}
+function formatThreshold(threshold) {
+	if (typeof threshold === "number") return `min ${threshold}`;
+	return Object.entries(threshold).map(([key, value]) => `${key} ${value}`).join(", ");
+}
+const ANSI_PATTERN = /\u001b\[[0-9;]*m/g;
+function stripAnsi(text) {
+	return text.replace(ANSI_PATTERN, "");
+}
+//#endregion
+export { EvalPassRateError, MastraEvalsReporter, evalMatchers, expectEval, expectEvals, registerEvalMatchers, toEvalMeta };
+
+//# sourceMappingURL=index.js.map

package/dist/vitest/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":[],"sources":["../../src/vitest/meta.ts","../../src/vitest/expect-evals.ts","../../src/vitest/reporter.ts"],"sourcesContent":["import type { EvalVerdict, GateResult, RunEvalsResult, ThresholdConfig, TurnResult } from '@mastra/core/evals';\n\n/** Per-threshold-scorer result, as reported in `RunEvalsResult.thresholdResults`. */\nexport type EvalThresholdResult = {\n  id: string;\n  passed: boolean;\n  averageScore: number;\n  threshold: ThresholdConfig;\n};\n\n/**\n * Serializable projection of a `RunEvalsResult` stored in `task.meta.mastraEval`.\n *\n * This is the contract between the test worker (which runs `runEvals`) and the\n * `MastraEvalsReporter` running in the main process. It intentionally excludes\n * raw agent/workflow payloads: `task.meta` is structured-cloned from worker to\n * main, so only lightweight, serializable data belongs here.\n */\nexport type MastraEvalMeta = {\n  /**\n   * Average score per scorer id across all data items. Categorized scorer\n   * configs are flattened into dot-separated keys (e.g. `agent.my-scorer`,\n   * `steps.step-1.my-scorer`).\n   */\n  scores: Record<string, number>;\n  /** Number of data items evaluated. */\n  totalItems: number;\n  /** Present when gates or threshold-bearing scorers were provided. */\n  verdict?: EvalVerdict;\n  /** Per-gate results (averaged across all data items). */\n  gateResults?: GateResult[];\n  /** Per-threshold-scorer results (averaged across all data items). */\n  thresholdResults?: EvalThresholdResult[];\n  /** Per-turn assertion results, present when data items use `turns`. */\n  turnResults?: TurnResult[];\n};\n\ndeclare module 'vitest' {\n  interface TaskMeta {\n    mastraEval?: MastraEvalMeta;\n  }\n}\n\n/**\n * Flattens the `scores` record of a `RunEvalsResult` into `Record<string, number>`.\n * Categorized results (`agent`, `workflow`, `steps`, `trajectory`) are flattened\n * into dot-separated keys, e.g. `agent.my-scorer` or `steps.step-1.my-scorer`.\n */\nfunction flattenScores(scores: Record<string, any>, prefix = '', out: Record<string, number> = {}) {\n  for (const [id, value] of Object.entries(scores)) {\n    const key = prefix ? `${prefix}.${id}` : id;\n    if (typeof value === 'number') {\n      out[key] = value;\n    } else if (value && typeof value === 'object') {\n      flattenScores(value, key, out);\n    }\n  }\n  return out;\n}\n\n/**\n * Projects a `RunEvalsResult` into the serializable `MastraEvalMeta` shape\n * stored in `task.meta.mastraEval`.\n */\nexport function toEvalMeta(result: RunEvalsResult): MastraEvalMeta {\n  const meta: MastraEvalMeta = {\n    scores: flattenScores(result.scores),\n    totalItems: result.summary.totalItems,\n  };\n  if (result.verdict !== undefined) meta.verdict = result.verdict;\n  if (result.gateResults) meta.gateResults = result.gateResults.map(g => ({ ...g }));\n  if (result.thresholdResults) meta.thresholdResults = result.thresholdResults.map(t => ({ ...t }));\n  if (result.turnResults) meta.turnResults = result.turnResults;\n  return meta;\n}\n","import type { RunEvalsConfig, RunEvalsResult } from '@mastra/core/evals';\nimport { runEvals } from '@mastra/core/evals';\n\nimport { toEvalMeta } from './meta';\n\n/**\n * Options accepted by `expectEvals`: exactly the `runEvals` configuration\n * (target, data items, scorers/gates/thresholds, etc.).\n */\nexport type ExpectEvalsOptions = RunEvalsConfig;\n\n/** A single eval data item, as accepted by `runEvals` in its `data` array. */\nexport type EvalDataItem = RunEvalsConfig['data'][number];\n\n/**\n * Options accepted by `expectEval`: same as `expectEvals`, but `data` is a single item.\n * Distributes over the config union so each target keeps its own data-item contract\n * (e.g. `turns` stays agent-only).\n */\nexport type ExpectEvalOptions = RunEvalsConfig extends infer T\n  ? T extends RunEvalsConfig\n    ? Omit<T, 'data'> & { data: T['data'][number] }\n    : never\n  : never;\n\n/** Thrown by `expectEval`/`expectEvals` `.toPass()` when the pass rate is not met. */\nexport class EvalPassRateError extends Error {\n  readonly result: RunEvalsResult;\n\n  constructor(message: string, result: RunEvalsResult) {\n    super(message);\n    this.name = 'EvalPassRateError';\n    this.result = result;\n  }\n}\n\n/** Fluent assertion returned by `expectEval` and `expectEvals`. */\nexport type EvalScoresAssertion = {\n  /**\n   * Runs the eval and asserts it passes.\n   *\n   * - Every gate's pass rate across data items must be `>= minPassRate`\n   *   (gates score 1 on pass and 0 on failure, so the averaged gate score\n   *   is exactly the fraction of items that passed the gate).\n   * - Every scorer threshold must pass (thresholds compare the average\n   *   score across items and are not relaxed by `minPassRate`).\n   *\n   * @param minPassRate Minimum per-gate pass rate in `[0, 1]`. Defaults to 1\n   *   (all items must pass every gate).\n   * @returns The full `RunEvalsResult` for further assertions.\n   */\n  toPass(minPassRate?: number): Promise<RunEvalsResult>;\n};\n\nasync function attachMetaToCurrentTest(result: RunEvalsResult): Promise<void> {\n  try {\n    const vitest = await import('vitest');\n    // Vitest >= 4.1\n    const task = (vitest as any).TestRunner?.getCurrentTest?.();\n    if (task?.meta) {\n      task.meta.mastraEval = toEvalMeta(result);\n      return;\n    }\n    // Vitest 3.x / < 4.1\n    const suite = await import('vitest/suite');\n    const legacyTask = (suite as any).getCurrentTest?.();\n    if (legacyTask?.meta) {\n      legacyTask.meta.mastraEval = toEvalMeta(result);\n    }\n  } catch {\n    // Outside a Vitest test context — assertion still works, reporter meta is skipped.\n  }\n}\n\nfunction assertPassRate(result: RunEvalsResult, minPassRate: number): void {\n  if (!Number.isFinite(minPassRate) || minPassRate < 0 || minPassRate > 1) {\n    throw new Error(`toPass(minPassRate): minPassRate must be within [0, 1], got ${minPassRate}`);\n  }\n\n  const lines: string[] = [];\n\n  for (const gate of result.gateResults ?? []) {\n    if (gate.score < minPassRate) {\n      lines.push(`  ✗ gate ${gate.id}: pass rate ${formatRate(gate.score)} < required ${formatRate(minPassRate)}`);\n    }\n  }\n\n  for (const t of result.thresholdResults ?? []) {\n    if (!t.passed) {\n      lines.push(`  ✗ threshold ${t.id}: average score ${t.averageScore} (threshold: ${JSON.stringify(t.threshold)})`);\n    }\n  }\n\n  for (const turn of result.turnResults ?? []) {\n    for (const gate of turn.gateResults ?? []) {\n      if (gate.score < minPassRate) {\n        lines.push(\n          `  ✗ turn ${turn.index} gate ${gate.id}: pass rate ${formatRate(gate.score)} < required ${formatRate(minPassRate)}`,\n        );\n      }\n    }\n    for (const t of turn.thresholdResults ?? []) {\n      if (!t.passed) {\n        lines.push(\n          `  ✗ turn ${turn.index} threshold ${t.id}: average score ${t.averageScore} (threshold: ${JSON.stringify(t.threshold)})`,\n        );\n      }\n    }\n  }\n\n  if (lines.length > 0) {\n    throw new EvalPassRateError(\n      [`Eval did not pass (required pass rate: ${formatRate(minPassRate)}).`, ...lines].join('\\n'),\n      result,\n    );\n  }\n}\n\nfunction formatRate(rate: number): string {\n  return `${Math.round(rate * 1000) / 10}%`;\n}\n\nfunction makeAssertion(options: ExpectEvalsOptions): EvalScoresAssertion {\n  return {\n    async toPass(minPassRate = 1): Promise<RunEvalsResult> {\n      const result = await runEvals(options as Parameters<typeof runEvals>[0]);\n      await attachMetaToCurrentTest(result);\n      assertPassRate(result, minPassRate);\n      return result;\n    },\n  };\n}\n\n/**\n * Fluent eval assertion over a dataset, for use inside a regular `test()`.\n *\n * Runs `runEvals` with the given options when awaited and asserts the outcome.\n * The run's scores are attached to the current test's meta so\n * `MastraEvalsReporter` displays them in the runner output.\n *\n * Must be awaited — otherwise the test finishes before the eval runs.\n *\n * @example\n * test('capitals agent answers with the expected city', async () => {\n *   await expectEvals({\n *     target: capitalsAgent,\n *     data: [{ input: 'What is the capital of France?', groundTruth: 'Paris' }],\n *     gates: [containsGroundTruth],\n *     scorers: [{ scorer: createKeywordCoverageScorer(), threshold: 0.4 }],\n *   }).toPass(0.8);\n * });\n */\nexport function expectEvals(options: ExpectEvalsOptions): EvalScoresAssertion {\n  return makeAssertion(options);\n}\n\n/**\n * Single-item variant of `expectEvals`: `data` is one item instead of an array.\n *\n * Use it with `test.each`/`it.each` for matrix testing when you want one test\n * (and one reporter entry) per data item instead of one per dataset run.\n *\n * @example\n * test.each([\n *   { input: 'What is the capital of France?', groundTruth: 'Paris' },\n *   { input: 'What is the capital of Japan?', groundTruth: 'Tokyo' },\n * ])('capitals agent: $input', async item => {\n *   await expectEval({\n *     target: capitalsAgent,\n *     data: item,\n *     gates: [containsGroundTruth],\n *   }).toPass();\n * });\n */\nexport function expectEval(options: ExpectEvalOptions): EvalScoresAssertion {\n  return makeAssertion({ ...options, data: [options.data] } as ExpectEvalsOptions);\n}\n","import type { ThresholdConfig } from '@mastra/core/evals';\nimport type { Reporter, TestCase, TestModule } from 'vitest/node';\n\nimport type { MastraEvalMeta } from './meta';\n\ntype EvalEntry = {\n  fullName: string;\n  meta: MastraEvalMeta;\n};\n\nconst GREEN = '\\u001b[32m';\nconst RED = '\\u001b[31m';\nconst YELLOW = '\\u001b[33m';\nconst BOLD = '\\u001b[1m';\nconst DIM = '\\u001b[2m';\nconst RESET = '\\u001b[0m';\n\nfunction supportsColor(): boolean {\n  return process.stdout.isTTY === true && process.env.NO_COLOR === undefined;\n}\n\n/**\n * Vitest reporter that prints a per-test score table for every eval run\n * using `expectEval`/`expectEvals` (or any test that populates `task.meta.mastraEval`).\n *\n * Usage in `vitest.config.ts`:\n *   test: { reporters: ['default', new MastraEvalsReporter()] }\n */\nexport class MastraEvalsReporter implements Reporter {\n  private readonly color: boolean;\n\n  constructor(options: { color?: boolean } = {}) {\n    this.color = options.color ?? supportsColor();\n  }\n\n  onTestRunEnd(testModules: ReadonlyArray<TestModule>): void {\n    const entries: EvalEntry[] = [];\n    for (const module of testModules) {\n      for (const test of module.children.allTests()) {\n        const meta = ((test as TestCase).meta() as { mastraEval?: MastraEvalMeta }).mastraEval;\n        if (meta) {\n          entries.push({ fullName: test.fullName, meta });\n        }\n      }\n    }\n\n    if (entries.length === 0) return;\n\n    this.print(this.render(entries));\n  }\n\n  /** Renders the score report as plain lines. Exposed for testing. */\n  render(entries: EvalEntry[]): string {\n    const lines: string[] = [];\n    lines.push('');\n    lines.push(this.style(BOLD, ' Mastra Evals '));\n\n    for (const { fullName, meta } of entries) {\n      lines.push('');\n      lines.push(\n        `${this.verdictBadge(meta.verdict)} ${this.style(BOLD, fullName)} ${this.style(DIM, `(${meta.totalItems} item${meta.totalItems === 1 ? '' : 's'})`)}`,\n      );\n\n      const rows: Array<[string, string, string]> = [];\n      for (const gate of meta.gateResults ?? []) {\n        rows.push([`${gate.id} ${this.style(DIM, '(gate)')}`, formatScore(gate.score), this.passMark(gate.passed)]);\n      }\n      for (const t of meta.thresholdResults ?? []) {\n        rows.push([\n          `${t.id} ${this.style(DIM, `(threshold: ${formatThreshold(t.threshold)})`)}`,\n          formatScore(t.averageScore),\n          this.passMark(t.passed),\n        ]);\n      }\n      const covered = new Set([\n        ...(meta.gateResults ?? []).map(g => g.id),\n        ...(meta.thresholdResults ?? []).map(t => t.id),\n      ]);\n      for (const [id, score] of Object.entries(meta.scores)) {\n        if (covered.has(id)) continue;\n        rows.push([id, formatScore(score), '']);\n      }\n\n      lines.push(...this.table(rows));\n\n      for (const turn of meta.turnResults ?? []) {\n        const turnRows: Array<[string, string, string]> = [];\n        for (const gate of turn.gateResults ?? []) {\n          turnRows.push([\n            `${gate.id} ${this.style(DIM, '(gate)')}`,\n            formatScore(gate.score),\n            this.passMark(gate.passed),\n          ]);\n        }\n        for (const t of turn.thresholdResults ?? []) {\n          turnRows.push([\n            `${t.id} ${this.style(DIM, `(threshold: ${formatThreshold(t.threshold)})`)}`,\n            formatScore(t.averageScore),\n            this.passMark(t.passed),\n          ]);\n        }\n        const turnCovered = new Set([\n          ...(turn.gateResults ?? []).map(g => g.id),\n          ...(turn.thresholdResults ?? []).map(t => t.id),\n        ]);\n        for (const [id, score] of Object.entries(turn.scores ?? {})) {\n          if (turnCovered.has(id)) continue;\n          turnRows.push([id, formatScore(score), '']);\n        }\n        if (turnRows.length === 0) continue;\n        lines.push(`  ${this.style(DIM, `turn ${turn.index}`)}`);\n        lines.push(...this.table(turnRows, '  '));\n      }\n    }\n\n    const withVerdict = entries.filter(e => e.meta.verdict !== undefined);\n    const passed = withVerdict.filter(e => e.meta.verdict === 'passed').length;\n    const failed = withVerdict.filter(e => e.meta.verdict === 'failed').length;\n    const scored = entries.length - passed - failed;\n    lines.push('');\n    const parts: string[] = [];\n    if (passed > 0) parts.push(this.style(GREEN, `${passed} passed`));\n    if (failed > 0) parts.push(this.style(RED, `${failed} failed`));\n    if (scored > 0) parts.push(this.style(YELLOW, `${scored} scored`));\n    lines.push(` Eval runs: ${entries.length} ${this.style(DIM, `(${parts.join(', ')})`)}`);\n    lines.push('');\n\n    return lines.join('\\n');\n  }\n\n  private table(rows: Array<[string, string, string]>, indent = ''): string[] {\n    if (rows.length === 0) return [];\n    const nameWidth = Math.max(...rows.map(([name]) => stripAnsi(name).length));\n    const scoreWidth = Math.max(5, ...rows.map(([, score]) => score.length));\n    return rows.map(([name, score, mark]) => {\n      const namePad = ' '.repeat(nameWidth - stripAnsi(name).length);\n      return `${indent}   ${name}${namePad}  ${score.padStart(scoreWidth)}  ${mark}`.trimEnd();\n    });\n  }\n\n  private verdictBadge(verdict: MastraEvalMeta['verdict']): string {\n    if (verdict === 'passed') return this.style(GREEN, '✓');\n    if (verdict === 'failed') return this.style(RED, '✗');\n    return this.style(YELLOW, '•');\n  }\n\n  private passMark(passed: boolean): string {\n    return passed ? this.style(GREEN, '✓') : this.style(RED, '✗');\n  }\n\n  private style(code: string, text: string): string {\n    return this.color ? `${code}${text}${RESET}` : text;\n  }\n\n  private print(output: string): void {\n    process.stdout.write(`${output}\\n`);\n  }\n}\n\nfunction formatScore(score: number): string {\n  return Number.isInteger(score) ? score.toFixed(1) : score.toFixed(3).replace(/0+$/, '').replace(/\\.$/, '.0');\n}\n\nfunction formatThreshold(threshold: ThresholdConfig): string {\n  if (typeof threshold === 'number') return `min ${threshold}`;\n  return Object.entries(threshold)\n    .map(([key, value]) => `${key} ${value}`)\n    .join(', ');\n}\n\n// eslint-disable-next-line no-control-regex\nconst ANSI_PATTERN = /\\u001b\\[[0-9;]*m/g;\n\nfunction stripAnsi(text: string): string {\n  return text.replace(ANSI_PATTERN, '');\n}\n"],"mappings":";;;;;;;;AAgDA,SAAS,cAAc,QAA6B,SAAS,IAAI,MAA8B,CAAC,GAAG;CACjG,KAAK,MAAM,CAAC,IAAI,UAAU,OAAO,QAAQ,MAAM,GAAG;EAChD,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,OAAO;EACzC,IAAI,OAAO,UAAU,UACnB,IAAI,OAAO;OACN,IAAI,SAAS,OAAO,UAAU,UACnC,cAAc,OAAO,KAAK,GAAG;CAEjC;CACA,OAAO;AACT;;;;;AAMA,SAAgB,WAAW,QAAwC;CACjE,MAAM,OAAuB;EAC3B,QAAQ,cAAc,OAAO,MAAM;EACnC,YAAY,OAAO,QAAQ;CAC7B;CACA,IAAI,OAAO,YAAY,KAAA,GAAW,KAAK,UAAU,OAAO;CACxD,IAAI,OAAO,aAAa,KAAK,cAAc,OAAO,YAAY,KAAI,OAAM,EAAE,GAAG,EAAE,EAAE;CACjF,IAAI,OAAO,kBAAkB,KAAK,mBAAmB,OAAO,iBAAiB,KAAI,OAAM,EAAE,GAAG,EAAE,EAAE;CAChG,IAAI,OAAO,aAAa,KAAK,cAAc,OAAO;CAClD,OAAO;AACT;;;;AChDA,IAAa,oBAAb,cAAuC,MAAM;CAC3C;CAEA,YAAY,SAAiB,QAAwB;EACnD,MAAM,OAAO;EACb,KAAK,OAAO;EACZ,KAAK,SAAS;CAChB;AACF;AAoBA,eAAe,wBAAwB,QAAuC;CAC5E,IAAI;EAGF,MAAM,QAAQ,MAFO,OAAO,UAAA,CAEC,YAAY,iBAAiB;EAC1D,IAAI,MAAM,MAAM;GACd,KAAK,KAAK,aAAa,WAAW,MAAM;GACxC;EACF;EAGA,MAAM,cAAc,MADA,OAAO,gBAAA,CACO,iBAAiB;EACnD,IAAI,YAAY,MACd,WAAW,KAAK,aAAa,WAAW,MAAM;CAElD,QAAQ,CAER;AACF;AAEA,SAAS,eAAe,QAAwB,aAA2B;CACzE,IAAI,CAAC,OAAO,SAAS,WAAW,KAAK,cAAc,KAAK,cAAc,GACpE,MAAM,IAAI,MAAM,+DAA+D,aAAa;CAG9F,MAAM,QAAkB,CAAC;CAEzB,KAAK,MAAM,QAAQ,OAAO,eAAe,CAAC,GACxC,IAAI,KAAK,QAAQ,aACf,MAAM,KAAK,YAAY,KAAK,GAAG,cAAc,WAAW,KAAK,KAAK,EAAE,cAAc,WAAW,WAAW,GAAG;CAI/G,KAAK,MAAM,KAAK,OAAO,oBAAoB,CAAC,GAC1C,IAAI,CAAC,EAAE,QACL,MAAM,KAAK,iBAAiB,EAAE,GAAG,kBAAkB,EAAE,aAAa,eAAe,KAAK,UAAU,EAAE,SAAS,EAAE,EAAE;CAInH,KAAK,MAAM,QAAQ,OAAO,eAAe,CAAC,GAAG;EAC3C,KAAK,MAAM,QAAQ,KAAK,eAAe,CAAC,GACtC,IAAI,KAAK,QAAQ,aACf,MAAM,KACJ,YAAY,KAAK,MAAM,QAAQ,KAAK,GAAG,cAAc,WAAW,KAAK,KAAK,EAAE,cAAc,WAAW,WAAW,GAClH;EAGJ,KAAK,MAAM,KAAK,KAAK,oBAAoB,CAAC,GACxC,IAAI,CAAC,EAAE,QACL,MAAM,KACJ,YAAY,KAAK,MAAM,aAAa,EAAE,GAAG,kBAAkB,EAAE,aAAa,eAAe,KAAK,UAAU,EAAE,SAAS,EAAE,EACvH;CAGN;CAEA,IAAI,MAAM,SAAS,GACjB,MAAM,IAAI,kBACR,CAAC,0CAA0C,WAAW,WAAW,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,IAAI,GAC3F,MACF;AAEJ;AAEA,SAAS,WAAW,MAAsB;CACxC,OAAO,GAAG,KAAK,MAAM,OAAO,GAAI,IAAI,GAAG;AACzC;AAEA,SAAS,cAAc,SAAkD;CACvE,OAAO,EACL,MAAM,OAAO,cAAc,GAA4B;EACrD,MAAM,SAAS,MAAM,SAAS,OAAyC;EACvE,MAAM,wBAAwB,MAAM;EACpC,eAAe,QAAQ,WAAW;EAClC,OAAO;CACT,EACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,YAAY,SAAkD;CAC5E,OAAO,cAAc,OAAO;AAC9B;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,WAAW,SAAiD;CAC1E,OAAO,cAAc;EAAE,GAAG;EAAS,MAAM,CAAC,QAAQ,IAAI;CAAE,CAAuB;AACjF;;;ACtKA,MAAM,QAAQ;AACd,MAAM,MAAM;AACZ,MAAM,SAAS;AACf,MAAM,OAAO;AACb,MAAM,MAAM;AACZ,MAAM,QAAQ;AAEd,SAAS,gBAAyB;CAChC,OAAO,QAAQ,OAAO,UAAU,QAAQ,QAAQ,IAAI,aAAa,KAAA;AACnE;;;;;;;;AASA,IAAa,sBAAb,MAAqD;CACnD;CAEA,YAAY,UAA+B,CAAC,GAAG;EAC7C,KAAK,QAAQ,QAAQ,SAAS,cAAc;CAC9C;CAEA,aAAa,aAA8C;EACzD,MAAM,UAAuB,CAAC;EAC9B,KAAK,MAAM,UAAU,aACnB,KAAK,MAAM,QAAQ,OAAO,SAAS,SAAS,GAAG;GAC7C,MAAM,OAAS,KAAkB,KAAK,CAAC,CAAqC;GAC5E,IAAI,MACF,QAAQ,KAAK;IAAE,UAAU,KAAK;IAAU;GAAK,CAAC;EAElD;EAGF,IAAI,QAAQ,WAAW,GAAG;EAE1B,KAAK,MAAM,KAAK,OAAO,OAAO,CAAC;CACjC;;CAGA,OAAO,SAA8B;EACnC,MAAM,QAAkB,CAAC;EACzB,MAAM,KAAK,EAAE;EACb,MAAM,KAAK,KAAK,MAAM,MAAM,gBAAgB,CAAC;EAE7C,KAAK,MAAM,EAAE,UAAU,UAAU,SAAS;GACxC,MAAM,KAAK,EAAE;GACb,MAAM,KACJ,GAAG,KAAK,aAAa,KAAK,OAAO,EAAE,GAAG,KAAK,MAAM,MAAM,QAAQ,EAAE,GAAG,KAAK,MAAM,KAAK,IAAI,KAAK,WAAW,OAAO,KAAK,eAAe,IAAI,KAAK,IAAI,EAAE,GACpJ;GAEA,MAAM,OAAwC,CAAC;GAC/C,KAAK,MAAM,QAAQ,KAAK,eAAe,CAAC,GACtC,KAAK,KAAK;IAAC,GAAG,KAAK,GAAG,GAAG,KAAK,MAAM,KAAK,QAAQ;IAAK,YAAY,KAAK,KAAK;IAAG,KAAK,SAAS,KAAK,MAAM;GAAC,CAAC;GAE5G,KAAK,MAAM,KAAK,KAAK,oBAAoB,CAAC,GACxC,KAAK,KAAK;IACR,GAAG,EAAE,GAAG,GAAG,KAAK,MAAM,KAAK,eAAe,gBAAgB,EAAE,SAAS,EAAE,EAAE;IACzE,YAAY,EAAE,YAAY;IAC1B,KAAK,SAAS,EAAE,MAAM;GACxB,CAAC;GAEH,MAAM,0BAAU,IAAI,IAAI,CACtB,IAAI,KAAK,eAAe,CAAC,EAAA,CAAG,KAAI,MAAK,EAAE,EAAE,GACzC,IAAI,KAAK,oBAAoB,CAAC,EAAA,CAAG,KAAI,MAAK,EAAE,EAAE,CAChD,CAAC;GACD,KAAK,MAAM,CAAC,IAAI,UAAU,OAAO,QAAQ,KAAK,MAAM,GAAG;IACrD,IAAI,QAAQ,IAAI,EAAE,GAAG;IACrB,KAAK,KAAK;KAAC;KAAI,YAAY,KAAK;KAAG;IAAE,CAAC;GACxC;GAEA,MAAM,KAAK,GAAG,KAAK,MAAM,IAAI,CAAC;GAE9B,KAAK,MAAM,QAAQ,KAAK,eAAe,CAAC,GAAG;IACzC,MAAM,WAA4C,CAAC;IACnD,KAAK,MAAM,QAAQ,KAAK,eAAe,CAAC,GACtC,SAAS,KAAK;KACZ,GAAG,KAAK,GAAG,GAAG,KAAK,MAAM,KAAK,QAAQ;KACtC,YAAY,KAAK,KAAK;KACtB,KAAK,SAAS,KAAK,MAAM;IAC3B,CAAC;IAEH,KAAK,MAAM,KAAK,KAAK,oBAAoB,CAAC,GACxC,SAAS,KAAK;KACZ,GAAG,EAAE,GAAG,GAAG,KAAK,MAAM,KAAK,eAAe,gBAAgB,EAAE,SAAS,EAAE,EAAE;KACzE,YAAY,EAAE,YAAY;KAC1B,KAAK,SAAS,EAAE,MAAM;IACxB,CAAC;IAEH,MAAM,8BAAc,IAAI,IAAI,CAC1B,IAAI,KAAK,eAAe,CAAC,EAAA,CAAG,KAAI,MAAK,EAAE,EAAE,GACzC,IAAI,KAAK,oBAAoB,CAAC,EAAA,CAAG,KAAI,MAAK,EAAE,EAAE,CAChD,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,UAAU,OAAO,QAAQ,KAAK,UAAU,CAAC,CAAC,GAAG;KAC3D,IAAI,YAAY,IAAI,EAAE,GAAG;KACzB,SAAS,KAAK;MAAC;MAAI,YAAY,KAAK;MAAG;KAAE,CAAC;IAC5C;IACA,IAAI,SAAS,WAAW,GAAG;IAC3B,MAAM,KAAK,KAAK,KAAK,MAAM,KAAK,QAAQ,KAAK,OAAO,GAAG;IACvD,MAAM,KAAK,GAAG,KAAK,MAAM,UAAU,IAAI,CAAC;GAC1C;EACF;EAEA,MAAM,cAAc,QAAQ,QAAO,MAAK,EAAE,KAAK,YAAY,KAAA,CAAS;EACpE,MAAM,SAAS,YAAY,QAAO,MAAK,EAAE,KAAK,YAAY,QAAQ,CAAC,CAAC;EACpE,MAAM,SAAS,YAAY,QAAO,MAAK,EAAE,KAAK,YAAY,QAAQ,CAAC,CAAC;EACpE,MAAM,SAAS,QAAQ,SAAS,SAAS;EACzC,MAAM,KAAK,EAAE;EACb,MAAM,QAAkB,CAAC;EACzB,IAAI,SAAS,GAAG,MAAM,KAAK,KAAK,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC;EAChE,IAAI,SAAS,GAAG,MAAM,KAAK,KAAK,MAAM,KAAK,GAAG,OAAO,QAAQ,CAAC;EAC9D,IAAI,SAAS,GAAG,MAAM,KAAK,KAAK,MAAM,QAAQ,GAAG,OAAO,QAAQ,CAAC;EACjE,MAAM,KAAK,eAAe,QAAQ,OAAO,GAAG,KAAK,MAAM,KAAK,IAAI,MAAM,KAAK,IAAI,EAAE,EAAE,GAAG;EACtF,MAAM,KAAK,EAAE;EAEb,OAAO,MAAM,KAAK,IAAI;CACxB;CAEA,MAAc,MAAuC,SAAS,IAAc;EAC1E,IAAI,KAAK,WAAW,GAAG,OAAO,CAAC;EAC/B,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,KAAK,CAAC,UAAU,UAAU,IAAI,CAAC,CAAC,MAAM,CAAC;EAC1E,MAAM,aAAa,KAAK,IAAI,GAAG,GAAG,KAAK,KAAK,GAAG,WAAW,MAAM,MAAM,CAAC;EACvE,OAAO,KAAK,KAAK,CAAC,MAAM,OAAO,UAAU;GAEvC,OAAO,GAAG,OAAO,KAAK,OADN,IAAI,OAAO,YAAY,UAAU,IAAI,CAAC,CAAC,MACpB,EAAE,IAAI,MAAM,SAAS,UAAU,EAAE,IAAI,OAAO,QAAQ;EACzF,CAAC;CACH;CAEA,aAAqB,SAA4C;EAC/D,IAAI,YAAY,UAAU,OAAO,KAAK,MAAM,OAAO,GAAG;EACtD,IAAI,YAAY,UAAU,OAAO,KAAK,MAAM,KAAK,GAAG;EACpD,OAAO,KAAK,MAAM,QAAQ,GAAG;CAC/B;CAEA,SAAiB,QAAyB;EACxC,OAAO,SAAS,KAAK,MAAM,OAAO,GAAG,IAAI,KAAK,MAAM,KAAK,GAAG;CAC9D;CAEA,MAAc,MAAc,MAAsB;EAChD,OAAO,KAAK,QAAQ,GAAG,OAAO,OAAO,UAAU;CACjD;CAEA,MAAc,QAAsB;EAClC,QAAQ,OAAO,MAAM,GAAG,OAAO,GAAG;CACpC;AACF;AAEA,SAAS,YAAY,OAAuB;CAC1C,OAAO,OAAO,UAAU,KAAK,IAAI,MAAM,QAAQ,CAAC,IAAI,MAAM,QAAQ,CAAC,CAAC,CAAC,QAAQ,OAAO,EAAE,CAAC,CAAC,QAAQ,OAAO,IAAI;AAC7G;AAEA,SAAS,gBAAgB,WAAoC;CAC3D,IAAI,OAAO,cAAc,UAAU,OAAO,OAAO;CACjD,OAAO,OAAO,QAAQ,SAAS,CAAC,CAC7B,KAAK,CAAC,KAAK,WAAW,GAAG,IAAI,GAAG,OAAO,CAAC,CACxC,KAAK,IAAI;AACd;AAGA,MAAM,eAAe;AAErB,SAAS,UAAU,MAAsB;CACvC,OAAO,KAAK,QAAQ,cAAc,EAAE;AACtC"}

package/dist/vitest/matchers.d.ts

@@ -0,0 +1,49 @@
+import type { EvalVerdict } from '@mastra/core/evals';
+/**
+ * Custom Vitest matchers operating on a `RunEvalsResult` (the return value of
+ * `runEvals`). Registered via `registerEvalMatchers()` or by adding
+ * `@mastra/evals/vitest/setup` to `setupFiles`.
+ */
+export interface EvalMatchers<R = unknown> {
+    /** Asserts the run's verdict (`passed`, `scored`, or `failed`). */
+    toHaveVerdict(verdict: EvalVerdict): R;
+    /** Asserts the average score of a scorer is strictly above `min`. Supports dot-paths for categorized configs (e.g. `agent.my-scorer`). */
+    toHaveScoreAbove(scorerName: string, min: number): R;
+    /** Asserts the average score of a scorer is strictly below `max`. Supports dot-paths for categorized configs (e.g. `agent.my-scorer`). */
+    toHaveScoreBelow(scorerName: string, max: number): R;
+    /** Asserts that all gates passed (fails when no gates were configured). */
+    toPassGates(): R;
+    /** Asserts that all scorer thresholds passed (fails when no thresholds were configured). */
+    toPassThresholds(): R;
+}
+declare module 'vitest' {
+    interface Assertion<T = any> extends EvalMatchers<T> {
+    }
+    interface AsymmetricMatchersContaining extends EvalMatchers {
+    }
+}
+export declare const evalMatchers: {
+    toHaveVerdict(received: unknown, verdict: EvalVerdict): {
+        pass: boolean;
+        message: () => string;
+    };
+    toHaveScoreAbove(received: unknown, scorerName: string, min: number): {
+        pass: boolean;
+        message: () => string;
+    };
+    toHaveScoreBelow(received: unknown, scorerName: string, max: number): {
+        pass: boolean;
+        message: () => string;
+    };
+    toPassGates(received: unknown): {
+        pass: boolean;
+        message: () => string;
+    };
+    toPassThresholds(received: unknown): {
+        pass: boolean;
+        message: () => string;
+    };
+};
+/** Registers the Mastra eval matchers on Vitest's `expect`. */
+export declare function registerEvalMatchers(): void;
+//# sourceMappingURL=matchers.d.ts.map

package/dist/vitest/matchers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/vitest/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAkB,MAAM,oBAAoB,CAAC;AAGtE;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,OAAO;IACvC,mEAAmE;IACnE,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,CAAC,CAAC;IACvC,0IAA0I;IAC1I,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,CAAC;IACrD,0IAA0I;IAC1I,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,CAAC;IACrD,2EAA2E;IAC3E,WAAW,IAAI,CAAC,CAAC;IACjB,4FAA4F;IAC5F,gBAAgB,IAAI,CAAC,CAAC;CACvB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,SAAS,CAAC,CAAC,GAAG,GAAG,CAAE,SAAQ,YAAY,CAAC,CAAC,CAAC;KAAG;IACvD,UAAU,4BAA6B,SAAQ,YAAY;KAAG;CAC/D;AAuCD,eAAO,MAAM,YAAY;IACvB,aAAa,WAAW,OAAO,WAAW,WAAW;;;;IAYrD,gBAAgB,WAAW,OAAO,cAAc,MAAM,OAAO,MAAM;;;;IAmBnE,gBAAgB,WAAW,OAAO,cAAc,MAAM,OAAO,MAAM;;;;IAmBnE,WAAW,WAAW,OAAO;;;;IAwB7B,gBAAgB,WAAW,OAAO;;;;CAyBnC,CAAC;AAEF,+DAA+D;AAC/D,wBAAgB,oBAAoB,SAEnC"}

package/dist/vitest/meta.d.ts

@@ -0,0 +1,45 @@
+import type { EvalVerdict, GateResult, RunEvalsResult, ThresholdConfig, TurnResult } from '@mastra/core/evals';
+/** Per-threshold-scorer result, as reported in `RunEvalsResult.thresholdResults`. */
+export type EvalThresholdResult = {
+    id: string;
+    passed: boolean;
+    averageScore: number;
+    threshold: ThresholdConfig;
+};
+/**
+ * Serializable projection of a `RunEvalsResult` stored in `task.meta.mastraEval`.
+ *
+ * This is the contract between the test worker (which runs `runEvals`) and the
+ * `MastraEvalsReporter` running in the main process. It intentionally excludes
+ * raw agent/workflow payloads: `task.meta` is structured-cloned from worker to
+ * main, so only lightweight, serializable data belongs here.
+ */
+export type MastraEvalMeta = {
+    /**
+     * Average score per scorer id across all data items. Categorized scorer
+     * configs are flattened into dot-separated keys (e.g. `agent.my-scorer`,
+     * `steps.step-1.my-scorer`).
+     */
+    scores: Record<string, number>;
+    /** Number of data items evaluated. */
+    totalItems: number;
+    /** Present when gates or threshold-bearing scorers were provided. */
+    verdict?: EvalVerdict;
+    /** Per-gate results (averaged across all data items). */
+    gateResults?: GateResult[];
+    /** Per-threshold-scorer results (averaged across all data items). */
+    thresholdResults?: EvalThresholdResult[];
+    /** Per-turn assertion results, present when data items use `turns`. */
+    turnResults?: TurnResult[];
+};
+declare module 'vitest' {
+    interface TaskMeta {
+        mastraEval?: MastraEvalMeta;
+    }
+}
+/**
+ * Projects a `RunEvalsResult` into the serializable `MastraEvalMeta` shape
+ * stored in `task.meta.mastraEval`.
+ */
+export declare function toEvalMeta(result: RunEvalsResult): MastraEvalMeta;
+//# sourceMappingURL=meta.d.ts.map

package/dist/vitest/meta.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"meta.d.ts","sourceRoot":"","sources":["../../src/vitest/meta.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE/G,qFAAqF;AACrF,MAAM,MAAM,mBAAmB,GAAG;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,eAAe,CAAC;CAC5B,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B;;;;OAIG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,yDAAyD;IACzD,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACzC,uEAAuE;IACvE,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;CAC5B,CAAC;AAEF,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,QAAQ;QAChB,UAAU,CAAC,EAAE,cAAc,CAAC;KAC7B;CACF;AAmBD;;;GAGG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,CAUjE"}

package/dist/vitest/reporter.d.ts

@@ -0,0 +1,29 @@
+import type { Reporter, TestModule } from 'vitest/node';
+import type { MastraEvalMeta } from './meta.js';
+type EvalEntry = {
+    fullName: string;
+    meta: MastraEvalMeta;
+};
+/**
+ * Vitest reporter that prints a per-test score table for every eval run
+ * using `expectEval`/`expectEvals` (or any test that populates `task.meta.mastraEval`).
+ *
+ * Usage in `vitest.config.ts`:
+ *   test: { reporters: ['default', new MastraEvalsReporter()] }
+ */
+export declare class MastraEvalsReporter implements Reporter {
+    private readonly color;
+    constructor(options?: {
+        color?: boolean;
+    });
+    onTestRunEnd(testModules: ReadonlyArray<TestModule>): void;
+    /** Renders the score report as plain lines. Exposed for testing. */
+    render(entries: EvalEntry[]): string;
+    private table;
+    private verdictBadge;
+    private passMark;
+    private style;
+    private print;
+}
+export {};
+//# sourceMappingURL=reporter.d.ts.map

package/dist/vitest/reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/vitest/reporter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAY,UAAU,EAAE,MAAM,aAAa,CAAC;AAElE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAE7C,KAAK,SAAS,GAAG;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,cAAc,CAAC;CACtB,CAAC;AAaF;;;;;;GAMG;AACH,qBAAa,mBAAoB,YAAW,QAAQ;IAClD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAEhC,YAAY,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAO,EAE5C;IAED,YAAY,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,IAAI,CAczD;IAED,oEAAoE;IACpE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,CA4EnC;IAED,OAAO,CAAC,KAAK;IAUb,OAAO,CAAC,YAAY;IAMpB,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,KAAK;IAIb,OAAO,CAAC,KAAK;CAGd"}

package/dist/vitest/setup.cjs

@@ -0,0 +1,11 @@
+//#region src/vitest/setup.ts
+/**
+* Vitest setup file that auto-registers the Mastra eval matchers.
+*
+* Usage in `vitest.config.ts`:
+*   test: { setupFiles: ['@mastra/evals/vitest/setup'] }
+*/
+require("../matchers-DzWDR4_h.cjs").registerEvalMatchers();
+//#endregion
+
+//# sourceMappingURL=setup.cjs.map

package/dist/vitest/setup.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"setup.cjs","names":["registerEvalMatchers"],"sources":["../../src/vitest/setup.ts"],"sourcesContent":["/**\n * Vitest setup file that auto-registers the Mastra eval matchers.\n *\n * Usage in `vitest.config.ts`:\n *   test: { setupFiles: ['@mastra/evals/vitest/setup'] }\n */\nimport { registerEvalMatchers } from './matchers';\n\nregisterEvalMatchers();\n"],"mappings":";;;;;;;kCAQAA,CAAAA,CAAAA,qBAAqB"}

package/dist/vitest/setup.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=setup.d.ts.map

package/dist/vitest/setup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../../src/vitest/setup.ts"],"names":[],"mappings":""}

package/dist/vitest/setup.js

@@ -0,0 +1,13 @@
+import { n as registerEvalMatchers } from "../matchers-D3c6f71B.js";
+//#region src/vitest/setup.ts
+/**
+* Vitest setup file that auto-registers the Mastra eval matchers.
+*
+* Usage in `vitest.config.ts`:
+*   test: { setupFiles: ['@mastra/evals/vitest/setup'] }
+*/
+registerEvalMatchers();
+//#endregion
+export {};
+
+//# sourceMappingURL=setup.js.map

package/dist/vitest/setup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"setup.js","names":[],"sources":["../../src/vitest/setup.ts"],"sourcesContent":["/**\n * Vitest setup file that auto-registers the Mastra eval matchers.\n *\n * Usage in `vitest.config.ts`:\n *   test: { setupFiles: ['@mastra/evals/vitest/setup'] }\n */\nimport { registerEvalMatchers } from './matchers';\n\nregisterEvalMatchers();\n"],"mappings":";;;;;;;;AAQA,qBAAqB"}

package/package.json

@@ -1,11 +1,10 @@
 {
   "name": "@mastra/evals",
-  "version": "1.9.0",
+  "version": "1.10.0-alpha.1",
   "description": "",
   "type": "module",
   "files": [
-    "dist",
-    "CHANGELOG.md"
+    "dist"
   ],
   "homepage": "https://mastra.ai",
   "repository": {
@@ -59,6 +58,26 @@
         "default": "./dist/checks.cjs"
       }
     },
+    "./vitest": {
+      "import": {
+        "types": "./dist/vitest/index.d.ts",
+        "default": "./dist/vitest/index.js"
+      },
+      "require": {
+        "types": "./dist/vitest/index.d.ts",
+        "default": "./dist/vitest/index.cjs"
+      }
+    },
+    "./vitest/setup": {
+      "import": {
+        "types": "./dist/vitest/setup.d.ts",
+        "default": "./dist/vitest/setup.js"
+      },
+      "require": {
+        "types": "./dist/vitest/setup.d.ts",
+        "default": "./dist/vitest/setup.cjs"
+      }
+    },
     "./package.json": "./package.json"
   },
   "keywords": [],
@@ -71,7 +90,13 @@
     "string-similarity": "^4.0.4"
   },
   "peerDependencies": {
-    "@mastra/core": ">=1.0.0-0 <2.0.0-0"
+    "@mastra/core": ">=1.0.0-0 <2.0.0-0",
+    "vitest": ">=3.0.0 <5.0.0"
+  },
+  "peerDependenciesMeta": {
+    "vitest": {
+      "optional": true
+    }
   },
   "devDependencies": {
     "@ai-sdk/openai": "^1.3.24",
@@ -85,15 +110,15 @@
     "eslint": "^10.7.0",
     "tsdown": "0.22.9",
     "tsx": "^4.23.1",
-    "typescript": "^6.0.3",
+    "typescript": "^7.0.2",
     "vitest": "4.1.10",
     "zod": "^4.4.3",
-    "@internal/ai-sdk-v5": "0.0.72",
-    "@internal/lint": "0.0.125",
-    "@internal/llm-recorder": "0.0.61",
-    "@internal/test-utils": "0.0.61",
-    "@internal/types-builder": "0.0.100",
-    "@mastra/core": "1.61.0"
+    "@internal/ai-sdk-v5": "0.0.76",
+    "@internal/test-utils": "0.0.65",
+    "@internal/lint": "0.0.129",
+    "@internal/llm-recorder": "0.0.65",
+    "@internal/types-builder": "0.0.104",
+    "@mastra/core": "1.64.0-alpha.7"
   },
   "engines": {
     "node": ">=22.13.0"