@mastra/evals 1.6.0 → 1.7.0

package/CHANGELOG.md

@@ -1,5 +1,63 @@
 # @mastra/evals
 
+## 1.7.0
+
+### Minor Changes
+
+- Added a `summarization` scorer to `@mastra/evals`. It grades a summary on two axes and returns the lower score, so a summary cannot pass by being faithful but empty, or thorough but wrong. ([#20293](https://github.com/mastra-ai/mastra/pull/20293))
+
+  **Alignment** checks that every claim in the summary is supported by the source text. **Coverage** draws closed-ended questions from the source and answers them using the summary alone, in a separate call that never receives the source, so a missing fact cannot be answered from the source instead. The final score is `min(alignment, coverage) × scale`, and the reason names the axis that produced it.
+
+  The source text defaults to the user message of the run input. Pass `source` or `sourceExtractor` when the text being summarized comes from somewhere else, such as a tool result. `maxQuestions` bounds the coverage questions so cost does not grow with document length.
+
+  ```ts
+  import { createSummarizationScorer } from '@mastra/evals/scorers/prebuilt';
+
+  const scorer = createSummarizationScorer({
+    model: 'openai/gpt-5.5',
+    options: { maxQuestions: 10 },
+  });
+
+  const result = await scorer.run(run);
+  result.score;
+  ```
+
+  This restores the summarization metric that was removed with the legacy evals system, rebuilt on the scorers pipeline.
+
+### Patch Changes
+
+- Updated dependencies [[`4844167`](https://github.com/mastra-ai/mastra/commit/4844167cff2d5ec5004e94edd34970833040fa3f), [`c5e56ff`](https://github.com/mastra-ai/mastra/commit/c5e56ff3bcabdf062708f2d48744fec304df6792), [`594f7b2`](https://github.com/mastra-ai/mastra/commit/594f7b28f5263fb9982fd50d95c471fb971ea984), [`7f4e26d`](https://github.com/mastra-ai/mastra/commit/7f4e26dd57bd9b23c278ea21235ab823a3810a6c), [`311f943`](https://github.com/mastra-ai/mastra/commit/311f943bee60e8fdf5c84499ea50e884276c936c), [`322daa6`](https://github.com/mastra-ai/mastra/commit/322daa6d90552909204044790d850958f6745fed), [`db4e6ff`](https://github.com/mastra-ai/mastra/commit/db4e6ff744503112eb64deeaf6c2b54bf26a54c7), [`5faf93f`](https://github.com/mastra-ai/mastra/commit/5faf93f03e19daea394b9e2a923f2e4f833407f2), [`82201f7`](https://github.com/mastra-ai/mastra/commit/82201f75fae8e050a8de2df08b74875ee74c6b83), [`cadaa13`](https://github.com/mastra-ai/mastra/commit/cadaa1372e1077c8e85eb64c5499ba8803caa323), [`0c89896`](https://github.com/mastra-ai/mastra/commit/0c8989673fb7d106837098398131e570c6023b68), [`6d19a65`](https://github.com/mastra-ai/mastra/commit/6d19a6517f5da3911023d446b7e2d5dad8adb1cb), [`23b4238`](https://github.com/mastra-ai/mastra/commit/23b423844ad0bcf2a502a68dd62866d6160f9f6d), [`80ad891`](https://github.com/mastra-ai/mastra/commit/80ad891f8cd10379aa5b5af7510c763783b2ab56), [`fb18da5`](https://github.com/mastra-ai/mastra/commit/fb18da56fc35689ae370621a8f10b5b0d8606e20), [`fb18da5`](https://github.com/mastra-ai/mastra/commit/fb18da56fc35689ae370621a8f10b5b0d8606e20), [`e320a76`](https://github.com/mastra-ai/mastra/commit/e320a763feaf65c6be3cebecf746defcbde161b3), [`03b4918`](https://github.com/mastra-ai/mastra/commit/03b4918c80d188ce375334c393e131c6e94bd7eb), [`14ef73a`](https://github.com/mastra-ai/mastra/commit/14ef73a4bbd73e7808414816eb0628ce1d80b5d7), [`b582f7f`](https://github.com/mastra-ai/mastra/commit/b582f7fa2f9c1f87d19efc63d344fbe5dda2608c), [`0a6598b`](https://github.com/mastra-ai/mastra/commit/0a6598bde80bde008986ad6616bed9632b9294cb), [`06000d7`](https://github.com/mastra-ai/mastra/commit/06000d73712911572e913b8a83339270296d0a22), [`1d677d5`](https://github.com/mastra-ai/mastra/commit/1d677d5f99d7db403f7828585e8c25f299f72628), [`9e1dad8`](https://github.com/mastra-ai/mastra/commit/9e1dad8f7b1cab2bb7ade90e5b7561f24577b88a), [`2f43145`](https://github.com/mastra-ai/mastra/commit/2f4314504c03cbba280414ac81ba3197448ee6b0), [`4e35a56`](https://github.com/mastra-ai/mastra/commit/4e35a56cdf8d74a5ff6d5eda01f2c1deaf6cc7be), [`d94b8e1`](https://github.com/mastra-ai/mastra/commit/d94b8e1cee67416d518a8c30099040061bef6a1c), [`93e28ec`](https://github.com/mastra-ai/mastra/commit/93e28ecce9031c02397e0ae8406593e5c7a95883), [`729dab4`](https://github.com/mastra-ai/mastra/commit/729dab408faccfaef0cbb048e5a4338f9172847e), [`484003d`](https://github.com/mastra-ai/mastra/commit/484003d33ff59330c86b19863e4a38732d7e4155), [`3de0188`](https://github.com/mastra-ai/mastra/commit/3de0188bfaf9a9c09c95fe322b53838cf52c70b6), [`34d34d8`](https://github.com/mastra-ai/mastra/commit/34d34d8c811df512fef4dd5459f79b7821be1866), [`b582f7f`](https://github.com/mastra-ai/mastra/commit/b582f7fa2f9c1f87d19efc63d344fbe5dda2608c), [`933d291`](https://github.com/mastra-ai/mastra/commit/933d291146b789c19442ad206f94da3e4be90c64), [`a1cb98d`](https://github.com/mastra-ai/mastra/commit/a1cb98d11990b560b98482292a1f34aa1a2d9092), [`598ad82`](https://github.com/mastra-ai/mastra/commit/598ad82d41c41389a686338a1d0e50b7400e1938), [`1fd6aad`](https://github.com/mastra-ai/mastra/commit/1fd6aad1ea4a9d32f65efa832307c35e981a4c0a)]:
+  - @mastra/core@1.56.0
+
+## 1.7.0-alpha.0
+
+### Minor Changes
+
+- Added a `summarization` scorer to `@mastra/evals`. It grades a summary on two axes and returns the lower score, so a summary cannot pass by being faithful but empty, or thorough but wrong. ([#20293](https://github.com/mastra-ai/mastra/pull/20293))
+
+  **Alignment** checks that every claim in the summary is supported by the source text. **Coverage** draws closed-ended questions from the source and answers them using the summary alone, in a separate call that never receives the source, so a missing fact cannot be answered from the source instead. The final score is `min(alignment, coverage) × scale`, and the reason names the axis that produced it.
+
+  The source text defaults to the user message of the run input. Pass `source` or `sourceExtractor` when the text being summarized comes from somewhere else, such as a tool result. `maxQuestions` bounds the coverage questions so cost does not grow with document length.
+
+  ```ts
+  import { createSummarizationScorer } from '@mastra/evals/scorers/prebuilt';
+
+  const scorer = createSummarizationScorer({
+    model: 'openai/gpt-5.5',
+    options: { maxQuestions: 10 },
+  });
+
+  const result = await scorer.run(run);
+  result.score;
+  ```
+
+  This restores the summarization metric that was removed with the legacy evals system, rebuilt on the scorers pipeline.
+
+### Patch Changes
+
+- Updated dependencies [[`594f7b2`](https://github.com/mastra-ai/mastra/commit/594f7b28f5263fb9982fd50d95c471fb971ea984), [`311f943`](https://github.com/mastra-ai/mastra/commit/311f943bee60e8fdf5c84499ea50e884276c936c), [`0c89896`](https://github.com/mastra-ai/mastra/commit/0c8989673fb7d106837098398131e570c6023b68), [`23b4238`](https://github.com/mastra-ai/mastra/commit/23b423844ad0bcf2a502a68dd62866d6160f9f6d), [`e320a76`](https://github.com/mastra-ai/mastra/commit/e320a763feaf65c6be3cebecf746defcbde161b3), [`03b4918`](https://github.com/mastra-ai/mastra/commit/03b4918c80d188ce375334c393e131c6e94bd7eb), [`14ef73a`](https://github.com/mastra-ai/mastra/commit/14ef73a4bbd73e7808414816eb0628ce1d80b5d7), [`1d677d5`](https://github.com/mastra-ai/mastra/commit/1d677d5f99d7db403f7828585e8c25f299f72628), [`93e28ec`](https://github.com/mastra-ai/mastra/commit/93e28ecce9031c02397e0ae8406593e5c7a95883), [`729dab4`](https://github.com/mastra-ai/mastra/commit/729dab408faccfaef0cbb048e5a4338f9172847e), [`484003d`](https://github.com/mastra-ai/mastra/commit/484003d33ff59330c86b19863e4a38732d7e4155), [`933d291`](https://github.com/mastra-ai/mastra/commit/933d291146b789c19442ad206f94da3e4be90c64)]:
+  - @mastra/core@1.56.0-alpha.3
+
 ## 1.6.0
 
 ### Minor Changes

package/dist/checks-64AonnEK.js

@@ -0,0 +1,379 @@
+import { extractToolCalls, getTextContentFromMastraDBMessage } from "./scorers/utils.js";
+import { createScorer } from "@mastra/core/evals";
+import stringSimilarity from "string-similarity";
+//#region src/scorers/code/checks/index.ts
+/**
+* Scores 1 if the agent's output text contains the expected substring, 0 otherwise.
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.includes('sunny');
+* ```
+*/
+function includes(expected, options = {}) {
+	const { ignoreCase = true } = options;
+	return createScorer({
+		id: "check-includes",
+		name: "Includes Check",
+		description: `Checks if output includes "${expected}"`,
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		let output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join(" ");
+		let target = expected;
+		if (ignoreCase) {
+			output = output.toLowerCase();
+			target = target.toLowerCase();
+		}
+		return {
+			output,
+			target,
+			found: output.includes(target)
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.found ? 1 : 0;
+	});
+}
+/**
+* Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.excludes('error');
+* ```
+*/
+function excludes(unwanted, options = {}) {
+	const { ignoreCase = true } = options;
+	return createScorer({
+		id: "check-excludes",
+		name: "Excludes Check",
+		description: `Checks that output does not include "${unwanted}"`,
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		let output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join(" ");
+		let target = unwanted;
+		if (ignoreCase) {
+			output = output.toLowerCase();
+			target = target.toLowerCase();
+		}
+		return {
+			output,
+			target,
+			excluded: !output.includes(target)
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.excluded ? 1 : 0;
+	});
+}
+/**
+* Scores 1 if the output text exactly equals the expected string (after optional normalization).
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.equals('Hello, world!');
+* ```
+*/
+function equals(expected, options = {}) {
+	const { ignoreCase = true } = options;
+	return createScorer({
+		id: "check-equals",
+		name: "Equals Check",
+		description: `Checks if output equals "${expected}"`,
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		let output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join("");
+		let target = expected;
+		if (ignoreCase) {
+			output = output.toLowerCase();
+			target = target.toLowerCase();
+		}
+		return {
+			output,
+			target,
+			isEqual: output === target
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.isEqual ? 1 : 0;
+	});
+}
+/**
+* Scores 1 if the output matches the given regular expression, 0 otherwise.
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.matches(/\d{1,3}°[FC]/);
+* ```
+*/
+function matches(pattern, options = {}) {
+	const { exact = false } = options;
+	return createScorer({
+		id: "check-matches",
+		name: "Matches Check",
+		description: `Checks if output matches pattern ${pattern}`,
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		const output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join("");
+		const matched = (exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern).test(output);
+		return {
+			output,
+			pattern: pattern.toString(),
+			matched
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.matched ? 1 : 0;
+	});
+}
+/**
+* Returns the string similarity score (0-1) between the output and an expected string.
+* Useful for fuzzy matching when exact equality is too strict.
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.similarity('Sunny, 72°F');
+* ```
+*/
+function similarity(expected, options = {}) {
+	const { ignoreCase = true, threshold } = options;
+	return createScorer({
+		id: "check-similarity",
+		name: "Similarity Check",
+		description: `Checks string similarity to "${expected}"`,
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		let output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join(" ");
+		let target = expected;
+		if (ignoreCase) {
+			output = output.toLowerCase();
+			target = target.toLowerCase();
+		}
+		const score = stringSimilarity.compareTwoStrings(output, target);
+		return {
+			output,
+			target,
+			score,
+			threshold
+		};
+	}).generateScore(({ results }) => {
+		const score = results.preprocessStepResult?.score ?? 0;
+		const t = results.preprocessStepResult?.threshold;
+		return t !== void 0 ? score >= t ? 1 : 0 : score;
+	});
+}
+/**
+* Scores 1 if the agent called the specified tool (at least `times` times).
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.calledTool('get_weather');
+* const twice = checks.calledTool('search', { times: 2 });
+* ```
+*/
+function calledTool(toolName, options = {}) {
+	const { times = 1 } = options;
+	return createScorer({
+		id: "check-called-tool",
+		name: "Called Tool Check",
+		description: `Checks that "${toolName}" was called${times > 1 ? ` at least ${times} times` : ""}`,
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		const { tools } = extractToolCalls(run.output);
+		const count = tools.filter((t) => t === toolName).length;
+		return {
+			toolName,
+			expectedTimes: times,
+			actualCount: count,
+			passed: count >= times
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.passed ? 1 : 0;
+	});
+}
+/**
+* Scores 1 if the agent did NOT call the specified tool.
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.didNotCall('delete_user');
+* ```
+*/
+function didNotCall(toolName) {
+	return createScorer({
+		id: "check-did-not-call",
+		name: "Did Not Call Check",
+		description: `Checks that "${toolName}" was NOT called`,
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		const { tools } = extractToolCalls(run.output);
+		const count = tools.filter((t) => t === toolName).length;
+		return {
+			toolName,
+			count,
+			passed: count === 0
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.passed ? 1 : 0;
+	});
+}
+/**
+* Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.toolOrder(['search', 'summarize', 'respond']);
+* ```
+*/
+function toolOrder(expectedOrder) {
+	return createScorer({
+		id: "check-tool-order",
+		name: "Tool Order Check",
+		description: `Checks tool call order: [${expectedOrder.join(" → ")}]`,
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		const { tools } = extractToolCalls(run.output);
+		let orderIndex = 0;
+		for (const tool of tools) if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) orderIndex++;
+		return {
+			actualTools: tools,
+			expectedOrder,
+			passed: orderIndex === expectedOrder.length
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.passed ? 1 : 0;
+	});
+}
+/**
+* Scores 1 if the agent used no more than `max` tool calls.
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.maxToolCalls(5);
+* ```
+*/
+function maxToolCalls(max) {
+	return createScorer({
+		id: "check-max-tool-calls",
+		name: "Max Tool Calls Check",
+		description: `Checks that no more than ${max} tool calls were made`,
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		const { tools } = extractToolCalls(run.output);
+		return {
+			count: tools.length,
+			max,
+			passed: tools.length <= max
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.passed ? 1 : 0;
+	});
+}
+/**
+* Scores 1 if the agent made no tool calls at all.
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.usedNoTools();
+* ```
+*/
+function usedNoTools() {
+	return createScorer({
+		id: "check-used-no-tools",
+		name: "Used No Tools Check",
+		description: "Checks that no tools were called",
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		const { tools } = extractToolCalls(run.output);
+		return {
+			count: tools.length,
+			passed: tools.length === 0
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.passed ? 1 : 0;
+	});
+}
+/**
+* Scores 1 if none of the tool invocations resulted in an error state.
+* Checks for tool invocations with state other than 'result' (i.e., missing results).
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+* const scorer = checks.noToolErrors();
+* ```
+*/
+function noToolErrors() {
+	return createScorer({
+		id: "check-no-tool-errors",
+		name: "No Tool Errors Check",
+		description: "Checks that no tool calls resulted in errors",
+		type: "agent"
+	}).preprocess(async ({ run }) => {
+		const invocations = extractRawInvocations(run.output);
+		const errorCount = invocations.filter((inv) => inv.state === "call" || inv.result && inv.result.error).length;
+		return {
+			errorCount,
+			totalCalls: invocations.length,
+			passed: errorCount === 0
+		};
+	}).generateScore(({ results }) => {
+		return results.preprocessStepResult?.passed ? 1 : 0;
+	});
+}
+function extractRawInvocations(output) {
+	const invocations = [];
+	for (const message of output) {
+		const legacy = message?.content?.toolInvocations;
+		const fromParts = legacy ? void 0 : (message?.content)?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
+		for (const inv of legacy ?? fromParts ?? []) if (inv) invocations.push(inv);
+	}
+	return invocations;
+}
+/**
+* Quick Checks — composable micro-scorers for common assertions.
+*
+* These are zero-LLM, zero-ceremony scorers that plug into the existing
+* `scorers: [...]` array anywhere scorers are used. Internally they are
+* standard `createScorer()` instances with the same observability, storage,
+* and pipeline integration as any other scorer.
+*
+* @example
+* ```ts
+* import { checks } from '@mastra/evals';
+*
+* await runEvals({
+*   data: [...],
+*   target: myAgent,
+*   scorers: [
+*     checks.includes('sunny'),
+*     checks.calledTool('get_weather'),
+*     checks.toolOrder(['search', 'summarize']),
+*     checks.noToolErrors(),
+*   ],
+* });
+* ```
+*/
+const checks = {
+	includes,
+	excludes,
+	equals,
+	matches,
+	similarity,
+	calledTool,
+	didNotCall,
+	toolOrder,
+	maxToolCalls,
+	usedNoTools,
+	noToolErrors
+};
+//#endregion
+export { excludes as a, maxToolCalls as c, toolOrder as d, usedNoTools as f, equals as i, noToolErrors as l, checks as n, includes as o, didNotCall as r, matches as s, calledTool as t, similarity as u };
+
+//# sourceMappingURL=checks-64AonnEK.js.map

package/dist/checks-64AonnEK.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checks-64AonnEK.js","names":[],"sources":["../src/scorers/code/checks/index.ts"],"sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error state.\n * Checks for tool invocations with state other than 'result' (i.e., missing results).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(inv => inv.state === 'call' || (inv.result && inv.result.error)).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    const legacy = message?.content?.toolInvocations;\n    const fromParts = legacy\n      ? undefined\n      : (message?.content as any)?.parts\n          ?.filter((p: any) => p.type === 'tool-invocation')\n          .map((p: any) => p.toolInvocation);\n    for (const inv of legacy ?? fromParts ?? []) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"],"mappings":";;;;;;;;;;;;;AAoBA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,8BAA8B,SAAS;EACpD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,OAAO,OAAO,SAAS,MAAM;EAAE;CAC1D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,QAAQ,IAAI;CACnD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,wCAAwC,SAAS;EAC9D,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,UAAU,CAAC,OAAO,SAAS,MAAM;EAAE;CAC9D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,WAAW,IAAI;CACtD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,OAAO,UAAkB,UAA2B,CAAC,GAAG;CACtE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,SAAS;EAClD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,SAAS,WAAW;EAAO;CACtD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;AAgBA,SAAgB,QAAQ,SAAiB,UAA0B,CAAC,GAAG;CACrE,MAAM,EAAE,QAAQ,UAAU;CAC1B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,oCAAoC;EACjD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,SAAS,IAAI,OAChB,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EAEV,MAAM,WADQ,QAAQ,IAAI,OAAO,IAAI,QAAQ,OAAO,IAAI,QAAQ,KAAK,IAAI,QAAA,CACnD,KAAK,MAAM;EACjC,OAAO;GAAE;GAAQ,SAAS,QAAQ,SAAS;GAAG;EAAQ;CACxD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,aAAa,MAAM,cAAc;CACzC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gCAAgC,SAAS;EACtD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,MAAM,QAAQ,iBAAiB,kBAAkB,QAAQ,MAAM;EAC/D,OAAO;GAAE;GAAQ;GAAQ;GAAO;EAAU;CAC5C,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,QAAQ,QAAQ,sBAAsB,SAAS;EACrD,MAAM,IAAI,QAAQ,sBAAsB;EACxC,OAAO,MAAM,KAAA,IAAa,SAAS,IAAI,IAAI,IAAK;CAClD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,QAAQ,MAAM;CACtB,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS,cAAc,QAAQ,IAAI,aAAa,MAAM,UAAU;EAC7F,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU,eAAe;GAAO,aAAa;GAAO,QAAQ,SAAS;EAAM;CACtF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,WAAW,UAAkB;CAC3C,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS;EACtC,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU;GAAO,QAAQ,UAAU;EAAE;CAChD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,UAAU,eAAyB;CACjD,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,cAAc,KAAK,KAAK,EAAE;EACnE,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAE7C,IAAI,aAAa;EACjB,KAAK,MAAM,QAAQ,OACjB,IAAI,aAAa,cAAc,UAAU,SAAS,cAAc,aAC9D;EAIJ,OAAO;GAAE,aAAa;GAAO;GAAe,QAD7B,eAAe,cAAc;EACO;CACrD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,aAAa,KAAa;CACxC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,IAAI;EAC7C,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ;GAAK,QAAQ,MAAM,UAAU;EAAI;CACjE,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,cAAc;CAC5B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ,QAAQ,MAAM,WAAW;EAAE;CAC3D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;;AAYA,SAAgB,eAAe;CAC7B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,cAAc,sBAAsB,IAAI,MAAM;EACpD,MAAM,aAAa,YAAY,QAAO,QAAO,IAAI,UAAU,UAAW,IAAI,UAAU,IAAI,OAAO,KAAM,CAAC,CAAC;EACvG,OAAO;GAAE;GAAY,YAAY,YAAY;GAAQ,QAAQ,eAAe;EAAE;CAChF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;AAIA,SAAS,sBAAsB,QAAgD;CAC7E,MAAM,cAAqB,CAAC;CAC5B,KAAK,MAAM,WAAW,QAAQ;EAC5B,MAAM,SAAS,SAAS,SAAS;EACjC,MAAM,YAAY,SACd,KAAA,KACC,SAAS,QAAA,EAAiB,OACvB,QAAQ,MAAW,EAAE,SAAS,iBAAiB,CAAC,CACjD,KAAK,MAAW,EAAE,cAAc;EACvC,KAAK,MAAM,OAAO,UAAU,aAAa,CAAC,GACxC,IAAI,KAAK,YAAY,KAAK,GAAG;CAEjC;CACA,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAa,SAAS;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}