@mastra/evals 1.5.1 → 1.6.0

package/CHANGELOG.md

@@ -1,5 +1,55 @@
 # @mastra/evals
 
+## 1.6.0
+
+### Minor Changes
+
+- Added context-recall LLM scorer that evaluates how well retrieved context covers the claims in a ground-truth reference answer. Complements the existing context-precision scorer by measuring retrieval completeness rather than relevance ranking. ([#19733](https://github.com/mastra-ai/mastra/pull/19733))
+
+  ```typescript
+  import { createContextRecallScorer } from '@mastra/evals/scorers/prebuilt';
+
+  const scorer = createContextRecallScorer({
+    model: 'openai/gpt-5-mini',
+    options: {
+      context: [
+        'Einstein was born on 14 March 1879 in Ulm, Germany.',
+        'Einstein developed the theory of special relativity in 1905.',
+      ],
+    },
+  });
+  ```
+
+### Patch Changes
+
+- Updated dependencies [[`ec857fc`](https://github.com/mastra-ai/mastra/commit/ec857fc79c264b53b38e16478c789b7177f2ad59), [`d7385ad`](https://github.com/mastra-ai/mastra/commit/d7385ad9e88f9e4f33d15c0ec0bfebedde0cbc2e), [`41a5392`](https://github.com/mastra-ai/mastra/commit/41a5392d9f6c5e18d6b227f0fc0ddf49c50774e9), [`3d6e539`](https://github.com/mastra-ai/mastra/commit/3d6e539272eb2ea0407034605ee1906b3be06b39), [`1426af2`](https://github.com/mastra-ai/mastra/commit/1426af24975879c000d13ac75673f630fcc970c1), [`a40adeb`](https://github.com/mastra-ai/mastra/commit/a40adeb222b961a56a58af56a106106525721b74), [`8a0d145`](https://github.com/mastra-ai/mastra/commit/8a0d145aadbdf7278665aceaaec364b35dd9bd94), [`bd2f1d2`](https://github.com/mastra-ai/mastra/commit/bd2f1d274d05e60e2366f005ea0d94d5cea0d5ff), [`e1f2fae`](https://github.com/mastra-ai/mastra/commit/e1f2faebaf048c3d4c2e2c01d293767c195d5794), [`63aa799`](https://github.com/mastra-ai/mastra/commit/63aa799c6b44eacc7806cda6846b7c5bbee06b37), [`b7e79c3`](https://github.com/mastra-ai/mastra/commit/b7e79c3c02ac5cd415db34ba0975ceafc1464333), [`675fbff`](https://github.com/mastra-ai/mastra/commit/675fbff84d3274391b33e852f76083c38a5514e5), [`da009e1`](https://github.com/mastra-ai/mastra/commit/da009e1aacd89ed94b8d1b2af09c9d4fe7c4db49), [`3b77e77`](https://github.com/mastra-ai/mastra/commit/3b77e7704936522e4769d29de1b5ea6901f302bd), [`c7d30cd`](https://github.com/mastra-ai/mastra/commit/c7d30cd86009c407df91105591f03cd6e3d2854d), [`21a0eb8`](https://github.com/mastra-ai/mastra/commit/21a0eb86746ba0b703acea360d4f84c6a5a493f2), [`8b20926`](https://github.com/mastra-ai/mastra/commit/8b20926cd59e2ba3d66458e062fa0e6e2ada3e68), [`975295d`](https://github.com/mastra-ai/mastra/commit/975295d418552f0d46a59edfef4c3ee555f9930a), [`73db8db`](https://github.com/mastra-ai/mastra/commit/73db8db90d69ab6153c7942749f624db0d96952d), [`6b1bf3b`](https://github.com/mastra-ai/mastra/commit/6b1bf3b9494bd51aa8f654c68c9355d6046fa2a1), [`35c2181`](https://github.com/mastra-ai/mastra/commit/35c2181e6a50e47c90ba36260db7c9723d54696f), [`0a2c22c`](https://github.com/mastra-ai/mastra/commit/0a2c22c902604439ec490319e14c17f331e0c84c), [`4cfdd64`](https://github.com/mastra-ai/mastra/commit/4cfdd645794feaea0c4ea711e70ecdfbef0c5b8e), [`b75d749`](https://github.com/mastra-ai/mastra/commit/b75d749621ff5d17e86bcb4ee809d301fb4f7cf3), [`821648b`](https://github.com/mastra-ai/mastra/commit/821648bf2871ef840100c7bacbecf676010bd12a), [`de86fd7`](https://github.com/mastra-ai/mastra/commit/de86fd7119f0438381d1a642e3d258143c0b9c29), [`2745031`](https://github.com/mastra-ai/mastra/commit/2745031d1d4a4978f037092da371428c32e2842a), [`b4b7ea8`](https://github.com/mastra-ai/mastra/commit/b4b7ea8733f033fc441ea47ed03f6afb17ec2248), [`3a8024c`](https://github.com/mastra-ai/mastra/commit/3a8024ce615f8aa89479c0d71fe61d10bb0040be), [`35865a5`](https://github.com/mastra-ai/mastra/commit/35865a53e194aa9634d6a70a97010e7a6b9d58b1), [`74faf8b`](https://github.com/mastra-ai/mastra/commit/74faf8bd9c1018f2492653c06b1e25fc8300e9e6), [`ef03fbc`](https://github.com/mastra-ai/mastra/commit/ef03fbcc556bcbc04c9b3d06fab88771ecaa043c), [`675fbff`](https://github.com/mastra-ai/mastra/commit/675fbff84d3274391b33e852f76083c38a5514e5), [`70687f7`](https://github.com/mastra-ai/mastra/commit/70687f7e495a322a02070b4a67cb0c77a5ca91ec), [`1fadac4`](https://github.com/mastra-ai/mastra/commit/1fadac44537caeefe81f9f775ae2f2f3d94e9069), [`73db8db`](https://github.com/mastra-ai/mastra/commit/73db8db90d69ab6153c7942749f624db0d96952d), [`76b7181`](https://github.com/mastra-ai/mastra/commit/76b71810366e6d90b9d3973149d1c7ba3659ffb9), [`792ec9a`](https://github.com/mastra-ai/mastra/commit/792ec9a0869bab8274cf5e0ed2840738737a1607), [`712b864`](https://github.com/mastra-ai/mastra/commit/712b864aa1ed12b14c54390ec17b69de163c37f7), [`85e4fb5`](https://github.com/mastra-ai/mastra/commit/85e4fb50087a81c74df3a762f53b56373db0b912), [`0c0e8d7`](https://github.com/mastra-ai/mastra/commit/0c0e8d7becd4d1445c656b78d5d845f606c1ff9d), [`a7bbe77`](https://github.com/mastra-ai/mastra/commit/a7bbe773577f60bc4761b534ef7ec6b476332dad), [`72e437c`](https://github.com/mastra-ai/mastra/commit/72e437c515942c80b9def5b026e0bdee61b469d9), [`8f7a5de`](https://github.com/mastra-ai/mastra/commit/8f7a5dedc246cdc938bb65516703cf9b27b03756), [`a7bbe77`](https://github.com/mastra-ai/mastra/commit/a7bbe773577f60bc4761b534ef7ec6b476332dad), [`11f6cd9`](https://github.com/mastra-ai/mastra/commit/11f6cd96fe42582403416608beb212cc1a2cc79e), [`ef03c0c`](https://github.com/mastra-ai/mastra/commit/ef03c0cfc62367a458e4cc56462e2148b35681c5), [`4fb4d88`](https://github.com/mastra-ai/mastra/commit/4fb4d881bc107acee13890ad4d78661016c510ed), [`4e68363`](https://github.com/mastra-ai/mastra/commit/4e683634f94ebd062d26a3bb6093a8dfc7263d37), [`c328769`](https://github.com/mastra-ai/mastra/commit/c3287698ff8ef98dba86d415faa566fa3e5f4d56), [`9f7c67a`](https://github.com/mastra-ai/mastra/commit/9f7c67abeeb52c41c51a9b5edee60b62afe7cd8d), [`3b65e68`](https://github.com/mastra-ai/mastra/commit/3b65e68d7f1c771c7a70eea42d83fefdd28cad88), [`4eba27a`](https://github.com/mastra-ai/mastra/commit/4eba27adcf60f991df0e62f94b3e75b4e67f3b4b), [`c701be3`](https://github.com/mastra-ai/mastra/commit/c701be32d7d9aa94a66da8c6cc38dcac6856f464), [`db650ce`](https://github.com/mastra-ai/mastra/commit/db650ce490348914e85b93651d83acdf8f2a4c31), [`232fcbc`](https://github.com/mastra-ai/mastra/commit/232fcbc14fce625dd672ba043329c0b732c62be2), [`6354eeb`](https://github.com/mastra-ai/mastra/commit/6354eeb32efa9f5f68f51dda394e90e2ee76f1fb), [`a8799bb`](https://github.com/mastra-ai/mastra/commit/a8799bb8e44f4a60d01e4e2acd3448ff80bf14f8), [`3d6e539`](https://github.com/mastra-ai/mastra/commit/3d6e539272eb2ea0407034605ee1906b3be06b39), [`e3868e2`](https://github.com/mastra-ai/mastra/commit/e3868e22babfffd0133771669ca724501c2dd58e), [`9251370`](https://github.com/mastra-ai/mastra/commit/9251370ad413af464aa22d7566338bec5613e8de), [`3491666`](https://github.com/mastra-ai/mastra/commit/34916663c4fdd43b48c21f4ab2d5fb6dcccc94f9), [`c0bec73`](https://github.com/mastra-ai/mastra/commit/c0bec732c93d1a22ae5e51ed66cf8cacca8bd6a6)]:
+  - @mastra/core@1.52.0
+
+## 1.6.0-alpha.0
+
+### Minor Changes
+
+- Added context-recall LLM scorer that evaluates how well retrieved context covers the claims in a ground-truth reference answer. Complements the existing context-precision scorer by measuring retrieval completeness rather than relevance ranking. ([#19733](https://github.com/mastra-ai/mastra/pull/19733))
+
+  ```typescript
+  import { createContextRecallScorer } from '@mastra/evals/scorers/prebuilt';
+
+  const scorer = createContextRecallScorer({
+    model: 'openai/gpt-5-mini',
+    options: {
+      context: [
+        'Einstein was born on 14 March 1879 in Ulm, Germany.',
+        'Einstein developed the theory of special relativity in 1905.',
+      ],
+    },
+  });
+  ```
+
+### Patch Changes
+
+- Updated dependencies [[`0a2c22c`](https://github.com/mastra-ai/mastra/commit/0a2c22c902604439ec490319e14c17f331e0c84c), [`3a8024c`](https://github.com/mastra-ai/mastra/commit/3a8024ce615f8aa89479c0d71fe61d10bb0040be)]:
+  - @mastra/core@1.52.0-alpha.9
+
 ## 1.5.1
 
 ### Patch Changes

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-evals
 description: Documentation for @mastra/evals. Use when working with @mastra/evals APIs, configuration, or implementation.
 metadata:
   package: "@mastra/evals"
-  version: "1.5.1"
+  version: "1.6.0"
 ---
 
 ## When to use
@@ -29,6 +29,7 @@ Read the individual reference documents for detailed explanations and code examp
 - [Reference: Completeness scorer](references/reference-evals-completeness.md) - Documentation for the Completeness Scorer in Mastra, which evaluates how thoroughly LLM outputs cover key elements present in the input.
 - [Reference: Content similarity scorer](references/reference-evals-content-similarity.md) - Documentation for the Content Similarity Scorer in Mastra, which measures textual similarity between strings and provides a matching score.
 - [Reference: Context precision scorer](references/reference-evals-context-precision.md) - Documentation for the Context Precision Scorer in Mastra. Evaluates the relevance and precision of retrieved context for generating expected outputs using Mean Average Precision.
+- [Reference: Context recall scorer](references/reference-evals-context-recall.md) - Documentation for the Context Recall Scorer in Mastra. Evaluates how well retrieved context covers the claims in a ground-truth reference answer.
 - [Reference: Context relevance scorer](references/reference-evals-context-relevance.md) - Documentation for the Context Relevance Scorer in Mastra. Evaluates the relevance and utility of provided context for generating agent responses using weighted relevance scoring.
 - [Reference: Faithfulness scorer](references/reference-evals-faithfulness.md) - Documentation for the Faithfulness Scorer in Mastra, which evaluates the factual accuracy of LLM outputs compared to the provided context.
 - [Reference: Hallucination scorer](references/reference-evals-hallucination.md) - Documentation for the Hallucination Scorer in Mastra, which evaluates the factual correctness of LLM outputs by identifying contradictions with provided context.

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.5.1",
+  "version": "1.6.0",
   "package": "@mastra/evals",
   "exports": {},
   "modules": {}

package/dist/docs/references/docs-evals-built-in-scorers.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Built-in scorers
 
 Mastra provides a comprehensive set of built-in scorers for evaluating AI outputs. These scorers are optimized for common evaluation scenarios and are ready to use in your agents and workflows.
@@ -52,12 +54,12 @@ These scorers evaluate adherence to format, style, and safety requirements:
 Zero-LLM micro-scorers for fast, deterministic assertions. See the [Quick Checks](https://mastra.ai/docs/evals/quick-checks) guide for usage details.
 
 - [`checks.includes`](https://mastra.ai/reference/evals/checks): Scores 1 if output contains a substring (`0` or `1`)
-- [`checks.excludes`](https://mastra.ai/reference/evals/checks): Scores 1 if output does not contain a substring (`0` or `1`)
+- [`checks.excludes`](https://mastra.ai/reference/evals/checks): Scores 1 if output doesn't contain a substring (`0` or `1`)
 - [`checks.equals`](https://mastra.ai/reference/evals/checks): Scores 1 if output exactly matches a string (`0` or `1`)
 - [`checks.matches`](https://mastra.ai/reference/evals/checks): Scores 1 if output matches a regex (`0` or `1`)
 - [`checks.similarity`](https://mastra.ai/reference/evals/checks): String similarity via Dice coefficient (`0-1`, or binary with threshold)
 - [`checks.calledTool`](https://mastra.ai/reference/evals/checks): Scores 1 if a tool was called at least N times (`0` or `1`)
-- [`checks.didNotCall`](https://mastra.ai/reference/evals/checks): Scores 1 if a tool was not called (`0` or `1`)
+- [`checks.didNotCall`](https://mastra.ai/reference/evals/checks): Scores 1 if a tool wasn't called (`0` or `1`)
 - [`checks.toolOrder`](https://mastra.ai/reference/evals/checks): Scores 1 if tools were called in order (`0` or `1`)
 - [`checks.maxToolCalls`](https://mastra.ai/reference/evals/checks): Scores 1 if tool call count is within limit (`0` or `1`)
 - [`checks.usedNoTools`](https://mastra.ai/reference/evals/checks): Scores 1 if no tools were called (`0` or `1`)

package/dist/docs/references/docs-evals-overview.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Scorers overview
 
 While traditional software tests have clear pass/fail conditions, AI outputs are non-deterministic — they can vary with the same input. **Scorers** help bridge this gap by providing quantifiable metrics for measuring agent quality.
@@ -6,6 +8,8 @@ Scorers are automated tests that evaluate Agents outputs using model-graded, rul
 
 Scorers can be run in the cloud, capturing real-time results. But scorers can also be part of your CI/CD pipeline, allowing you to test and monitor your agents over time.
 
+> **📹 Watch:** Watch [Mastra evals overview](https://www.youtube.com/watch?v=12WN6u2DrBk) for an introduction to evals and how to reason about agent quality.
+
 ## Types of scorers
 
 Mastra provides different kinds of scorers, each serving a specific purpose. Here are some common types:
@@ -55,6 +59,7 @@ import { Agent } from '@mastra/core/agent'
 import { createAnswerRelevancyScorer, createToxicityScorer } from '@mastra/evals/scorers/prebuilt'
 
 export const evaluatedAgent = new Agent({
+  id: 'evaluated-agent',
   scorers: {
     relevancy: {
       scorer: createAnswerRelevancyScorer({ model: 'openai/gpt-5-mini' }),
@@ -137,6 +142,8 @@ Once registered, you can score traces interactively within Studio under the **Ob
 
 - Use [Quick Checks](https://mastra.ai/docs/evals/quick-checks) for fast, deterministic assertions on text and tool usage
 - Add [gates and verdicts](https://mastra.ai/docs/evals/gates-and-verdicts) to enforce hard requirements and quality thresholds
+- Test [multi-turn conversations](https://mastra.ai/docs/evals/multi-turn) where behavior emerges across sequential turns
 - Learn how to create your own scorers in the [Creating Custom Scorers](https://mastra.ai/docs/evals/custom-scorers) guide
 - Explore built-in scorers in the [Built-in Scorers](https://mastra.ai/docs/evals/built-in-scorers) section
-- Test scorers with [Studio](https://mastra.ai/docs/studio/overview)
+- Test scorers with [Studio](https://mastra.ai/docs/studio/overview)
+- 📹 [Mastra evals workshop](https://www.youtube.com/watch?v=OHOZ5PgPj5M\&t=3578s)

package/dist/docs/references/docs-evals-quick-checks.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Quick Checks
 
 Quick Checks are composable micro-scorers for common assertions like "output contains X" or "agent called tool Y." They require no LLM, run instantly, and plug into the same `scorers: [...]` array as any other scorer.
@@ -42,7 +44,7 @@ These scorers evaluate the agent's text output:
 | Check                    | What it does                          | Score                            |
 | ------------------------ | ------------------------------------- | -------------------------------- |
 | `checks.includes(str)`   | Output contains substring             | 1 or 0                           |
-| `checks.excludes(str)`   | Output does not contain substring     | 1 or 0                           |
+| `checks.excludes(str)`   | Output doesn't contain substring      | 1 or 0                           |
 | `checks.equals(str)`     | Output exactly equals string          | 1 or 0                           |
 | `checks.matches(regex)`  | Output matches regular expression     | 1 or 0                           |
 | `checks.similarity(str)` | Dice coefficient similarity to string | 0-1 (or binary with `threshold`) |
@@ -54,7 +56,7 @@ These scorers evaluate tool usage from the agent's run:
 | Check                     | What it does                     | Score  |
 | ------------------------- | -------------------------------- | ------ |
 | `checks.calledTool(name)` | Tool was called at least N times | 1 or 0 |
-| `checks.didNotCall(name)` | Tool was not called              | 1 or 0 |
+| `checks.didNotCall(name)` | Tool wasn't called               | 1 or 0 |
 | `checks.toolOrder([...])` | Tools called in expected order   | 1 or 0 |
 | `checks.maxToolCalls(n)`  | No more than N tool calls total  | 1 or 0 |
 | `checks.usedNoTools()`    | No tools called at all           | 1 or 0 |
@@ -100,6 +102,7 @@ import { Agent } from '@mastra/core/agent'
 import { checks } from '@mastra/evals/checks'
 
 export const weatherAgent = new Agent({
+  id: 'weather-agent',
   name: 'Weather Agent',
   instructions: 'Answer weather questions using the get_weather tool.',
   model: 'openai/gpt-5.5',
@@ -124,9 +127,7 @@ Each check is a standard `createScorer()` instance with a `preprocess` step and
 1. **preprocess**: Extracts and normalizes relevant data from the agent run (text content, tool calls)
 2. **generateScore**: Converts the preprocessed result into a score (typically binary 1 or 0)
 
-Because checks skip the `analyze` and `generateReason` steps and make no LLM calls, they run in microseconds.
-
-> **Note:** Visit the [Quick Checks reference](https://mastra.ai/reference/evals/checks) for the full API, including all parameters and options for each check.
+Because checks skip the `analyze` and `generateReason` steps and make no LLM calls, they run in microseconds. Visit the [Quick Checks reference](https://mastra.ai/reference/evals/checks) for the full API, including all parameters and options for each check.
 
 ## Related
 

package/dist/docs/references/reference-evals-answer-relevancy.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Answer relevancy scorer
 
 The `createAnswerRelevancyScorer()` function accepts a single options object with the following properties:

package/dist/docs/references/reference-evals-answer-similarity.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Answer similarity scorer
 
 The `createAnswerSimilarityScorer()` function creates a scorer that evaluates how similar an agent's output is to a ground truth answer. This scorer is specifically designed for CI/CD testing scenarios where you have expected answers and want to ensure consistency over time.

package/dist/docs/references/reference-evals-bias.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Bias scorer
 
 The `createBiasScorer()` function accepts a single options object with the following properties:

package/dist/docs/references/reference-evals-checks.md

@@ -1,8 +1,10 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Quick Checks
 
 Quick Checks are zero-LLM, composable micro-scorers for common assertions. They plug into the existing `scorers: [...]` array anywhere scorers are used — in `runEvals`, live scoring, experiments, and Studio.
 
-Internally they are standard `createScorer()` instances, so they have the same observability, storage, and pipeline integration as any other scorer.
+Internally they're standard `createScorer()` instances, so they have the same observability, storage, and pipeline integration as any other scorer.
 
 ## Usage example
 
@@ -44,7 +46,7 @@ Returns: `1` if found, `0` otherwise.
 
 ### `checks.excludes(unwanted, options?)`
 
-Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.
+Scores 1 if the agent's output text doesn't contain the substring, 0 otherwise.
 
 ```typescript
 checks.excludes('error')
@@ -123,7 +125,7 @@ Returns: `1` if called at least `times` times, `0` otherwise.
 
 ### `checks.didNotCall(toolName)`
 
-Scores 1 if the agent did NOT call the specified tool.
+Scores 1 if the agent didn't call the specified tool.
 
 ```typescript
 checks.didNotCall('delete_user')
@@ -131,7 +133,7 @@ checks.didNotCall('delete_user')
 
 **toolName** (`string`): Name of the tool that must not appear.
 
-Returns: `1` if the tool was not called, `0` otherwise.
+Returns: `1` if the tool wasn't called, `0` otherwise.
 
 ### `checks.toolOrder(expectedOrder)`
 

package/dist/docs/references/reference-evals-completeness.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Completeness scorer
 
 The `createCompletenessScorer()` function evaluates how thoroughly an LLM's output covers the key elements present in the input. It analyzes nouns, verbs, topics, and terms to determine coverage and provides a detailed completeness score.

package/dist/docs/references/reference-evals-content-similarity.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Content similarity scorer
 
 The `createContentSimilarityScorer()` function measures the textual similarity between two strings, providing a score that indicates how closely they match. It supports configurable options for case sensitivity and whitespace handling.

package/dist/docs/references/reference-evals-context-precision.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Context precision scorer
 
 The `createContextPrecisionScorer()` function creates a scorer that evaluates how relevant and well-positioned retrieved context pieces are for generating expected outputs. It uses **Mean Average Precision (MAP)** to reward systems that place relevant context earlier in the sequence.

package/dist/docs/references/reference-evals-context-recall.md

@@ -0,0 +1,205 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
+# Context recall scorer
+
+The `createContextRecallScorer()` function creates a scorer that evaluates how well retrieved context covers the claims in a ground-truth reference answer. It measures retrieval completeness by checking what fraction of the ground truth's claims are attributable to the retrieved context.
+
+This scorer requires a ground-truth reference answer, making it suitable for labeled datasets in CI or test environments. When `groundTruth` is not provided in the run, the scorer returns a score of 0 rather than throwing an error.
+
+## RAG retrieval evaluation
+
+Ideal for evaluating retrieval completeness in RAG pipelines where:
+
+- You need to verify the retriever fetches all necessary information
+- You have labeled datasets with known-correct answers
+- You want to catch regressions in what the retriever returns
+
+## Dataset-driven testing
+
+Use when running evaluations against curated test sets:
+
+- CI pipelines with ground-truth labeled questions
+- A/B testing retrieval strategies
+- Benchmarking embedding models for coverage
+
+## Parameters
+
+**model** (`MastraModelConfig`): The language model to use for evaluating claim attribution
+
+**options** (`ContextRecallMetricOptions`): Configuration options for the scorer
+
+**Note**: Either `context` or `contextExtractor` must be provided. When both are provided, `contextExtractor` is used only if the run input and output are agent-shaped (`MastraDBMessage[]`); otherwise the scorer falls back to `context`.
+
+## `.run()` returns
+
+**score** (`number`): Recall score between 0 and scale (default 0-1), representing the fraction of ground-truth claims covered by the context
+
+**reason** (`string`): Human-readable explanation of which ground-truth claims were and were not found in the context
+
+## Scoring details
+
+### Claim attribution
+
+Context Recall uses a two-step LLM evaluation followed by a deterministic score calculation:
+
+1. **Claim extraction**: The ground-truth answer is decomposed into atomic claims
+2. **Attribution check**: Each claim is checked against the retrieval context for support
+
+The score is then calculated as the ratio of attributed claims to total claims, multiplied by scale.
+
+### Scoring formula
+
+```text
+Context Recall = attributed_claims / total_claims × scale
+
+Where:
+- attributed_claims = number of ground-truth claims supported by the context
+- total_claims = total number of claims extracted from the ground truth
+- Attribution is binary: a claim is either supported (yes) or not (no)
+```
+
+### Score interpretation
+
+These ranges assume the default `scale` of 1. When using a custom scale, multiply accordingly.
+
+- **0.9-1.0**: Excellent recall — context covers nearly all ground-truth claims
+- **0.7-0.8**: Good recall — most claims are covered, minor gaps
+- **0.4-0.6**: Moderate recall — significant information missing from context
+- **0.1-0.3**: Poor recall — most ground-truth claims not found in context
+- **0.0**: No recall — none of the ground-truth claims are in the context
+
+### Reason analysis
+
+The reason field explains:
+
+- Which ground-truth claims were found in the context
+- Which claims were missing and what information gaps exist
+- Specific context pieces that supported attributed claims
+
+### Optimization insights
+
+Use results to:
+
+- **Improve retrieval**: Identify what types of information the retriever misses
+- **Tune chunk size**: Ensure chunks contain enough detail to cover ground-truth claims
+- **Evaluate embeddings**: Test different embedding models for better information coverage
+- **Expand knowledge base**: Add documents that cover frequently missed claims
+
+### Example calculation
+
+Given ground truth: "Einstein was born in 1879. He developed relativity. He won the Nobel Prize."
+
+Claims extracted: 3
+
+- "Einstein was born in 1879" → attributed (context mentions birthdate)
+- "Einstein developed relativity" → attributed (context covers relativity)
+- "Einstein won the Nobel Prize" → not attributed (context doesn't mention Nobel Prize)
+
+Recall = 2/3 = 0.67
+
+## Scorer configuration
+
+### Dynamic context extraction
+
+```typescript
+const scorer = createContextRecallScorer({
+  model: 'openai/gpt-5.5',
+  options: {
+    contextExtractor: (input, output) => {
+      const query = input?.inputMessages?.[0]?.content || ''
+      const searchResults = vectorDB.search(query, { limit: 10 })
+      return searchResults.map(result => result.content)
+    },
+    scale: 1,
+  },
+})
+```
+
+### Static context evaluation
+
+```typescript
+const scorer = createContextRecallScorer({
+  model: 'openai/gpt-5.5',
+  options: {
+    context: [
+      'Document 1: Einstein was born on 14 March 1879 in Ulm, Germany.',
+      'Document 2: Einstein published the theory of special relativity in 1905.',
+      'Document 3: Einstein moved to the United States in 1933.',
+    ],
+  },
+})
+```
+
+## Example
+
+Evaluate RAG retrieval completeness against a labeled dataset:
+
+```typescript
+import { runEvals } from '@mastra/core/evals'
+import { createContextRecallScorer } from '@mastra/evals/scorers/prebuilt'
+import { myAgent } from './agent'
+
+const scorer = createContextRecallScorer({
+  model: 'openai/gpt-5.5',
+  options: {
+    contextExtractor: (input, output) => {
+      // Extract context from tool invocation results in the agent output
+      return output
+        .filter(msg => msg?.role === 'assistant')
+        .flatMap(msg => msg?.content?.toolInvocations ?? [])
+        .filter((tool: any) => tool.state === 'result')
+        .map((tool: any) => JSON.stringify(tool.result))
+    },
+  },
+})
+
+const result = await runEvals({
+  data: [
+    {
+      input: 'What are the health benefits of green tea?',
+      groundTruth:
+        'Green tea contains antioxidants that reduce inflammation, L-theanine that improves focus, and catechins that boost metabolism.',
+    },
+    {
+      input: 'How does photosynthesis work?',
+      groundTruth:
+        'Photosynthesis converts sunlight into chemical energy using chlorophyll in chloroplasts, producing glucose and oxygen from carbon dioxide and water.',
+    },
+  ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+      reason: scorerResults[scorer.id].reason,
+    })
+  },
+})
+
+console.log(result.scores)
+```
+
+For more details on `runEvals`, see the [runEvals reference](https://mastra.ai/reference/evals/run-evals).
+
+To add this scorer to an agent, see the [Scorers overview](https://mastra.ai/docs/evals/overview) guide.
+
+## Comparison with context precision
+
+Choose the right scorer for your needs:
+
+| Use case                  | Context Recall           | Context Precision             |
+| ------------------------- | ------------------------ | ----------------------------- |
+| **What it measures**      | Coverage of ground truth | Relevance of retrieved chunks |
+| **Direction**             | Ground truth → context   | Context → ground truth        |
+| **Position sensitive**    | No                       | Yes (rewards early placement) |
+| **Requires ground truth** | Yes                      | Yes                           |
+| **Failure mode caught**   | Missing information      | Irrelevant noise              |
+
+Use both together for a complete picture of retrieval quality: precision catches junk in the context, recall catches gaps.
+
+## Related
+
+- [Context Precision Scorer](https://mastra.ai/reference/evals/context-precision): Evaluates if retrieved context is relevant and well-ranked
+- [Context Relevance Scorer](https://mastra.ai/reference/evals/context-relevance): Evaluates context usage and quality
+- [Faithfulness Scorer](https://mastra.ai/reference/evals/faithfulness): Measures answer groundedness in context
+- [Custom Scorers](https://mastra.ai/docs/evals/custom-scorers): Creating your own evaluation metrics

package/dist/docs/references/reference-evals-context-relevance.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Context relevance scorer
 
 The `createContextRelevanceScorerLLM()` function creates a scorer that evaluates how relevant and useful provided context was for generating agent responses. It uses weighted relevance levels and applies penalties for unused high-relevance context and missing information.

package/dist/docs/references/reference-evals-faithfulness.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Faithfulness scorer
 
 The `createFaithfulnessScorer()` function evaluates how factually accurate an LLM's output is compared to the provided context. It extracts claims from the output and verifies them against the context, making it essential to measure RAG pipeline responses' reliability.

package/dist/docs/references/reference-evals-hallucination.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Hallucination scorer
 
 The `createHallucinationScorer()` function evaluates whether an LLM generates factually correct information by comparing its output against the provided context. This scorer measures hallucination by identifying direct contradictions between the context and the output.
@@ -155,6 +157,7 @@ const hallucinationScorer = createHallucinationScorer({
 })
 
 const agent = new Agent({
+  id: 'my-agent',
   name: 'my-agent',
   model: 'openai/gpt-5.5',
   instructions: 'You are a helpful assistant.',

package/dist/docs/references/reference-evals-keyword-coverage.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Keyword coverage scorer
 
 The `createKeywordCoverageScorer()` function evaluates how well an LLM's output covers the important keywords from the input. It analyzes keyword presence and matches while ignoring common words and stop words.

package/dist/docs/references/reference-evals-noise-sensitivity.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Noise sensitivity scorer
 
 The `createNoiseSensitivityScorerLLM()` function creates a **CI/testing scorer** that evaluates how robust an agent is when exposed to irrelevant, distracting, or misleading information. Unlike live scorers that evaluate single production runs, this scorer requires predetermined test data including both baseline responses and noisy variations.

package/dist/docs/references/reference-evals-prompt-alignment.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Prompt alignment scorer
 
 The `createPromptAlignmentScorerLLM()` function creates a scorer that evaluates how well agent responses align with user prompts across multiple dimensions: intent understanding, requirement fulfillment, response completeness, and format appropriateness.
@@ -219,6 +221,7 @@ Measure how well your AI agents follow user instructions:
 
 ```typescript
 const agent = new Agent({
+  id: 'coding-assistant',
   name: 'CodingAssistant',
   instructions: 'You are a helpful coding assistant. Always provide working code examples.',
   model: 'openai/gpt-5.5',

package/dist/docs/references/reference-evals-rubric.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Rubric scorer
 
 **Added in:** `@mastra/evals@1.3.0`
@@ -10,7 +12,7 @@ This scorer is designed to drop into [`isTaskComplete`](https://mastra.ai/refere
 
 **model** (`MastraModelConfig`): The language model used to grade the output against the rubric. A smaller, cheaper model is usually sufficient for grading.
 
-**criteria** (`RubricCriterion[] | string`): The rubric to grade against. A string is treated as a newline-delimited checklist (each line becomes a required criterion). If omitted, the rubric is read at run time from a \`rubric\` value on request/additional context; if none resolves, the scorer is a no-op and returns 1.
+**criteria** (`RubricCriterion[] | string`): The rubric to grade against. A string is treated as a newline-delimited checklist (each line becomes a required criterion). If omitted, the rubric is read at run time from a rubric value on request/additional context; if none resolves, the scorer is a no-op and returns 1.
 
 **options** (`RubricScorerOptions`): Configuration options for the scorer
 

package/dist/docs/references/reference-evals-scorer-utils.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Scorer utils
 
 Mastra provides utility functions to help extract and process data from scorer run inputs and outputs. These utilities are particularly useful in the `preprocess` step of custom scorers.

package/dist/docs/references/reference-evals-textual-difference.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Textual difference scorer
 
 The `createTextualDifferenceScorer()` function uses sequence matching to measure the textual differences between two strings. It provides detailed information about changes, including the number of operations needed to transform one text into another.

package/dist/docs/references/reference-evals-tone-consistency.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Tone consistency scorer
 
 The `createToneScorer()` function evaluates the text's emotional tone and sentiment consistency. It can operate in two modes: comparing tone between input/output pairs or analyzing tone stability within a single text.

package/dist/docs/references/reference-evals-tool-call-accuracy.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Tool call accuracy scorers
 
 Mastra provides two tool call accuracy scorers for evaluating whether an LLM selects the correct tools from available options:

package/dist/docs/references/reference-evals-toxicity.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Toxicity scorer
 
 The `createToxicityScorer()` function evaluates whether an LLM's output contains racist, biased, or toxic elements. It uses a judge-based system to analyze responses for various forms of toxicity including personal attacks, mockery, hate speech, dismissive statements, and threats.

package/dist/docs/references/reference-evals-trajectory-accuracy.md

@@ -1,3 +1,5 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
 # Trajectory accuracy scorers
 
 Mastra provides two trajectory accuracy scorers for evaluating whether an agent or workflow follows an expected sequence of actions:
@@ -111,7 +113,7 @@ Omit `stepType` entirely to match any step by name only.
 
 **stepType** (`TrajectoryStepType`): Step type discriminant. When set, enables autocomplete for that variant's fields. If omitted, matches any step type with the given name.
 
-**(variant fields)** (`varies`): Type-specific fields from the corresponding TrajectoryStep variant. For example, \`toolArgs\` and \`toolResult\` for \`tool\_call\`, \`modelId\` for \`model\_generation\`, \`output\` for \`workflow\_step\`. All optional — only specified fields are compared.
+**(variant fields)** (`varies`): Type-specific fields from the corresponding TrajectoryStep variant. For example, toolArgs and toolResult for tool\_call, modelId for model\_generation, output for workflow\_step. All optional — only specified fields are compared.
 
 **children** (`TrajectoryExpectation`): Nested expectation config for this step's children. Overrides the parent config for evaluating children of this step.
 

package/dist/scorers/llm/context-recall/index.d.ts

@@ -0,0 +1,20 @@
+import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/evals';
+import type { MastraModelConfig } from '@mastra/core/llm';
+import type { ScorerRunInputForLLMJudge, ScorerRunOutputForLLMJudge } from '../../utils.js';
+export interface ContextRecallMetricOptions {
+    scale?: number;
+    context?: string[];
+    contextExtractor?: (input: ScorerRunInputForAgent, output: ScorerRunOutputForAgent) => string[];
+}
+export declare function createContextRecallScorer({ model, options, }: {
+    model: MastraModelConfig;
+    options: ContextRecallMetricOptions;
+}): import("@mastra/core/evals").MastraScorer<string, ScorerRunInputForLLMJudge, ScorerRunOutputForLLMJudge, Record<"preprocessStepResult", {
+    claims: string[];
+}> & Record<"analyzeStepResult", {
+    verdicts: {
+        verdict: string;
+        reason: string;
+    }[];
+}> & Record<"generateScoreStepResult", number> & Record<"generateReasonStepResult", string>>;
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/llm/context-recall/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/context-recall/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC1F,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAQzF,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,uBAAuB,KAAK,MAAM,EAAE,CAAC;CACjG;AAmCD,wBAAgB,yBAAyB,CAAC,EACxC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,EAAE,0BAA0B,CAAC;CACrC;;;;;;;6FAkFA"}