@mastra/evals 1.7.0 → 1.8.0

package/CHANGELOG.md

@@ -1,5 +1,81 @@
 # @mastra/evals
 
+## 1.8.0
+
+### Minor Changes
+
+- Added an `includeConversationHistory` option to the Prompt Alignment scorer so multi-turn agent runs are scored in context. ([#21683](https://github.com/mastra-ai/mastra/pull/21683))
+
+  Previously the scorer only saw the current turn. In a conversation a short reply like `"A"` has no meaning on its own, so the judge could not tell what the user asked for and scored a perfectly good response as misaligned. The scorer now optionally includes the prior turns from the agent's memory, uses them to interpret the current prompt, and still scores only the current response.
+
+  **Before**
+
+  ```typescript
+  const scorer = createPromptAlignmentScorerLLM({
+    model: 'openai/gpt-5-mini',
+    options: { evaluationMode: 'user' },
+  });
+  ```
+
+  **After**
+
+  ```typescript
+  const scorer = createPromptAlignmentScorerLLM({
+    model: 'openai/gpt-5-mini',
+    options: {
+      evaluationMode: 'user',
+      includeConversationHistory: true, // or { maxMessages: 6 }
+    },
+  });
+  ```
+
+  The option is off by default, so existing scores do not change. It only applies to agent runs, which are the runs that carry remembered messages. Fixes https://github.com/mastra-ai/mastra/issues/21638
+
+### Patch Changes
+
+- Fixed LLM-judge scorers scoring an intermediate reply instead of the agent's final answer. When agent output contains multiple assistant messages (multi-step runs), scorers such as Prompt Alignment now evaluate the last assistant response that contains text. Fixes #21645 ([#21686](https://github.com/mastra-ai/mastra/pull/21686))
+
+- Updated dependencies [[`587f6ef`](https://github.com/mastra-ai/mastra/commit/587f6efcfc25880b93760a8607d1cd381ec612fe), [`7e096f0`](https://github.com/mastra-ai/mastra/commit/7e096f02f0dddbf09b85d306458351245ed2f886), [`d7e6745`](https://github.com/mastra-ai/mastra/commit/d7e67456954863c55440ea9c49bc6ceb9949972d), [`6223446`](https://github.com/mastra-ai/mastra/commit/6223446ddce6166e96e0ba5e00d628b615dee8ca), [`15101bb`](https://github.com/mastra-ai/mastra/commit/15101bb53c0d934f31af6b8813b88191e382a5e5), [`4e7a421`](https://github.com/mastra-ai/mastra/commit/4e7a421dce8a48742f785d1e93ad2f43a572b282), [`c2c3deb`](https://github.com/mastra-ai/mastra/commit/c2c3debcf670c7082d0a5e553aa99818a864698c), [`d8308a2`](https://github.com/mastra-ai/mastra/commit/d8308a2be3c07e777393d1017a381dcae3890d30), [`b0a2a07`](https://github.com/mastra-ai/mastra/commit/b0a2a07800d42bd9823292e7db832374ed084c9c), [`74e5bd3`](https://github.com/mastra-ai/mastra/commit/74e5bd315b8b3a1e04cb6cf480bb0f5fc4951dc8), [`242e324`](https://github.com/mastra-ai/mastra/commit/242e3241e73cbd5c9bb86a31ebb49ca0256488d4), [`217e967`](https://github.com/mastra-ai/mastra/commit/217e9672d8b3160eb729d8e9f0044949e88da239), [`d774e89`](https://github.com/mastra-ai/mastra/commit/d774e8930c781df8c9effe3763e6b501c099b6cc), [`9c27a53`](https://github.com/mastra-ai/mastra/commit/9c27a53cd9d3de4f3f025bc387d94ce371c33f95), [`8f0a332`](https://github.com/mastra-ai/mastra/commit/8f0a3321bf180368d76fe7b36aa1a8f60f00b6de), [`0b4f108`](https://github.com/mastra-ai/mastra/commit/0b4f1089aa8d92e67c2a8e99726822c5ee410784), [`9acb50f`](https://github.com/mastra-ai/mastra/commit/9acb50f71cec9c362f06820033f90ae6b1f8282f), [`46e9e3f`](https://github.com/mastra-ai/mastra/commit/46e9e3f73babe1bc70080a596cf2ac0b9da48519), [`3f9a190`](https://github.com/mastra-ai/mastra/commit/3f9a19057c027155867b9317294ee4ca7bd0581a), [`dff25a1`](https://github.com/mastra-ai/mastra/commit/dff25a1103fa72ee082a9b6f805ebeb5ce400753), [`6db7a5d`](https://github.com/mastra-ai/mastra/commit/6db7a5dd3dd2b6f7ef75dcd804fcffef5fa83963), [`217e967`](https://github.com/mastra-ai/mastra/commit/217e9672d8b3160eb729d8e9f0044949e88da239), [`583e235`](https://github.com/mastra-ai/mastra/commit/583e23519c13af16c1746f9c49722d011216611b), [`b098de9`](https://github.com/mastra-ai/mastra/commit/b098de9d7cb9f672e0883a5c716465a3a689693d), [`e8808e3`](https://github.com/mastra-ai/mastra/commit/e8808e3d8eb585a2565be53e56a7e0e1477352a4), [`a77f8d4`](https://github.com/mastra-ai/mastra/commit/a77f8d4740d2178a74c41e4bf678b4fcd8fa0bb2), [`7f78585`](https://github.com/mastra-ai/mastra/commit/7f785857e401570e2ffb316911f126ed363aa537), [`33374ba`](https://github.com/mastra-ai/mastra/commit/33374ba359e4fb13eaa918ae925fe167a3c55414), [`940bf5c`](https://github.com/mastra-ai/mastra/commit/940bf5ccf04f2c9ebd8a1390431733222a03b1cd), [`c549e2f`](https://github.com/mastra-ai/mastra/commit/c549e2f40edc1cac5d9e74e82f90da22b48df084), [`58c43d3`](https://github.com/mastra-ai/mastra/commit/58c43d3f7cb2eeaeb8ac733ae71dde822348e588), [`ef6e295`](https://github.com/mastra-ai/mastra/commit/ef6e295b59bc25a5b61b633a89c97bcfce9fb465), [`208e1b3`](https://github.com/mastra-ai/mastra/commit/208e1b39f30f4b386e494394e9d71d96f0f90241), [`c938d34`](https://github.com/mastra-ai/mastra/commit/c938d34739936c8ecbabd67ad6a4a4396f41c4c6), [`88ddc7c`](https://github.com/mastra-ai/mastra/commit/88ddc7ce01d40175f13a3228b789a906779680bd), [`f2a4afd`](https://github.com/mastra-ai/mastra/commit/f2a4afd7e37e809669001ed17724b341a5c1f45e), [`d438148`](https://github.com/mastra-ai/mastra/commit/d438148e222c1e2fb3c652725ce75680962ebec4), [`ba05fe0`](https://github.com/mastra-ai/mastra/commit/ba05fe0738f70cb686777546e968237d09269142), [`40d358e`](https://github.com/mastra-ai/mastra/commit/40d358e29d55543803e64b49241122f598ffabc7), [`d26a8d4`](https://github.com/mastra-ai/mastra/commit/d26a8d4281f28414715b333c85bedaf70d0b2890), [`e80cd7e`](https://github.com/mastra-ai/mastra/commit/e80cd7e7683e7d732e1cc6784bcac1d2640d2ce3), [`ccbbcd9`](https://github.com/mastra-ai/mastra/commit/ccbbcd974eedff4367a54ed0e24c9ee742ab2f61), [`1d9a0ea`](https://github.com/mastra-ai/mastra/commit/1d9a0ea4a9901baee6cd56737243bd6d1f631ac0), [`677cdc6`](https://github.com/mastra-ai/mastra/commit/677cdc6af564dec29a13464d12b7ab2a4efc22e9), [`c549e2f`](https://github.com/mastra-ai/mastra/commit/c549e2f40edc1cac5d9e74e82f90da22b48df084), [`a7dd322`](https://github.com/mastra-ai/mastra/commit/a7dd32247d95afc539f483ca37f4594af0387f59), [`3f5c6f7`](https://github.com/mastra-ai/mastra/commit/3f5c6f728ea35da344248de9aa070f12849f3aa0), [`a318490`](https://github.com/mastra-ai/mastra/commit/a318490e17da32f338d50929c770d901a9b3dd72), [`b860493`](https://github.com/mastra-ai/mastra/commit/b86049391100e665d579f700c8a2034c036defc3), [`d4be8c1`](https://github.com/mastra-ai/mastra/commit/d4be8c1739d22d621e3f78790e1dd5eb5ecc3589), [`a5d2eb1`](https://github.com/mastra-ai/mastra/commit/a5d2eb10347eade1ae2816d88f466c25186c54a5), [`3667679`](https://github.com/mastra-ai/mastra/commit/3667679db057edfb086846d13369fdda4902ad65), [`49696e8`](https://github.com/mastra-ai/mastra/commit/49696e8e42f870674a0a58f5abcd22cc54dd2864), [`2ef2f23`](https://github.com/mastra-ai/mastra/commit/2ef2f230a7aed342e7dc3b2000cd42e4c43e08a7), [`763e0c6`](https://github.com/mastra-ai/mastra/commit/763e0c61e04d76ad9a9efd301aa57525ca0cbea9), [`20504b2`](https://github.com/mastra-ai/mastra/commit/20504b2ecebd0e077acda3d457ab57480a98ed3e), [`77e6b1b`](https://github.com/mastra-ai/mastra/commit/77e6b1bc4c46ce94fe501023fb4393c812ec6be3), [`c5f964d`](https://github.com/mastra-ai/mastra/commit/c5f964d3f77064e978f8066ec506eed77ba5c63c), [`23e0be2`](https://github.com/mastra-ai/mastra/commit/23e0be261381e49534b4ff3101c60ee64a946cbf), [`7fc8806`](https://github.com/mastra-ai/mastra/commit/7fc880627d3cbf995d31ea0e8b807bf15417e651), [`0e02eac`](https://github.com/mastra-ai/mastra/commit/0e02eacdb2e30e1697a41910b41163742a181dc1), [`4df174c`](https://github.com/mastra-ai/mastra/commit/4df174c32bddf093a82f273070b8380aef7c9e90), [`f7c25b5`](https://github.com/mastra-ai/mastra/commit/f7c25b5106ddfb48e591f98df7a51e0f2dd01dba), [`7aad631`](https://github.com/mastra-ai/mastra/commit/7aad631b43bc10db77d5b8c66b200d7a49d18bf2), [`512100a`](https://github.com/mastra-ai/mastra/commit/512100a7d8b7e9c920f2590c6b3612f5de0d3cff), [`e81744c`](https://github.com/mastra-ai/mastra/commit/e81744cd13c46619c142dc521dc0baac47607a84), [`f8f653f`](https://github.com/mastra-ai/mastra/commit/f8f653f10980d01a73706cc3c8689ca5e40ce808), [`dc09cc1`](https://github.com/mastra-ai/mastra/commit/dc09cc1083d861cde192c1cd235324dc75b8c731), [`9ef432b`](https://github.com/mastra-ai/mastra/commit/9ef432b6faa534b57b0d182a610e13dd9a7123ff), [`36b4649`](https://github.com/mastra-ai/mastra/commit/36b4649045a3a380cbab8ceca866db4086223aff), [`b9cf308`](https://github.com/mastra-ai/mastra/commit/b9cf30846f97f99ac1906ee8a68f4f2d117b0378), [`2e1d098`](https://github.com/mastra-ai/mastra/commit/2e1d0984e325fd319d32ea182f596b3170be3847), [`377eb81`](https://github.com/mastra-ai/mastra/commit/377eb81ce43b964e3a6b541df172da74a8ff3716), [`1794a79`](https://github.com/mastra-ai/mastra/commit/1794a79178c418004a7261b1ad9114066f7ef01d), [`0cdc5dc`](https://github.com/mastra-ai/mastra/commit/0cdc5dc69024957815da4f51acc4119eb4f447d7), [`5740ec6`](https://github.com/mastra-ai/mastra/commit/5740ec60c760ffdfbfaa59d603d03b847c864e05)]:
+  - @mastra/core@1.60.0
+
+## 1.8.0-alpha.0
+
+### Minor Changes
+
+- Added an `includeConversationHistory` option to the Prompt Alignment scorer so multi-turn agent runs are scored in context. ([#21683](https://github.com/mastra-ai/mastra/pull/21683))
+
+  Previously the scorer only saw the current turn. In a conversation a short reply like `"A"` has no meaning on its own, so the judge could not tell what the user asked for and scored a perfectly good response as misaligned. The scorer now optionally includes the prior turns from the agent's memory, uses them to interpret the current prompt, and still scores only the current response.
+
+  **Before**
+
+  ```typescript
+  const scorer = createPromptAlignmentScorerLLM({
+    model: 'openai/gpt-5-mini',
+    options: { evaluationMode: 'user' },
+  });
+  ```
+
+  **After**
+
+  ```typescript
+  const scorer = createPromptAlignmentScorerLLM({
+    model: 'openai/gpt-5-mini',
+    options: {
+      evaluationMode: 'user',
+      includeConversationHistory: true, // or { maxMessages: 6 }
+    },
+  });
+  ```
+
+  The option is off by default, so existing scores do not change. It only applies to agent runs, which are the runs that carry remembered messages. Fixes https://github.com/mastra-ai/mastra/issues/21638
+
+### Patch Changes
+
+- Fixed LLM-judge scorers scoring an intermediate reply instead of the agent's final answer. When agent output contains multiple assistant messages (multi-step runs), scorers such as Prompt Alignment now evaluate the last assistant response that contains text. Fixes #21645 ([#21686](https://github.com/mastra-ai/mastra/pull/21686))
+
+- Updated dependencies [[`4e7a421`](https://github.com/mastra-ai/mastra/commit/4e7a421dce8a48742f785d1e93ad2f43a572b282), [`242e324`](https://github.com/mastra-ai/mastra/commit/242e3241e73cbd5c9bb86a31ebb49ca0256488d4), [`217e967`](https://github.com/mastra-ai/mastra/commit/217e9672d8b3160eb729d8e9f0044949e88da239), [`d774e89`](https://github.com/mastra-ai/mastra/commit/d774e8930c781df8c9effe3763e6b501c099b6cc), [`9c27a53`](https://github.com/mastra-ai/mastra/commit/9c27a53cd9d3de4f3f025bc387d94ce371c33f95), [`dff25a1`](https://github.com/mastra-ai/mastra/commit/dff25a1103fa72ee082a9b6f805ebeb5ce400753), [`217e967`](https://github.com/mastra-ai/mastra/commit/217e9672d8b3160eb729d8e9f0044949e88da239), [`7f78585`](https://github.com/mastra-ai/mastra/commit/7f785857e401570e2ffb316911f126ed363aa537), [`f2a4afd`](https://github.com/mastra-ai/mastra/commit/f2a4afd7e37e809669001ed17724b341a5c1f45e), [`d438148`](https://github.com/mastra-ai/mastra/commit/d438148e222c1e2fb3c652725ce75680962ebec4), [`ba05fe0`](https://github.com/mastra-ai/mastra/commit/ba05fe0738f70cb686777546e968237d09269142), [`d26a8d4`](https://github.com/mastra-ai/mastra/commit/d26a8d4281f28414715b333c85bedaf70d0b2890), [`677cdc6`](https://github.com/mastra-ai/mastra/commit/677cdc6af564dec29a13464d12b7ab2a4efc22e9), [`a318490`](https://github.com/mastra-ai/mastra/commit/a318490e17da32f338d50929c770d901a9b3dd72), [`763e0c6`](https://github.com/mastra-ai/mastra/commit/763e0c61e04d76ad9a9efd301aa57525ca0cbea9), [`23e0be2`](https://github.com/mastra-ai/mastra/commit/23e0be261381e49534b4ff3101c60ee64a946cbf), [`7fc8806`](https://github.com/mastra-ai/mastra/commit/7fc880627d3cbf995d31ea0e8b807bf15417e651), [`0e02eac`](https://github.com/mastra-ai/mastra/commit/0e02eacdb2e30e1697a41910b41163742a181dc1), [`4df174c`](https://github.com/mastra-ai/mastra/commit/4df174c32bddf093a82f273070b8380aef7c9e90), [`f7c25b5`](https://github.com/mastra-ai/mastra/commit/f7c25b5106ddfb48e591f98df7a51e0f2dd01dba), [`dc09cc1`](https://github.com/mastra-ai/mastra/commit/dc09cc1083d861cde192c1cd235324dc75b8c731), [`36b4649`](https://github.com/mastra-ai/mastra/commit/36b4649045a3a380cbab8ceca866db4086223aff), [`377eb81`](https://github.com/mastra-ai/mastra/commit/377eb81ce43b964e3a6b541df172da74a8ff3716)]:
+  - @mastra/core@1.60.0-alpha.8
+
 ## 1.7.0
 
 ### Minor Changes

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-evals
 description: Documentation for @mastra/evals. Use when working with @mastra/evals APIs, configuration, or implementation.
 metadata:
   package: "@mastra/evals"
-  version: "1.7.0"
+  version: "1.8.0"
 ---
 
 ## When to use
@@ -17,7 +17,7 @@ Read the individual reference documents for detailed explanations and code examp
 ### Docs
 
 - [Built-in scorers](references/docs-evals-built-in-scorers.md) - Overview of Mastra's ready-to-use scorers for evaluating AI outputs across quality, safety, and performance dimensions.
-- [Scorers overview](references/docs-evals-overview.md) - Overview of scorers in Mastra, detailing their capabilities for evaluating AI outputs and measuring performance.
+- [Evals](references/docs-evals-overview.md) - Overview of evaluations and scorers in Mastra, including Quick Checks, live evaluations, and trace evaluations.
 - [Quick Checks](references/docs-evals-quick-checks.md) - Zero-LLM micro-scorers for fast, deterministic assertions on agent output text and tool usage.
 
 ### Reference
@@ -44,6 +44,7 @@ Read the individual reference documents for detailed explanations and code examp
 - [Reference: Tool call accuracy scorers](references/reference-evals-tool-call-accuracy.md) - Documentation for the Tool Call Accuracy Scorers in Mastra, which evaluate whether LLM outputs call the correct tools from available options.
 - [Reference: Toxicity scorer](references/reference-evals-toxicity.md) - Documentation for the Toxicity Scorer in Mastra, which evaluates LLM outputs for racist, biased, or toxic elements.
 - [Reference: Trajectory accuracy scorers](references/reference-evals-trajectory-accuracy.md) - Documentation for the Trajectory Accuracy Scorers in Mastra, which evaluate whether an agent or workflow follows the expected sequence of actions.
+- [Evals and scorers](references/reference-migrations-upgrade-to-v1-evals.md) - Learn how to migrate evals and scorers changes when upgrading to v1.
 
 
 Read [assets/SOURCE_MAP.json](assets/SOURCE_MAP.json) for source code references.

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.7.0",
+  "version": "1.8.0",
   "package": "@mastra/evals",
   "exports": {},
   "modules": {}

package/dist/docs/references/docs-evals-overview.md

@@ -1,12 +1,12 @@
 > Discover all available pages from the documentation index: https://mastra.ai/llms.txt
 
-# Scorers overview
+# Evals
 
 While traditional software tests have clear pass/fail conditions, AI outputs are non-deterministic, they can vary with the same input. **Scorers** help bridge this gap by providing quantifiable metrics for measuring agent quality.
 
-Scorers are automated tests that evaluate Agents outputs using model-graded, rule-based, and statistical methods. Scorers return **scores**: numerical values (typically between 0 and 1) that quantify how well an output meets your evaluation criteria. These scores enable you to objectively track performance, compare different approaches, and identify areas for improvement in your AI systems. Scorers can be customized with your own prompts and scoring functions.
+Scorers are automated tests that evaluate Agents outputs using model-graded, rule-based, and statistical methods. Scorers return **scores**: numerical values (typically between 0 and 1) that quantify how well an output meets your evaluation criteria. These scores enable you to objectively track performance and compare different approaches while identifying areas for improvement in your AI systems. Scorers can be customized with your own prompts and scoring functions.
 
-Scorers can be run in the cloud, capturing real-time results. But scorers can also be part of your CI/CD pipeline, allowing you to test and monitor your agents over time.
+Scorers can run in the cloud to capture real-time results. Scorers can also be part of your CI/CD pipeline for testing and monitoring agents over time.
 
 > **📹 Watch:** Watch [Mastra evals overview](https://www.youtube.com/watch?v=12WN6u2DrBk) for an introduction to evals and how to reason about agent quality.
 
@@ -75,7 +75,7 @@ export const evaluatedAgent = new Agent({
 
 ### Adding scorers to workflow steps
 
-You can also add scorers to individual workflow steps to evaluate outputs at specific points in your process:
+You can also add scorers to individual workflow steps to evaluate outputs at specific points in your process. Each scorer receives that step's own input and output, so you can measure quality at each step instead of only scoring the final answer:
 
 ```typescript
 import { createWorkflow, createStep } from "@mastra/core/workflows";
@@ -83,25 +83,37 @@ import { z } from "zod";
 import { customStepScorer } from "../scorers/custom-step-scorer";
 
 const contentStep = createStep({
+  id: "content-step",
+  inputSchema: z.object({ topic: z.string() }),
+  outputSchema: z.object({ content: z.string() }),
   scorers: {
     customStepScorer: {
       scorer: customStepScorer(),
       sampling: {
         type: "ratio",
         rate: 1, // Score every step execution
-      }
-    }
+      },
+    },
+  },
+  execute: async ({ inputData }) => {
+    return { content: await generateContent(inputData.topic) };
   },
 });
 
-export const contentWorkflow = createWorkflow({ ... })
+export const contentWorkflow = createWorkflow({
+  id: "content-workflow",
+  inputSchema: z.object({ topic: z.string() }),
+  outputSchema: z.object({ content: z.string() }),
+})
   .then(contentStep)
   .commit();
 ```
 
+For the step-level `scorers` API, see the [Step class reference](https://mastra.ai/reference/workflows/step).
+
 ### How live evaluations work
 
-**Asynchronous execution**: Live evaluations run in the background without blocking your agent responses or workflow execution. This ensures your AI systems maintain their performance while still being monitored.
+**Asynchronous execution**: Live evaluations run in the background without blocking your agent responses or workflow execution. Your AI systems remain responsive while live evaluations monitor them.
 
 **Sampling control**: The `sampling.rate` parameter (0-1) controls what percentage of outputs get scored:
 

package/dist/docs/references/docs-evals-quick-checks.md

@@ -1,6 +1,6 @@
 > Discover all available pages from the documentation index: https://mastra.ai/llms.txt
 
-# Quick Checks
+# Quick checks
 
 Quick Checks are composable micro-scorers for common assertions like "output contains X" or "agent called tool Y." They require no LLM, run instantly, and plug into the same `scorers: [...]` array as any other scorer.
 

package/dist/docs/references/reference-evals-completeness.md

@@ -67,7 +67,7 @@ The scorer evaluates completeness through linguistic element coverage analysis.
 
 1. Extracts key elements:
 
-   - Nouns and named entities
+   - Nouns and entities such as names
    - Action verbs
    - Topic-specific terms
    - Normalized word forms

package/dist/docs/references/reference-evals-noise-sensitivity.md

@@ -177,7 +177,7 @@ Include in your test harness to:
 
 - Benchmark different models' noise resistance before deployment
 - Identify agents vulnerable to manipulation during development
-- Create detailed test coverage for various noise types
+- Create detailed test coverage for multiple noise types
 - Ensure consistent behavior across updates
 
 ### Security Testing

package/dist/docs/references/reference-evals-prompt-alignment.md

@@ -70,6 +70,24 @@ const scorer = createPromptAlignmentScorerLLM({
 })
 ```
 
+### Multi-turn conversations
+
+By default the scorer only sees the current turn. In a conversation, a reply like `"A"` is meaningless on its own, so the judge can't tell what the user asked for and scores the response as misaligned.
+
+Set `includeConversationHistory` to give the judge the prior turns from the agent's memory. The judge uses them to interpret the current prompt, but still scores only the current response.
+
+```typescript
+const scorer = createPromptAlignmentScorerLLM({
+  model: 'openai/gpt-5.6-sol',
+  options: {
+    evaluationMode: 'user',
+    includeConversationHistory: { maxMessages: 6 }, // or `true` for the last 10 messages
+  },
+})
+```
+
+This only affects agent runs, where the scorer receives the remembered messages. Runs scored from a plain prompt string are unchanged.
+
 ### Multi-Dimensional Analysis
 
 Prompt Alignment evaluates responses across four key dimensions with weighted scoring that adapts based on the evaluation mode:

package/dist/docs/references/reference-evals-rubric.md

@@ -111,5 +111,5 @@ The `reason` summarizes the result and lists each criterion with its verdict, so
 ## Related
 
 - [isTaskComplete on stream()](https://mastra.ai/reference/streaming/agents/stream)
-- [Supervisor agents](https://mastra.ai/docs/agents/supervisor-agents)
+- [Supervisor agents](https://mastra.ai/docs/subagents)
 - [createScorer](https://mastra.ai/reference/evals/create-scorer)

package/dist/docs/references/reference-evals-summarization.md

@@ -2,7 +2,7 @@
 
 # Summarization scorer
 
-The `createSummarizationScorer()` function creates a scorer that evaluates a summary on two axes: whether every claim it makes is supported by the source text, and whether it preserves the information the source states. The final score is the lower of the two, so a summary can't pass by being faithful but empty, or thorough but wrong.
+The `createSummarizationScorer()` function creates a scorer that evaluates a summary on two axes: whether every claim has source-text support and whether the summary preserves the source information. The final score is the lower of the two. A summary can't pass by being faithful but empty or by being thorough but wrong.
 
 The summary is the agent's last message that carries text, and the source text defaults to the first user message of the run input. Pass `source` or `sourceExtractor` when the text being summarized lives somewhere else, such as a tool result.
 

package/dist/docs/references/reference-migrations-upgrade-to-v1-evals.md

@@ -0,0 +1,241 @@
+> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
+
+# Evals and scorers
+
+The evaluation API has been consolidated on the new scorers system with updated naming conventions and configuration requirements.
+
+## Changed
+
+### `getScorers` to `listScorers`
+
+The `getScorers()` method has been renamed to `listScorers()`. This change aligns with the naming convention used across the API where plural getter methods use the `list` prefix.
+
+To migrate, replace all calls to `getScorers()` with `listScorers()`.
+
+```diff
+- const scorers = mastra.getScorers();
++ const scorers = mastra.listScorers();
+```
+
+> **Codemod:** You can use Mastra's codemod CLI to update your code automatically:
+>
+> ```bash
+> npx @mastra/codemod@latest v1/mastra-plural-apis .
+> ```
+
+### `runExperiment` to `runEvals`
+
+The `runExperiment()` function has been renamed to `runEvals()` to identify that it runs evaluations.
+
+To migrate, update function calls from `runExperiment` to `runEvals`.
+
+```diff
+- import { createScorer, runExperiment } from '@mastra/core/evals';
++ import { createScorer, runEvals } from '@mastra/core/evals';
+  import { myAgent } from './agents/my-agent';
+
+  const scorer = createScorer({
+    id: 'helpfulness-scorer',
+    // ...
+  });
+
+- const result = await runExperiment({ target: myAgent, scorers: [scorer], data: inputs });
++ const result = await runEvals({ target: myAgent, scorers: [scorer], data: inputs });
+```
+
+> **Codemod:** You can use Mastra's codemod CLI to update your code automatically:
+>
+> ```bash
+> npx @mastra/codemod@latest v1/evals-run-experiment .
+> ```
+
+### `getScorerByName` to `getScorerById`
+
+The `getScorerByName()` method has been renamed to `getScorerById()`. Scorers now require an `id` field instead of `name`. This change aligns with the broader API pattern of using `id` for entity identification.
+
+To migrate, update method calls and scorer configuration to use `id` instead of `name`.
+
+```diff
+const scorer = createScorer({
+-   name: 'helpfulness-scorer',
++   id: 'helpfulness-scorer',
+    // ...
+  });
+
+- const scorer = mastra.getScorerByName('helpfulness-scorer');
++ const scorer = mastra.getScorerById('helpfulness-scorer');
+```
+
+> **Codemod:** You can use Mastra's codemod CLI to update your code automatically:
+>
+> ```bash
+> npx @mastra/codemod@latest v1/evals-scorer-by-name .
+> ```
+
+### Scorer configuration from `name` to `id`
+
+Scorers now require an `id` field instead of `name`. The `name` field is now optional. The new field aligns scorers with other Mastra entities.
+
+To migrate, update scorer definitions to use `id` as the required field.
+
+```diff
+const scorer = createScorer({
+-   name: 'helpfulness-scorer',
++   id: 'helpfulness-scorer',
++   name: 'Helpfulness Scorer', // optional
+    // ...
+  });
+```
+
+### Storage score APIs to `listScoresBy*` pattern
+
+Score storage APIs have been renamed to follow the `listScoresBy*` pattern. This change aligns with the broader storage API naming conventions.
+
+To migrate, update score query methods to use the new naming pattern.
+
+```diff
+- const scores = await storage.getScores({ scorerName: 'helpfulness-scorer' });
++ const scores = await storage.listScoresByScorerId({
++   scorerId: 'helpfulness-scorer',
++ });
+
+// Also available:
+// - listScoresByRunId
+// - listScoresByEntityId
+// - listScoresBySpan
+```
+
+### Prebuilt scorer imports to `scorers/prebuilt` path
+
+Prebuilt scorer imports have been consolidated under a single `@mastra/evals/scorers/prebuilt` path instead of separate `scorers/llm` and `scorers/code` paths. This change simplifies imports and provides a clearer organization of prebuilt scorers.
+
+To migrate, update import statements to use the new `scorers/prebuilt` path.
+
+```diff
+// LLM-based scorers
+- import { createHallucinationScorer } from '@mastra/evals/scorers/llm';
+- import { createFaithfulnessScorer } from '@mastra/evals/scorers/llm';
++ import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt';
++ import { createFaithfulnessScorer } from '@mastra/evals/scorers/prebuilt';
+
+  // Code-based scorers
+- import { createContentSimilarityScorer } from '@mastra/evals/scorers/code';
+- import { createCompletenessScorer } from '@mastra/evals/scorers/code';
++ import { createContentSimilarityScorer } from '@mastra/evals/scorers/prebuilt';
++ import { createCompletenessScorer } from '@mastra/evals/scorers/prebuilt';
+```
+
+> **Codemod:** You can use Mastra's codemod CLI to update your code automatically:
+>
+> ```bash
+> npx @mastra/codemod@latest v1/evals-prebuilt-imports .
+> ```
+
+### Scorer message types from `UIMessage` to `MastraDBMessage`
+
+Scorer input and output types now use `MastraDBMessage[]` instead of `UIMessage`. This change aligns scorers with the database-persisted message format for consistency across the framework.
+
+To migrate, update scorer implementations to use `MastraDBMessage` types and access message content through the nested `content` structure.
+
+```diff
+import type {
+    ScorerRunInputForAgent,
+    ScorerRunOutputForAgent
+  } from '@mastra/core/evals';
+
+- // ScorerRunInputForAgent uses UIMessage[]
+- const inputMessages: UIMessage[] = run.input.inputMessages;
++ // ScorerRunInputForAgent now uses MastraDBMessage[]
++ import type { MastraDBMessage } from '@mastra/core/agent';
++ const inputMessages: MastraDBMessage[] = run.input.inputMessages;
+```
+
+### Message content structure to nested format
+
+Tool invocations and text content are now accessed through a nested `content` object instead of flat message properties. The nested structure matches the database message format and its types.
+
+To migrate, access tool invocations via `message.content.toolInvocations` and text via `message.content.content` or use the `getTextContentFromMastraDBMessage()` helper.
+
+```diff
++ import { getTextContentFromMastraDBMessage } from '@mastra/evals';
++
+  const run = await scorer.run(testRun);
+
+  // Accessing text content
+- const text = message.content;
++ const text = getTextContentFromMastraDBMessage(message);
++ // or directly: message.content.content
+
+  // Accessing tool invocations
+- const toolCalls = message.toolInvocations;
++ const toolCalls = message.content.toolInvocations;
+```
+
+## Removed
+
+### Legacy evals code
+
+Legacy evals code has been removed from `@mastra/core`. This includes legacy evaluation metrics, scorer/judge modules, and hook-based automatic evaluation code. This change simplifies the codebase by removing outdated evaluation approaches.
+
+To migrate, use the new evals/scorers API in `@mastra/core/evals` or `@mastra/evals`.
+
+```diff
+- // Legacy evals APIs
++ import { createScorer, runEvals } from '@mastra/core/evals';
++
++ const scorer = createScorer({
++   id: 'my-scorer',
++   // Use new scorer API
++ });
+```
+
+### Agent `TMetrics` generic parameter
+
+The `TMetrics` generic parameter has been removed from `AgentConfig` and the `Agent` constructor. Metrics/scorers are now configured using the scorers API instead of being part of the Agent type system. This change simplifies the Agent type signature.
+
+To migrate, remove the `TMetrics` generic parameter and configure scorers using the scorers API.
+
+```diff
+- const agent = new Agent<AgentId, Tools, Metrics>({
++ const agent = new Agent<AgentId, Tools>({
+    // ...
+  });
+```
+
+### Evals-related type exports
+
+Several evals-related type exports have been removed including `DeprecatedOutputOptions`, `Metric`, and processor option types. These types are now internal or have been replaced by the new scorers API. This change reduces API surface area.
+
+To migrate, remove references to these removed types and use the new scorers API.
+
+```diff
+- import type {
+-   DeprecatedOutputOptions,
+-   Metric,
+-   LanguageDetectorOptions,
+-   ModerationOptions,
+- } from '@mastra/core';
++ // Use new scorers API types
++ import type { Scorer } from '@mastra/core/evals';
+```
+
+### `createUIMessage` test helper
+
+The `createUIMessage()` test helper has been removed and replaced with `createTestMessage()`. The new helper creates `MastraDBMessage` objects with the nested content structure and supports optional tool invocations. This change aligns test utilities with the new message format.
+
+To migrate, replace `createUIMessage()` calls with `createTestMessage()` and update to use `MastraDBMessage` types.
+
+```diff
+- import { createUIMessage } from '@mastra/evals';
++ import { createTestMessage } from '@mastra/evals';
+
+  // Creating test messages
+- const message = createUIMessage({
+-   id: 'test-1',
++ const message = createTestMessage({
++   id: 'test-1', // optional, defaults to 'test-message'
+    role: 'user',
+    content: 'Hello',
+    toolInvocations: [], // optional
+  });
+```

package/dist/scorers/llm/prompt-alignment/index.d.ts

@@ -3,6 +3,16 @@ import type { ScorerRunInputForLLMJudge, ScorerRunOutputForLLMJudge } from '../.
 export interface PromptAlignmentOptions {
     scale?: number;
     evaluationMode?: 'user' | 'system' | 'both';
+    /**
+     * Include prior conversation turns from the agent's remembered messages in the judge prompt.
+     *
+     * Off by default. Enable it for multi-turn agents so short, referential replies (e.g. "A",
+     * "yes") are judged against what was previously asked instead of in isolation. Only applies
+     * to agent runs that carry `rememberedMessages`.
+     */
+    includeConversationHistory?: boolean | {
+        maxMessages?: number;
+    };
 }
 export declare function createPromptAlignmentScorerLLM({ model, options, }: {
     model: MastraModelConfig;

package/dist/scorers/llm/prompt-alignment/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/prompt-alignment/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAQ1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAGzF,MAAM,WAAW,sBAAsB;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC7C;AAyDD,wBAAgB,8BAA8B,CAAC,EAC7C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAClC;;;;;;;;;;;;;;;;;;;;;;;;;;;6FAqHA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/prompt-alignment/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAS1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAGzF,MAAM,WAAW,sBAAsB;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC5C;;;;;;OAMG;IACH,0BAA0B,CAAC,EAAE,OAAO,GAAG;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACjE;AAgED,wBAAgB,8BAA8B,CAAC,EAC7C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAClC;;;;;;;;;;;;;;;;;;;;;;;;;;;6FAwHA"}

package/dist/scorers/llm/prompt-alignment/prompts.d.ts

@@ -1,9 +1,10 @@
 export declare const PROMPT_ALIGNMENT_INSTRUCTIONS = "You are an expert prompt-response alignment evaluator. Your job is to analyze how well an agent's response aligns with the user's prompt in terms of intent, requirements, completeness, and appropriateness.\n\nKey Evaluation Dimensions:\n1. **Intent Alignment**: Does the response address the core purpose of the prompt?\n2. **Requirements Fulfillment**: Are all explicit and implicit requirements met?\n3. **Completeness**: Is the response comprehensive and thorough?\n4. **Response Appropriateness**: Does the format, tone, and style match expectations?\n\nEvaluation Guidelines:\n- Identify the primary intent and any secondary intents in the prompt\n- Extract all explicit requirements (specific tasks, constraints, formats)\n- Consider implicit requirements based on context and standard expectations\n- Assess whether the response fully addresses the prompt or leaves gaps\n- Evaluate if the response format and tone are appropriate for the request\n- Be objective and focus on alignment rather than response quality\n\nScore each dimension from 0.0 (completely misaligned) to 1.0 (perfectly aligned).";
-export declare function createAnalyzePrompt({ userPrompt, systemPrompt, agentResponse, evaluationMode, }: {
+export declare function createAnalyzePrompt({ userPrompt, systemPrompt, agentResponse, evaluationMode, conversationHistory, }: {
     userPrompt: string;
     systemPrompt?: string;
     agentResponse: string;
     evaluationMode: 'user' | 'system' | 'both';
+    conversationHistory?: string;
 }): string;
 export type AnalysisResult = {
     intentAlignment: {
@@ -33,12 +34,13 @@ export type AnalysisResult = {
     };
     overallAssessment: string;
 };
-export declare function createReasonPrompt({ userPrompt, systemPrompt, score, scale, analysis, evaluationMode, }: {
+export declare function createReasonPrompt({ userPrompt, systemPrompt, score, scale, analysis, evaluationMode, conversationHistory, }: {
     userPrompt: string;
     systemPrompt?: string;
     score: number;
     scale: number;
     analysis: AnalysisResult;
     evaluationMode: 'user' | 'system' | 'both';
+    conversationHistory?: string;
 }): string;
 //# sourceMappingURL=prompts.d.ts.map

package/dist/scorers/llm/prompt-alignment/prompts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/prompt-alignment/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,6BAA6B,ulCAgBwC,CAAC;AAEnF,wBAAgB,mBAAmB,CAAC,EAClC,UAAU,EACV,YAAY,EACZ,aAAa,EACb,cAAc,GACf,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC5C,UA6KA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,eAAe,EAAE;QACf,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,OAAO,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,uBAAuB,EAAE;QACvB,YAAY,EAAE,KAAK,CAAC;YAClB,WAAW,EAAE,MAAM,CAAC;YACpB,WAAW,EAAE,OAAO,CAAC;YACrB,SAAS,EAAE,MAAM,CAAC;SACnB,CAAC,CAAC;QACH,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,YAAY,EAAE;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,uBAAuB,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;QACd,eAAe,EAAE,OAAO,CAAC;QACzB,aAAa,EAAE,OAAO,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,EACjC,UAAU,EACV,YAAY,EACZ,KAAK,EACL,KAAK,EACL,QAAQ,EACR,cAAc,GACf,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,cAAc,CAAC;IACzB,cAAc,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC5C,UAkEA"}
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/prompt-alignment/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,6BAA6B,ulCAgBwC,CAAC;AAgBnF,wBAAgB,mBAAmB,CAAC,EAClC,UAAU,EACV,YAAY,EACZ,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC3C,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,UA8KA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,eAAe,EAAE;QACf,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,OAAO,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,uBAAuB,EAAE;QACvB,YAAY,EAAE,KAAK,CAAC;YAClB,WAAW,EAAE,MAAM,CAAC;YACpB,WAAW,EAAE,OAAO,CAAC;YACrB,SAAS,EAAE,MAAM,CAAC;SACnB,CAAC,CAAC;QACH,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,YAAY,EAAE;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,uBAAuB,EAAE;QACvB,KAAK,EAAE,MAAM,CAAC;QACd,eAAe,EAAE,OAAO,CAAC;QACzB,aAAa,EAAE,OAAO,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,EACjC,UAAU,EACV,YAAY,EACZ,KAAK,EACL,KAAK,EACL,QAAQ,EACR,cAAc,EACd,mBAAmB,GACpB,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,cAAc,CAAC;IACzB,cAAc,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC3C,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,UAmEA"}