@mastra/evals 1.6.0 → 1.7.0

package/dist/docs/references/reference-evals-trajectory-accuracy.md

@@ -15,7 +15,7 @@ The `runEvals` pipeline uses two extraction strategies, depending on whether obs
 
 ### Trace-based extraction (preferred)
 
-When the target's `Mastra` instance has storage configured, the pipeline fetches the full execution trace from the observability store and calls `extractTrajectoryFromTrace()`. This produces a hierarchical trajectory with nested `children`, capturing the complete execution tree — including nested agent runs, tool calls within workflow steps, and model generations.
+When the target's `Mastra` instance has storage configured, the pipeline fetches the full execution trace from the observability store and calls `extractTrajectoryFromTrace()`. This produces a hierarchical trajectory with nested `children` that captures the complete execution tree. The tree includes nested agent runs and tool calls within workflow steps. It also includes model generations.
 
 For example, a workflow that calls an agent, which in turn calls tools, produces:
 
@@ -35,8 +35,8 @@ workflow_run
 
 When storage isn't available, the pipeline falls back to:
 
-- **Agents:** `extractTrajectory()` — Extracts `ToolCallStep` entries from `toolInvocations` in the agent's message output. Produces a flat list of tool calls.
-- **Workflows:** `extractWorkflowTrajectory()` — Extracts `WorkflowStepStep` entries from `stepResults`. Produces a flat list of workflow steps.
+- **Agents:** `extractTrajectory()`, Extracts `ToolCallStep` entries from `toolInvocations` in the agent's message output. Produces a flat list of tool calls.
+- **Workflows:** `extractWorkflowTrajectory()`, Extracts `WorkflowStepStep` entries from `stepResults`. Produces a flat list of workflow steps.
 
 These fallbacks don't capture nested execution or non-tool-call spans.
 
@@ -105,7 +105,7 @@ All step types share the base properties `name`, `durationMs`, `metadata`, and `
 
 ## Expected steps
 
-When defining expected trajectories, use `ExpectedStep` instead of the full `TrajectoryStep` discriminated union. `ExpectedStep` is a discriminated union that mirrors `TrajectoryStep` — when you specify a `stepType`, you get autocomplete for that variant's fields (e.g., `toolArgs` for `tool_call`, `modelId` for `model_generation`). All variant-specific fields are optional, so you only assert against what you care about.
+When defining expected trajectories, use `ExpectedStep` instead of the full `TrajectoryStep` discriminated union. `ExpectedStep` is a discriminated union that mirrors `TrajectoryStep`: when you specify a `stepType`, you get autocomplete for that variant's fields (e.g., `toolArgs` for `tool_call`, `modelId` for `model_generation`). All variant-specific fields are optional, so you only assert against what you care about.
 
 Omit `stepType` entirely to match any step by name only.
 
@@ -199,8 +199,8 @@ This function returns an instance of the MastraScorer class. See the [MastraScor
 
 The code-based scorer resolves `expectedTrajectory` from two sources, in order of priority:
 
-1. **Constructor option** — A static trajectory passed when creating the scorer. Used for all dataset items.
-2. **Dataset item** — An `expectedTrajectory` field on the dataset item, passed through the `runEvals` pipeline. Allows different expected trajectories per item.
+1. **Constructor option**: A static trajectory passed when creating the scorer. Used for all dataset items.
+2. **Dataset item**: An `expectedTrajectory` field on the dataset item, passed through the `runEvals` pipeline. Allows different expected trajectories per item.
 
 ```typescript
 // Static: same expected trajectory for all items
@@ -474,10 +474,10 @@ The `createTrajectoryScorerCode()` function from `@mastra/evals/scorers/prebuilt
 
 The unified scorer evaluates four dimensions:
 
-1. **Accuracy** — Matches actual steps against expected steps (if `steps` is configured). Uses the `ordering` mode.
-2. **Efficiency** — Checks step budgets (`maxSteps`, `maxTotalTokens`, `maxTotalDurationMs`) and redundant calls (`noRedundantCalls`).
-3. **Blacklist** — Checks for forbidden tools or sequences. Any violation immediately results in a score of **0.0** regardless of other dimensions.
-4. **Tool failures** — Detects retry patterns, fallback patterns, and argument correction patterns.
+1. **Accuracy**: Matches actual steps against expected steps (if `steps` is configured). Uses the `ordering` mode.
+2. **Efficiency**: Checks step budgets (`maxSteps`, `maxTotalTokens`, `maxTotalDurationMs`) and redundant calls (`noRedundantCalls`).
+3. **Blacklist**: Checks for forbidden tools or sequences. Any violation immediately results in a score of **0.0** regardless of other dimensions.
+4. **Tool failures**: Detects retry and fallback patterns. It also detects argument correction patterns.
 
 The final score is a weighted combination of active dimensions, normalized by which dimensions are active. Default weights are accuracy 0.4, efficiency 0.3, tool failures 0.2, blacklist 0.1, but you can customize them via the `weights` option. Blacklist violations override everything to 0. When nested evaluations are present, the score is 70% top-level and 30% nested average.
 

package/dist/index.cjs

@@ -1,4 +1,14 @@
-'use strict';
+//#region src/index.ts
+/**
+* @mastra/evals - Evaluation framework for AI agents
+*
+* This package uses subpath exports. Import from specific paths:
+*
+* @example
+* ```ts
+* import { createToolCallAccuracyScorerCode } from '@mastra/evals/scorers/prebuilt';
+* import { getUserMessageFromRunInput } from '@mastra/evals/scorers/utils';
+* */
+//#endregion
 
-//# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":[],"names":[],"mappings":"","file":"index.cjs"}
+{"version":3,"file":"index.cjs","names":[],"sources":["../src/index.ts"],"sourcesContent":["/**\n * @mastra/evals - Evaluation framework for AI agents\n *\n * This package uses subpath exports. Import from specific paths:\n *\n * @example\n * ```ts\n * import { createToolCallAccuracyScorerCode } from '@mastra/evals/scorers/prebuilt';\n * import { getUserMessageFromRunInput } from '@mastra/evals/scorers/utils';\n * */\n\n// This package uses subpath exports - see package.json for available paths\n"],"mappings":""}

package/dist/index.js

@@ -1,3 +1,15 @@
+//#region src/index.ts
+/**
+* @mastra/evals - Evaluation framework for AI agents
+*
+* This package uses subpath exports. Import from specific paths:
+*
+* @example
+* ```ts
+* import { createToolCallAccuracyScorerCode } from '@mastra/evals/scorers/prebuilt';
+* import { getUserMessageFromRunInput } from '@mastra/evals/scorers/utils';
+* */
+//#endregion
+export {};
 
-//# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"names":[],"mappings":"","file":"index.js"}
+{"version":3,"file":"index.js","names":[],"sources":["../src/index.ts"],"sourcesContent":["/**\n * @mastra/evals - Evaluation framework for AI agents\n *\n * This package uses subpath exports. Import from specific paths:\n *\n * @example\n * ```ts\n * import { createToolCallAccuracyScorerCode } from '@mastra/evals/scorers/prebuilt';\n * import { getUserMessageFromRunInput } from '@mastra/evals/scorers/utils';\n * */\n\n// This package uses subpath exports - see package.json for available paths\n"],"mappings":""}

package/dist/scorers/llm/index.d.ts

@@ -11,5 +11,6 @@ export * from './context-recall/index.js';
 export * from './noise-sensitivity/index.js';
 export * from './prompt-alignment/index.js';
 export * from './rubric/index.js';
+export * from './summarization/index.js';
 export * from './trajectory/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/scorers/llm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scorers/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,QAAQ,CAAC;AACvB,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scorers/llm/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,QAAQ,CAAC;AACvB,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC"}

package/dist/scorers/llm/summarization/index.d.ts

@@ -0,0 +1,55 @@
+import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/evals';
+import type { MastraModelConfig } from '@mastra/core/llm';
+import type { ScorerRunInputForLLMJudge, ScorerRunOutputForLLMJudge } from '../../utils.js';
+export interface SummarizationMetricOptions {
+    scale?: number;
+    /** Text the summary is judged against. Defaults to the user message of the run input. */
+    source?: string;
+    sourceExtractor?: (input: ScorerRunInputForAgent, output: ScorerRunOutputForAgent) => string;
+    /** Upper bound on the coverage questions drawn from the source. Defaults to 10. */
+    maxQuestions?: number;
+}
+/**
+ * Creates a scorer that evaluates a summary against the text it condenses.
+ *
+ * The scorer judges two axes and returns the lower one, so a summary cannot pass
+ * by being faithful but empty, or thorough but wrong:
+ * - Alignment: supported claims / total claims
+ * - Coverage: answered questions / total questions
+ *
+ * Coverage is judged in a separate call that never receives the source text, so
+ * the judge cannot answer a question from information the summary left out.
+ *
+ * @param model - Language model used as the judge
+ * @param options - Source resolution, question cap, and score scaling
+ * @returns A scorer producing `min(alignment, coverage) × scale`
+ * @throws When `maxQuestions` is below 1
+ *
+ * @example
+ * ```ts
+ * const scorer = createSummarizationScorer({
+ *   model: 'openai/gpt-5.5',
+ *   options: { maxQuestions: 10 },
+ * });
+ *
+ * const result = await scorer.run(run);
+ * ```
+ */
+export declare function createSummarizationScorer({ model, options, }: {
+    model: MastraModelConfig;
+    options?: SummarizationMetricOptions;
+}): import("@mastra/core/evals").MastraScorer<string, ScorerRunInputForLLMJudge, ScorerRunOutputForLLMJudge, Record<"preprocessStepResult", {
+    alignment: {
+        claim: string;
+        supported: boolean;
+        reason: string;
+    }[];
+    questions: string[];
+}> & Record<"analyzeStepResult", {
+    coverage: {
+        question: string;
+        answered: boolean;
+        reason: string;
+    }[];
+}> & Record<"generateScoreStepResult", number> & Record<"generateReasonStepResult", string>>;
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/llm/summarization/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/summarization/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC1F,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAU1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAQzF,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,uBAAuB,KAAK,MAAM,CAAC;IAC7F,mFAAmF;IACnF,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAmID;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,yBAAyB,CAAC,EACxC,KAAK,EACL,OAAY,GACb,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,0BAA0B,CAAC;CACtC;;;;;;;;;;;;;6FA4EA"}

package/dist/scorers/llm/summarization/prompts.d.ts

@@ -0,0 +1,48 @@
+export declare const SUMMARIZATION_AGENT_INSTRUCTIONS = "You are a strict summarization evaluator. Your job is to determine whether a summary is factually faithful to its source text and whether it preserves the information the source states.\n\nKey Principles:\n1. Judge only against the text you are given, never against prior knowledge\n2. A claim is supported only when the source states or directly implies it\n3. Treat approximations as deviations when the source is precise: \"about 2020\" does not match \"2020\"\n4. Allow semantically equivalent phrasing: \"founded in Boston\" matches \"based in Boston\"\n5. Judge each claim and each question independently\n6. Factual accuracy and information coverage are separate concerns and are judged separately";
+/**
+ * Builds the preprocess prompt, which covers everything that needs the source text.
+ *
+ * Claims are extracted from the summary and judged against the source in one pass,
+ * so every verdict carries the claim it belongs to. Questions are drawn from the
+ * source for the coverage step that follows.
+ *
+ * @param maxQuestions - Upper bound on the generated questions
+ * @returns A prompt asking for `{ alignment, questions }`
+ */
+export declare function createSourceJudgementPrompt({ sourceText, summary, maxQuestions, }: {
+    sourceText: string;
+    summary: string;
+    maxQuestions: number;
+}): string;
+/**
+ * Builds the analyze prompt, which answers the coverage questions.
+ *
+ * The source text is deliberately absent. A judge that could see it would answer
+ * from the source instead of the summary, which would hide exactly the omissions
+ * this axis exists to measure.
+ *
+ * @returns A prompt asking for `{ coverage }`
+ */
+export declare function createCoveragePrompt({ summary, questions }: {
+    summary: string;
+    questions: string[];
+}): string;
+/**
+ * Builds the prompt that explains a finished score.
+ *
+ * Both axis scores and the failing claims and questions are passed in so the
+ * explanation names the axis that decided the result instead of recalculating it.
+ *
+ * @returns A prompt asking for a one-sentence explanation of the score
+ */
+export declare function createSummarizationReasonPrompt({ sourceText, summary, score, scale, alignmentScore, coverageScore, unsupportedClaims, missingQuestions, }: {
+    sourceText: string;
+    summary: string;
+    score: number;
+    scale: number;
+    alignmentScore: number;
+    coverageScore: number;
+    unsupportedClaims: string[];
+    missingQuestions: string[];
+}): string;
+//# sourceMappingURL=prompts.d.ts.map

package/dist/scorers/llm/summarization/prompts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/summarization/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,gCAAgC,qsBAQgD,CAAC;AAE9F;;;;;;;;;GASG;AACH,wBAAgB,2BAA2B,CAAC,EAC1C,UAAU,EACV,OAAO,EACP,YAAY,GACb,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB,UAqEA;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,EAAE,CAAA;CAAE,UAuDpG;AAED;;;;;;;GAOG;AACH,wBAAgB,+BAA+B,CAAC,EAC9C,UAAU,EACV,OAAO,EACP,KAAK,EACL,KAAK,EACL,cAAc,EACd,aAAa,EACb,iBAAiB,EACjB,gBAAgB,GACjB,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,UAmCA"}