@mastra/mcp-docs-server 1.0.0-beta.4 → 1.0.0-beta.6

package/.docs/raw/reference/evals/scorer-utils.mdx

@@ -0,0 +1,362 @@
+---
+title: "Reference: Scorer Utils | Evals"
+description: Utility functions for extracting data from scorer run inputs and outputs, including text content, reasoning, system messages, and tool calls.
+---
+
+# Scorer Utils
+
+Mastra provides utility functions to help extract and process data from scorer run inputs and outputs. These utilities are particularly useful in the `preprocess` step of custom scorers.
+
+## Import
+
+```typescript
+import {
+  getAssistantMessageFromRunOutput,
+  getReasoningFromRunOutput,
+  getUserMessageFromRunInput,
+  getSystemMessagesFromRunInput,
+  getCombinedSystemPrompt,
+  extractToolCalls,
+  extractInputMessages,
+  extractAgentResponseMessages,
+} from "@mastra/evals/scorers/utils";
+```
+
+## Message Extraction
+
+### getAssistantMessageFromRunOutput
+
+Extracts the text content from the first assistant message in the run output.
+
+```typescript
+const scorer = createScorer({
+  id: "my-scorer",
+  description: "My scorer",
+  type: "agent",
+})
+  .preprocess(({ run }) => {
+    const response = getAssistantMessageFromRunOutput(run.output);
+    return { response };
+  })
+  .generateScore(({ results }) => {
+    return results.preprocessStepResult?.response ? 1 : 0;
+  });
+```
+
+<PropertiesTable
+  content={[
+    {
+      name: "output",
+      type: "ScorerRunOutputForAgent",
+      isOptional: true,
+      description: "The scorer run output (array of MastraDBMessage)",
+    },
+  ]}
+/>
+
+**Returns:** `string | undefined` - The assistant message text, or undefined if no assistant message is found.
+
+### getUserMessageFromRunInput
+
+Extracts the text content from the first user message in the run input.
+
+```typescript
+.preprocess(({ run }) => {
+  const userMessage = getUserMessageFromRunInput(run.input);
+  return { userMessage };
+})
+```
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "ScorerRunInputForAgent",
+      isOptional: true,
+      description: "The scorer run input containing input messages",
+    },
+  ]}
+/>
+
+**Returns:** `string | undefined` - The user message text, or undefined if no user message is found.
+
+### extractInputMessages
+
+Extracts text content from all input messages as an array.
+
+```typescript
+.preprocess(({ run }) => {
+  const allUserMessages = extractInputMessages(run.input);
+  return { conversationHistory: allUserMessages.join("\n") };
+})
+```
+
+**Returns:** `string[]` - Array of text strings from each input message.
+
+### extractAgentResponseMessages
+
+Extracts text content from all assistant response messages as an array.
+
+```typescript
+.preprocess(({ run }) => {
+  const allResponses = extractAgentResponseMessages(run.output);
+  return { allResponses };
+})
+```
+
+**Returns:** `string[]` - Array of text strings from each assistant message.
+
+## Reasoning Extraction
+
+### getReasoningFromRunOutput
+
+Extracts reasoning text from the run output. This is particularly useful when evaluating responses from reasoning models like `deepseek-reasoner` that produce chain-of-thought reasoning.
+
+Reasoning can be stored in two places:
+1. `content.reasoning` - a string field on the message content
+2. `content.parts` - as parts with `type: 'reasoning'` containing `details`
+
+```typescript
+import { 
+  getReasoningFromRunOutput, 
+  getAssistantMessageFromRunOutput 
+} from "@mastra/evals/scorers/utils";
+
+const reasoningQualityScorer = createScorer({
+  id: "reasoning-quality",
+  name: "Reasoning Quality",
+  description: "Evaluates the quality of model reasoning",
+  type: "agent",
+})
+  .preprocess(({ run }) => {
+    const reasoning = getReasoningFromRunOutput(run.output);
+    const response = getAssistantMessageFromRunOutput(run.output);
+    return { reasoning, response };
+  })
+  .analyze(({ results }) => {
+    const { reasoning } = results.preprocessStepResult || {};
+    return {
+      hasReasoning: !!reasoning,
+      reasoningLength: reasoning?.length || 0,
+      hasStepByStep: reasoning?.includes("step") || false,
+    };
+  })
+  .generateScore(({ results }) => {
+    const { hasReasoning, reasoningLength } = results.analyzeStepResult || {};
+    if (!hasReasoning) return 0;
+    // Score based on reasoning length (normalized to 0-1)
+    return Math.min(reasoningLength / 500, 1);
+  })
+  .generateReason(({ results, score }) => {
+    const { hasReasoning, reasoningLength } = results.analyzeStepResult || {};
+    if (!hasReasoning) {
+      return "No reasoning was provided by the model.";
+    }
+    return `Model provided ${reasoningLength} characters of reasoning. Score: ${score}`;
+  });
+```
+
+<PropertiesTable
+  content={[
+    {
+      name: "output",
+      type: "ScorerRunOutputForAgent",
+      isOptional: true,
+      description: "The scorer run output (array of MastraDBMessage)",
+    },
+  ]}
+/>
+
+**Returns:** `string | undefined` - The reasoning text, or undefined if no reasoning is present.
+
+## System Message Extraction
+
+### getSystemMessagesFromRunInput
+
+Extracts all system messages from the run input, including both standard system messages and tagged system messages (specialized prompts like memory instructions).
+
+```typescript
+.preprocess(({ run }) => {
+  const systemMessages = getSystemMessagesFromRunInput(run.input);
+  return { 
+    systemPromptCount: systemMessages.length,
+    systemPrompts: systemMessages 
+  };
+})
+```
+
+**Returns:** `string[]` - Array of system message strings.
+
+### getCombinedSystemPrompt
+
+Combines all system messages into a single prompt string, joined with double newlines.
+
+```typescript
+.preprocess(({ run }) => {
+  const fullSystemPrompt = getCombinedSystemPrompt(run.input);
+  return { fullSystemPrompt };
+})
+```
+
+**Returns:** `string` - Combined system prompt string.
+
+## Tool Call Extraction
+
+### extractToolCalls
+
+Extracts information about all tool calls from the run output, including tool names, call IDs, and their positions in the message array.
+
+```typescript
+const toolUsageScorer = createScorer({
+  id: "tool-usage",
+  description: "Evaluates tool usage patterns",
+  type: "agent",
+})
+  .preprocess(({ run }) => {
+    const { tools, toolCallInfos } = extractToolCalls(run.output);
+    return {
+      toolsUsed: tools,
+      toolCount: tools.length,
+      toolDetails: toolCallInfos,
+    };
+  })
+  .generateScore(({ results }) => {
+    const { toolCount } = results.preprocessStepResult || {};
+    // Score based on appropriate tool usage
+    return toolCount > 0 ? 1 : 0;
+  });
+```
+
+**Returns:**
+
+```typescript
+{
+  tools: string[];           // Array of tool names
+  toolCallInfos: ToolCallInfo[];  // Detailed tool call information
+}
+```
+
+Where `ToolCallInfo` is:
+
+```typescript
+type ToolCallInfo = {
+  toolName: string;      // Name of the tool
+  toolCallId: string;    // Unique call identifier
+  messageIndex: number;  // Index in the output array
+  invocationIndex: number; // Index within message's tool invocations
+};
+```
+
+## Test Utilities
+
+These utilities help create test data for scorer development.
+
+### createTestMessage
+
+Creates a `MastraDBMessage` object for testing purposes.
+
+```typescript
+import { createTestMessage } from "@mastra/evals/scorers/utils";
+
+const userMessage = createTestMessage({
+  content: "What is the weather?",
+  role: "user",
+});
+
+const assistantMessage = createTestMessage({
+  content: "The weather is sunny.",
+  role: "assistant",
+  toolInvocations: [
+    {
+      toolCallId: "call-1",
+      toolName: "weatherTool",
+      args: { location: "London" },
+      result: { temp: 20 },
+      state: "result",
+    },
+  ],
+});
+```
+
+### createAgentTestRun
+
+Creates a complete test run object for testing scorers.
+
+```typescript
+import { createAgentTestRun, createTestMessage } from "@mastra/evals/scorers/utils";
+
+const testRun = createAgentTestRun({
+  inputMessages: [
+    createTestMessage({ content: "Hello", role: "user" }),
+  ],
+  output: [
+    createTestMessage({ content: "Hi there!", role: "assistant" }),
+  ],
+});
+
+// Run your scorer with the test data
+const result = await myScorer.run({
+  input: testRun.input,
+  output: testRun.output,
+});
+```
+
+## Complete Example
+
+Here's a complete example showing how to use multiple utilities together:
+
+```typescript
+import { createScorer } from "@mastra/core/evals";
+import {
+  getAssistantMessageFromRunOutput,
+  getReasoningFromRunOutput,
+  getUserMessageFromRunInput,
+  getCombinedSystemPrompt,
+  extractToolCalls,
+} from "@mastra/evals/scorers/utils";
+
+const comprehensiveScorer = createScorer({
+  id: "comprehensive-analysis",
+  name: "Comprehensive Analysis",
+  description: "Analyzes all aspects of an agent response",
+  type: "agent",
+})
+  .preprocess(({ run }) => {
+    // Extract all relevant data
+    const userMessage = getUserMessageFromRunInput(run.input);
+    const response = getAssistantMessageFromRunOutput(run.output);
+    const reasoning = getReasoningFromRunOutput(run.output);
+    const systemPrompt = getCombinedSystemPrompt(run.input);
+    const { tools, toolCallInfos } = extractToolCalls(run.output);
+
+    return {
+      userMessage,
+      response,
+      reasoning,
+      systemPrompt,
+      toolsUsed: tools,
+      toolCount: tools.length,
+    };
+  })
+  .generateScore(({ results }) => {
+    const { response, reasoning, toolCount } = results.preprocessStepResult || {};
+    
+    let score = 0;
+    if (response && response.length > 0) score += 0.4;
+    if (reasoning) score += 0.3;
+    if (toolCount > 0) score += 0.3;
+    
+    return score;
+  })
+  .generateReason(({ results, score }) => {
+    const { response, reasoning, toolCount } = results.preprocessStepResult || {};
+    
+    const parts = [];
+    if (response) parts.push("provided a response");
+    if (reasoning) parts.push("included reasoning");
+    if (toolCount > 0) parts.push(`used ${toolCount} tool(s)`);
+    
+    return `Score: ${score}. The agent ${parts.join(", ")}.`;
+  });
+```
+

package/.docs/raw/reference/evals/textual-difference.mdx

@@ -83,118 +83,45 @@ A textual difference score between 0 and 1:
 - **0.1–0.3**: Major differences – extensive changes needed.
 - **0.0**: Completely different texts.
 
-## Examples
+## Example
 
-### No differences example
+Measure textual differences between expected and actual agent outputs:
 
-In this example, the texts are exactly the same. The scorer identifies complete similarity with a perfect score and no detected changes.
-
-```typescript title="src/example-no-differences.ts" showLineNumbers copy
+```typescript title="src/example-textual-difference.ts" showLineNumbers copy
+import { runEvals } from "@mastra/core/evals";
 import { createTextualDifferenceScorer } from "@mastra/evals/scorers/prebuilt";
+import { myAgent } from "./agent";
 
 const scorer = createTextualDifferenceScorer();
 
-const input = "The quick brown fox jumps over the lazy dog";
-const output = "The quick brown fox jumps over the lazy dog";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: input }],
-  output: { role: "assistant", text: output },
-});
-
-console.log("Score:", result.score);
-console.log("AnalyzeStepResult:", result.analyzeStepResult);
-```
-
-#### No differences output
-
-The scorer returns a high score, indicating the texts are identical. The detailed info confirms zero changes and no length difference.
-
-```typescript
-{
-  score: 1,
-  analyzeStepResult: {
-    confidence: 1,
-    ratio: 1,
-    changes: 0,
-    lengthDiff: 0,
+const result = await runEvals({
+  data: [
+    {
+      input: "Summarize the concept of recursion",
+      groundTruth:
+        "Recursion is when a function calls itself to solve a problem by breaking it into smaller subproblems.",
+    },
+    {
+      input: "What is the capital of France?",
+      groundTruth: "The capital of France is Paris.",
+    },
+  ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+      groundTruth: scorerResults[scorer.id].groundTruth,
+    });
   },
-}
-```
-
-### Minor differences example
-
-In this example, the texts have small variations. The scorer detects these minor differences and returns a moderate similarity score.
-
-```typescript title="src/example-minor-differences.ts" showLineNumbers copy
-import { createTextualDifferenceScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createTextualDifferenceScorer();
-
-const input = "Hello world! How are you?";
-const output = "Hello there! How is it going?";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: input }],
-  output: { role: "assistant", text: output },
 });
 
-console.log("Score:", result.score);
-console.log("AnalyzeStepResult:", result.analyzeStepResult);
+console.log(result.scores);
 ```
 
-#### Minor differences output
+For more details on `runEvals`, see the [runEvals reference](/reference/v1/evals/run-evals).
 
-The scorer returns a moderate score reflecting the small variations between the texts. The detailed info includes the number of changes and length difference observed.
-
-```typescript
-{
-  score: 0.5925925925925926,
-  analyzeStepResult: {
-    confidence: 0.8620689655172413,
-    ratio: 0.5925925925925926,
-    changes: 5,
-    lengthDiff: 0.13793103448275862
-  }
-}
-```
-
-### Major differences example
-
-In this example, the texts differ significantly. The scorer detects extensive changes and returns a low similarity score.
-
-```typescript title="src/example-major-differences.ts" showLineNumbers copy
-import { createTextualDifferenceScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createTextualDifferenceScorer();
-
-const input = "Python is a high-level programming language";
-const output = "JavaScript is used for web development";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: input }],
-  output: { role: "assistant", text: output },
-});
-
-console.log("Score:", result.score);
-console.log("AnalyzeStepResult:", result.analyzeStepResult);
-```
-
-#### Major differences output
-
-The scorer returns a low score due to significant differences between the texts. The detailed `analyzeStepResult` shows numerous changes and a notable length difference.
-
-```typescript
-{
-  score: 0.3170731707317073,
-  analyzeStepResult: {
-    confidence: 0.8636363636363636,
-    ratio: 0.3170731707317073,
-    changes: 8,
-    lengthDiff: 0.13636363636363635
-  }
-}
-```
+To add this scorer to an agent, see the [Scorers overview](/docs/v1/evals/overview#adding-scorers-to-agents) guide.
 
 ## Related
 

package/.docs/raw/reference/evals/tone-consistency.mdx

@@ -94,116 +94,43 @@ Object with tone metrics:
 - **avgSentiment**: Average sentiment across sentences (stability mode).
 - **sentimentVariance**: Variance of sentiment across sentences (stability mode).
 
-## Examples
+## Example
 
-### Positive tone example
+Evaluate tone consistency between related agent responses:
 
-In this example, the texts exhibit a similar positive sentiment. The scorer measures the consistency between the tones, resulting in a high score.
-
-```typescript title="src/example-positive-tone.ts" showLineNumbers copy
-import { createToneScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createToneScorer();
-
-const input = "This product is fantastic and amazing!";
-const output = "The product is excellent and wonderful!";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: input }],
-  output: { role: "assistant", text: output },
-});
-
-console.log("Score:", result.score);
-console.log("AnalyzeStepResult:", result.analyzeStepResult);
-```
-
-#### Positive tone output
-
-The scorer returns a high score reflecting strong sentiment alignment. The `analyzeStepResult` field provides sentiment values and the difference between them.
-
-```typescript
-{
-  score: 0.8333333333333335,
-  analyzeStepResult: {
-    responseSentiment: 1.3333333333333333,
-    referenceSentiment: 1.1666666666666667,
-    difference: 0.16666666666666652,
-  },
-}
-```
-
-### Stable tone example
-
-In this example, the text’s internal tone consistency is analyzed by passing an empty response. This signals the scorer to evaluate sentiment stability within the single input text, resulting in a score reflecting how uniform the tone is throughout.
-
-```typescript title="src/example-stable-tone.ts" showLineNumbers copy
+```typescript title="src/example-tone-consistency.ts" showLineNumbers copy
+import { runEvals } from "@mastra/core/evals";
 import { createToneScorer } from "@mastra/evals/scorers/prebuilt";
+import { myAgent } from "./agent";
 
 const scorer = createToneScorer();
 
-const input = "Great service! Friendly staff. Perfect atmosphere.";
-const output = "";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: input }],
-  output: { role: "assistant", text: output },
-});
-
-console.log("Score:", result.score);
-console.log("AnalyzeStepResult:", result.analyzeStepResult);
-```
-
-#### Stable tone output
-
-The scorer returns a high score indicating consistent sentiment throughout the input text. The `analyzeStepResult` field includes the average sentiment and sentiment variance, reflecting tone stability.
-
-```typescript
-{
-  score: 0.9444444444444444,
-  analyzeStepResult: {
-    avgSentiment: 1.3333333333333333,
-    sentimentVariance: 0.05555555555555556,
+const result = await runEvals({
+  data: [
+    {
+      input: "How was your experience with our service?",
+      groundTruth: "The service was excellent and exceeded expectations!",
+    },
+    {
+      input: "Tell me about the customer support",
+      groundTruth: "The support team was friendly and very helpful.",
+    },
+  ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+    });
   },
-}
-```
-
-### Mixed tone example
-
-In this example, the input and response have different emotional tones. The scorer picks up on these variations and gives a lower consistency score.
-
-```typescript title="src/example-mixed-tone.ts" showLineNumbers copy
-import { createToneScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createToneScorer();
-
-const input =
-  "The interface is frustrating and confusing, though it has potential.";
-const output =
-  "The design shows promise but needs significant improvements to be usable.";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: input }],
-  output: { role: "assistant", text: output },
 });
 
-console.log("Score:", result.score);
-console.log("AnalyzeStepResult:", result.analyzeStepResult);
+console.log(result.scores);
 ```
 
-#### Mixed tone output
-
-The scorer returns a low score due to the noticeable differences in emotional tone. The `analyzeStepResult` field highlights the sentiment values and the degree of variation between them.
+For more details on `runEvals`, see the [runEvals reference](/reference/v1/evals/run-evals).
 
-```typescript
-{
-  score: 0.4181818181818182,
-  analyzeStepResult: {
-    responseSentiment: -0.4,
-    referenceSentiment: 0.18181818181818182,
-    difference: 0.5818181818181818,
-  },
-}
-```
+To add this scorer to an agent, see the [Scorers overview](/docs/v1/evals/overview#adding-scorers-to-agents) guide.
 
 ## Related
 

package/.docs/raw/reference/evals/tool-call-accuracy.mdx

@@ -349,7 +349,7 @@ The LLM-based scorer provides:
 ```typescript showLineNumbers copy
 // Basic configuration
 const basicLLMScorer = createLLMScorer({
-  model: 'openai/gpt-4o-mini',
+  model: 'openai/gpt-5.1',
   availableTools: [
     { name: 'tool1', description: 'Description 1' },
     { name: 'tool2', description: 'Description 2' }
@@ -358,7 +358,7 @@ const basicLLMScorer = createLLMScorer({
 
 // With different model
 const customModelScorer = createLLMScorer({
-  model: openai('gpt-4'), // More powerful model for complex evaluations
+  model: 'openai/gpt-5', // More powerful model for complex evaluations
   availableTools: [...]
 });
 ```
@@ -389,7 +389,7 @@ The LLM-based scorer uses AI to evaluate whether tool selections are appropriate
 
 ```typescript title="src/example-llm-basic.ts" showLineNumbers copy
 const llmScorer = createToolCallAccuracyScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   availableTools: [
     {
       name: "weather-tool",
@@ -510,9 +510,9 @@ console.log(result.reason); // "The agent appropriately asked for clarification
 Here's an example using both scorers on the same data:
 
 ```typescript title="src/example-comparison.ts" showLineNumbers copy
-import { 
-  createToolCallAccuracyScorerCode as createCodeScorer, 
-  createToolCallAccuracyScorerLLM as createLLMScorer 
+import {
+  createToolCallAccuracyScorerCode as createCodeScorer,
+  createToolCallAccuracyScorerLLM as createLLMScorer
 } from "@mastra/evals/scorers/prebuilt";
 
 // Setup both scorers
@@ -522,7 +522,7 @@ const codeScorer = createCodeScorer({
 });
 
 const llmScorer = createLLMScorer({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   availableTools: [
     { name: "weather-tool", description: "Get weather information" },
     { name: "search-tool", description: "Search the web" },