@mastra/mcp-docs-server 0.13.7-alpha.0 → 0.13.7-alpha.2

package/.docs/raw/scorers/custom-scorers.mdx

@@ -0,0 +1,319 @@
+## Creating scorers
+
+Mastra provides two approaches for creating custom scorers:
+
+**Code scorers** use programmatic logic and algorithms. They're ideal for deterministic evaluations, performance-critical scenarios, and cases where you have clear algorithmic criteria.
+
+**LLM scorers** use language models as judges. They're perfect for subjective evaluations, complex criteria that are difficult to code algorithmically, and cases where human-like judgment is needed.
+
+### Code-based scorers
+
+Code scorers use `createScorer` to build evaluation logic with programmatic algorithms. They're ideal for deterministic evaluations, performance-critical scenarios, and cases where you have clear algorithmic criteria or need integration with existing libraries.
+
+Code scorers follow Mastra's three-step evaluation pipeline: 
+- an optional **extract** step for preprocessing complex data
+- a required **analyze** step for core evaluation and scoring
+- and an optional **reason** step for generating explanations.
+
+For the complete API reference, see [`createScorer`](/reference/scorers/custom-code-scorer), and for a detailed explanation of the pipeline, see [evaluation process](/docs/scorers/overview#evaluation-pipeline).
+
+#### Extract Step
+
+This optional step preprocesses input/output data when you need to evaluate multiple distinct elements, filter content, or focus analysis on specific parts of complex data.
+
+- **Receives:**
+  - `input`: User messages (when used with agents) or workflow step input (when used with workflow steps)
+  - `output`: Agent's response (when used with agents) or workflow step output (when used with workflow steps)
+  - `runtimeContext`: Runtime context from the agent or workflow step being evaluated
+- **Must return:** `{ results: any }`
+- **Data flow:** The `results` value is passed to the analyze step as `extractStepResult`
+
+```typescript filename="src/mastra/scorers/keyword-coverage-scorer.ts" showLineNumbers copy
+import { createScorer } from "@mastra/core/scores";
+import keywordExtractor from "keyword-extractor";
+
+export const keywordCoverageScorer = createScorer({
+  name: "Keyword Coverage",
+  description: "Evaluates how well the output covers keywords from the input",
+
+  // Step 1: Extract keywords from input and output
+  extract: async ({ input, output }) => {
+    const inputText = input?.map(i => i.content).join(", ") || "";
+    const outputText = output.text;
+
+    const extractKeywords = (text: string) => {
+      return keywordExtractor.extract(text);
+    };
+
+    const inputKeywords = new Set(extractKeywords(inputText));
+    const outputKeywords = new Set(extractKeywords(outputText));
+    
+    return {
+      results: {
+        inputKeywords,
+        outputKeywords,
+      },
+    };
+  },
+
+  // ... analyze and reason steps
+});
+```
+
+#### Analyze Step
+
+This required step performs the core evaluation and generates the numerical score for all scorers.
+
+- **Receives:** Everything from extract step, plus:
+  - `extractStepResult`: Results from the extract step (if extract step was defined)
+- **Must return:** `{ score: number, results?: any }`
+- **Data flow:** The `score` and optional `results` are passed to the reason step
+
+```typescript filename="src/mastra/scorers/keyword-coverage-scorer.ts" showLineNumbers copy
+export const keywordCoverageScorer = createScorer({
+  // ... name, description, extract step
+
+  // Step 2: Analyze keyword coverage and calculate score
+  analyze: async ({ input, output, extractStepResult }) => {
+    const { inputKeywords, outputKeywords } = extractStepResult.results;
+    
+    if (inputKeywords.size === 0) {
+      return { score: 1, results: { coverage: 1, matched: 0, total: 0 } };
+    }
+
+    const matchedKeywords = [...inputKeywords].filter(keyword =>
+      outputKeywords.has(keyword)
+    );
+    
+    const coverage = matchedKeywords.length / inputKeywords.size;
+
+    return {
+      score: coverage,
+      results: {
+        coverage,
+        matched: matchedKeywords.length,
+        total: inputKeywords.size,
+        matchedKeywords,
+      },
+    };
+  },
+
+  // ... reason step
+});
+```
+
+#### Reason Step
+
+This optional step generates human-readable explanations for scores, useful for actionable feedback, debugging transparency, or compliance documentation.
+
+- **Receives:** Everything from analyze step, plus:
+  - `score`: The numerical score (0-1) calculated by the analyze step
+  - `analyzeStepResult`: Results from the analyze step (contains the score and any additional results)
+- **Must return:** `{ reason: string }`
+
+```typescript filename="src/mastra/scorers/keyword-coverage-scorer.ts" showLineNumbers copy
+export const keywordCoverageScorer = createScorer({
+  // ... name, description, extract and analyze steps
+
+  // Step 3: Generate explanation for the score
+  reason: async ({ score, analyzeStepResult, extractStepResult }) => {
+    const { matched, total, matchedKeywords } = analyzeStepResult.results;
+    const { inputKeywords } = extractStepResult.results;
+    
+    const percentage = Math.round(score * 100);
+    const missedKeywords = [...inputKeywords].filter(
+      keyword => !matchedKeywords.includes(keyword)
+    );
+
+    let reason = `The output achieved ${percentage}% keyword coverage (${matched}/${total} keywords).`;
+    
+    if (matchedKeywords.length > 0) {
+      reason += ` Covered keywords: ${matchedKeywords.join(", ")}.`;
+    }
+    
+    if (missedKeywords.length > 0) {
+      reason += ` Missing keywords: ${missedKeywords.join(", ")}.`;
+    }
+
+    return { reason };
+  },
+});
+```
+
+**Examples and Resources:**
+- [Custom Native JavaScript Scorer Example](/examples/scorers/custom-native-javascript-eval) - Example walkthrough.
+- [Built-in Code Scorers](https://github.com/mastra-ai/mastra/tree/main/packages/evals/src/scorers/code) - Real implementations for reference
+  
+### LLM-based scorers
+
+LLM scorers use `createLLMScorer` to build evaluations that leverage language models as judges. They're perfect for subjective evaluations that require understanding context, complex criteria that are difficult to code algorithmically, natural language understanding tasks, and cases where human-like judgment is needed.
+
+LLM scorers follow the same evaluation pipeline as code scorers with an additional `calculateScore` function:
+- an optional **extract** step where the LLM processes input/output and returns structured data
+- a required **analyze** step where the LLM performs evaluation and returns structured analysis  
+- a required **calculateScore** function that converts LLM analysis into numerical score
+- and an optional **reason** step where the LLM generates human-readable explanations
+
+The `calculateScore` function leverages the best of both approaches: LLMs excel at qualitative analysis and understanding, while deterministic functions ensure precise and consistent numerical scoring.
+
+For the complete API reference, see [`createLLMScorer`](/reference/scorers/llm-scorer), and for a detailed explanation of the pipeline, see [evaluation process](/docs/scorers/overview#evaluation-pipeline).
+
+#### Judge Configuration
+
+All LLM scorer steps share this required configuration that defines the model and system instructions.
+
+- **Configuration:** `judge` object containing:
+  - **model:** The LLM model instance for evaluation
+  - **instructions:** System prompt that guides the LLM's behavior
+
+```typescript filename="src/mastra/scorers/tone-scorer.ts" showLineNumbers copy
+import { openai } from "@ai-sdk/openai";
+import { createLLMScorer } from "@mastra/core/scores";
+
+export const toneScorer = createLLMScorer({
+  name: 'Tone Scorer',
+  description: 'Evaluates the tone of the output',
+  
+  // Shared judge configuration
+  judge: {
+    model: openai('gpt-4o'),
+    instructions: 'You are an expert in analyzing text tone and communication style.',
+  },
+  
+  // ... other steps
+});
+```
+
+#### Extract Step
+
+This optional step uses an LLM to preprocess input/output data when you need to evaluate multiple distinct elements, filter content, or focus analysis on specific parts of complex data.
+
+- **Configuration:** `{ description, outputSchema, createPrompt }`
+- **Data flow:** The structured output (defined by outputSchema) is passed to the analyze step as `extractStepResult`
+
+```typescript filename="src/mastra/scorers/content-scorer.ts" showLineNumbers copy
+export const contentScorer = createLLMScorer({
+  // ... judge configuration
+  
+  extract: {
+    description: 'Extract key themes and topics from the content',
+    outputSchema: z.object({
+      themes: z.array(z.string()),
+      topics: z.array(z.string()),
+      keyPhrases: z.array(z.string())
+    }),
+    createPrompt: ({ run }) => `
+      Analyze this content and extract:
+      1. Main themes (3-5 high-level concepts)
+      2. Specific topics mentioned
+      3. Key phrases that capture the essence
+      
+      Content: ${run.output.text}
+      
+      Return a JSON object with themes, topics, and keyPhrases arrays.
+    `,
+  },
+  
+  // ... other steps
+});
+```
+
+#### Analyze Step
+
+This required step uses an LLM to perform the core evaluation and return structured analysis that will be converted to a numerical score.
+
+- **Configuration:** `{ description, outputSchema, createPrompt }`
+- **Data flow:** The structured output is passed to the calculateScore function and then to the reason step
+
+```typescript filename="src/mastra/scorers/quality-scorer.ts" showLineNumbers copy
+export const qualityScorer = createLLMScorer({
+  // ... judge configuration
+  
+  analyze: {
+    description: 'Evaluate content quality across multiple dimensions',
+    outputSchema: z.object({
+      clarity: z.number().min(1).max(5),
+      accuracy: z.number().min(1).max(5),
+      completeness: z.number().min(1).max(5),
+      relevance: z.number().min(1).max(5)
+    }),
+    createPrompt: ({ run }) => `
+      Evaluate this content on a scale of 1-5 for:
+      - Clarity: How clear and understandable is it?
+      - Accuracy: How factually correct does it appear?
+      - Completeness: How thorough is the response?
+      - Relevance: How well does it address the input?
+      
+      Input: ${run.input.map(i => i.content).join(', ')}
+      Output: ${run.output.text}
+      
+      Return a JSON object with numeric scores for each dimension.
+    `,
+  },
+  
+  // ... other steps
+});
+```
+
+#### Calculate Score Step
+
+This required function converts the LLM's structured analysis into a numerical score, providing deterministic scoring logic since LLMs aren't reliable for consistent numerical outputs.
+
+- **Configuration:** `calculateScore` function that receives `{ run }` and returns a number
+- **Data flow:** Converts the analyze step's structured output into a numerical score (0-1 range)
+
+```typescript filename="src/mastra/scorers/quality-scorer.ts" showLineNumbers copy
+export const qualityScorer = createLLMScorer({
+  // ... previous steps
+  
+  calculateScore: ({ run }) => {
+    const { clarity, accuracy, completeness, relevance } = run.analyzeStepResult;
+    
+    // Calculate weighted average (scale of 1-5 to 0-1)
+    const weights = { clarity: 0.3, accuracy: 0.3, completeness: 0.2, relevance: 0.2 };
+    const weightedSum = (clarity * weights.clarity) + 
+                       (accuracy * weights.accuracy) + 
+                       (completeness * weights.completeness) + 
+                       (relevance * weights.relevance);
+    
+    // Convert from 1-5 scale to 0-1 scale
+    return (weightedSum - 1) / 4;
+  },
+  
+  // ... other steps
+});
+```
+
+#### Reason Step
+
+This optional step uses an LLM to generate human-readable explanations for scores, useful for actionable feedback, debugging transparency, or compliance documentation.
+
+- **Configuration:** `{ description, createPrompt }`
+- **Data flow:** Receives all previous step results and score, returns a string explanation
+
+```typescript filename="src/mastra/scorers/quality-scorer.ts" showLineNumbers copy
+export const qualityScorer = createLLMScorer({
+  // ... previous steps
+  
+  reason: {
+    createPrompt: ({ run }) => {
+      const { clarity, accuracy, completeness, relevance } = run.analyzeStepResult;
+      const percentage = Math.round(run.score * 100);
+      
+      return `
+        The content received a ${percentage}% quality score based on:
+        - Clarity: ${clarity}/5
+        - Accuracy: ${accuracy}/5  
+        - Completeness: ${completeness}/5
+        - Relevance: ${relevance}/5
+        
+        Provide a brief explanation of what contributed to this score.
+      `;
+    },
+  },
+});
+```
+
+**Examples and Resources:**
+- [Custom LLM Judge Scorer Example](/examples/scorers/custom-llm-judge-eval) - Example Walkthrough with gluten checker
+- [Built-in LLM Scorers](https://github.com/mastra-ai/mastra/tree/main/packages/evals/src/scorers/llm) - Real implementations for reference

package/.docs/raw/scorers/off-the-shelf-scorers.mdx

@@ -0,0 +1,30 @@
+---
+title: "Built-in Scorers"
+description: "Overview of Mastra's ready-to-use scorers for evaluating AI outputs across quality, safety, and performance dimensions."
+---
+
+# Built-in Scorers
+
+Mastra provides a comprehensive set of built-in scorers for evaluating AI outputs. These scorers are optimized for common evaluation scenarios and are ready to use in your agents and workflows.
+
+## Available Scorers
+
+### Accuracy and Reliability
+
+These scorers evaluate how correct, truthful, and complete your agent's answers are:
+
+- [`answer-relevancy`](/reference/scorers/answer-relevancy): Evaluates how well responses address the input query (`0-1`, higher is better)
+- [`faithfulness`](/reference/scorers/faithfulness): Measures how accurately responses represent provided context (`0-1`, higher is better)
+- [`hallucination`](/reference/scorers/hallucination): Detects factual contradictions and unsupported claims (`0-1`, lower is better)
+- [`completeness`](/reference/scorers/completeness): Checks if responses include all necessary information (`0-1`, higher is better)
+- [`content-similarity`](/reference/scorers/content-similarity): Measures textual similarity using character-level matching (`0-1`, higher is better)
+- [`textual-difference`](/reference/scorers/textual-difference): Measures textual differences between strings (`0-1`, higher means more similar)
+
+### Output Quality
+
+These scorers evaluate adherence to format, style, and safety requirements:
+
+- [`tone-consistency`](/reference/scorers/tone-consistency): Measures consistency in formality, complexity, and style (`0-1`, higher is better)
+- [`toxicity`](/reference/scorers/toxicity): Detects harmful or inappropriate content (`0-1`, lower is better)
+- [`bias`](/reference/scorers/bias): Detects potential biases in the output (`0-1`, lower is better)
+- [`keyword-coverage`](/reference/scorers/keyword-coverage): Assesses technical terminology usage (`0-1`, higher is better)

package/.docs/raw/scorers/overview.mdx

@@ -0,0 +1,124 @@
+---
+title: "Overview"
+description: Overview of scorers in Mastra, detailing their capabilities for evaluating AI outputs and measuring performance.
+---
+
+# Scorers overview
+
+**Scorers** are evaluation tools that measure the quality, accuracy, or performance of AI-generated outputs. Scorers provide an automated way to assess whether your agents, workflows, or language models are producing the desired results by analyzing their responses against specific criteria.
+
+**Scores** are numerical values (typically between 0 and 1) that quantify how well an output meets your evaluation criteria. These scores enable you to objectively track performance, compare different approaches, and identify areas for improvement in your AI systems.
+
+## Evaluation pipeline
+
+Mastra scorers follow an optional three-step pipeline that allows for evaluation workflows:
+
+1. **Extract** (Optional): Identify and isolate relevant content for focused evaluation
+2. **Analyze** (Required): Perform the core evaluation and generate a score
+3. **Reason** (Optional): Provide explanations or justifications for the score
+
+This modular structure enables both simple single-step evaluations and complex multi-stage analysis workflows, allowing you to build evaluations that match your specific needs.
+
+### When to use each step
+
+**Extract step** - Use when your content is complex or needs preprocessing:
+- Separating facts from opinions in mixed responses
+- Focusing evaluation on specific sections of long outputs
+- Parsing multiple claims that need individual evaluation
+- Example: Bias detection that first identifies opinion statements
+
+**Analyze step** - Always required for core evaluation:
+- Straightforward scenarios: Direct scoring of input/output pairs
+- Complex scenarios: Evaluate preprocessed content and generate detailed results
+- Applies your scoring criteria and calculates the numerical score
+
+**Reason step** - Use when explanations are important:
+- Users need to understand why a score was assigned
+- Debugging and transparency are critical
+- Compliance or auditing requires explanations
+- Providing actionable feedback for improvement
+
+To learn how to create your own Scorers, see [Creating Custom Scorers](/docs/scorers/custom-scorers).
+
+## Live evaluations
+
+**Live evaluations** allow you to automatically score AI outputs in real-time as your agents and workflows operate. Instead of running evaluations manually or in batches, scorers run asynchronously alongside your AI systems, providing continuous quality monitoring.
+
+### Adding scorers to agents
+
+You can add built-in scorers to your agents to automatically evaluate their outputs. See the [full list of built-in scorers](/docs/scorers/off-the-shelf-scorers) for all available options.
+
+```typescript filename="src/mastra/agents/evaluated-agent.ts" showLineNumbers copy
+import { Agent } from "@mastra/core/agent";
+import { openai } from "@ai-sdk/openai";
+import { 
+  createAnswerRelevancyScorer,
+  createToxicityScorer 
+} from "@mastra/evals/scorers/llm";
+
+export const evaluatedAgent = new Agent({
+  // ...
+  scorers: {
+    relevancy: {
+      scorer: createAnswerRelevancyScorer({ model: openai("gpt-4o-mini") }),
+      sampling: { type: "ratio", rate: 0.5 }
+    },
+    safety: {
+      scorer: createToxicityScorer({ model: openai("gpt-4o-mini") }),
+      sampling: { type: "ratio", rate: 1 }
+    }
+  }
+});
+```
+
+### Adding scorers to workflow steps
+
+You can also add scorers to individual workflow steps to evaluate outputs at specific points in your process:
+
+```typescript filename="src/mastra/workflows/content-generation.ts" showLineNumbers copy
+import { createWorkflow, createStep } from "@mastra/core/workflows";
+import { z } from "zod";
+import { customStepScorer } from "../scorers/custom-step-scorer";
+
+const contentStep = createStep({
+  // ...
+  scorers: {
+    customStepScorer: {
+      scorer: customStepScorer(),
+      sampling: {
+        type: "ratio",
+        rate: 1, // Score every step execution
+      }
+    }
+  },
+});
+
+export const contentWorkflow = createWorkflow({ ... })
+  .then(contentStep)
+  .commit();
+```
+
+### How live evaluations work
+
+**Asynchronous execution**: Live evaluations run in the background without blocking your agent responses or workflow execution. This ensures your AI systems maintain their performance while still being monitored.
+
+**Sampling control**: The `sampling.rate` parameter (0-1) controls what percentage of outputs get scored:
+- `1.0`: Score every single response (100%)
+- `0.5`: Score half of all responses (50%) 
+- `0.1`: Score 10% of responses
+- `0.0`: Disable scoring
+
+**Automatic storage**: All scoring results are automatically stored in the `mastra_scorers` table in your configured database, allowing you to analyze performance trends over time.
+
+## Testing scorers locally
+
+Mastra provides a CLI command `mastra dev` to test your scorers. The playground includes a scorers section where you can run individual scorers against test inputs and view detailed results.
+
+For more details, see the [Local Dev Playground](/docs/server-db/local-dev-playground) docs.
+
+## Next steps
+
+- Learn how to create your own scorers in the [Creating Custom Scorers](/docs/scorers/custom-scorers) guide
+- Explore built-in scorers in the [Off-the-shelf Scorers](/docs/scorers/off-the-shelf-scorers) section
+- Test scorers with the [Local Dev Playground](/docs/server-db/local-dev-playground)
+- See example scorers in the [Examples Overview](/examples) section

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/mcp-docs-server",
-  "version": "0.13.7-alpha.0",
+  "version": "0.13.7-alpha.2",
   "description": "MCP server for accessing Mastra.ai documentation, changelogs, and news.",
   "type": "module",
   "main": "dist/index.js",
@@ -32,7 +32,7 @@
     "uuid": "^11.1.0",
     "zod": "^3.25.67",
     "zod-to-json-schema": "^3.24.5",
-    "@mastra/core": "0.12.0-alpha.0",
+    "@mastra/core": "0.12.0-alpha.2",
     "@mastra/mcp": "^0.10.7"
   },
   "devDependencies": {
@@ -43,13 +43,13 @@
     "@wong2/mcp-cli": "^1.10.0",
     "cross-env": "^7.0.3",
     "eslint": "^9.30.1",
-    "hono": "^4.8.4",
+    "hono": "^4.8.9",
     "tsup": "^8.5.0",
     "tsx": "^4.19.4",
     "typescript": "^5.8.3",
     "vitest": "^3.2.4",
     "@internal/lint": "0.0.23",
-    "@mastra/core": "0.12.0-alpha.0"
+    "@mastra/core": "0.12.0-alpha.2"
   },
   "scripts": {
     "prepare-docs": "cross-env PREPARE=true node dist/prepare-docs/prepare.js",