@mastra/mcp-docs-server 0.13.10 → 0.13.11-alpha.0

package/.docs/raw/reference/rag/chunk.mdx

@@ -49,7 +49,7 @@ The following parameters are available for all chunking strategies.
   content={[
     {
       name: "strategy",
-      type: "'recursive' | 'character' | 'token' | 'markdown' | 'html' | 'json' | 'latex' | 'sentence'",
+      type: "'recursive' | 'character' | 'token' | 'markdown' | 'semantic-markdown' | 'html' | 'json' | 'latex' | 'sentence'",
       isOptional: true,
       description:
         "The chunking strategy to use. If not specified, defaults based on document type. Depending on the chunking strategy, there are additional optionals. Defaults: .md files → 'markdown', .html/.htm → 'html', .json → 'json', .tex → 'latex', others → 'recursive'",
@@ -106,11 +106,13 @@ The following parameters are available for all chunking strategies.
       type: "ExtractParams",
       isOptional: true,
       description:
-        "Metadata extraction configuration. See [ExtractParams reference](/reference/rag/extract-params) for details.",
+        "Metadata extraction configuration.",
     },
   ]}
 />
 
+See [ExtractParams reference](/reference/rag/extract-params.mdx) for details on the `extract` parameter.
+
 ## Strategy-Specific Options
 
 Strategy-specific options are passed as top-level parameters alongside the strategy parameter. For example:
@@ -161,6 +163,13 @@ const chunks = await doc.chunk({
   stripHeaders: true, // Markdown-specific option
 });
 
+// Semantic Markdown strategy example
+const chunks = await doc.chunk({
+  strategy: "semantic-markdown",
+  joinThreshold: 500, // Semantic Markdown-specific option
+  modelName: "gpt-3.5-turbo", // Semantic Markdown-specific option
+});
+
 // Token strategy example
 const chunks = await doc.chunk({
   strategy: "token",
@@ -319,6 +328,46 @@ The options documented below are passed directly at the top level of the configu
 
 **Important:** When using the `headers` option, the markdown strategy ignores all general options and content is split based on the markdown header structure. To use size-based chunking with markdown, omit the `headers` parameter.
 
+### Semantic Markdown
+
+<PropertiesTable
+  content={[
+    {
+      name: "joinThreshold",
+      type: "number",
+      isOptional: true,
+      defaultValue: "500",
+      description: "Maximum token count for merging related sections. Sections exceeding this limit individually are left intact, but smaller sections are merged with siblings or parents if the combined size stays under this threshold.",
+    },
+    {
+      name: "modelName",
+      type: "string",
+      isOptional: true,
+      description: "Name of the model for tokenization. If provided, the model's underlying tokenization `encodingName` will be used.",
+    },
+    {
+      name: "encodingName",
+      type: "string",
+      isOptional: true,
+      defaultValue: "cl100k_base",
+      description: "Name of the token encoding to use. Derived from `modelName` if available.",
+    },
+    {
+      name: "allowedSpecial",
+      type: "Set<string> | 'all'",
+      isOptional: true,
+      description: "Set of special tokens allowed during tokenization, or 'all' to allow all special tokens",
+    },
+    {
+      name: "disallowedSpecial",
+      type: "Set<string> | 'all'",
+      isOptional: true,
+      defaultValue: "all",
+      description: "Set of special tokens to disallow during tokenization, or 'all' to disallow all special tokens",
+    },
+  ]}
+/>
+
 ### Token
 
 <PropertiesTable

package/.docs/raw/reference/scorers/answer-relevancy.mdx

@@ -53,12 +53,12 @@ This function returns an instance of the MastraScorer class. The `.run()` method
       description: "Relevancy score (0 to scale, default 0-1)",
     },
     {
-      name: "extractPrompt",
+      name: "preprocessPrompt",
       type: "string",
-      description: "The prompt sent to the LLM for the extract step (optional).",
+      description: "The prompt sent to the LLM for the preprocess step (optional).",
     },
     {
-      name: "extractStepResult",
+      name: "preprocessStepResult",
       type: "object",
       description: "Object with extracted statements: { statements: string[] }",
     },
@@ -73,7 +73,7 @@ This function returns an instance of the MastraScorer class. The `.run()` method
       description: "Object with results: { results: Array<{ result: 'yes' | 'unsure' | 'no', reason: string }> }",
     },
     {
-      name: "reasonPrompt",
+      name: "generateReasonPrompt",
       type: "string",
       description: "The prompt sent to the LLM for the reason step (optional).",
     },
@@ -91,7 +91,7 @@ The scorer evaluates relevancy through query-answer alignment, considering compl
 
 ### Scoring Process
 
-1. **Statement Extraction:**
+1. **Statement Preprocess:**
    - Breaks output into meaningful statements while preserving context.
 2. **Relevance Analysis:**
    - Each statement is evaluated as:
@@ -111,4 +111,4 @@ The scorer evaluates relevancy through query-answer alignment, considering compl
 
 ## Related
 
-- [Faithfulness Scorer](./faithfulness)
+- [Faithfulness Scorer](./faithfulness)

package/.docs/raw/reference/scorers/bias.mdx

@@ -41,14 +41,14 @@ This function returns an instance of the MastraScorer class. The `.run()` method
       description: "The id of the run (optional).",
     },
     {
-      name: "extractStepResult",
+      name: "preprocessStepResult",
       type: "object",
       description: "Object with extracted opinions: { opinions: string[] }",
     },
     {
-      name: "extractPrompt",
+      name: "preprocessPrompt",
       type: "string",
-      description: "The prompt sent to the LLM for the extract step (optional).",
+      description: "The prompt sent to the LLM for the preprocess step (optional).",
     },
     {
       name: "analyzeStepResult",
@@ -71,9 +71,9 @@ This function returns an instance of the MastraScorer class. The `.run()` method
       description: "Explanation of the score.",
     },
     {
-      name: "reasonPrompt",
+      name: "generateReasonPrompt",
       type: "string",
-      description: "The prompt sent to the LLM for the reason step (optional).",
+      description: "The prompt sent to the LLM for the generateReason step (optional).",
     },
   ]}
 />
@@ -124,4 +124,4 @@ Final score: `(biased_opinions / total_opinions) * scale`
 
 - [Toxicity Scorer](./toxicity)
 - [Faithfulness Scorer](./faithfulness)
-- [Hallucination Scorer](./hallucination)
+- [Hallucination Scorer](./hallucination)

package/.docs/raw/reference/scorers/completeness.mdx

@@ -25,7 +25,7 @@ This function returns an instance of the MastraScorer class. See the [MastraScor
       description: "The id of the run (optional).",
     },
     {
-      name: "extractStepResult",
+      name: "preprocessStepResult",
       type: "object",
       description: "Object with extracted elements and coverage details: { inputElements: string[], outputElements: string[], missingElements: string[], elementCounts: { input: number, output: number } }",
     },
@@ -86,4 +86,4 @@ Final score: `(covered_elements / total_input_elements) * scale`
 - [Answer Relevancy Scorer](./answer-relevancy)
 - [Content Similarity Scorer](./content-similarity)
 - [Textual Difference Scorer](./textual-difference)
-- [Keyword Coverage Scorer](./keyword-coverage)
+- [Keyword Coverage Scorer](./keyword-coverage)

package/.docs/raw/reference/scorers/content-similarity.mdx

@@ -44,7 +44,7 @@ This function returns an instance of the MastraScorer class. See the [MastraScor
       description: "The id of the run (optional).",
     },
     {
-      name: "extractStepResult",
+      name: "preprocessStepResult",
       type: "object",
       description: "Object with processed input and output: { processedInput: string, processedOutput: string }",
     },

package/.docs/raw/reference/scorers/create-scorer.mdx

@@ -0,0 +1,445 @@
+---
+title: "Reference: Create Custom Scorer | Scorers | Mastra Docs"
+description: Documentation for creating custom scorers in Mastra, allowing users to define their own evaluation logic using either JavaScript functions or LLM-based prompts.
+---
+
+# createScorer
+
+Mastra provides a unified `createScorer` factory that allows you to define custom scorers for evaluating input/output pairs. You can use either native JavaScript functions or LLM-based prompt objects for each evaluation step. Custom scorers can be added to Agents and Workflow steps.
+
+## How to Create a Custom Scorer
+
+Use the `createScorer` factory to define your scorer with a name, description, and optional judge configuration. Then chain step methods to build your evaluation pipeline. You must provide at least a `generateScore` step.
+
+```typescript
+const scorer = createScorer({
+  name: "My Custom Scorer",
+  description: "Evaluates responses based on custom criteria",
+  judge: {
+    model: myModel,
+    instructions: "You are an expert evaluator..."
+  }
+})
+.preprocess({ /* step config */ })
+.analyze({ /* step config */ })
+.generateScore(({ run, results }) => {
+  // Return a number
+})
+.generateReason({ /* step config */ });
+```
+
+## createScorer Options
+
+<PropertiesTable
+  content={[
+    {
+      name: "name",
+      type: "string",
+      required: true,
+      description: "Name of the scorer.",
+    },
+    {
+      name: "description",
+      type: "string",
+      required: true,
+      description: "Description of what the scorer does.",
+    },
+    {
+      name: "judge",
+      type: "object",
+      required: false,
+      description: "Optional judge configuration for LLM-based steps. See Judge Object section below.",
+    },
+  ]}
+/>
+
+This function returns a scorer builder that you can chain step methods onto. See the [MastraScorer reference](./mastra-scorer) for details on the `.run()` method and its input/output.
+
+## Judge Object
+
+<PropertiesTable
+  content={[
+    {
+      name: "model",
+      type: "LanguageModel",
+      required: true,
+      description: "The LLM model instance to use for evaluation.",
+    },
+    {
+      name: "instructions",
+      type: "string",
+      required: true,
+      description: "System prompt/instructions for the LLM.",
+    },
+  ]}
+/>
+
+## Type Safety
+
+For better type inference and IntelliSense support, you can specify input/output types when creating scorers:
+
+```typescript
+import { createScorer, ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core';
+
+// For agent evaluation with full type safety
+const agentScorer = createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({
+  name: 'Agent Response Quality',
+  description: 'Evaluates agent responses'
+})
+.preprocess(({ run }) => {
+  // run.input is typed as ScorerRunInputForAgent
+  const userMessage = run.input.inputMessages[0]?.content;
+  return { userMessage };
+})
+.generateScore(({ run, results }) => {
+  // run.output is typed as ScorerRunOutputForAgent  
+  const response = run.output[0]?.content;
+  return response.length > 10 ? 1.0 : 0.5;
+});
+
+// For custom input/output types
+type CustomInput = { query: string; context: string[] };
+type CustomOutput = { answer: string; confidence: number };
+
+const customScorer = createScorer<CustomInput, CustomOutput>({
+  name: 'Custom Scorer',
+  description: 'Evaluates custom data'
+})
+.generateScore(({ run }) => run.output.confidence);
+```
+
+### Built-in Agent Types
+
+- **`ScorerRunInputForAgent`** - Contains `inputMessages`, `rememberedMessages`, `systemMessages`, and `taggedSystemMessages` for agent evaluation
+- **`ScorerRunOutputForAgent`** - Array of agent response messages
+
+Using these types provides autocomplete, compile-time validation, and better documentation for your scoring logic.
+
+## Step Method Signatures
+
+### preprocess
+
+Optional preprocessing step that can extract or transform data before analysis.
+
+**Function Mode:**
+Function: `({ run, results }) => any`
+
+<PropertiesTable
+  content={[
+    {
+      name: "run.input",
+      type: "any",
+      required: true,
+      description: "Input records provided to the scorer. If the scorer is added to an agent, this will be an array of user messages, e.g. `[{ role: 'user', content: 'hello world' }]`. If the scorer is used in a workflow, this will be the input of the workflow.",
+    },
+    {
+      name: "run.output",
+      type: "any",
+      required: true,
+      description: "Output record provided to the scorer. For agents, this is usually the agent's response. For workflows, this is the workflow's output.",
+    },
+    {
+      name: "run.runId",
+      type: "string",
+      required: true,
+      description: "Unique identifier for this scoring run.",
+    },
+    {
+      name: "run.runtimeContext",
+      type: "object",
+      required: false,
+      description: "Runtime context from the agent or workflow step being evaluated (optional).",
+    },
+    {
+      name: "results",
+      type: "object",
+      required: true,
+      description: "Empty object (no previous steps).",
+    },
+  ]}
+/>
+
+Returns: `any`  
+The method can return any value. The returned value will be available to subsequent steps as `preprocessStepResult`.
+
+**Prompt Object Mode:**
+<PropertiesTable
+  content={[
+    {
+      name: "description",
+      type: "string",
+      required: true,
+      description: "Description of what this preprocessing step does.",
+    },
+    {
+      name: "outputSchema",
+      type: "ZodSchema",
+      required: true,
+      description: "Zod schema for the expected output of the preprocess step.",
+    },
+    {
+      name: "createPrompt",
+      type: "function",
+      required: true,
+      description: "Function: ({ run, results }) => string. Returns the prompt for the LLM.",
+    },
+    {
+      name: "judge",
+      type: "object",
+      required: false,
+      description: "(Optional) LLM judge for this step (can override main judge). See Judge Object section.",
+    },
+  ]}
+/>
+
+### analyze
+
+Optional analysis step that processes the input/output and any preprocessed data.
+
+**Function Mode:**
+Function: `({ run, results }) => any`
+
+<PropertiesTable
+  content={[
+    {
+      name: "run.input",
+      type: "any",
+      required: true,
+      description: "Input records provided to the scorer. If the scorer is added to an agent, this will be an array of user messages, e.g. `[{ role: 'user', content: 'hello world' }]`. If the scorer is used in a workflow, this will be the input of the workflow.",
+    },
+    {
+      name: "run.output",
+      type: "any",
+      required: true,
+      description: "Output record provided to the scorer. For agents, this is usually the agent's response. For workflows, this is the workflow's output.",
+    },
+    {
+      name: "run.runId",
+      type: "string",
+      required: true,
+      description: "Unique identifier for this scoring run.",
+    },
+    {
+      name: "run.runtimeContext",
+      type: "object",
+      required: false,
+      description: "Runtime context from the agent or workflow step being evaluated (optional).",
+    },
+    {
+      name: "results.preprocessStepResult",
+      type: "any",
+      required: false,
+      description: "Result from preprocess step, if defined (optional).",
+    },
+  ]}
+/>
+
+Returns: `any`  
+The method can return any value. The returned value will be available to subsequent steps as `analyzeStepResult`.
+
+**Prompt Object Mode:**
+<PropertiesTable
+  content={[
+    {
+      name: "description",
+      type: "string",
+      required: true,
+      description: "Description of what this analysis step does.",
+    },
+    {
+      name: "outputSchema",
+      type: "ZodSchema",
+      required: true,
+      description: "Zod schema for the expected output of the analyze step.",
+    },
+    {
+      name: "createPrompt",
+      type: "function",
+      required: true,
+      description: "Function: ({ run, results }) => string. Returns the prompt for the LLM.",
+    },
+    {
+      name: "judge",
+      type: "object",
+      required: false,
+      description: "(Optional) LLM judge for this step (can override main judge). See Judge Object section.",
+    },
+  ]}
+/>
+
+### generateScore
+
+**Required** step that computes the final numerical score.
+
+**Function Mode:**
+Function: `({ run, results }) => number`
+
+<PropertiesTable
+  content={[
+    {
+      name: "run.input",
+      type: "any",
+      required: true,
+      description: "Input records provided to the scorer. If the scorer is added to an agent, this will be an array of user messages, e.g. `[{ role: 'user', content: 'hello world' }]`. If the scorer is used in a workflow, this will be the input of the workflow.",
+    },
+    {
+      name: "run.output",
+      type: "any",
+      required: true,
+      description: "Output record provided to the scorer. For agents, this is usually the agent's response. For workflows, this is the workflow's output.",
+    },
+    {
+      name: "run.runId",
+      type: "string",
+      required: true,
+      description: "Unique identifier for this scoring run.",
+    },
+    {
+      name: "run.runtimeContext",
+      type: "object",
+      required: false,
+      description: "Runtime context from the agent or workflow step being evaluated (optional).",
+    },
+    {
+      name: "results.preprocessStepResult",
+      type: "any",
+      required: false,
+      description: "Result from preprocess step, if defined (optional).",
+    },
+    {
+      name: "results.analyzeStepResult",
+      type: "any",
+      required: false,
+      description: "Result from analyze step, if defined (optional).",
+    },
+  ]}
+/>
+
+Returns: `number`  
+The method must return a numerical score.
+
+**Prompt Object Mode:**
+<PropertiesTable
+  content={[
+    {
+      name: "description",
+      type: "string",
+      required: true,
+      description: "Description of what this scoring step does.",
+    },
+    {
+      name: "outputSchema",
+      type: "ZodSchema",
+      required: true,
+      description: "Zod schema for the expected output of the generateScore step.",
+    },
+    {
+      name: "createPrompt",
+      type: "function",
+      required: true,
+      description: "Function: ({ run, results }) => string. Returns the prompt for the LLM.",
+    },
+    {
+      name: "judge",
+      type: "object",
+      required: false,
+      description: "(Optional) LLM judge for this step (can override main judge). See Judge Object section.",
+    },
+  ]}
+/>
+
+When using prompt object mode, you must also provide a `calculateScore` function to convert the LLM output to a numerical score:
+
+<PropertiesTable
+  content={[
+    {
+      name: "calculateScore",
+      type: "function",
+      required: true,
+      description: "Function: ({ run, results, analyzeStepResult }) => number. Converts the LLM's structured output into a numerical score.",
+    },
+  ]}
+/>
+
+### generateReason
+
+Optional step that provides an explanation for the score.
+
+**Function Mode:**
+Function: `({ run, results, score }) => string`
+
+<PropertiesTable
+  content={[
+    {
+      name: "run.input",
+      type: "any",
+      required: true,
+      description: "Input records provided to the scorer. If the scorer is added to an agent, this will be an array of user messages, e.g. `[{ role: 'user', content: 'hello world' }]`. If the scorer is used in a workflow, this will be the input of the workflow.",
+    },
+    {
+      name: "run.output",
+      type: "any",
+      required: true,
+      description: "Output record provided to the scorer. For agents, this is usually the agent's response. For workflows, this is the workflow's output.",
+    },
+    {
+      name: "run.runId",
+      type: "string",
+      required: true,
+      description: "Unique identifier for this scoring run.",
+    },
+    {
+      name: "run.runtimeContext",
+      type: "object",
+      required: false,
+      description: "Runtime context from the agent or workflow step being evaluated (optional).",
+    },
+    {
+      name: "results.preprocessStepResult",
+      type: "any",
+      required: false,
+      description: "Result from preprocess step, if defined (optional).",
+    },
+    {
+      name: "results.analyzeStepResult",
+      type: "any",
+      required: false,
+      description: "Result from analyze step, if defined (optional).",
+    },
+    {
+      name: "score",
+      type: "number",
+      required: true,
+      description: "Score computed by the generateScore step.",
+    },
+  ]}
+/>
+
+Returns: `string`  
+The method must return a string explaining the score.
+
+**Prompt Object Mode:**
+<PropertiesTable
+  content={[
+    {
+      name: "description",
+      type: "string",
+      required: true,
+      description: "Description of what this reasoning step does.",
+    },
+    {
+      name: "createPrompt",
+      type: "function",
+      required: true,
+      description: "Function: ({ run, results, score }) => string. Returns the prompt for the LLM.",
+    },
+    {
+      name: "judge",
+      type: "object",
+      required: false,
+      description: "(Optional) LLM judge for this step (can override main judge). See Judge Object section.",
+    },
+  ]}
+/>
+
+All step functions can be async.

package/.docs/raw/reference/scorers/faithfulness.mdx

@@ -49,14 +49,14 @@ This function returns an instance of the MastraScorer class. The `.run()` method
       description: "The id of the run (optional).",
     },
     {
-      name: "extractStepResult",
+      name: "preprocessStepResult",
       type: "string[]",
       description: "Array of extracted claims from the output.",
     },
     {
-      name: "extractPrompt",
+      name: "preprocessPrompt",
       type: "string",
-      description: "The prompt sent to the LLM for the extract step (optional).",
+      description: "The prompt sent to the LLM for the preprocess step (optional).",
     },
     {
       name: "analyzeStepResult",
@@ -79,9 +79,9 @@ This function returns an instance of the MastraScorer class. The `.run()` method
       description: "A detailed explanation of the score, including which claims were supported, contradicted, or marked as unsure.",
     },
     {
-      name: "reasonPrompt",
+      name: "generateReasonPrompt",
       type: "string",
-      description: "The prompt sent to the LLM for the reason step (optional).",
+      description: "The prompt sent to the LLM for the generateReason step (optional).",
     },
   ]}
 />
@@ -119,4 +119,4 @@ Final score: `(supported_claims / total_claims) * scale`
 ## Related
 
 - [Answer Relevancy Scorer](./answer-relevancy)
-- [Hallucination Scorer](./hallucination)
+- [Hallucination Scorer](./hallucination)