@mastra/mcp-docs-server 0.0.1-alpha.1

package/.docs/raw/reference/rag/chunk.mdx

@@ -0,0 +1,237 @@
+---
+title: "Reference: .chunk() | Document Processing | RAG | Mastra Docs"
+description: Documentation for the chunk function in Mastra, which splits documents into smaller segments using various strategies.
+---
+
+# Reference: .chunk()
+
+The `.chunk()` function splits documents into smaller segments using various strategies and options.
+
+## Example
+
+```typescript
+import { Document } from '@mastra/core';
+
+const doc = new Document(`
+# Introduction
+This is a sample document that we want to split into chunks.
+
+## Section 1
+Here is the first section with some content.
+
+## Section 2 
+Here is another section with different content.
+`);
+
+// Basic chunking with defaults
+const chunks = await doc.chunk();
+
+// Markdown-specific chunking with header extraction
+const chunksWithMetadata = await doc.chunk({
+  strategy: 'markdown',
+  headers: [['#', 'title'], ['##', 'section']],
+  extract: {
+    fields: [
+      { name: 'summary', description: 'A brief summary of the chunk content' },
+      { name: 'keywords', description: 'Key terms found in the chunk' }
+    ]
+  }
+});
+```
+
+## Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "strategy",
+      type: "'recursive' | 'character' | 'token' | 'markdown' | 'html' | 'json' | 'latex'",
+      isOptional: true,
+      description:
+        "The chunking strategy to use. If not specified, defaults based on document type. Depending on the chunking strategy, there are additional optionals. Defaults: .md files → 'markdown', .html/.htm → 'html', .json → 'json', .tex → 'latex', others → 'recursive'",
+    },
+     {
+      name: "size",
+      type: "number",
+      isOptional: true,
+      defaultValue: "512",
+      description: "Maximum size of each chunk",
+    },
+    {
+      name: "overlap",
+      type: "number",
+      isOptional: true,
+      defaultValue: "50",
+      description: "Number of characters/tokens that overlap between chunks.",
+    },
+    {
+      name: "separator",
+      type: "string",
+      isOptional: true,
+      defaultValue: "\\n\\n",
+      description: "Character(s) to split on. Defaults to double newline for text content.",
+    },
+    {
+      name: "isSeparatorRegex",
+      type: "boolean",
+      isOptional: true,
+      defaultValue: "false",
+      description: "Whether the separator is a regex pattern",
+    },
+    {
+      name: "keepSeparator",
+      type: "'start' | 'end'",
+      isOptional: true,
+      description:
+        "Whether to keep the separator at the start or end of chunks",
+    },
+    {
+      name: "extract",
+      type: "ExtractParams",
+      isOptional: true,
+      description: "Metadata extraction configuration. See [ExtractParams reference](./extract-params) for details.",
+    },
+  ]}
+/>
+
+## Strategy-Specific Options
+
+Strategy-specific options are passed as top-level parameters alongside the strategy parameter. For example:
+
+```typescript showLineNumbers copy
+// HTML strategy example
+const chunks = await doc.chunk({
+  strategy: 'html',
+  headers: [['h1', 'title'], ['h2', 'subtitle']], // HTML-specific option
+  sections: [['div.content', 'main']], // HTML-specific option
+  size: 500 // general option
+});
+
+// Markdown strategy example
+const chunks = await doc.chunk({
+  strategy: 'markdown',
+  headers: [['#', 'title'], ['##', 'section']], // Markdown-specific option
+  stripHeaders: true, // Markdown-specific option
+  overlap: 50 // general option
+});
+
+// Token strategy example
+const chunks = await doc.chunk({
+  strategy: 'token',
+  encodingName: 'gpt2', // Token-specific option
+  modelName: 'gpt-3.5-turbo', // Token-specific option
+  size: 1000 // general option
+});
+```
+
+The options documented below are passed directly at the top level of the configuration object, not nested within a separate options object.
+
+### HTML
+
+<PropertiesTable
+  content={[
+    {
+      name: "headers",
+      type: "Array<[string, string]>",
+      description:
+        "Array of [selector, metadata key] pairs for header-based splitting",
+    },
+    {
+      name: "sections",
+      type: "Array<[string, string]>",
+      description:
+        "Array of [selector, metadata key] pairs for section-based splitting",
+    },
+    {
+      name: "returnEachLine",
+      type: "boolean",
+      isOptional: true,
+      description: "Whether to return each line as a separate chunk",
+    },
+  ]}
+/>
+
+### Markdown
+
+<PropertiesTable
+  content={[
+    {
+      name: "headers",
+      type: "Array<[string, string]>",
+      description: "Array of [header level, metadata key] pairs",
+    },
+    {
+      name: "stripHeaders",
+      type: "boolean",
+      isOptional: true,
+      description: "Whether to remove headers from the output",
+    },
+    {
+      name: "returnEachLine",
+      type: "boolean",
+      isOptional: true,
+      description: "Whether to return each line as a separate chunk",
+    },
+  ]}
+/>
+
+### Token
+
+<PropertiesTable
+  content={[
+    {
+      name: "encodingName",
+      type: "string",
+      isOptional: true,
+      description: "Name of the token encoding to use",
+    },
+    {
+      name: "modelName",
+      type: "string",
+      isOptional: true,
+      description: "Name of the model for tokenization",
+    },
+  ]}
+/>
+
+### JSON
+
+<PropertiesTable
+  content={[
+    {
+      name: "maxSize",
+      type: "number",
+      description: "Maximum size of each chunk",
+    },
+    {
+      name: "minSize",
+      type: "number",
+      isOptional: true,
+      description: "Minimum size of each chunk",
+    },
+    {
+      name: "ensureAscii",
+      type: "boolean",
+      isOptional: true,
+      description: "Whether to ensure ASCII encoding",
+    },
+    {
+      name: "convertLists",
+      type: "boolean",
+      isOptional: true,
+      description: "Whether to convert lists in the JSON",
+    },
+  ]}
+/>
+
+## Return Value
+
+Returns a `MDocument` instance containing the chunked documents. Each chunk includes:
+
+```typescript
+interface DocumentNode {
+  text: string;
+  metadata: Record<string, any>;
+  embedding?: number[];
+}
+```

package/.docs/raw/reference/rag/document.mdx

@@ -0,0 +1,129 @@
+---
+title: "Reference: MDocument | Document Processing | RAG | Mastra Docs"
+description: Documentation for the MDocument class in Mastra, which handles document processing and chunking.
+---
+
+# MDocument
+
+The MDocument class processes documents for RAG applications. The main methods are `.chunk()` and `.extractMetadata()`.
+
+## Constructor
+
+<PropertiesTable
+  content={[
+    {
+      name: "docs",
+      type: "Array<{ text: string, metadata?: Record<string, any> }>",
+      description: "Array of document chunks with their text content and optional metadata",
+    },
+    {
+      name: "type",
+      type: "'text' | 'html' | 'markdown' | 'json' | 'latex'",
+      description: "Type of document content",
+    }
+  ]}
+/>
+
+## Static Methods
+
+### fromText()
+
+Creates a document from plain text content.
+
+```typescript
+static fromText(text: string, metadata?: Record<string, any>): MDocument
+```
+
+### fromHTML()
+
+Creates a document from HTML content.
+
+```typescript
+static fromHTML(html: string, metadata?: Record<string, any>): MDocument
+```
+
+### fromMarkdown() 
+
+Creates a document from Markdown content.
+
+```typescript
+static fromMarkdown(markdown: string, metadata?: Record<string, any>): MDocument
+```
+
+### fromJSON()
+
+Creates a document from JSON content.
+
+```typescript
+static fromJSON(json: string, metadata?: Record<string, any>): MDocument
+```
+
+## Instance Methods
+
+### chunk()
+
+Splits document into chunks and optionally extracts metadata.
+
+```typescript
+async chunk(params?: ChunkParams): Promise<Chunk[]>
+```
+
+See [chunk() reference](./chunk) for detailed options.
+
+### getDocs()
+
+Returns array of processed document chunks.
+
+```typescript
+getDocs(): Chunk[]
+```
+
+### getText()
+
+Returns array of text strings from chunks.
+
+```typescript
+getText(): string[]
+```
+
+### getMetadata()
+
+Returns array of metadata objects from chunks.
+
+```typescript
+getMetadata(): Record<string, any>[]
+```
+
+### extractMetadata()
+
+Extracts metadata using specified extractors. See [ExtractParams reference](./extract-params) for details.
+
+```typescript
+async extractMetadata(params: ExtractParams): Promise<MDocument>
+```
+
+## Examples
+
+```typescript
+import { MDocument } from '@mastra/rag';
+
+// Create document from text
+const doc = MDocument.fromText('Your content here');
+
+// Split into chunks with metadata extraction
+const chunks = await doc.chunk({
+  strategy: 'markdown',
+  headers: [['#', 'title'], ['##', 'section']],
+  extract: {
+    fields: [
+      { name: 'summary', description: 'A brief summary' },
+      { name: 'keywords', description: 'Key terms' }
+    ]
+  }
+});
+
+// Get processed chunks
+const docs = doc.getDocs();
+const texts = doc.getText();
+const metadata = doc.getMetadata();
+```

package/.docs/raw/reference/rag/embeddings.mdx

@@ -0,0 +1,160 @@
+---
+title: "Reference: embed() | Document Embedding | RAG | Mastra Docs"
+description: Documentation for embedding functionality in Mastra using the AI SDK.
+---
+
+# Embed
+
+Mastra uses the AI SDK's `embed` and `embedMany` functions to generate vector embeddings for text inputs, enabling similarity search and RAG workflows.
+
+## Single Embedding
+
+The `embed` function generates a vector embedding for a single text input:
+
+```typescript
+import { embed } from 'ai';
+
+const result = await embed({
+  model: openai.embedding('text-embedding-3-small'),
+  value: "Your text to embed",
+  maxRetries: 2  // optional, defaults to 2
+});
+```
+
+### Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "model",
+      type: "EmbeddingModel",
+      description: "The embedding model to use (e.g. openai.embedding('text-embedding-3-small'))"
+    },
+    {
+      name: "value",
+      type: "string | Record<string, any>",
+      description: "The text content or object to embed"
+    },
+    {
+      name: "maxRetries",
+      type: "number",
+      description: "Maximum number of retries per embedding call. Set to 0 to disable retries.",
+      isOptional: true,
+      defaultValue: "2"
+    },
+    {
+      name: "abortSignal",
+      type: "AbortSignal",
+      description: "Optional abort signal to cancel the request",
+      isOptional: true
+    },
+    {
+      name: "headers",
+      type: "Record<string, string>",
+      description: "Additional HTTP headers for the request (only for HTTP-based providers)",
+      isOptional: true
+    }
+  ]}
+/>
+
+### Return Value
+
+<PropertiesTable
+  content={[
+    {
+      name: "embedding",
+      type: "number[]",
+      description: "The embedding vector for the input"
+    }
+  ]}
+/>
+
+## Multiple Embeddings
+
+For embedding multiple texts at once, use the `embedMany` function:
+
+```typescript
+import { embedMany } from 'ai';
+
+const result = await embedMany({
+  model: openai.embedding('text-embedding-3-small'),
+  values: ["First text", "Second text", "Third text"],
+  maxRetries: 2  // optional, defaults to 2
+});
+```
+
+### Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "model",
+      type: "EmbeddingModel",
+      description: "The embedding model to use (e.g. openai.embedding('text-embedding-3-small'))"
+    },
+    {
+      name: "values",
+      type: "string[] | Record<string, any>[]",
+      description: "Array of text content or objects to embed"
+    },
+    {
+      name: "maxRetries",
+      type: "number",
+      description: "Maximum number of retries per embedding call. Set to 0 to disable retries.",
+      isOptional: true,
+      defaultValue: "2"
+    },
+    {
+      name: "abortSignal",
+      type: "AbortSignal",
+      description: "Optional abort signal to cancel the request",
+      isOptional: true
+    },
+    {
+      name: "headers",
+      type: "Record<string, string>",
+      description: "Additional HTTP headers for the request (only for HTTP-based providers)",
+      isOptional: true
+    }
+  ]}
+/>
+
+### Return Value
+
+<PropertiesTable
+  content={[
+    {
+      name: "embeddings",
+      type: "number[][]",
+      description: "Array of embedding vectors corresponding to the input values"
+    }
+  ]}
+/>
+
+## Example Usage
+
+```typescript
+import { embed, embedMany } from 'ai';
+import { openai } from '@ai-sdk/openai';
+
+// Single embedding
+const singleResult = await embed({
+  model: openai.embedding('text-embedding-3-small'),
+  value: "What is the meaning of life?",
+});
+
+// Multiple embeddings
+const multipleResult = await embedMany({
+  model: openai.embedding('text-embedding-3-small'),
+  values: [
+    "First question about life",
+    "Second question about universe",
+    "Third question about everything"
+  ],
+});
+```
+
+For more detailed information about embeddings in the Vercel AI SDK, see:
+- [AI SDK Embeddings Overview](https://sdk.vercel.ai/docs/ai-sdk-core/embeddings)
+- [embed()](https://sdk.vercel.ai/docs/reference/ai-sdk-core/embed)
+- [embedMany()](https://sdk.vercel.ai/docs/reference/ai-sdk-core/embed-many)

package/.docs/raw/reference/rag/extract-params.mdx

@@ -0,0 +1,72 @@
+---
+title: "Reference: ExtractParams | Document Processing | RAG | Mastra Docs"
+description: Documentation for metadata extraction configuration in Mastra.
+---
+
+# ExtractParams
+
+ExtractParams configures metadata extraction from document chunks.
+
+## Example
+
+## ExtractParams
+
+`ExtractParams` configures automatic metadata extraction from chunks using LLM analysis.
+
+```typescript showLineNumbers copy
+const doc = new Document(text);
+const chunks = await doc.chunk({
+  extract: {
+    fields: [
+      { 
+        name: 'summary', 
+        description: 'A 1-2 sentence summary of the main points' 
+      },
+      { 
+        name: 'entities', 
+        description: 'List of companies, people, and locations mentioned' 
+      },
+      {
+        name: 'custom_field',
+        description: 'Any other metadata you want to extract, guided by this description'
+      }
+    ],
+    model: 'gpt-4o-mini' // Optional: specify a different model
+  }
+});
+```
+
+## Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "fields",
+      type: "Array<{ name: string, description: string }>",
+      description: "Array of fields to extract from each chunk",
+      isOptional: false
+    },
+    {
+      name: "model",
+      type: "string",
+      description: "OpenAI model to use for extraction",
+      defaultValue: "gpt-3.5-turbo",
+      isOptional: true
+    }
+  ]}
+/> 
+
+## Field Types
+
+The fields are flexible - you can define any metadata fields you want to extract. Common field types include:
+
+- `summary`: Brief overview of chunk content
+- `keywords`: Key terms or concepts
+- `topics`: Main subjects discussed
+- `entities`: Named entities (people, places, organizations)
+- `sentiment`: Emotional tone
+- `language`: Detected language
+- `timestamp`: Temporal references
+- `categories`: Content classification
+
+Example: