@mastra/mcp-docs-server 1.0.0-beta.5 → 1.0.0-beta.7

package/.docs/raw/reference/evals/scorer-utils.mdx

@@ -0,0 +1,362 @@
+---
+title: "Reference: Scorer Utils | Evals"
+description: Utility functions for extracting data from scorer run inputs and outputs, including text content, reasoning, system messages, and tool calls.
+---
+
+# Scorer Utils
+
+Mastra provides utility functions to help extract and process data from scorer run inputs and outputs. These utilities are particularly useful in the `preprocess` step of custom scorers.
+
+## Import
+
+```typescript
+import {
+  getAssistantMessageFromRunOutput,
+  getReasoningFromRunOutput,
+  getUserMessageFromRunInput,
+  getSystemMessagesFromRunInput,
+  getCombinedSystemPrompt,
+  extractToolCalls,
+  extractInputMessages,
+  extractAgentResponseMessages,
+} from "@mastra/evals/scorers/utils";
+```
+
+## Message Extraction
+
+### getAssistantMessageFromRunOutput
+
+Extracts the text content from the first assistant message in the run output.
+
+```typescript
+const scorer = createScorer({
+  id: "my-scorer",
+  description: "My scorer",
+  type: "agent",
+})
+  .preprocess(({ run }) => {
+    const response = getAssistantMessageFromRunOutput(run.output);
+    return { response };
+  })
+  .generateScore(({ results }) => {
+    return results.preprocessStepResult?.response ? 1 : 0;
+  });
+```
+
+<PropertiesTable
+  content={[
+    {
+      name: "output",
+      type: "ScorerRunOutputForAgent",
+      isOptional: true,
+      description: "The scorer run output (array of MastraDBMessage)",
+    },
+  ]}
+/>
+
+**Returns:** `string | undefined` - The assistant message text, or undefined if no assistant message is found.
+
+### getUserMessageFromRunInput
+
+Extracts the text content from the first user message in the run input.
+
+```typescript
+.preprocess(({ run }) => {
+  const userMessage = getUserMessageFromRunInput(run.input);
+  return { userMessage };
+})
+```
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "ScorerRunInputForAgent",
+      isOptional: true,
+      description: "The scorer run input containing input messages",
+    },
+  ]}
+/>
+
+**Returns:** `string | undefined` - The user message text, or undefined if no user message is found.
+
+### extractInputMessages
+
+Extracts text content from all input messages as an array.
+
+```typescript
+.preprocess(({ run }) => {
+  const allUserMessages = extractInputMessages(run.input);
+  return { conversationHistory: allUserMessages.join("\n") };
+})
+```
+
+**Returns:** `string[]` - Array of text strings from each input message.
+
+### extractAgentResponseMessages
+
+Extracts text content from all assistant response messages as an array.
+
+```typescript
+.preprocess(({ run }) => {
+  const allResponses = extractAgentResponseMessages(run.output);
+  return { allResponses };
+})
+```
+
+**Returns:** `string[]` - Array of text strings from each assistant message.
+
+## Reasoning Extraction
+
+### getReasoningFromRunOutput
+
+Extracts reasoning text from the run output. This is particularly useful when evaluating responses from reasoning models like `deepseek-reasoner` that produce chain-of-thought reasoning.
+
+Reasoning can be stored in two places:
+1. `content.reasoning` - a string field on the message content
+2. `content.parts` - as parts with `type: 'reasoning'` containing `details`
+
+```typescript
+import { 
+  getReasoningFromRunOutput, 
+  getAssistantMessageFromRunOutput 
+} from "@mastra/evals/scorers/utils";
+
+const reasoningQualityScorer = createScorer({
+  id: "reasoning-quality",
+  name: "Reasoning Quality",
+  description: "Evaluates the quality of model reasoning",
+  type: "agent",
+})
+  .preprocess(({ run }) => {
+    const reasoning = getReasoningFromRunOutput(run.output);
+    const response = getAssistantMessageFromRunOutput(run.output);
+    return { reasoning, response };
+  })
+  .analyze(({ results }) => {
+    const { reasoning } = results.preprocessStepResult || {};
+    return {
+      hasReasoning: !!reasoning,
+      reasoningLength: reasoning?.length || 0,
+      hasStepByStep: reasoning?.includes("step") || false,
+    };
+  })
+  .generateScore(({ results }) => {
+    const { hasReasoning, reasoningLength } = results.analyzeStepResult || {};
+    if (!hasReasoning) return 0;
+    // Score based on reasoning length (normalized to 0-1)
+    return Math.min(reasoningLength / 500, 1);
+  })
+  .generateReason(({ results, score }) => {
+    const { hasReasoning, reasoningLength } = results.analyzeStepResult || {};
+    if (!hasReasoning) {
+      return "No reasoning was provided by the model.";
+    }
+    return `Model provided ${reasoningLength} characters of reasoning. Score: ${score}`;
+  });
+```
+
+<PropertiesTable
+  content={[
+    {
+      name: "output",
+      type: "ScorerRunOutputForAgent",
+      isOptional: true,
+      description: "The scorer run output (array of MastraDBMessage)",
+    },
+  ]}
+/>
+
+**Returns:** `string | undefined` - The reasoning text, or undefined if no reasoning is present.
+
+## System Message Extraction
+
+### getSystemMessagesFromRunInput
+
+Extracts all system messages from the run input, including both standard system messages and tagged system messages (specialized prompts like memory instructions).
+
+```typescript
+.preprocess(({ run }) => {
+  const systemMessages = getSystemMessagesFromRunInput(run.input);
+  return { 
+    systemPromptCount: systemMessages.length,
+    systemPrompts: systemMessages 
+  };
+})
+```
+
+**Returns:** `string[]` - Array of system message strings.
+
+### getCombinedSystemPrompt
+
+Combines all system messages into a single prompt string, joined with double newlines.
+
+```typescript
+.preprocess(({ run }) => {
+  const fullSystemPrompt = getCombinedSystemPrompt(run.input);
+  return { fullSystemPrompt };
+})
+```
+
+**Returns:** `string` - Combined system prompt string.
+
+## Tool Call Extraction
+
+### extractToolCalls
+
+Extracts information about all tool calls from the run output, including tool names, call IDs, and their positions in the message array.
+
+```typescript
+const toolUsageScorer = createScorer({
+  id: "tool-usage",
+  description: "Evaluates tool usage patterns",
+  type: "agent",
+})
+  .preprocess(({ run }) => {
+    const { tools, toolCallInfos } = extractToolCalls(run.output);
+    return {
+      toolsUsed: tools,
+      toolCount: tools.length,
+      toolDetails: toolCallInfos,
+    };
+  })
+  .generateScore(({ results }) => {
+    const { toolCount } = results.preprocessStepResult || {};
+    // Score based on appropriate tool usage
+    return toolCount > 0 ? 1 : 0;
+  });
+```
+
+**Returns:**
+
+```typescript
+{
+  tools: string[];           // Array of tool names
+  toolCallInfos: ToolCallInfo[];  // Detailed tool call information
+}
+```
+
+Where `ToolCallInfo` is:
+
+```typescript
+type ToolCallInfo = {
+  toolName: string;      // Name of the tool
+  toolCallId: string;    // Unique call identifier
+  messageIndex: number;  // Index in the output array
+  invocationIndex: number; // Index within message's tool invocations
+};
+```
+
+## Test Utilities
+
+These utilities help create test data for scorer development.
+
+### createTestMessage
+
+Creates a `MastraDBMessage` object for testing purposes.
+
+```typescript
+import { createTestMessage } from "@mastra/evals/scorers/utils";
+
+const userMessage = createTestMessage({
+  content: "What is the weather?",
+  role: "user",
+});
+
+const assistantMessage = createTestMessage({
+  content: "The weather is sunny.",
+  role: "assistant",
+  toolInvocations: [
+    {
+      toolCallId: "call-1",
+      toolName: "weatherTool",
+      args: { location: "London" },
+      result: { temp: 20 },
+      state: "result",
+    },
+  ],
+});
+```
+
+### createAgentTestRun
+
+Creates a complete test run object for testing scorers.
+
+```typescript
+import { createAgentTestRun, createTestMessage } from "@mastra/evals/scorers/utils";
+
+const testRun = createAgentTestRun({
+  inputMessages: [
+    createTestMessage({ content: "Hello", role: "user" }),
+  ],
+  output: [
+    createTestMessage({ content: "Hi there!", role: "assistant" }),
+  ],
+});
+
+// Run your scorer with the test data
+const result = await myScorer.run({
+  input: testRun.input,
+  output: testRun.output,
+});
+```
+
+## Complete Example
+
+Here's a complete example showing how to use multiple utilities together:
+
+```typescript
+import { createScorer } from "@mastra/core/evals";
+import {
+  getAssistantMessageFromRunOutput,
+  getReasoningFromRunOutput,
+  getUserMessageFromRunInput,
+  getCombinedSystemPrompt,
+  extractToolCalls,
+} from "@mastra/evals/scorers/utils";
+
+const comprehensiveScorer = createScorer({
+  id: "comprehensive-analysis",
+  name: "Comprehensive Analysis",
+  description: "Analyzes all aspects of an agent response",
+  type: "agent",
+})
+  .preprocess(({ run }) => {
+    // Extract all relevant data
+    const userMessage = getUserMessageFromRunInput(run.input);
+    const response = getAssistantMessageFromRunOutput(run.output);
+    const reasoning = getReasoningFromRunOutput(run.output);
+    const systemPrompt = getCombinedSystemPrompt(run.input);
+    const { tools, toolCallInfos } = extractToolCalls(run.output);
+
+    return {
+      userMessage,
+      response,
+      reasoning,
+      systemPrompt,
+      toolsUsed: tools,
+      toolCount: tools.length,
+    };
+  })
+  .generateScore(({ results }) => {
+    const { response, reasoning, toolCount } = results.preprocessStepResult || {};
+    
+    let score = 0;
+    if (response && response.length > 0) score += 0.4;
+    if (reasoning) score += 0.3;
+    if (toolCount > 0) score += 0.3;
+    
+    return score;
+  })
+  .generateReason(({ results, score }) => {
+    const { response, reasoning, toolCount } = results.preprocessStepResult || {};
+    
+    const parts = [];
+    if (response) parts.push("provided a response");
+    if (reasoning) parts.push("included reasoning");
+    if (toolCount > 0) parts.push(`used ${toolCount} tool(s)`);
+    
+    return `Score: ${score}. The agent ${parts.join(", ")}.`;
+  });
+```
+

package/.docs/raw/reference/index.mdx

@@ -1,6 +1,7 @@
 ---
 title: "Reference: Overview"
 description: "Reference documentation on Mastra's APIs and tools"
+showCopyButton: false
 ---
 
 import { ReferenceCards } from "@site/src/components/ReferenceCards";

package/.docs/raw/reference/observability/tracing/bridges/otel.mdx

@@ -0,0 +1,177 @@
+---
+title: "Reference: OtelBridge | Observability"
+description: OpenTelemetry bridge for Tracing
+---
+
+import PropertiesTable from "@site/src/components/PropertiesTable";
+
+# OtelBridge
+
+:::warning
+
+The OpenTelemetry Bridge is currently **experimental**. APIs and configuration options may change in future releases.
+
+:::
+
+Enables bidirectional integration between Mastra tracing and OpenTelemetry infrastructure. Creates native OTEL spans for Mastra operations and inherits context from active OTEL spans.
+
+## Constructor
+
+```typescript
+new OtelBridge()
+```
+
+## Methods
+
+### executeInContext
+
+```typescript
+executeInContext<T>(spanId: string, fn: () => Promise<T>): Promise<T>
+```
+
+Executes an async function within the OTEL context of a Mastra span. OTEL-instrumented code running inside the function will have correct parent relationships.
+
+<PropertiesTable
+  props={[
+    {
+      name: "spanId",
+      type: "string",
+      description: "The ID of the Mastra span to use as context",
+      required: true,
+    },
+    {
+      name: "fn",
+      type: "() => Promise<T>",
+      description: "The async function to execute within the span context",
+      required: true,
+    },
+  ]}
+/>
+
+**Returns:** `Promise<T>` - The result of the function execution.
+
+### executeInContextSync
+
+```typescript
+executeInContextSync<T>(spanId: string, fn: () => T): T
+```
+
+Executes a synchronous function within the OTEL context of a Mastra span.
+
+<PropertiesTable
+  props={[
+    {
+      name: "spanId",
+      type: "string",
+      description: "The ID of the Mastra span to use as context",
+      required: true,
+    },
+    {
+      name: "fn",
+      type: "() => T",
+      description: "The synchronous function to execute within the span context",
+      required: true,
+    },
+  ]}
+/>
+
+**Returns:** `T` - The result of the function execution.
+
+### shutdown
+
+```typescript
+async shutdown(): Promise<void>
+```
+
+Shuts down the bridge and cleans up resources. Ends any spans that were not properly closed.
+
+## Usage Examples
+
+### Basic Usage
+
+```typescript
+import { Mastra } from "@mastra/core";
+import { Observability } from "@mastra/observability";
+import { OtelBridge } from "@mastra/otel-bridge";
+
+const mastra = new Mastra({
+  observability: new Observability({
+    configs: {
+      default: {
+        serviceName: "my-service",
+        bridge: new OtelBridge(),
+      },
+    },
+  }),
+  agents: { myAgent },
+});
+```
+
+### Combined with Exporters
+
+The bridge can be used alongside exporters. The bridge handles OTEL context, while exporters send data to additional destinations:
+
+```typescript
+import { Mastra } from "@mastra/core";
+import { Observability, DefaultExporter } from "@mastra/observability";
+import { OtelBridge } from "@mastra/otel-bridge";
+import { LangfuseExporter } from "@mastra/langfuse";
+
+const mastra = new Mastra({
+  observability: new Observability({
+    configs: {
+      default: {
+        serviceName: "my-service",
+        bridge: new OtelBridge(), // Handles OTEL context
+        exporters: [
+          new DefaultExporter(), // Studio access
+          new LangfuseExporter({
+            // Additional destination
+            publicKey: process.env.LANGFUSE_PUBLIC_KEY,
+            secretKey: process.env.LANGFUSE_SECRET_KEY,
+          }),
+        ],
+      },
+    },
+  }),
+});
+```
+
+## OpenTelemetry Setup Requirements
+
+The OtelBridge requires an active OpenTelemetry SDK to function. The bridge reads from OTEL's ambient context.
+
+See the [OtelBridge Guide](/docs/v1/observability/tracing/bridges/otel#configuration) for complete setup instructions, including how to configure OTEL instrumentation and run your application.
+
+## Tags Support
+
+The OtelBridge supports trace tagging for categorization and filtering. Tags are only applied to root spans and are included as the `mastra.tags` attribute on native OTEL spans.
+
+### Usage
+
+```typescript
+const result = await agent.generate({
+  messages: [{ role: "user", content: "Hello" }],
+  tracingOptions: {
+    tags: ["production", "experiment-v2", "user-request"],
+  },
+});
+```
+
+### How Tags Are Stored
+
+Tags are stored as a JSON-stringified array in the `mastra.tags` span attribute:
+
+```json
+{
+  "mastra.tags": "[\"production\",\"experiment-v2\",\"user-request\"]"
+}
+```
+
+This format ensures compatibility with all OTEL-compatible backends and collectors.
+
+## Related
+
+- [OtelBridge Guide](/docs/v1/observability/tracing/bridges/otel) - Setup guide with examples
+- [Tracing Overview](/docs/v1/observability/tracing/overview) - General tracing concepts
+- [OtelExporter Reference](/reference/v1/observability/tracing/exporters/otel) - OTEL exporter for sending traces

package/.docs/raw/reference/observability/tracing/configuration.mdx

@@ -232,10 +232,6 @@ Shuts down all observability instances and clears the registry.
 - [Interfaces](/reference/v1/observability/tracing/interfaces) - Type definitions
 - [Spans Reference](/reference/v1/observability/tracing/spans) - Span lifecycle
 
-### Examples
-
-- [Basic Tracing](/examples/v1/observability/basic-ai-tracing) - Getting started
-
 ### Exporters
 
 - [DefaultExporter](/reference/v1/observability/tracing/exporters/default-exporter) - Storage configuration

package/.docs/raw/reference/observability/tracing/exporters/arize.mdx

@@ -38,6 +38,10 @@ Inherits from `OtelExporterConfig` (excluding `provider`), which includes:
 - `logLevel?: LogLevel | 'debug' | 'info' | 'warn' | 'error'` - Log level (default: WARN)
 - `resourceAttributes?: Record<string, any>` - Custom resource attributes
 
+### Metadata passthrough
+
+Non-reserved span attributes are serialized into the OpenInference `metadata` payload. Add them via `tracingOptions.metadata` (e.g., `companyId`, `tier`). Reserved fields such as `input`, `output`, `sessionId`, thread/user IDs, and OpenInference IDs are excluded automatically.
+
 <PropertiesTable
   props={[
     {
@@ -157,6 +161,31 @@ const exporter = new ArizeExporter({
 
 The ArizeExporter implements [OpenInference Semantic Conventions](https://github.com/Arize-ai/openinference/tree/main/spec) for generative AI applications, providing standardized trace structure across different observability platforms.
 
+## Tags Support
+
+The ArizeExporter supports trace tagging for categorization and filtering. Tags are only applied to root spans and are mapped to the native OpenInference `tag.tags` semantic convention.
+
+### Usage
+
+```typescript
+const result = await agent.generate({
+  messages: [{ role: "user", content: "Hello" }],
+  tracingOptions: {
+    tags: ["production", "experiment-v2", "user-request"],
+  },
+});
+```
+
+### How Tags Are Stored
+
+Tags are stored using the OpenInference `tag.tags` attribute:
+
+```json
+{
+  "tag.tags": ["production", "experiment-v2", "user-request"]
+}
+```
+
 ## Related
 
 - [ArizeExporter Documentation](/docs/v1/observability/tracing/exporters/arize)

package/.docs/raw/reference/observability/tracing/exporters/langfuse.mdx

@@ -117,3 +117,46 @@ const exporter = new LangfuseExporter({
 - `MODEL_GENERATION` spans → Langfuse generations
 - All other spans → Langfuse spans
 - Event spans → Langfuse events
+
+## Prompt Linking
+
+Link LLM generations to [Langfuse Prompt Management](https://langfuse.com/docs/prompt-management) using the `withLangfusePrompt` helper:
+
+```typescript
+import { buildTracingOptions } from "@mastra/observability";
+import { withLangfusePrompt } from "@mastra/langfuse";
+import { Langfuse } from "langfuse";
+
+const langfuse = new Langfuse({
+  publicKey: process.env.LANGFUSE_PUBLIC_KEY!,
+  secretKey: process.env.LANGFUSE_SECRET_KEY!,
+});
+
+const prompt = await langfuse.getPrompt("customer-support");
+
+const agent = new Agent({
+  name: "support-agent",
+  instructions: prompt.prompt,
+  model: openai("gpt-4o"),
+  defaultGenerateOptions: {
+    tracingOptions: buildTracingOptions(withLangfusePrompt(prompt)),
+  },
+});
+```
+
+### Helper Functions
+
+#### `withLangfusePrompt(prompt)`
+
+Adds Langfuse prompt metadata to tracing options.
+
+```typescript
+// With Langfuse SDK prompt object
+withLangfusePrompt(prompt)
+
+// With manual fields
+withLangfusePrompt({ name: "my-prompt", version: 1 })
+withLangfusePrompt({ id: "prompt-uuid" })
+```
+
+When `metadata.langfuse.prompt` is set on a `MODEL_GENERATION` span (with either `id` alone, or `name` + `version`), the exporter automatically links the generation to the prompt in Langfuse.

package/.docs/raw/reference/observability/tracing/exporters/langsmith.mdx

@@ -20,6 +20,7 @@ new LangSmithExporter(config: LangSmithExporterConfig)
 ```typescript
 interface LangSmithExporterConfig extends ClientConfig, BaseExporterConfig {
   client?: Client;
+  projectName?: string;
 }
 ```
 
@@ -32,7 +33,13 @@ Extends both `ClientConfig` (from LangSmith SDK) and `BaseExporterConfig`:
     {
       name: "apiKey",
       type: "string",
-      description: "LangSmith API key",
+      description: "LangSmith API key. Defaults to LANGSMITH_API_KEY env var.",
+      required: false,
+    },
+    {
+      name: "projectName",
+      type: "string",
+      description: "The LangSmith project to send traces to. Overrides LANGCHAIN_PROJECT env var. Defaults to 'default'.",
       required: false,
     },
     {
@@ -99,11 +106,20 @@ import { LangSmithExporter } from "@mastra/langsmith";
 
 const exporter = new LangSmithExporter({
   apiKey: process.env.LANGSMITH_API_KEY,
+  projectName: "my-project", // Optional: specify which project to send traces to
   apiUrl: "https://api.smith.langchain.com",
   logLevel: "info",
 });
 ```
 
+## Environment Variables
+
+| Variable | Description |
+|----------|-------------|
+| `LANGSMITH_API_KEY` | Your LangSmith API key |
+| `LANGCHAIN_PROJECT` | Default project name for traces (used if `projectName` not specified) |
+| `LANGSMITH_BASE_URL` | API URL for self-hosted instances |
+
 ## Span Type Mapping
 
 | Span Type       | LangSmith Type |