@mastra/mcp-docs-server 0.13.24 → 0.13.25-alpha.1

package/.docs/raw/reference/scorers/run-experiment.mdx

@@ -0,0 +1,216 @@
+---
+title: "Reference: runExperiment | Scorers | Mastra Docs"
+description: "Documentation for the runExperiment function in Mastra, which enables batch evaluation of agents and workflows using multiple scorers."
+---
+
+# runExperiment
+
+The `runExperiment` function enables batch evaluation of agents and workflows by running multiple test cases against scorers concurrently. This is essential for systematic testing, performance analysis, and validation of AI systems.
+
+## Usage Example
+
+```typescript
+import { runExperiment } from '@mastra/core/scores';
+import { myAgent } from './agents/my-agent';
+import { myScorer1, myScorer2 } from './scorers';
+
+const result = await runExperiment({
+  target: myAgent,
+  data: [
+    { input: "What is machine learning?" },
+    { input: "Explain neural networks" },
+    { input: "How does AI work?" }
+  ],
+  scorers: [myScorer1, myScorer2],
+  concurrency: 2,
+  onItemComplete: ({ item, targetResult, scorerResults }) => {
+    console.log(`Completed: ${item.input}`);
+    console.log(`Scores:`, scorerResults);
+  }
+});
+
+console.log(`Average scores:`, result.scores);
+console.log(`Processed ${result.summary.totalItems} items`);
+```
+
+## Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "target",
+      type: "Agent | Workflow",
+      description: "The agent or workflow to evaluate.",
+      isOptional: false,
+    },
+    {
+      name: "data",
+      type: "RunExperimentDataItem[]",
+      description: "Array of test cases with input data and optional ground truth.",
+      isOptional: false,
+    },
+    {
+      name: "scorers",
+      type: "MastraScorer[] | WorkflowScorerConfig",
+      description: "Array of scorers for agents, or configuration object for workflows specifying scorers for the workflow and individual steps.",
+      isOptional: false,
+    },
+    {
+      name: "concurrency",
+      type: "number",
+      description: "Number of test cases to run concurrently.",
+      isOptional: true,
+      defaultValue: "1",
+    },
+    {
+      name: "onItemComplete",
+      type: "function",
+      description: "Callback function called after each test case completes. Receives item, target result, and scorer results.",
+      isOptional: true,
+    },
+  ]}
+/>
+
+## Data Item Structure
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "string | string[] | CoreMessage[] | any",
+      description: "Input data for the target. For agents: messages or strings. For workflows: workflow input data.",
+      isOptional: false,
+    },
+    {
+      name: "groundTruth",
+      type: "any",
+      description: "Expected or reference output for comparison during scoring.",
+      isOptional: true,
+    },
+    {
+      name: "runtimeContext",
+      type: "RuntimeContext",
+      description: "Runtime context to pass to the target during execution.",
+      isOptional: true,
+    },
+    {
+      name: "tracingContext",
+      type: "TracingContext",
+      description: "Tracing context for observability and debugging.",
+      isOptional: true,
+    },
+  ]}
+/>
+
+## Workflow Scorer Configuration
+
+For workflows, you can specify scorers at different levels using `WorkflowScorerConfig`:
+
+<PropertiesTable
+  content={[
+    {
+      name: "workflow",
+      type: "MastraScorer[]",
+      description: "Array of scorers to evaluate the entire workflow output.",
+      isOptional: true,
+    },
+    {
+      name: "steps",
+      type: "Record<string, MastraScorer[]>",
+      description: "Object mapping step IDs to arrays of scorers for evaluating individual step outputs.",
+      isOptional: true,
+    },
+  ]}
+/>
+
+## Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: "scores",
+      type: "Record<string, any>",
+      description: "Average scores across all test cases, organized by scorer name.",
+    },
+    {
+      name: "summary",
+      type: "object",
+      description: "Summary information about the experiment execution.",
+    },
+    {
+      name: "summary.totalItems",
+      type: "number",
+      description: "Total number of test cases processed.",
+    },
+  ]}
+/>
+
+## Examples
+
+### Agent Evaluation
+
+```typescript
+import { runExperiment } from '@mastra/core/scores';
+import { createScorer } from '@mastra/core/scores';
+
+const myScorer = createScorer({
+  name: 'My Scorer',
+  description: "Check if Agent's response contains ground truth",
+  type: 'agent'
+}).generateScore(({ run }) => {
+  const response = run.output[0]?.content || '';
+  const expectedResponse = run.groundTruth
+  return response.includes(expectedResponse) ? 1 : 0
+});
+
+const result = await runExperiment({
+  target: chatAgent,
+  data: [
+    { 
+      input: "What is AI?",
+      groundTruth: "AI is a field of computer science that creates intelligent machines."
+    },
+    {
+      input: "How does machine learning work?",
+      groundTruth: "Machine learning uses algorithms to learn patterns from data."
+    }
+  ],
+  scorers: [relevancyScorer],
+  concurrency: 3
+});
+```
+
+### Workflow Evaluation
+
+```typescript
+const workflowResult = await runExperiment({
+  target: myWorkflow,
+  data: [
+    { input: { query: "Process this data", priority: "high" } },
+    { input: { query: "Another task", priority: "low" } }
+  ],
+  scorers: {
+    workflow: [outputQualityScorer],
+    steps: {
+      'validation-step': [validationScorer],
+      'processing-step': [processingScorer]
+    }
+  },
+  onItemComplete: ({ item, targetResult, scorerResults }) => {
+    console.log(`Workflow completed for: ${item.input.query}`);
+    if (scorerResults.workflow) {
+      console.log('Workflow scores:', scorerResults.workflow);
+    }
+    if (scorerResults.steps) {
+      console.log('Step scores:', scorerResults.steps);
+    }
+  }
+});
+```
+
+## Related
+
+- [createScorer()](../../reference/scorers/create-scorer) - Create custom scorers for experiments
+- [MastraScorer](../../reference/scorers/mastra-scorer) - Learn about scorer structure and methods
+- [Custom Scorers](../../docs/scorers/custom-scorers) - Guide to building evaluation logic
+- [Scorers Overview](../../docs/scorers/overview) - Understanding scorer concepts

package/.docs/raw/reference/streaming/ChunkType.mdx

@@ -853,5 +853,6 @@ for await (const chunk of stream.fullStream) {
 
 ## Related Types
 
-- [MastraModelOutput](./MastraModelOutput.mdx) - The stream object that emits these chunks
-- [.streamVNext()](./streamVNext.mdx) - Method that returns streams emitting these chunks
+- [MastraModelOutput](./agents/MastraModelOutput.mdx) - The stream object that emits these chunks
+- [agent.streamVNext()](./agents/streamVNext.mdx) - Method that returns streams emitting these chunks for agents
+- [workflow.streamVNext()](./workflows/streamVNext.mdx) - Method that returns streams emitting these chunks for workflows

package/.docs/raw/reference/streaming/agents/MastraModelOutput.mdx

@@ -318,5 +318,5 @@ if (stream.error) {
 
 ## Related Types
 
-- [ChunkType](./ChunkType.mdx) - All possible chunk types in the full stream
+- [ChunkType](../ChunkType.mdx) - All possible chunk types in the full stream
 - [.streamVNext()](./streamVNext.mdx) - Method that returns MastraModelOutput

package/.docs/raw/reference/streaming/agents/stream.mdx

@@ -182,7 +182,7 @@ await agent.stream("message for agent");
       type: "TelemetrySettings",
       isOptional: true,
       description:
-        "Settings for telemetry collection during streaming.",
+        "Settings for OTLP telemetry collection during streaming (not AI tracing).",
       properties: [
         {
           parameters: [{
@@ -341,6 +341,38 @@ await agent.stream("message for agent");
       isOptional: true,
       description: "Runtime context for dependency injection and contextual information.",
     },
+    {
+      name: "tracingContext",
+      type: "TracingContext",
+      isOptional: true,
+      description: "AI tracing context for creating child spans and adding metadata. Automatically injected when using Mastra's tracing system.",
+      properties: [
+        {
+          parameters: [{
+            name: "currentSpan",
+            type: "AISpan",
+            isOptional: true,
+            description: "Current AI span for creating child spans and adding metadata. Use this to create custom child spans or update span attributes during execution."
+          }]
+        }
+      ]
+    },
+    {
+      name: "tracingOptions",
+      type: "TracingOptions",
+      isOptional: true,
+      description: "Options for AI tracing configuration.",
+      properties: [
+        {
+          parameters: [{
+            name: "metadata",
+            type: "Record<string, any>",
+            isOptional: true,
+            description: "Metadata to add to the root trace span. Useful for adding custom attributes like user IDs, session IDs, or feature flags."
+          }]
+        }
+      ]
+    },
     {
       name: "maxTokens",
       type: "number",
@@ -456,6 +488,12 @@ await agent.stream("message for agent");
         }
       ]
     },
+    {
+      name: "traceId",
+      type: "string",
+      isOptional: true,
+      description: "The trace ID associated with this execution when AI tracing is enabled. Use this to correlate logs and debug execution flow.",
+    },
   ]}
 />
 
@@ -475,5 +513,5 @@ await agent.stream("message for agent", {
 
 ## Related
 
-- [Generating responses](../../docs/agents/overview.mdx#generating-responses)
-- [Streaming responses](../../docs/agents/overview.mdx#streaming-responses)
+- [Generating responses](../../../../docs/agents/overview.mdx#generating-responses)
+- [Streaming responses](../../../../docs/agents/overview.mdx#streaming-responses)

package/.docs/raw/reference/streaming/agents/streamVNext.mdx

@@ -105,6 +105,32 @@ const aiSdkStream = await agent.streamVNext("message for agent", {
       type: "TracingContext",
       isOptional: true,
       description: "AI tracing context for span hierarchy and metadata.",
+      properties: [
+        {
+          parameters: [{
+            name: "currentSpan",
+            type: "AISpan",
+            isOptional: true,
+            description: "Current AI span for creating child spans and adding metadata."
+          }]
+        }
+      ]
+    },
+    {
+      name: "tracingOptions",
+      type: "TracingOptions",
+      isOptional: true,
+      description: "Options for AI tracing configuration.",
+      properties: [
+        {
+          parameters: [{
+            name: "metadata",
+            type: "Record<string, any>",
+            isOptional: true,
+            description: "Metadata to add to the root trace span."
+          }]
+        }
+      ]
     },
     {
       name: "returnScorerData",
@@ -292,7 +318,7 @@ const aiSdkStream = await agent.streamVNext("message for agent", {
       type: "TelemetrySettings",
       isOptional: true,
       description:
-        "Settings for telemetry collection during streaming.",
+        "Settings for OTLP telemetry collection during streaming (not AI tracing).",
       properties: [
         {
           parameters: [{
@@ -525,7 +551,13 @@ const aiSdkStream = await agent.streamVNext("message for agent", {
     {
       name: "stream",
       type: "MastraModelOutput<Output> | AISDKV5OutputStream<Output>",
-      description: "Returns a streaming interface based on the format parameter. When format is 'mastra' (default), returns MastraModelOutput. When format is 'aisdk', returns AISDKV5OutputStream for AI SDK v5 compatibility.",
+      description: "Returns a streaming interface based on the format parameter. When format is 'mastra' (default), returns MastraModelOutput with traceId property. When format is 'aisdk', returns AISDKV5OutputStream for AI SDK v5 compatibility.",
+    },
+    {
+      name: "traceId",
+      type: "string",
+      isOptional: true,
+      description: "The trace ID associated with this execution when AI tracing is enabled. Available via stream.traceId for Mastra format.",
     },
   ]}
 />

package/.docs/raw/reference/streaming/workflows/resumeStreamVNext.mdx

@@ -53,6 +53,22 @@ if (result.status === "suspended") {
       description: "The step to resume execution from",
       isOptional: true,
     },
+    {
+      name: "tracingOptions",
+      type: "TracingOptions",
+      isOptional: true,
+      description: "Options for AI tracing configuration.",
+      properties: [
+        {
+          parameters: [{
+            name: "metadata",
+            type: "Record<string, any>",
+            isOptional: true,
+            description: "Metadata to add to the root trace span. Useful for adding custom attributes like user IDs, session IDs, or feature flags."
+          }]
+        }
+      ]
+    },
   ]}
 />
 
@@ -96,5 +112,5 @@ The stream emits various event types during workflow execution. Each event has a
 ## Related
 
 - [Workflows overview](../../../docs/workflows/overview.mdx#run-workflow)
-- [Workflow.createRunAsync()](../create-run.mdx)
+- [Workflow.createRunAsync()](../../../reference/workflows/workflow-methods/create-run.mdx)
 - [Run.streamVNext()](./streamVNext.mdx)

package/.docs/raw/reference/streaming/workflows/stream.mdx

@@ -35,6 +35,38 @@ const stream = await run.stream({
       description: "Runtime context data to use during workflow execution",
       isOptional: true,
     },
+    {
+      name: "tracingContext",
+      type: "TracingContext",
+      isOptional: true,
+      description: "AI tracing context for creating child spans and adding metadata. Automatically injected when using Mastra's tracing system.",
+      properties: [
+        {
+          parameters: [{
+            name: "currentSpan",
+            type: "AISpan",
+            isOptional: true,
+            description: "Current AI span for creating child spans and adding metadata. Use this to create custom child spans or update span attributes during execution."
+          }]
+        }
+      ]
+    },
+    {
+      name: "tracingOptions",
+      type: "TracingOptions",
+      isOptional: true,
+      description: "Options for AI tracing configuration.",
+      properties: [
+        {
+          parameters: [{
+            name: "metadata",
+            type: "Record<string, any>",
+            isOptional: true,
+            description: "Metadata to add to the root trace span. Useful for adding custom attributes like user IDs, session IDs, or feature flags."
+          }]
+        }
+      ]
+    },
   ]}
 />
 
@@ -52,6 +84,12 @@ const stream = await run.stream({
       type: "() => Promise<WorkflowResult<TOutput, TSteps>>",
       description: "A function that returns a promise resolving to the final workflow result",
     },
+    {
+      name: "traceId",
+      type: "string",
+      isOptional: true,
+      description: "The trace ID associated with this execution when AI tracing is enabled. Use this to correlate logs and debug execution flow.",
+    },
   ]}
 />
 
@@ -84,4 +122,4 @@ The stream emits various event types during workflow execution. Each event has a
 ## Related
 
 - [Workflows overview](../../../docs/workflows/overview.mdx#run-workflow)
-- [Workflow.createRunAsync()](../create-run.mdx)
+- [Workflow.createRunAsync()](../../../reference/workflows/workflow-methods/create-run.mdx)

package/.docs/raw/reference/streaming/workflows/streamVNext.mdx

@@ -39,6 +39,38 @@ const stream = run.streamVNext({
       description: "Runtime context data to use during workflow execution",
       isOptional: true,
     },
+    {
+      name: "tracingContext",
+      type: "TracingContext",
+      isOptional: true,
+      description: "AI tracing context for creating child spans and adding metadata.",
+      properties: [
+        {
+          parameters: [{
+            name: "currentSpan",
+            type: "AISpan",
+            isOptional: true,
+            description: "Current AI span for creating child spans and adding metadata."
+          }]
+        }
+      ]
+    },
+    {
+      name: "tracingOptions",
+      type: "TracingOptions",
+      isOptional: true,
+      description: "Options for AI tracing configuration.",
+      properties: [
+        {
+          parameters: [{
+            name: "metadata",
+            type: "Record<string, any>",
+            isOptional: true,
+            description: "Metadata to add to the root trace span."
+          }]
+        }
+      ]
+    },
     {
       name: "closeOnSuspend",
       type: "boolean",
@@ -72,6 +104,12 @@ const stream = run.streamVNext({
       type: "Promise<{ inputTokens: number; outputTokens: number; totalTokens: number, reasoningTokens?: number, cacheInputTokens?: number }>",
       description: "A promise that resolves to token usage statistics",
     },
+    {
+      name: "stream.traceId",
+      type: "string",
+      isOptional: true,
+      description: "The trace ID associated with this execution when AI tracing is enabled.",
+    },
   ]}
 />
 
@@ -102,5 +140,5 @@ The stream emits various event types during workflow execution. Each event has a
 ## Related
 
 - [Workflows overview](../../../docs/workflows/overview.mdx#run-workflow)
-- [Workflow.createRunAsync()](../create-run.mdx)
+- [Workflow.createRunAsync()](../../../reference/workflows/workflow-methods/create-run.mdx)
 - [Run.resumeStreamVNext()](./resumeStreamVNext.mdx)

package/.docs/raw/reference/tools/create-tool.mdx

@@ -68,8 +68,41 @@ export const tool = createTool({
       name: "execute",
       type: "function",
       description:
-        "The function that contains the tool's logic. It receives an object with `context` (the parsed input based on `inputSchema`), `runtimeContext`, and an object containing `abortSignal`.",
+        "The function that contains the tool's logic. It receives an object with `context` (the parsed input based on `inputSchema`), `runtimeContext`, `tracingContext`, and an object containing `abortSignal`.",
       isOptional: false,
+      properties: [
+        {
+          parameters: [{
+            name: "context",
+            type: "z.infer<TInput>",
+            description: "The parsed input based on inputSchema"
+          }]
+        },
+        {
+          parameters: [{
+            name: "runtimeContext",
+            type: "RuntimeContext",
+            isOptional: true,
+            description: "Runtime context for accessing shared state and dependencies"
+          }]
+        },
+        {
+          parameters: [{
+            name: "tracingContext",
+            type: "TracingContext",
+            isOptional: true,
+            description: "AI tracing context for creating child spans and adding metadata. Automatically injected when the tool is called within a traced operation."
+          }]
+        },
+        {
+          parameters: [{
+            name: "abortSignal",
+            type: "AbortSignal",
+            isOptional: true,
+            description: "Signal for aborting the tool execution"
+          }]
+        }
+      ]
     },
   ]}
 />

package/.docs/raw/reference/workflows/run-methods/resume.mdx

@@ -48,6 +48,38 @@ if (result.status === "suspended") {
       description: "Optional run count for nested workflow execution",
       isOptional: true,
     },
+    {
+      name: "tracingContext",
+      type: "TracingContext",
+      isOptional: true,
+      description: "AI tracing context for creating child spans and adding metadata. Automatically injected when using Mastra's tracing system.",
+      properties: [
+        {
+          parameters: [{
+            name: "currentSpan",
+            type: "AISpan",
+            isOptional: true,
+            description: "Current AI span for creating child spans and adding metadata. Use this to create custom child spans or update span attributes during execution."
+          }]
+        }
+      ]
+    },
+    {
+      name: "tracingOptions",
+      type: "TracingOptions",
+      isOptional: true,
+      description: "Options for AI tracing configuration.",
+      properties: [
+        {
+          parameters: [{
+            name: "metadata",
+            type: "Record<string, any>",
+            isOptional: true,
+            description: "Metadata to add to the root trace span. Useful for adding custom attributes like user IDs, session IDs, or feature flags."
+          }]
+        }
+      ]
+    },
   ]}
 />
 
@@ -60,6 +92,12 @@ if (result.status === "suspended") {
       type: "Promise<WorkflowResult<TOutput, TSteps>>",
       description: "A promise that resolves to the workflow execution result containing step outputs and status",
     },
+    {
+      name: "traceId",
+      type: "string",
+      isOptional: true,
+      description: "The trace ID associated with this execution when AI tracing is enabled. Use this to correlate logs and debug execution flow.",
+    },
   ]}
 />
 

package/.docs/raw/reference/workflows/run-methods/start.mdx

@@ -41,6 +41,38 @@ const result = await run.start({
       description: "Optional writable stream for streaming workflow output",
       isOptional: true,
     },
+    {
+      name: "tracingContext",
+      type: "TracingContext",
+      isOptional: true,
+      description: "AI tracing context for creating child spans and adding metadata. Automatically injected when using Mastra's tracing system.",
+      properties: [
+        {
+          parameters: [{
+            name: "currentSpan",
+            type: "AISpan",
+            isOptional: true,
+            description: "Current AI span for creating child spans and adding metadata. Use this to create custom child spans or update span attributes during execution."
+          }]
+        }
+      ]
+    },
+    {
+      name: "tracingOptions",
+      type: "TracingOptions",
+      isOptional: true,
+      description: "Options for AI tracing configuration.",
+      properties: [
+        {
+          parameters: [{
+            name: "metadata",
+            type: "Record<string, any>",
+            isOptional: true,
+            description: "Metadata to add to the root trace span. Useful for adding custom attributes like user IDs, session IDs, or feature flags."
+          }]
+        }
+      ]
+    },
   ]}
 />
 
@@ -53,6 +85,12 @@ const result = await run.start({
       type: "Promise<WorkflowResult<TOutput, TSteps>>",
       description: "A promise that resolves to the workflow execution result containing step outputs and status",
     },
+    {
+      name: "traceId",
+      type: "string",
+      isOptional: true,
+      description: "The trace ID associated with this execution when AI tracing is enabled. Use this to correlate logs and debug execution flow.",
+    },
   ]}
 />
 

package/.docs/raw/scorers/custom-scorers.mdx

@@ -31,7 +31,7 @@ You can mix and match approaches within a single scorer - for example, use a fun
 
 ### Initializing a Scorer
 
-Every scorer starts with the `createScorer` factory function, which requires a name and description, and optionally accepts a judge configuration for LLM-based steps.
+Every scorer starts with the `createScorer` factory function, which requires a name and description, and optionally accepts a type specification and judge configuration.
 
 ```typescript
 import { createScorer } from '@mastra/core/scores';
@@ -54,6 +54,21 @@ const glutenCheckerScorer = createScorer({
 
 The judge configuration is only needed if you plan to use prompt objects in any step. Individual steps can override this default configuration with their own judge settings.
 
+#### Agent Type for Agent Evaluation
+
+For type safety and compatibility with both live agent scoring and trace scoring, use `type: 'agent'` when creating scorers for agent evaluation. This allows you to use the same scorer for an agent and also use it to score traces:
+
+```typescript
+const myScorer = createScorer({
+  // ...
+  type: 'agent', // Automatically handles agent input/output types
+})
+.generateScore(({ run, results }) => {
+  // run.output is automatically typed as ScorerRunOutputForAgent
+  // run.input is automatically typed as ScorerRunInputForAgent
+});
+```
+
 ### Step-by-Step Breakdown
 
 #### preprocess Step (Optional)