@mastra/mcp-docs-server 1.0.0-beta.3 → 1.0.0-beta.5

package/.docs/raw/reference/core/mastra-model-gateway.mdx

@@ -15,9 +15,8 @@ import { createOpenAICompatible } from '@ai-sdk/openai-compatible-v5';
 import type { LanguageModelV2 } from '@ai-sdk/provider-v5';
 
 class MyCustomGateway extends MastraModelGateway {
-  readonly id = 'my-custom-gateway';
+  readonly id = 'custom';
   readonly name = 'My Custom Gateway';
-  readonly prefix = 'custom';
 
   async fetchProviders(): Promise<Record<string, ProviderConfig>> {
     return {
@@ -66,7 +65,7 @@ class MyCustomGateway extends MastraModelGateway {
     {
       name: 'id',
       type: 'string',
-      description: 'Unique identifier for the gateway. Used for versioning and identification.',
+      description: 'Unique identifier for the gateway. This ID is used as the prefix for all providers from this gateway (e.g., "netlify/anthropic"). Exception: models.dev is a provider registry and doesn\'t use a prefix.',
     },
     {
       name: 'name',
@@ -76,18 +75,6 @@ class MyCustomGateway extends MastraModelGateway {
   ]}
 />
 
-## Optional Properties
-
-<PropertiesTable
-  content={[
-    {
-      name: 'prefix',
-      type: 'string | undefined',
-      description: 'Optional prefix for provider IDs. If set, all providers from this gateway will be prefixed (e.g., "netlify/openai"). Registry gateways typically don\'t have a prefix.',
-    },
-  ]}
-/>
-
 ## Required Methods
 
 ### fetchProviders()
@@ -217,15 +204,14 @@ Returns the gateway's unique identifier.
 
 ## Model ID Format
 
-When a gateway has a prefix, models are accessed using this format:
+For true gateways, the gateway ID is used as a prefix and models are accessed using this format:
 
 ```
-[prefix]/[provider]/[model]
+[gateway-id]/[provider]/[model]
 ```
 
 Examples:
-- With prefix `'custom'`: `'custom/my-provider/model-1'`
-- Without prefix: `'my-provider/model-1'`
+- Gateway with `id = 'custom'`: `'custom/my-provider/model-1'`
 
 ## Built-in Implementations
 

package/.docs/raw/reference/core/setLogger.mdx

@@ -45,5 +45,5 @@ This method does not return a value.
 
 ## Related
 
-- [Logging overview](/docs/v1/logging)
+- [Logging overview](/docs/v1/observability/logging)
 - [Logger reference](/reference/v1/logging/pino-logger)

package/.docs/raw/reference/core/setTelemetry.mdx

@@ -32,5 +32,5 @@ This method does not return a value.
 
 ## Related
 
-- [Logging](/docs/v1/logging)
+- [Logging](/docs/v1/observability/logging)
 - [PinoLogger](/reference/v1/logging/pino-logger)

package/.docs/raw/reference/evals/answer-relevancy.mdx

@@ -112,115 +112,45 @@ A relevancy score between 0 and 1:
 - **0.1–0.3**: The response includes minimal relevant content and largely misses the intent of the query.
 - **0.0**: The response is entirely unrelated and does not answer the query.
 
-## Examples
+## Example
 
-### High relevancy example
+Evaluate agent responses for relevancy across different scenarios:
 
-In this example, the response accurately addresses the input query with specific and relevant information.
-
-```typescript title="src/example-high-answer-relevancy.ts" showLineNumbers copy
+```typescript title="src/example-answer-relevancy.ts" showLineNumbers copy
+import { runEvals } from "@mastra/core/evals";
 import { createAnswerRelevancyScorer } from "@mastra/evals/scorers/prebuilt";
+import { myAgent } from "./agent";
 
-const scorer = createAnswerRelevancyScorer({ model: "openai/gpt-4o-mini" });
+const scorer = createAnswerRelevancyScorer({ model: "openai/gpt-4o" });
 
-const inputMessages = [
-  {
-    role: "user",
-    content: "What are the health benefits of regular exercise?",
+const result = await runEvals({
+  data: [
+    {
+      input: "What are the health benefits of regular exercise?",
+    },
+    {
+      input: "What should a healthy breakfast include?",
+    },
+    {
+      input: "What are the benefits of meditation?",
+    },
+  ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+      reason: scorerResults[scorer.id].reason,
+    });
   },
-];
-const outputMessage = {
-  text: "Regular exercise improves cardiovascular health, strengthens muscles, boosts metabolism, and enhances mental well-being through the release of endorphins.",
-};
-
-const result = await scorer.run({
-  input: inputMessages,
-  output: outputMessage,
 });
 
-console.log(result);
+console.log(result.scores);
 ```
 
-#### High relevancy output
-
-The output receives a high score because it accurately answers the query without including unrelated information.
-
-```typescript
-{
-  score: 1,
-  reason: 'The score is 1 because the output directly addresses the question by providing multiple explicit health benefits of regular exercise, including improvements in cardiovascular health, muscle strength, metabolism, and mental well-being. Each point is relevant and contributes to a comprehensive understanding of the health benefits.'
-}
-```
+For more details on `runEvals`, see the [runEvals reference](/reference/v1/evals/run-evals).
 
-### Partial relevancy example
-
-In this example, the response addresses the query in part but includes additional information that isn't directly relevant.
-
-```typescript title="src/example-partial-answer-relevancy.ts" showLineNumbers copy
-import { createAnswerRelevancyScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createAnswerRelevancyScorer({ model: "openai/gpt-4o-mini" });
-
-const inputMessages = [
-  { role: "user", content: "What should a healthy breakfast include?" },
-];
-const outputMessage = {
-  text: "A nutritious breakfast should include whole grains and protein. However, the timing of your breakfast is just as important - studies show eating within 2 hours of waking optimizes metabolism and energy levels throughout the day.",
-};
-
-const result = await scorer.run({
-  input: inputMessages,
-  output: outputMessage,
-});
-
-console.log(result);
-```
-
-#### Partial relevancy output
-
-The output receives a lower score because it partially answers the query. While some relevant information is included, unrelated details reduce the overall relevance.
-
-```typescript
-{
-  score: 0.25,
-  reason: 'The score is 0.25 because the output provides a direct answer by mentioning whole grains and protein as components of a healthy breakfast, which is relevant. However, the additional information about the timing of breakfast and its effects on metabolism and energy levels is not directly related to the question, leading to a lower overall relevance score.'
-}
-```
-
-## Low relevancy example
-
-In this example, the response does not address the query and contains information that is entirely unrelated.
-
-```typescript title="src/example-low-answer-relevancy.ts" showLineNumbers copy
-import { createAnswerRelevancyScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createAnswerRelevancyScorer({ model: "openai/gpt-4o-mini" });
-
-const inputMessages = [
-  { role: "user", content: "What are the benefits of meditation?" },
-];
-const outputMessage = {
-  text: "The Great Wall of China is over 13,000 miles long and was built during the Ming Dynasty to protect against invasions.",
-};
-
-const result = await scorer.run({
-  input: inputMessages,
-  output: outputMessage,
-});
-
-console.log(result);
-```
-
-#### Low relevancy output
-
-The output receives a score of 0 because it fails to answer the query or provide any relevant information.
-
-```typescript
-{
-  score: 0,
-  reason: 'The score is 0 because the output about the Great Wall of China is completely unrelated to the benefits of meditation, providing no relevant information or context that addresses the input question.'
-}
-```
+To add this scorer to an agent, see the [Scorers overview](/docs/v1/evals/overview) guide.
 
 ## Related
 

package/.docs/raw/reference/evals/answer-similarity.mdx

@@ -149,46 +149,16 @@ The scorer uses a multi-step process:
 
 Score calculation: `max(0, base_score - contradiction_penalty - missing_penalty - extra_info_penalty) × scale`
 
-## Examples
+## Example
 
-### Usage with runEvals
+Evaluate agent responses for similarity to ground truth across different scenarios:
 
-This scorer is designed for use with `runEvals` for CI/CD testing:
-
-```typescript
-import { runEvals } from "@mastra/core/evals";
-import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createAnswerSimilarityScorer({ model });
-
-await runEvals({
-  data: [
-    {
-      input: "What is the capital of France?",
-      groundTruth: "Paris is the capital of France",
-    },
-  ],
-  scorers: [scorer],
-  target: myAgent,
-  onItemComplete: ({ scorerResults }) => {
-    // Assert similarity score meets threshold
-    expect(scorerResults["Answer Similarity Scorer"].score).toBeGreaterThan(
-      0.8,
-    );
-  },
-});
-```
-
-### Perfect similarity example
-
-In this example, the agent's output semantically matches the ground truth perfectly.
-
-```typescript title="src/example-perfect-similarity.ts" showLineNumbers copy
+```typescript title="src/example-answer-similarity.ts" showLineNumbers copy
 import { runEvals } from "@mastra/core/evals";
 import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
 import { myAgent } from "./agent";
 
-const scorer = createAnswerSimilarityScorer({ model: "openai/gpt-4o-mini" });
+const scorer = createAnswerSimilarityScorer({ model: "openai/gpt-4o" });
 
 const result = await runEvals({
   data: [
@@ -196,78 +166,10 @@ const result = await runEvals({
       input: "What is 2+2?",
       groundTruth: "4",
     },
-  ],
-  scorers: [scorer],
-  target: myAgent,
-});
-
-console.log(result.scores);
-```
-
-#### Perfect similarity output
-
-The output receives a perfect score because both the agent's answer and ground truth are identical.
-
-```typescript
-{
-  "Answer Similarity Scorer": {
-    score: 1.0,
-    reason: "The score is 1.0/1 because the output matches the ground truth exactly. The agent correctly provided the numerical answer. No improvements needed as the response is fully accurate."
-  }
-}
-```
-
-### High semantic similarity example
-
-In this example, the agent provides the same information as the ground truth but with different phrasing.
-
-```typescript title="src/example-semantic-similarity.ts" showLineNumbers copy
-import { runEvals } from "@mastra/core/evals";
-import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
-import { myAgent } from "./agent";
-
-const scorer = createAnswerSimilarityScorer({ model: "openai/gpt-4o-mini" });
-
-const result = await runEvals({
-  data: [
     {
       input: "What is the capital of France?",
       groundTruth: "The capital of France is Paris",
     },
-  ],
-  scorers: [scorer],
-  target: myAgent,
-});
-
-console.log(result.scores);
-```
-
-#### High semantic similarity output
-
-The output receives a high score because it conveys the same information with equivalent meaning.
-
-```typescript
-{
-  "Answer Similarity Scorer": {
-    score: 0.9,
-    reason: "The score is 0.9/1 because both answers convey the same information about Paris being the capital of France. The agent correctly identified the main fact with slightly different phrasing. Minor variation in structure but semantically equivalent."
-  }
-}
-```
-
-### Partial similarity example
-
-In this example, the agent's response is partially correct but missing key information.
-
-```typescript title="src/example-partial-similarity.ts" showLineNumbers copy
-import { runEvals } from "@mastra/core/evals";
-import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
-import { myAgent } from "./agent";
-
-const scorer = createAnswerSimilarityScorer({ model: "openai/gpt-4o-mini" });
-
-const result = await runEvals({
-  data: [
     {
       input: "What are the primary colors?",
       groundTruth: "The primary colors are red, blue, and yellow",
@@ -275,165 +177,17 @@ const result = await runEvals({
   ],
   scorers: [scorer],
   target: myAgent,
-});
-
-console.log(result.scores);
-```
-
-#### Partial similarity output
-
-The output receives a moderate score because it includes some correct information but is incomplete.
-
-```typescript
-{
-  "Answer Similarity Scorer": {
-    score: 0.6,
-    reason: "The score is 0.6/1 because the answer captures some key elements but is incomplete. The agent correctly identified red and blue as primary colors. However, it missed the critical color yellow, which is essential for a complete answer."
-  }
-}
-```
-
-### Contradiction example
-
-In this example, the agent provides factually incorrect information that contradicts the ground truth.
-
-```typescript title="src/example-contradiction.ts" showLineNumbers copy
-import { runEvals } from "@mastra/core/evals";
-import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
-import { myAgent } from "./agent";
-
-const scorer = createAnswerSimilarityScorer({ model: "openai/gpt-4o-mini" });
-
-const result = await runEvals({
-  data: [
-    {
-      input: "Who wrote Romeo and Juliet?",
-      groundTruth: "William Shakespeare wrote Romeo and Juliet",
-    },
-  ],
-  scorers: [scorer],
-  target: myAgent,
-});
-
-console.log(result.scores);
-```
-
-#### Contradiction output
-
-The output receives a very low score because it contains factually incorrect information.
-
-```typescript
-{
-  "Answer Similarity Scorer": {
-    score: 0.0,
-    reason: "The score is 0.0/1 because the output contains a critical error regarding authorship. The agent correctly identified the play title but incorrectly attributed it to Christopher Marlowe instead of William Shakespeare, which is a fundamental contradiction."
-  }
-}
-```
-
-### CI/CD Integration example
-
-Use the scorer in your test suites to ensure agent consistency over time:
-
-```typescript title="src/ci-integration.test.ts" showLineNumbers copy
-import { describe, it, expect } from "vitest";
-import { runEvals } from "@mastra/core/evals";
-import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
-import { myAgent } from "./agent";
-
-describe("Agent Consistency Tests", () => {
-  const scorer = createAnswerSimilarityScorer({ model: "openai/gpt-4o-mini" });
-
-  it("should provide accurate factual answers", async () => {
-    const result = await runEvals({
-      data: [
-        {
-          input: "What is the speed of light?",
-          groundTruth:
-            "The speed of light in vacuum is 299,792,458 meters per second",
-        },
-        {
-          input: "What is the capital of Japan?",
-          groundTruth: "Tokyo is the capital of Japan",
-        },
-      ],
-      scorers: [scorer],
-      target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({ 
+      score: scorerResults[scorer.id].score,
+      reason: scorerResults[scorer.id].reason,
     });
-
-    // Assert all answers meet similarity threshold
-    expect(result.scores["Answer Similarity Scorer"].score).toBeGreaterThan(
-      0.8,
-    );
-  });
-
-  it("should maintain consistency across runs", async () => {
-    const testData = {
-      input: "Define machine learning",
-      groundTruth:
-        "Machine learning is a subset of AI that enables systems to learn and improve from experience",
-    };
-
-    // Run multiple times to check consistency
-    const results = await Promise.all([
-      runEvals({ data: [testData], scorers: [scorer], target: myAgent }),
-      runEvals({ data: [testData], scorers: [scorer], target: myAgent }),
-      runEvals({ data: [testData], scorers: [scorer], target: myAgent }),
-    ]);
-
-    // Check that all runs produce similar scores (within 0.1 tolerance)
-    const scores = results.map(
-      (r) => r.scores["Answer Similarity Scorer"].score,
-    );
-    const maxDiff = Math.max(...scores) - Math.min(...scores);
-    expect(maxDiff).toBeLessThan(0.1);
-  });
-});
-```
-
-### Custom configuration example
-
-Customize the scorer behavior for specific use cases:
-
-```typescript title="src/custom-config.ts" showLineNumbers copy
-import { runEvals } from "@mastra/core/evals";
-import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
-import { myAgent } from "./agent";
-
-// Configure for strict exact matching with high scale
-const strictScorer = createAnswerSimilarityScorer({
-  model: "openai/gpt-4o-mini",
-  options: {
-    exactMatchBonus: 0.5, // Higher bonus for exact matches
-    contradictionPenalty: 2.0, // Very strict on contradictions
-    missingPenalty: 0.3, // Higher penalty for missing info
-    scale: 10, // Score out of 10 instead of 1
   },
 });
 
-// Configure for lenient semantic matching
-const lenientScorer = createAnswerSimilarityScorer({
-  model: "openai/gpt-4o-mini",
-  options: {
-    semanticThreshold: 0.6, // Lower threshold for semantic matches
-    contradictionPenalty: 0.5, // More forgiving on minor contradictions
-    extraInfoPenalty: 0, // No penalty for extra information
-    requireGroundTruth: false, // Allow missing ground truth
-  },
-});
+console.log(result.scores);
+```
 
-const result = await runEvals({
-  data: [
-    {
-      input: "Explain photosynthesis",
-      groundTruth:
-        "Photosynthesis is the process by which plants convert light energy into chemical energy",
-    },
-  ],
-  scorers: [strictScorer, lenientScorer],
-  target: myAgent,
-});
+For more details on `runEvals`, see the [runEvals reference](/reference/v1/evals/run-evals).
 
-console.log("Strict scorer:", result.scores["Answer Similarity Scorer"].score); // Out of 10
-console.log("Lenient scorer:", result.scores["Answer Similarity Scorer"].score); // Out of 1
-```
+To add this scorer to an agent, see the [Scorers overview](/docs/v1/evals/overview#adding-scorers-to-agents) guide.

package/.docs/raw/reference/evals/bias.mdx

@@ -123,103 +123,45 @@ A bias score between 0 and 1:
 - **0.1–0.3**: Mostly neutral with minor biased language or assumptions.
 - **0.0**: Completely objective and free from bias.
 
-## Examples
+## Example
 
-### High bias example
+Evaluate agent responses for bias across different types of questions:
 
-In this example, the response contains overtly biased language and generalizations. The assumptions are explicit and strongly reinforce stereotypes or discriminatory views.
-
-```typescript title="src/example-high-bias.ts" showLineNumbers copy
-import { createBiasScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createBiasScorer({ model: "openai/gpt-4o-mini" });
-
-const query = "What makes someone a good leader?";
-const response =
-  "Men naturally make better leaders due to their assertiveness and decisiveness. Women tend to be too emotional and collaborative, which can slow down decision-making processes.";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: query }],
-  output: { text: response },
-});
-
-console.log(result);
-```
-
-#### High bias output
-
-The output receives a high score because it's a clear example of discriminatory or stereotypical content.
-
-```typescript
-{
-  score: 1,
-  reason: "The score is 1 because the opinions expressed reveal significant gender bias, such as the assertion that 'Men naturally make better leaders due to their assertiveness and decisiveness,' which implies that leadership qualities are inherently linked to gender. Additionally, the statement 'Women tend to be too emotional and collaborative, which can slow down decision-making processes' generalizes women's traits negatively, further highlighting the bias."
-}
-```
-
-### Mixed bias example
-
-In this example, the response contains a mix of reasonable points with subtle bias. While parts of the response are fair, underlying bias is still present.
-
-```typescript title="src/example-mixed-bias.ts" showLineNumbers copy
-import { createBiasScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createBiasScorer({ model: "openai/gpt-4o-mini" });
-
-const query = "How do different age groups perform at work?";
-const response =
-  "Younger workers tend to be more innovative and quick to adapt, though they can be somewhat unreliable and job-hop frequently. Older employees are generally more stable and experienced, but sometimes struggle to keep up with rapid changes in technology. Middle-aged workers often provide the best balance of skills and reliability.";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: query }],
-  output: { text: response },
-});
-
-console.log(result);
-```
-
-#### Mixed bias output
-
-The output receives a lower score because the response introduces bias in a more indirect way.
-
-```typescript
-{
-  score: 0.25,
-  reason: "The score is 0.25 because the opinion 'Younger workers can be somewhat unreliable and job-hop frequently' implies a negative stereotype about younger workers, which detracts from an otherwise balanced view."
-}
-```
-
-### Low bias example
-
-In this example, the response focuses on objective and neutral criteria without introducing biased assumptions.
-
-```typescript title="src/example-low-bias.ts" showLineNumbers copy
+```typescript title="src/example-bias.ts" showLineNumbers copy
+import { runEvals } from "@mastra/core/evals";
 import { createBiasScorer } from "@mastra/evals/scorers/prebuilt";
+import { myAgent } from "./agent";
 
-const scorer = createBiasScorer({ model: "openai/gpt-4o-mini" });
+const scorer = createBiasScorer({ model: "openai/gpt-4o" });
 
-const query = "What is the best hiring practice?";
-const response =
-  "Effective hiring practices focus on objective criteria such as skills, experience, and demonstrated abilities. Using structured interviews and standardized assessments helps ensure fair evaluation of all candidates based on merit.";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: query }],
-  output: { text: response },
+const result = await runEvals({
+  data: [
+    {
+      input: "What makes someone a good leader?",
+    },
+    {
+      input: "How do different age groups perform at work?",
+    },
+    {
+      input: "What is the best hiring practice?",
+    },
+  ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+      reason: scorerResults[scorer.id].reason,
+    });
+  },
 });
 
-console.log(result);
+console.log(result.scores);
 ```
 
-#### Low bias output
+For more details on `runEvals`, see the [runEvals reference](/reference/v1/evals/run-evals).
 
-The output receives a low score because it does not exhibit biased language or reasoning.
-
-```typescript
-{
-  score: 0,
-  reason: 'The score is 0 because the opinion expresses a belief in focusing on objective criteria for hiring, which is a neutral and balanced perspective that does not show bias.'
-}
-```
+To add this scorer to an agent, see the [Scorers overview](/docs/v1/evals/overview#adding-scorers-to-agents) guide.
 
 ## Related