@mastra/mcp-docs-server 1.0.0-beta.0 → 1.0.0-beta.10

package/.docs/raw/reference/evals/content-similarity.mdx

@@ -1,5 +1,5 @@
 ---
-title: "Reference: Content Similarity Scorer | Evals | Mastra Docs"
+title: "Reference: Content Similarity Scorer | Evals"
 description: Documentation for the Content Similarity Scorer in Mastra, which measures textual similarity between strings and provides a matching score.
 ---
 
@@ -78,106 +78,46 @@ The scorer evaluates textual similarity through character-level matching and con
 
 Final score: `similarity_value * scale`
 
-## Examples
+## Example
 
-### High similarity example
+Evaluate textual similarity between expected and actual agent outputs:
 
-In this example, the response closely resembles the query in both structure and meaning. Minor differences in tense and phrasing do not significantly affect the overall similarity.
-
-```typescript title="src/example-high-similarity.ts" showLineNumbers copy
+```typescript title="src/example-content-similarity.ts" showLineNumbers copy
+import { runEvals } from "@mastra/core/evals";
 import { createContentSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
+import { myAgent } from "./agent";
 
 const scorer = createContentSimilarityScorer();
 
-const query = "The quick brown fox jumps over the lazy dog.";
-const response = "A quick brown fox jumped over a lazy dog.";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: query }],
-  output: { text: response },
-});
-
-console.log(result);
-```
-
-#### High similarity output
-
-The output receives a high score because the response preserves the intent and content of the query with only subtle wording changes.
-
-```typescript
-{
-  score: 0.7761194029850746,
-  analyzeStepResult: {
-    similarity: 0.7761194029850746
+const result = await runEvals({
+  data: [
+    {
+      input: "Summarize the benefits of TypeScript",
+      groundTruth:
+        "TypeScript provides static typing, better tooling support, and improved code maintainability.",
+    },
+    {
+      input: "What is machine learning?",
+      groundTruth:
+        "Machine learning is a subset of AI that enables systems to learn from data without explicit programming.",
+    },
+  ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+      groundTruth: scorerResults[scorer.id].groundTruth,
+    });
   },
-}
-```
-
-### Moderate similarity example
-
-In this example, the response shares some conceptual overlap with the query but diverges in structure and wording. Key elements remain present, but the phrasing introduces moderate variation.
-
-```typescript title="src/example-moderate-similarity.ts" showLineNumbers copy
-import { createContentSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createContentSimilarityScorer();
-
-const query = "A brown fox quickly leaps across a sleeping dog.";
-const response = "The quick brown fox jumps over the lazy dog.";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: query }],
-  output: { text: response },
 });
 
-console.log(result);
-```
-
-#### Moderate similarity output
-
-The output receives a mid-range score because the response captures the general idea of the query, though it differs enough in wording to reduce overall similarity.
-
-```typescript
-{
-  score: 0.40540540540540543,
-  analyzeStepResult: {
-    similarity: 0.40540540540540543
-  }
-}
+console.log(result.scores);
 ```
 
-### Low similarity example
-
-In this example, the response and query are unrelated in meaning, despite having a similar grammatical structure. There is little to no shared content overlap.
-
-```typescript title="src/example-low-similarity.ts" showLineNumbers copy
-import { createContentSimilarityScorer } from "@mastra/evals/scorers/prebuilt";
-
-const scorer = createContentSimilarityScorer();
+For more details on `runEvals`, see the [runEvals reference](/reference/v1/evals/run-evals).
 
-const query = "The cat sleeps on the windowsill.";
-const response = "The quick brown fox jumps over the lazy dog.";
-
-const result = await scorer.run({
-  input: [{ role: "user", content: query }],
-  output: { text: response },
-});
-
-console.log(result);
-```
-
-#### Low similarity output
-
-The output receives a low score because the response does not align with the content or intent of the query.
-
-```typescript
-{
-  score: 0.25806451612903225,
-  analyzeStepResult: {
-    similarity: 0.25806451612903225
-  },
-}
-```
+To add this scorer to an agent, see the [Scorers overview](/docs/v1/evals/overview#adding-scorers-to-agents) guide.
 
 ### Score interpretation
 

package/.docs/raw/reference/evals/context-precision.mdx

@@ -1,5 +1,5 @@
 ---
-title: "Reference: Context Precision Scorer | Evals | Mastra Docs"
+title: "Reference: Context Precision Scorer | Evals"
 description: Documentation for the Context Precision Scorer in Mastra. Evaluates the relevance and precision of retrieved context for generating expected outputs using Mean Average Precision.
 ---
 
@@ -152,7 +152,7 @@ MAP = (1.0 + 0.67) / 2 = 0.835 ≈ **0.83**
 
 ```typescript
 const scorer = createContextPrecisionScorer({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     contextExtractor: (input, output) => {
       // Extract context dynamically based on the query
@@ -171,7 +171,7 @@ const scorer = createContextPrecisionScorer({
 
 ```typescript
 const scorer = createContextPrecisionScorer({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     context: [
       // Simulate retrieved documents from vector database
@@ -186,152 +186,50 @@ const scorer = createContextPrecisionScorer({
 });
 ```
 
-## Examples
+## Example
 
-### High precision example
+Evaluate RAG system context retrieval precision for different queries:
 
-This example shows perfect context precision where all relevant context appears early:
-
-```typescript
-import { createContextPrecisionScorer } from "@mastra/evals";
+```typescript title="src/example-context-precision.ts" showLineNumbers copy
+import { runEvals } from "@mastra/core/evals";
+import { createContextPrecisionScorer } from "@mastra/evals/scorers/prebuilt";
+import { myAgent } from "./agent";
 
 const scorer = createContextPrecisionScorer({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-4o",
   options: {
-    context: [
-      "Photosynthesis is the process by which plants convert sunlight, carbon dioxide, and water into glucose and oxygen.",
-      "The process occurs in the chloroplasts of plant cells, specifically in the thylakoids.",
-      "Light-dependent reactions happen in the thylakoid membranes, while the Calvin cycle occurs in the stroma.",
-    ],
-    scale: 1,
+    contextExtractor: (input, output) => {
+      // Extract context from agent's retrieved documents
+      return output.metadata?.retrievedContext || [];
+    },
   },
 });
 
-const result = await scorer.run({
-  input: {
-    inputMessages: [
-      {
-        id: "1",
-        role: "user",
-        content: "How does photosynthesis work in plants?",
-      },
-    ],
-  },
-  output: [
+const result = await runEvals({
+  data: [
     {
-      id: "2",
-      role: "assistant",
-      content:
-        "Photosynthesis is the process where plants convert sunlight, CO2, and water into glucose and oxygen using chloroplasts.",
+      input: "How does photosynthesis work in plants?",
     },
-  ],
-});
-
-console.log(result);
-// Output:
-// {
-//   score: 1.0,
-//   reason: "The score is 1.0 because all context pieces are highly relevant to explaining photosynthesis and are optimally ordered to support the expected output."
-// }
-```
-
-### Mixed precision example
-
-This example shows moderate precision with both relevant and irrelevant context:
-
-```typescript
-import { createContextPrecisionScorer } from "@mastra/evals";
-
-const scorer = createContextPrecisionScorer({
-  model: "openai/gpt-4o-mini",
-  options: {
-    context: [
-      "Regular exercise improves cardiovascular health by strengthening the heart muscle.",
-      "A balanced diet should include fruits, vegetables, and whole grains.",
-      "Physical activity releases endorphins which improve mood and reduce stress.",
-      "The average person should drink 8 glasses of water per day.",
-      "Exercise also helps maintain healthy body weight and muscle mass.",
-    ],
-    scale: 1,
-  },
-});
-
-const result = await scorer.run({
-  input: {
-    inputMessages: [
-      {
-        id: "1",
-        role: "user",
-        content: "What are the mental and physical benefits of exercise?",
-      },
-    ],
-  },
-  output: [
     {
-      id: "2",
-      role: "assistant",
-      content:
-        "Exercise provides cardiovascular benefits, improves mood through endorphin release, and helps maintain healthy body composition.",
+      input: "What are the mental and physical benefits of exercise?",
     },
   ],
+  scorers: [scorer],
+  target: myAgent,
+  onItemComplete: ({ scorerResults }) => {
+    console.log({
+      score: scorerResults[scorer.id].score,
+      reason: scorerResults[scorer.id].reason,
+    });
+  },
 });
 
-console.log(result);
-// Output:
-// {
-//   score: 0.72,
-//   reason: "The score is 0.72 because contexts 1, 3, and 5 are relevant to exercise benefits, but irrelevant contexts about diet and hydration reduce the precision score."
-// }
+console.log(result.scores);
 ```
 
-### Low precision example
-
-This example shows poor context precision with mostly irrelevant context:
-
-```typescript
-import { createContextPrecisionScorer } from "@mastra/evals";
-
-const scorer = createContextPrecisionScorer({
-  model: "openai/gpt-4o-mini",
-  options: {
-    context: [
-      "The weather forecast shows sunny skies this weekend.",
-      "Coffee is one of the world's most popular beverages.",
-      "Machine learning requires large amounts of training data.",
-      "Cats typically sleep 12-16 hours per day.",
-      "The capital of France is Paris.",
-    ],
-    scale: 1,
-  },
-});
-
-const result = await scorer.run({
-  input: {
-    inputMessages: [
-      {
-        id: "1",
-        role: "user",
-        content: "How does photosynthesis work?",
-      },
-    ],
-  },
-  output: [
-    {
-      id: "2",
-      role: "assistant",
-      content:
-        "Photosynthesis is the process by which plants convert sunlight into energy using chlorophyll.",
-    },
-  ],
-});
+For more details on `runEvals`, see the [runEvals reference](/reference/v1/evals/run-evals).
 
-console.log(result);
-// Output:
-// {
-//   score: 0.0,
-//   reason: "The score is 0.0 because none of the retrieved context pieces are relevant to explaining photosynthesis."
-// }
-```
+To add this scorer to an agent, see the [Scorers overview](/docs/v1/evals/overview#adding-scorers-to-agents) guide.
 
 ## Comparison with Context Relevance
 

package/.docs/raw/reference/evals/context-relevance.mdx

@@ -1,5 +1,5 @@
 ---
-title: "Reference: Context Relevance Scorer | Evals | Mastra Docs"
+title: "Reference: Context Relevance Scorer | Evals"
 description: Documentation for the Context Relevance Scorer in Mastra. Evaluates the relevance and utility of provided context for generating agent responses using weighted relevance scoring.
 ---
 
@@ -198,7 +198,7 @@ import { createContextRelevanceScorerLLM } from "@mastra/evals";
 
 // Stricter penalty configuration
 const strictScorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     context: [
       "Einstein won the Nobel Prize for photoelectric effect",
@@ -216,7 +216,7 @@ const strictScorer = createContextRelevanceScorerLLM({
 
 // Lenient penalty configuration
 const lenientScorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     context: [
       "Einstein won the Nobel Prize for photoelectric effect",
@@ -263,7 +263,7 @@ console.log("Lenient penalties:", lenientResult.score); // Higher score, less pe
 
 ```typescript
 const scorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o",
+  model: "openai/gpt-5.1",
   options: {
     contextExtractor: (input, output) => {
       // Extract context based on the query
@@ -287,7 +287,7 @@ const scorer = createContextRelevanceScorerLLM({
 
 ```typescript
 const scorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     context: ["Relevant information...", "Supporting details..."],
     scale: 100, // Scale scores from 0-100 instead of 0-1
@@ -301,7 +301,7 @@ const scorer = createContextRelevanceScorerLLM({
 
 ```typescript
 const scorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     contextExtractor: (input, output) => {
       const query = input?.inputMessages?.[0]?.content || "";
@@ -328,7 +328,7 @@ This example shows excellent context relevance where all context directly suppor
 import { createContextRelevanceScorerLLM } from "@mastra/evals";
 
 const scorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     context: [
       "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1921.",
@@ -375,7 +375,7 @@ This example shows moderate relevance with some context being irrelevant or unus
 import { createContextRelevanceScorerLLM } from "@mastra/evals";
 
 const scorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     context: [
       "Solar eclipses occur when the Moon blocks the Sun.",
@@ -417,7 +417,7 @@ console.log(result);
 
 // With custom penalty configuration
 const customScorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     context: [
       "Solar eclipses occur when the Moon blocks the Sun.",
@@ -466,7 +466,7 @@ This example shows poor context relevance with mostly irrelevant information:
 import { createContextRelevanceScorerLLM } from "@mastra/evals";
 
 const scorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     context: [
       "The Great Barrier Reef is located in Australia.",
@@ -514,7 +514,7 @@ Extract context dynamically based on the run input:
 import { createContextRelevanceScorerLLM } from "@mastra/evals";
 
 const scorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     contextExtractor: (input, output) => {
       // Extract query from input
@@ -557,7 +557,7 @@ Integrate with RAG pipelines to evaluate retrieved context:
 import { createContextRelevanceScorerLLM } from "@mastra/evals";
 
 const scorer = createContextRelevanceScorerLLM({
-  model: "openai/gpt-4o-mini",
+  model: "openai/gpt-5.1",
   options: {
     contextExtractor: (input, output) => {
       // Extract from RAG retrieval results