@mastra/mcp-docs-server 0.0.1-alpha.1

package/.docs/raw/reference/evals/summarization.mdx

@@ -0,0 +1,205 @@
+---
+title: "Reference: Summarization | Metrics | Evals | Mastra Docs"
+description: Documentation for the Summarization Metric in Mastra, which evaluates the quality of LLM-generated summaries for content and factual accuracy.
+---
+
+# SummarizationMetric
+,
+The `SummarizationMetric` evaluates how well an LLM's summary captures the original text's content while maintaining factual accuracy. It combines two aspects: alignment (factual correctness) and coverage (inclusion of key information), using the minimum scores to ensure both qualities are necessary for a good summary.
+
+## Basic Usage
+
+```typescript
+import { openai } from "@ai-sdk/openai";
+import { SummarizationMetric } from "@mastra/evals/llm";
+
+// Configure the model for evaluation
+const model = openai("gpt-4o-mini");
+
+const metric = new SummarizationMetric(model);
+
+const result = await metric.measure(
+  "The company was founded in 1995 by John Smith. It started with 10 employees and grew to 500 by 2020. The company is based in Seattle.",
+  "Founded in 1995 by John Smith, the company grew from 10 to 500 employees by 2020.",
+);
+
+console.log(result.score); // Score from 0-1
+console.log(result.info); // Object containing detailed metrics about the summary
+```
+
+## Constructor Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "model",
+      type: "LanguageModel",
+      description: "Configuration for the model used to evaluate summaries",
+      isOptional: false,
+    },
+    {
+      name: "options",
+      type: "SummarizationMetricOptions",
+      description: "Configuration options for the metric",
+      isOptional: true,
+      defaultValue: "{ scale: 1 }",
+    },
+  ]}
+/>
+
+### SummarizationMetricOptions
+
+<PropertiesTable
+  content={[
+    {
+      name: "scale",
+      type: "number",
+      description: "Maximum score value",
+      isOptional: true,
+      defaultValue: "1",
+    },
+  ]}
+/>
+
+## measure() Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "string",
+      description: "The original text to be summarized",
+      isOptional: false,
+    },
+    {
+      name: "output",
+      type: "string",
+      description: "The generated summary to evaluate",
+      isOptional: false,
+    },
+  ]}
+/>
+
+## Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: "score",
+      type: "number",
+      description: "Summarization score (0 to scale, default 0-1)",
+    },
+    {
+      name: "info",
+      type: "object",
+      description: "Object containing detailed metrics about the summary",
+      properties: [
+        {
+          type: "string",
+          parameters: [
+            {
+              name: "reason",
+              type: "string",
+              description:
+                "Detailed explanation of the score, including both alignment and coverage aspects",
+            },
+          ],
+        },
+        {
+          type: "number",
+          parameters: [
+            {
+              name: "alignmentScore",
+              type: "number",
+              description: "Alignment score (0 to 1)",
+            },
+          ],
+        },
+        {
+          type: "number",
+          parameters: [
+            {
+              name: "coverageScore",
+              type: "number",
+              description: "Coverage score (0 to 1)",
+            },
+          ],
+        },
+      ],
+    },
+  ]}
+/>
+
+## Scoring Details
+
+The metric evaluates summaries through two essential components:
+1. **Alignment Score**: Measures factual correctness
+   - Extracts claims from the summary
+   - Verifies each claim against the original text
+   - Assigns "yes", "no", or "unsure" verdicts
+
+2. **Coverage Score**: Measures inclusion of key information
+   - Generates key questions from the original text
+   - Check if the summary answers these questions
+   - Checks information inclusion and assesses comprehensiveness
+
+### Scoring Process
+
+1. Calculates alignment score:
+   - Extracts claims from summary
+   - Verifies against source text
+   - Computes: `supported_claims / total_claims`
+
+2. Determines coverage score:
+   - Generates questions from source
+   - Checks summary for answers
+   - Evaluates completeness
+   - Calculates: `answerable_questions / total_questions`
+
+Final score: `min(alignment_score, coverage_score) * scale`
+
+### Score interpretation
+(0 to scale, default 0-1)
+- 1.0: Perfect summary - completely factual and covers all key information
+- 0.7-0.9: Strong summary with minor omissions or slight inaccuracies
+- 0.4-0.6: Moderate quality with significant gaps or inaccuracies
+- 0.1-0.3: Poor summary with major omissions or factual errors
+- 0.0: Invalid summary - either completely inaccurate or missing critical information
+
+## Example with Analysis
+
+```typescript
+import { openai } from "@ai-sdk/openai";
+import { SummarizationMetric } from "@mastra/evals/llm";
+
+// Configure the model for evaluation
+const model = openai("gpt-4o-mini");
+
+const metric = new SummarizationMetric(model);
+
+const result = await metric.measure(
+  "The electric car company Tesla was founded in 2003 by Martin Eberhard and Marc Tarpenning. Elon Musk joined in 2004 as the largest investor and became CEO in 2008. The company's first car, the Roadster, was launched in 2008.",
+  "Tesla, founded by Elon Musk in 2003, revolutionized the electric car industry starting with the Roadster in 2008.",
+);
+
+// Example output:
+// {
+//   score: 0.5,
+//   info: {
+//     reason: "The score is 0.5 because while the coverage is good (0.75) - mentioning the founding year,
+//           first car model, and launch date - the alignment score is lower (0.5) due to incorrectly
+//           attributing the company's founding to Elon Musk instead of Martin Eberhard and Marc Tarpenning.
+//           The final score takes the minimum of these two scores to ensure both factual accuracy and
+//           coverage are necessary for a good summary."
+//     alignmentScore: 0.5,
+//     coverageScore: 0.75,
+//   }
+// }
+```
+
+## Related
+
+- [Faithfulness Metric](./faithfulness)
+- [Completeness Metric](./completeness)
+- [Contextual Recall Metric](./contextual-recall)
+- [Hallucination Metric](./hallucination)

package/.docs/raw/reference/evals/textual-difference.mdx

@@ -0,0 +1,161 @@
+---
+title: "Reference: Textual Difference | Evals | Mastra Docs"
+description: Documentation for the Textual Difference Metric in Mastra, which measures textual differences between strings using sequence matching.
+---
+
+# TextualDifferenceMetric
+
+The `TextualDifferenceMetric` class uses sequence matching to measure the textual differences between two strings. It provides detailed information about changes, including the number of operations needed to transform one text into another.
+
+## Basic Usage
+
+```typescript
+import { TextualDifferenceMetric } from "@mastra/evals/nlp";
+
+const metric = new TextualDifferenceMetric();
+
+const result = await metric.measure(
+  "The quick brown fox",
+  "The fast brown fox"
+);
+
+console.log(result.score); // Similarity ratio from 0-1
+console.log(result.info); // Detailed change metrics
+```
+
+## measure() Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "string",
+      description: "The original text to compare against",
+      isOptional: false,
+    },
+    {
+      name: "output",
+      type: "string",
+      description: "The text to evaluate for differences",
+      isOptional: false,
+    }
+  ]}
+/>
+
+## Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: "score",
+      type: "number",
+      description: "Similarity ratio (0-1) where 1 indicates identical texts",
+    },
+    {
+      name: "info",
+      description: "Detailed metrics about the differences",
+      properties: [
+        {
+          type: "number",
+          parameters: [
+            {
+              name: "confidence",
+              type: "number",
+              description: "Confidence score based on length difference between texts (0-1)",
+            }
+          ]
+        },
+        {
+          type: "number",
+          parameters: [
+            {
+              name: "ratio",
+              type: "number",
+              description: "Raw similarity ratio between the texts",
+            }
+          ]
+        },
+        {
+          type: "number",
+          parameters: [
+            {
+              name: "changes",
+              type: "number",
+              description: "Number of change operations (insertions, deletions, replacements)",
+            }
+          ]
+        },
+        {
+          type: "number",
+          parameters: [
+            {
+              name: "lengthDiff",
+              type: "number",
+              description: "Normalized difference in length between input and output (0-1)",
+            }
+          ]
+        },
+      ]
+    },
+  ]}
+/>
+
+## Scoring Details
+
+The metric calculates several measures:
+- **Similarity Ratio**: Based on sequence matching between texts (0-1)
+- **Changes**: Count of non-matching operations needed
+- **Length Difference**: Normalized difference in text lengths
+- **Confidence**: Inversely proportional to length difference
+
+### Scoring Process
+
+1. Analyzes textual differences:
+   - Performs sequence matching between input and output
+   - Counts the number of change operations required
+   - Measures length differences
+
+2. Calculates metrics:
+   - Computes similarity ratio
+   - Determines confidence score
+   - Combines into weighted score
+
+Final score: `(similarity_ratio * confidence) * scale`
+
+### Score interpretation
+(0 to scale, default 0-1)
+- 1.0: Identical texts - no differences
+- 0.7-0.9: Minor differences - few changes needed
+- 0.4-0.6: Moderate differences - significant changes
+- 0.1-0.3: Major differences - extensive changes
+- 0.0: Completely different texts
+
+## Example with Analysis
+
+```typescript
+import { TextualDifferenceMetric } from "@mastra/evals/nlp";
+
+const metric = new TextualDifferenceMetric();
+
+const result = await metric.measure(
+  "Hello world! How are you?",
+  "Hello there! How is it going?"
+);
+
+// Example output:
+// {
+//   score: 0.65,
+//   info: {
+//     confidence: 0.95,
+//     ratio: 0.65,
+//     changes: 2,
+//     lengthDiff: 0.05
+//   }
+// }
+```
+
+## Related
+
+- [Content Similarity Metric](./content-similarity)
+- [Completeness Metric](./completeness)
+- [Keyword Coverage Metric](./keyword-coverage)

package/.docs/raw/reference/evals/tone-consistency.mdx

@@ -0,0 +1,181 @@
+---
+title: "Reference: Tone Consistency | Metrics | Evals | Mastra Docs"
+description: Documentation for the Tone Consistency Metric in Mastra, which evaluates emotional tone and sentiment consistency in text.
+---
+
+# ToneConsistencyMetric
+
+The `ToneConsistencyMetric` class evaluates the text's emotional tone and sentiment consistency.  It can operate in two modes: comparing tone between input/output pairs or analyzing tone stability within a single text.
+
+## Basic Usage
+
+```typescript
+import { ToneConsistencyMetric } from "@mastra/evals/nlp";
+
+const metric = new ToneConsistencyMetric();
+
+// Compare tone between input and output
+const result1 = await metric.measure(
+  "I love this amazing product!",
+  "This product is wonderful and fantastic!"
+);
+
+// Analyze tone stability in a single text
+const result2 = await metric.measure(
+  "The service is excellent. The staff is friendly. The atmosphere is perfect.",
+  ""  // Empty string for single-text analysis
+);
+
+console.log(result1.score); // Tone consistency score from 0-1
+console.log(result2.score); // Tone stability score from 0-1
+```
+
+## measure() Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "string",
+      description: "The text to analyze for tone",
+      isOptional: false,
+    },
+    {
+      name: "output",
+      type: "string",
+      description: "Reference text for tone comparison (empty string for stability analysis)",
+      isOptional: false,
+    }
+  ]}
+/>
+
+## Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: "score",
+      type: "number",
+      description: "Tone consistency/stability score (0-1)",
+    },
+    {
+      name: "info",
+      type: "object",
+      description: "Detailed tone info",
+    }
+  ]}
+/>
+
+### info Object (Tone Comparison)
+
+<PropertiesTable
+  content={[
+    {
+      name: "responseSentiment",
+      type: "number",
+      description: "Sentiment score for the input text",
+    },
+    {
+      name: "referenceSentiment",
+      type: "number",
+      description: "Sentiment score for the output text",
+    },
+    {
+      name: "difference",
+      type: "number",
+      description: "Absolute difference between sentiment scores",
+    }
+  ]}
+/>
+
+### info Object (Tone Stability)
+
+<PropertiesTable
+  content={[
+    {
+      name: "avgSentiment",
+      type: "number",
+      description: "Average sentiment score across sentences",
+    },
+    {
+      name: "sentimentVariance",
+      type: "number",
+      description: "Variance in sentiment between sentences",
+    }
+  ]}
+/>
+
+
+## Scoring Details
+
+The metric evaluates sentiment consistency through tone pattern analysis and mode-specific scoring.
+
+### Scoring Process
+
+1. Analyzes tone patterns:
+   - Extracts sentiment features
+   - Computes sentiment scores
+   - Measures tone variations
+
+2. Calculates mode-specific score:
+   **Tone Consistency** (input and output):
+   - Compares sentiment between texts
+   - Calculates sentiment difference
+   - Score = 1 - (sentiment_difference / max_difference)
+
+   **Tone Stability** (single input):
+   - Analyzes sentiment across sentences
+   - Calculates sentiment variance
+   - Score = 1 - (sentiment_variance / max_variance)
+
+Final score: `mode_specific_score * scale`
+
+### Score interpretation
+(0 to scale, default 0-1)
+- 1.0: Perfect tone consistency/stability
+- 0.7-0.9: Strong consistency with minor variations
+- 0.4-0.6: Moderate consistency with noticeable shifts
+- 0.1-0.3: Poor consistency with major tone changes
+- 0.0: No consistency - completely different tones
+
+## Example with Both Modes
+
+```typescript
+import { ToneConsistencyMetric } from "@mastra/evals/nlp";
+
+const metric = new ToneConsistencyMetric();
+
+// Tone Consistency Mode
+const consistencyResult = await metric.measure(
+  "This product is fantastic and amazing!",
+  "The product is excellent and wonderful!"
+);
+// Example output:
+// {
+//   score: 0.95,
+//   info: {
+//     responseSentiment: 0.8,
+//     referenceSentiment: 0.75,
+//     difference: 0.05
+//   }
+// }
+
+// Tone Stability Mode
+const stabilityResult = await metric.measure(
+  "Great service! Friendly staff. Perfect atmosphere.",
+  ""
+);
+// Example output:
+// {
+//   score: 0.9,
+//   info: {
+//     avgSentiment: 0.6,
+//     sentimentVariance: 0.1
+//   }
+// }
+```
+
+## Related
+
+- [Content Similarity Metric](./content-similarity)
+- [Toxicity Metric](./toxicity)

package/.docs/raw/reference/evals/toxicity.mdx

@@ -0,0 +1,165 @@
+---
+title: "Reference: Toxicity | Metrics | Evals | Mastra Docs"
+description: Documentation for the Toxicity Metric in Mastra, which evaluates LLM outputs for racist, biased, or toxic elements.
+---
+
+# ToxicityMetric
+
+The `ToxicityMetric` class evaluates whether an LLM's output contains racist, biased, or toxic elements. It uses a judge-based system to analyze responses for various forms of toxicity including personal attacks, mockery, hate speech, dismissive statements, and threats.
+
+## Basic Usage
+
+```typescript
+import { openai } from "@ai-sdk/openai";
+import { ToxicityMetric } from "@mastra/evals/llm";
+
+// Configure the model for evaluation
+const model = openai("gpt-4o-mini");
+
+const metric = new ToxicityMetric(model, {
+  scale: 1, // Default scale is 0-1
+});
+
+const result = await metric.measure(
+  "How is Sarah as a person?",
+  "Sarah is a dedicated team member who always delivers quality work.",
+);
+
+console.log(result.score); // Score from 0-1 (0 = not toxic, 1 = toxic)
+console.log(result.info.reason); // Explanation of the toxicity assessment
+```
+
+## Constructor Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "model",
+      type: "LanguageModel",
+      description: "Configuration for the model used to evaluate toxicity",
+      isOptional: false,
+    },
+    {
+      name: "options",
+      type: "ToxicityMetricOptions",
+      description: "Configuration options for the metric",
+      isOptional: true,
+      defaultValue: "{ scale: 1 }",
+    },
+  ]}
+/>
+
+### ToxicityMetricOptions
+
+<PropertiesTable
+  content={[
+    {
+      name: "scale",
+      type: "number",
+      description: "Maximum score value (default is 1)",
+      isOptional: true,
+      defaultValue: "1",
+    },
+  ]}
+/>
+
+## measure() Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "string",
+      description: "The original query or prompt",
+      isOptional: false,
+    },
+    {
+      name: "output",
+      type: "string",
+      description: "The LLM's response to evaluate",
+      isOptional: false,
+    },
+  ]}
+/>
+
+## Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: "score",
+      type: "number",
+      description: "Toxicity score (0 to scale, default 0-1)",
+    },
+    {
+      name: "info",
+      type: "object",
+      description: "Detailed toxicity info",
+      properties: [
+        {
+          type: "string",
+          parameters: [
+            {
+              name: "reason",
+              type: "string",
+              description: "Detailed explanation of the toxicity assessment",
+            },
+          ],
+        },
+      ],
+    },
+  ]}
+/>
+
+## Scoring Details
+
+The metric evaluates toxicity through multiple aspects:
+- Personal attacks
+- Mockery or sarcasm
+- Hate speech
+- Dismissive statements
+- Threats or intimidation
+
+### Scoring Process
+
+1. Analyzes toxic elements:
+   - Identifies personal attacks and mockery
+   - Detects hate speech and threats
+   - Evaluates dismissive statements
+   - Assesses severity levels
+
+2. Calculates toxicity score:
+   - Weighs detected elements
+   - Combines severity ratings
+   - Normalizes to scale
+
+Final score: `(toxicity_weighted_sum / max_toxicity) * scale`
+
+### Score interpretation
+(0 to scale, default 0-1)
+- 0.8-1.0: Severe toxicity
+- 0.4-0.7: Moderate toxicity
+- 0.1-0.3: Mild toxicity
+- 0.0: No toxic elements detected
+
+## Example with Custom Configuration
+
+```typescript
+import { openai } from "@ai-sdk/openai";
+
+const model = openai("gpt-4o-mini");
+
+const metric = new ToxicityMetric(model, {
+  scale: 10, // Use 0-10 scale instead of 0-1
+});
+
+const result = await metric.measure(
+  "What do you think about the new team member?",
+  "The new team member shows promise but needs significant improvement in basic skills.",
+);
+```
+
+## Related
+
+- [Tone Consistency Metric](./tone-consistency)
+- [Bias Metric](./bias)