@mastra/mcp-docs-server 0.0.0-commonjs-20250414101718

package/.docs/raw/reference/evals/hallucination.mdx

@@ -0,0 +1,219 @@
+---
+title: "Reference: Hallucination | Metrics | Evals | Mastra Docs"
+description: Documentation for the Hallucination Metric in Mastra, which evaluates the factual correctness of LLM outputs by identifying contradictions with provided context.
+---
+
+# HallucinationMetric
+
+The `HallucinationMetric` evaluates whether an LLM generates factually correct information by comparing its output against the provided context. This metric measures hallucination by identifying direct contradictions between the context and the output.
+
+## Basic Usage
+
+```typescript
+import { openai } from "@ai-sdk/openai";
+import { HallucinationMetric } from "@mastra/evals/llm";
+
+// Configure the model for evaluation
+const model = openai("gpt-4o-mini");
+
+const metric = new HallucinationMetric(model, {
+  context: [
+    "Tesla was founded in 2003 by Martin Eberhard and Marc Tarpenning in San Carlos, California.",
+  ],
+});
+
+const result = await metric.measure(
+  "Tell me about Tesla's founding.",
+  "Tesla was founded in 2004 by Elon Musk in California.",
+);
+
+console.log(result.score); // Score from 0-1
+console.log(result.info.reason); // Explanation of the score
+
+// Example output:
+// {
+//   score: 0.67,
+//   info: {
+//     reason: "The score is 0.67 because two out of three statements from the context
+//           (founding year and founders) were contradicted by the output, while the
+//           location statement was not contradicted."
+//   }
+// }
+```
+
+## Constructor Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "model",
+      type: "LanguageModel",
+      description: "Configuration for the model used to evaluate hallucination",
+      isOptional: false,
+    },
+    {
+      name: "options",
+      type: "HallucinationMetricOptions",
+      description: "Configuration options for the metric",
+      isOptional: false,
+    },
+  ]}
+/>
+
+### HallucinationMetricOptions
+
+<PropertiesTable
+  content={[
+    {
+      name: "scale",
+      type: "number",
+      description: "Maximum score value",
+      isOptional: true,
+      defaultValue: "1",
+    },
+    {
+      name: "context",
+      type: "string[]",
+      description: "Array of context pieces used as the source of truth",
+      isOptional: false,
+    },
+  ]}
+/>
+
+## measure() Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "string",
+      description: "The original query or prompt",
+      isOptional: false,
+    },
+    {
+      name: "output",
+      type: "string",
+      description: "The LLM's response to evaluate",
+      isOptional: false,
+    },
+  ]}
+/>
+
+## Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: "score",
+      type: "number",
+      description: "Hallucination score (0 to scale, default 0-1)",
+    },
+    {
+      name: "info",
+      type: "object",
+      description: "Object containing the reason for the score",
+      properties: [
+        {
+          type: "string",
+          parameters: [
+            {
+              name: "reason",
+              type: "string",
+              description:
+                "Detailed explanation of the score and identified contradictions",
+            },
+          ],
+        },
+      ],
+    },
+  ]}
+/>
+
+## Scoring Details
+
+The metric evaluates hallucination through contradiction detection and unsupported claim analysis.
+
+### Scoring Process
+
+1. Analyzes factual content:
+   - Extracts statements from context
+   - Identifies numerical values and dates
+   - Maps statement relationships
+
+2. Analyzes output for hallucinations:
+   - Compares against context statements
+   - Marks direct conflicts as hallucinations
+   - Identifies unsupported claims as hallucinations
+   - Evaluates numerical accuracy
+   - Considers approximation context
+
+3. Calculates hallucination score:
+   - Counts hallucinated statements (contradictions and unsupported claims)
+   - Divides by total statements
+   - Scales to configured range
+
+Final score: `(hallucinated_statements / total_statements) * scale`
+
+### Important Considerations
+
+- Claims not present in context are treated as hallucinations
+- Subjective claims are hallucinations unless explicitly supported
+- Speculative language ("might", "possibly") about facts IN context is allowed
+- Speculative language about facts NOT in context is treated as hallucination
+- Empty outputs result in zero hallucinations
+- Numerical evaluation considers:
+  - Scale-appropriate precision
+  - Contextual approximations
+  - Explicit precision indicators
+
+### Score interpretation
+(0 to scale, default 0-1)
+- 1.0: Complete hallucination - contradicts all context statements
+- 0.75: High hallucination - contradicts 75% of context statements
+- 0.5: Moderate hallucination - contradicts half of context statements
+- 0.25: Low hallucination - contradicts 25% of context statements
+- 0.0: No hallucination - output aligns with all context statements
+
+**Note:** The score represents the degree of hallucination - lower scores indicate better factual alignment with the provided context
+
+## Example with Analysis
+
+```typescript
+import { openai } from "@ai-sdk/openai";
+import { HallucinationMetric } from "@mastra/evals/llm";
+
+// Configure the model for evaluation
+const model = openai("gpt-4o-mini");
+
+const metric = new HallucinationMetric(model, {
+  context: [
+    "OpenAI was founded in December 2015 by Sam Altman, Greg Brockman, and others.",
+    "The company launched with a $1 billion investment commitment.",
+    "Elon Musk was an early supporter but left the board in 2018.",
+  ],
+});
+
+const result = await metric.measure({
+  input: "What are the key details about OpenAI?",
+  output:
+    "OpenAI was founded in 2015 by Elon Musk and Sam Altman with a $2 billion investment.",
+});
+
+// Example output:
+// {
+//   score: 0.33,
+//   info: {
+//     reason: "The score is 0.33 because one out of three statements from the context
+//           was contradicted (the investment amount was stated as $2 billion instead
+//           of $1 billion). The founding date was correct, and while the output's
+//           description of founders was incomplete, it wasn't strictly contradictory."
+//   }
+// }
+```
+
+## Related
+
+- [Faithfulness Metric](./faithfulness)
+- [Answer Relevancy Metric](./answer-relevancy)
+- [Context Precision Metric](./context-precision)
+- [Context Relevancy Metric](./context-relevancy)

package/.docs/raw/reference/evals/keyword-coverage.mdx

@@ -0,0 +1,176 @@
+---
+title: "Reference: Keyword Coverage | Metrics | Evals | Mastra Docs"
+description: Documentation for the Keyword Coverage Metric in Mastra, which evaluates how well LLM outputs cover important keywords from the input.
+---
+
+# KeywordCoverageMetric
+
+The `KeywordCoverageMetric` class evaluates how well an LLM's output covers the important keywords from the input. It analyzes keyword presence and matches while ignoring common words and stop words.
+
+## Basic Usage
+
+```typescript
+import { KeywordCoverageMetric } from "@mastra/evals/nlp";
+
+const metric = new KeywordCoverageMetric();
+
+const result = await metric.measure(
+  "What are the key features of Python programming language?",
+  "Python is a high-level programming language known for its simple syntax and extensive libraries."
+);
+
+console.log(result.score); // Coverage score from 0-1
+console.log(result.info); // Object containing detailed metrics about keyword coverage
+```
+
+## measure() Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "string",
+      description: "The original text containing keywords to be matched",
+      isOptional: false,
+    },
+    {
+      name: "output",
+      type: "string",
+      description: "The text to evaluate for keyword coverage",
+      isOptional: false,
+    }
+  ]}
+/>
+
+## Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: "score",
+      type: "number",
+      description: "Coverage score (0-1) representing the proportion of matched keywords",
+    },
+    {
+      name: "info",
+      type: "object",
+      description: "Object containing detailed metrics about keyword coverage",
+      properties: [
+        {
+          type: "number",
+          parameters: [
+            {
+              name: "matchedKeywords",
+              type: "number",
+              description: "Number of keywords found in the output",
+            }
+          ]
+        },
+        {
+          type: "number",
+          parameters: [
+            {
+              name: "totalKeywords",
+              type: "number",
+              description: "Total number of keywords from the input",
+            }
+          ]
+        }
+      ]
+    }
+  ]}
+/>
+
+## Scoring Details
+
+The metric evaluates keyword coverage by matching keywords with the following features:
+- Common word and stop word filtering (e.g., "the", "a", "and")
+- Case-insensitive matching
+- Word form variation handling
+- Special handling of technical terms and compound words
+
+### Scoring Process
+
+1. Processes keywords from input and output:
+   - Filters out common words and stop words
+   - Normalizes case and word forms
+   - Handles special terms and compounds
+
+2. Calculates keyword coverage:
+   - Matches keywords between texts
+   - Counts successful matches
+   - Computes coverage ratio
+
+Final score: `(matched_keywords / total_keywords) * scale`
+
+### Score interpretation
+(0 to scale, default 0-1)
+- 1.0: Perfect keyword coverage
+- 0.7-0.9: Good coverage with most keywords present
+- 0.4-0.6: Moderate coverage with some keywords missing
+- 0.1-0.3: Poor coverage with many keywords missing
+- 0.0: No keyword matches
+
+## Examples with Analysis
+
+```typescript
+import { KeywordCoverageMetric } from "@mastra/evals/nlp";
+
+const metric = new KeywordCoverageMetric();
+
+// Perfect coverage example
+const result1 = await metric.measure(
+  "The quick brown fox jumps over the lazy dog",
+  "A quick brown fox jumped over a lazy dog"
+);
+// {
+//   score: 1.0,
+//   info: {
+//     matchedKeywords: 6,
+//     totalKeywords: 6
+//   }
+// }
+
+// Partial coverage example
+const result2 = await metric.measure(
+  "Python features include easy syntax, dynamic typing, and extensive libraries",
+  "Python has simple syntax and many libraries"
+);
+// {
+//   score: 0.67,
+//   info: {
+//     matchedKeywords: 4,
+//     totalKeywords: 6
+//   }
+// }
+
+// Technical terms example
+const result3 = await metric.measure(
+  "Discuss React.js component lifecycle and state management",
+  "React components have lifecycle methods and manage state"
+);
+// {
+//   score: 1.0,
+//   info: {
+//     matchedKeywords: 4,
+//     totalKeywords: 4
+//   }
+// }
+```
+
+## Special Cases
+
+The metric handles several special cases:
+- Empty input/output: Returns score of 1.0 if both empty, 0.0 if only one is empty
+- Single word: Treated as a single keyword
+- Technical terms: Preserves compound technical terms (e.g., "React.js", "machine learning")
+- Case differences: "JavaScript" matches "javascript"
+- Common words: Ignored in scoring to focus on meaningful keywords
+
+## Related
+
+- [Completeness Metric](./completeness)
+- [Content Similarity Metric](./content-similarity)
+- [Answer Relevancy Metric](./answer-relevancy)
+- [Textual Difference Metric](./textual-difference)
+- [Context Relevancy Metric](./context-relevancy)

package/.docs/raw/reference/evals/prompt-alignment.mdx

@@ -0,0 +1,238 @@
+---
+title: "Reference: Prompt Alignment | Metrics | Evals | Mastra Docs"
+description: Documentation for the Prompt Alignment Metric in Mastra, which evaluates how well LLM outputs adhere to given prompt instructions.
+---
+
+# PromptAlignmentMetric
+
+The `PromptAlignmentMetric` class evaluates how strictly an LLM's output follows a set of given prompt instructions. It uses a judge-based system to verify each instruction is followed exactly and provides detailed reasoning for any deviations.
+
+## Basic Usage
+
+```typescript
+import { openai } from "@ai-sdk/openai";
+import { PromptAlignmentMetric } from "@mastra/evals/llm";
+
+// Configure the model for evaluation
+const model = openai("gpt-4o-mini");
+
+const instructions = [
+  "Start sentences with capital letters",
+  "End each sentence with a period",
+  "Use present tense",
+];
+
+const metric = new PromptAlignmentMetric(model, {
+  instructions,
+  scale: 1,
+});
+
+const result = await metric.measure(
+  "describe the weather",
+  "The sun is shining. Clouds float in the sky. A gentle breeze blows.",
+);
+
+console.log(result.score); // Alignment score from 0-1
+console.log(result.info.reason); // Explanation of the score
+```
+
+## Constructor Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "model",
+      type: "LanguageModel",
+      description:
+        "Configuration for the model used to evaluate instruction alignment",
+      isOptional: false,
+    },
+    {
+      name: "options",
+      type: "PromptAlignmentOptions",
+      description: "Configuration options for the metric",
+      isOptional: false,
+    },
+  ]}
+/>
+
+### PromptAlignmentOptions
+
+<PropertiesTable
+  content={[
+    {
+      name: "instructions",
+      type: "string[]",
+      description: "Array of instructions that the output should follow",
+      isOptional: false,
+    },
+    {
+      name: "scale",
+      type: "number",
+      description: "Maximum score value",
+      isOptional: true,
+      defaultValue: "1",
+    },
+  ]}
+/>
+
+## measure() Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: "input",
+      type: "string",
+      description: "The original prompt or query",
+      isOptional: false,
+    },
+    {
+      name: "output",
+      type: "string",
+      description: "The LLM's response to evaluate",
+      isOptional: false,
+    },
+  ]}
+/>
+
+## Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: "score",
+      type: "number",
+      description: "Alignment score (0 to scale, default 0-1)",
+    },
+    {
+      name: "info",
+      type: "object",
+      description:
+        "Object containing detailed metrics about instruction compliance",
+      properties: [
+        {
+          type: "string",
+          parameters: [
+            {
+              name: "reason",
+              type: "string",
+              description:
+                "Detailed explanation of the score and instruction compliance",
+            },
+          ],
+        },
+      ],
+    },
+  ]}
+/>
+
+## Scoring Details
+
+The metric evaluates instruction alignment through:
+- Applicability assessment for each instruction
+- Strict compliance evaluation for applicable instructions
+- Detailed reasoning for all verdicts
+- Proportional scoring based on applicable instructions
+
+### Instruction Verdicts
+
+Each instruction receives one of three verdicts:
+- "yes": Instruction is applicable and completely followed
+- "no": Instruction is applicable but not followed or only partially followed
+- "n/a": Instruction is not applicable to the given context
+
+### Scoring Process
+
+1. Evaluates instruction applicability:
+   - Determines if each instruction applies to the context
+   - Marks irrelevant instructions as "n/a"
+   - Considers domain-specific requirements
+
+2. Assesses compliance for applicable instructions:
+   - Evaluates each applicable instruction independently
+   - Requires complete compliance for "yes" verdict
+   - Documents specific reasons for all verdicts
+
+3. Calculates alignment score:
+   - Counts followed instructions ("yes" verdicts)
+   - Divides by total applicable instructions (excluding "n/a")
+   - Scales to configured range
+
+Final score: `(followed_instructions / applicable_instructions) * scale`
+
+### Important Considerations
+
+- Empty outputs:
+  - All formatting instructions are considered applicable
+  - Marked as "no" since they cannot satisfy requirements
+- Domain-specific instructions:
+  - Always applicable if about the queried domain
+  - Marked as "no" if not followed, not "n/a"
+- "n/a" verdicts:
+  - Only used for completely different domains
+  - Do not affect the final score calculation
+
+### Score interpretation
+(0 to scale, default 0-1)
+- 1.0: All applicable instructions followed perfectly
+- 0.7-0.9: Most applicable instructions followed
+- 0.4-0.6: Mixed compliance with applicable instructions
+- 0.1-0.3: Limited compliance with applicable instructions
+- 0.0: No applicable instructions followed
+
+## Example with Analysis
+
+```typescript
+import { openai } from "@ai-sdk/openai";
+import { PromptAlignmentMetric } from "@mastra/evals/llm";
+
+// Configure the model for evaluation
+const model = openai("gpt-4o-mini");
+
+const metric = new PromptAlignmentMetric(model, {
+  instructions: [
+    "Use bullet points for each item",
+    "Include exactly three examples",
+    "End each point with a semicolon"
+  ],
+  scale: 1
+});
+
+const result = await metric.measure(
+  "List three fruits",
+  "• Apple is red and sweet;
+• Banana is yellow and curved;
+• Orange is citrus and round."
+);
+
+// Example output:
+// {
+//   score: 1.0,
+//   info: {
+//     reason: "The score is 1.0 because all instructions were followed exactly:
+//           bullet points were used, exactly three examples were provided, and
+//           each point ends with a semicolon."
+//   }
+// }
+
+const result2 = await metric.measure(
+  "List three fruits",
+  "1. Apple
+2. Banana
+3. Orange and Grape"
+);
+
+// Example output:
+// {
+//   score: 0.33,
+//   info: {
+//     reason: "The score is 0.33 because: numbered lists were used instead of bullet points,
+//           no semicolons were used, and four fruits were listed instead of exactly three."
+//   }
+// }
+```
+
+## Related
+
+- [Answer Relevancy Metric](./answer-relevancy)
+- [Keyword Coverage Metric](./keyword-coverage)