@mastra/evals 0.0.1-alpha.0

package/src/metrics/context-precision/prompts.ts

@@ -0,0 +1,111 @@
+export const CONTEXT_PRECISION_AGENT_INSTRUCTIONS = `You are a balanced and nuanced context precision evaluator. Your job is to determine if retrieved context nodes are relevant to generating the expected output.
+
+Key Principles:
+1. Evaluate whether each context node was useful in generating the expected output
+2. Consider both direct and indirect relevance
+3. Prioritize usefulness over completeness
+4. Recognize that some nodes may be partially relevant
+5. Empty or error nodes should be marked as not relevant`;
+
+export function generateEvaluatePrompt({
+  input,
+  output,
+  context,
+}: {
+  input: string;
+  output: string;
+  context: string[];
+}) {
+  return `Given the input, output, and context, evaluate each context piece's relevance by generating a list of JSON objects.
+
+**
+IMPORTANT: Your response must be in JSON format with a 'verdicts' key containing a list. Each verdict must have only two fields: \`verdict\` with either 'yes' or 'no', and \`reason\` explaining the verdict. Your reason should include relevant quotes from the context.
+
+Example Context: ["The Sun is a star", "Stars produce their own light", "The Moon reflects sunlight"]
+Example Query: "What is the Sun?"
+Example Expected Response: "The Sun is a star that produces light."
+
+Example:
+{
+    "verdicts": [
+        {
+            "verdict": "yes",
+            "reason": "The context 'The Sun is a star' directly defines what the Sun is."
+        },
+        {
+            "verdict": "yes",
+            "reason": "The context 'Stars produce their own light' is relevant as it describes a key characteristic of stars, which includes the Sun."
+        },
+        {
+            "verdict": "no",
+            "reason": "The context 'The Moon reflects sunlight' is not relevant to defining what the Sun is."
+        }
+    ]  
+}
+
+Consider context relevant if it:
+- Directly addresses the query
+- Provides examples or instances that help explain the concept
+- Offers related information that helps build understanding
+- Contains partial information that contributes to the response
+
+The number of verdicts MUST MATCH the number of context pieces exactly.
+**
+
+Input:
+${input}
+
+Output:
+${output}
+
+Context:
+${context}
+
+JSON:
+`;
+}
+
+export function generateReasonPrompt({
+  input,
+  output,
+  verdicts,
+  score,
+}: {
+  input: string;
+  output: string;
+  verdicts: Array<{ verdict: string; reason: string }>;
+  score: number;
+}) {
+  return `Given the input, output, verdicts, and precision score, provide a BRIEF explanation for the score. Explain both its strengths and limitations.
+The retrieved contexts is a list containing \`verdict\` ('yes' or 'no' for relevance), \`reason\` (explaining the verdict) and \`node\` (the context text). Contexts are listed in their ranking order.
+
+**
+IMPORTANT: Return only JSON format with a single 'reason' key explaining the score.
+Example JSON:
+{
+    "reason": "The score is <score> because <explanation>."
+}
+
+Guidelines:
+- Don't mention 'verdict' - refer to relevant/irrelevant nodes instead
+- Use information from the \`reason\` field, not the field itself
+- Reference node positions (first, second, etc.) when explaining relevance
+- For perfect scores (10.0), emphasize both relevance and optimal ordering
+- Always reference the ranking order when discussing relevance
+**
+
+Precision Score:
+${score}
+
+Input:
+${input}
+
+Output:
+${output}
+
+Context:
+${verdicts}
+
+JSON:
+`;
+}

package/src/metrics/difference/index.test.ts

@@ -0,0 +1,116 @@
+import { it, expect } from '@jest/globals';
+
+import { DifferenceMetric } from './index';
+
+describe('DifferenceMetric', () => {
+  const metric = new DifferenceMetric();
+
+  it('should return perfect match for identical strings', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: 'The quick brown fox',
+    });
+    expect(result.score).toBe(1);
+    expect(result.confidence).toBe(1);
+    expect(result.metrics!.changes).toBe(0);
+  });
+
+  it('should handle small differences', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: 'The quick brown cat',
+    });
+    expect(result.score).toBeGreaterThan(0.8);
+    expect(result.metrics!.changes).toBe(1);
+  });
+
+  it('should handle word additions', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: 'The very quick brown fox',
+    });
+    expect(result.score).toBeGreaterThan(0.7);
+    expect(result.metrics!.changes).toBe(1);
+  });
+
+  it('should handle word deletions', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox jumps',
+      output: 'The quick fox jumps',
+    });
+    expect(result.score).toBeGreaterThan(0.7);
+    expect(result.metrics!.changes).toBe(1);
+  });
+
+  it('should handle multiple changes', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox jumps over the lazy dog',
+      output: 'The slow black fox runs under the active cat',
+    });
+    expect(result.score).toBeGreaterThan(0.4);
+    expect(result.score).toBeLessThan(0.7);
+    expect(result.metrics!.changes).toBeGreaterThan(3);
+  });
+
+  it('should handle completely different strings', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: 'Lorem ipsum dolor sit amet',
+    });
+    expect(result.score).toBeLessThan(0.3);
+    expect(result.metrics!.changes).toBeGreaterThan(3);
+  });
+
+  it('should handle empty strings', async () => {
+    const result = await metric.measure({
+      input: '',
+      output: '',
+    });
+    expect(result.score).toBe(1);
+    expect(result.confidence).toBe(1);
+    expect(result.metrics!.changes).toBe(0);
+    expect(result.metrics!.lengthDiff).toBe(0);
+  });
+
+  it('should handle one empty string', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: '',
+    });
+    expect(result.score).toBe(0);
+    expect(result.confidence).toBe(0);
+    expect(result.metrics!.changes).toBeGreaterThan(0);
+    expect(result.metrics!.lengthDiff).toBe(1);
+  });
+
+  it('should handle case sensitivity', async () => {
+    const result = await metric.measure({
+      input: 'The Quick Brown Fox',
+      output: 'the quick brown fox',
+    });
+    expect(result.score).toBeLessThan(1);
+    expect(result.metrics!.changes).toBeGreaterThan(0);
+  });
+
+  it('should handle whitespace sensitivity', async () => {
+    const result = await metric.measure({
+      input: 'The   quick\nbrown    fox',
+      output: 'The quick brown fox',
+    });
+    expect(result.score).toBeLessThan(1);
+    expect(result.metrics!.changes).toBeGreaterThan(0);
+  });
+
+  it('should include difference details in result', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: 'The quick brown fox',
+    });
+    expect(result.details).toBe('Difference score: 100.0% with 0 changes');
+    expect(result.metrics!).toEqual({
+      ratio: 1,
+      changes: 0,
+      lengthDiff: 0,
+    });
+  });
+});

package/src/metrics/difference/index.ts

@@ -0,0 +1,31 @@
+import { Metric } from '@mastra/core';
+import { SequenceMatcher } from 'difflib';
+
+import { MetricScoringResult } from '../types';
+
+export class DifferenceMetric extends Metric {
+  async measure({ input, output }: { input: string; output: string }): Promise<MetricScoringResult> {
+    const matcher = new SequenceMatcher(null, input, output);
+    const ratio = matcher.ratio();
+
+    // Get detailed operations
+    const ops = matcher.getOpcodes();
+    const changes = ops.filter(([op]) => op !== 'equal').length;
+
+    // Calculate confidence based on text length difference
+    const maxLength = Math.max(input.length, output.length);
+    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
+    const confidence = 1 - lengthDiff;
+
+    return {
+      score: ratio,
+      details: `Difference score: ${(ratio * 100).toFixed(1)}% with ${changes} changes`,
+      confidence,
+      metrics: {
+        ratio,
+        changes,
+        lengthDiff,
+      },
+    };
+  }
+}

package/src/metrics/index.ts

@@ -0,0 +1,9 @@
+export { AnswerRelevancyMetric } from './answer-relevancy';
+export { CompletenessMetric } from './completeness';
+export { ContentSimilarityMetric } from './content-similarity';
+export { ContextPositionMetric } from './context-position';
+export { ContextPrecisionMetric } from './context-precision';
+export { DifferenceMetric } from './difference';
+export { KeywordCoverageMetric } from './keyword-coverage';
+export { PromptAlignmentMetric } from './prompt-alignment';
+export { ToneConsistencyMetric } from './tone';

package/src/metrics/keyword-coverage/index.test.ts

@@ -0,0 +1,114 @@
+import { it, expect } from '@jest/globals';
+
+import { KeywordCoverageMetric } from './index';
+
+describe('KeywordCoverageMetric', () => {
+  const metric = new KeywordCoverageMetric();
+
+  it('should return perfect coverage for identical text', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox jumps over the lazy dog',
+      output: 'The quick brown fox jumps over the lazy dog',
+    });
+    expect(result.score).toBe(1);
+    expect(result.confidence).toBe(0.85);
+    const matched = result.metrics?.matchedKeywords as number;
+    const total = result.metrics?.totalKeywords as number;
+    expect(matched).toBeGreaterThan(0);
+    expect(total).toBeGreaterThan(0);
+    expect(matched).toBe(total);
+  });
+
+  it('should handle partial keyword coverage', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox jumps over the lazy dog',
+      output: 'A quick brown fox runs past a sleeping cat',
+    });
+    expect(result.score).toBeGreaterThan(0.3);
+    expect(result.score).toBeLessThan(0.7);
+    const matched = result.metrics?.matchedKeywords as number;
+    const total = result.metrics?.totalKeywords as number;
+    expect(matched).toBeLessThan(total);
+  });
+
+  it('should ignore common words and stop words', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: 'A quick brown fox',
+    });
+    expect(result.score).toBe(1); // "the" and "a" should be ignored
+    const matched = result.metrics?.matchedKeywords as number;
+    const total = result.metrics?.totalKeywords as number;
+    expect(matched).toBe(total);
+  });
+
+  it('should handle case differences', async () => {
+    const result = await metric.measure({
+      input: 'The Quick Brown Fox',
+      output: 'the quick brown fox',
+    });
+    expect(result.score).toBe(1);
+    const matched = result.metrics?.matchedKeywords as number;
+    const total = result.metrics?.totalKeywords as number;
+    expect(matched).toBe(total);
+  });
+
+  it('should handle empty strings', async () => {
+    const result = await metric.measure({
+      input: '',
+      output: '',
+    });
+    expect(result.score).toBe(1);
+    expect(result.metrics?.totalKeywords).toBe(0);
+    expect(result.metrics?.matchedKeywords).toBe(0);
+  });
+
+  it('should handle one empty string', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: '',
+    });
+    expect(result.score).toBe(0);
+    expect(result.metrics?.matchedKeywords).toBe(0);
+    expect(result.metrics?.totalKeywords).toBeGreaterThan(0);
+  });
+
+  it('should ignore numbers by default', async () => {
+    const result = await metric.measure({
+      input: 'The 123 quick 456 brown fox',
+      output: 'The quick brown fox',
+    });
+    expect(result.score).toBe(1);
+  });
+
+  it('should handle special characters', async () => {
+    const result = await metric.measure({
+      input: 'The quick-brown fox!',
+      output: 'The quick brown fox',
+    });
+    // Hyphenated words are treated as separate keywords
+    expect(result.score).toBeGreaterThanOrEqual(0.5);
+    expect(result.score).toBeLessThan(1);
+  });
+
+  it('should handle completely different content', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox jumps over the lazy dog',
+      output: 'Lorem ipsum dolor sit amet',
+    });
+    expect(result.score).toBe(0);
+    expect(result.metrics?.matchedKeywords).toBe(0);
+  });
+
+  it('should include coverage details in result', async () => {
+    const result = await metric.measure({
+      input: 'quick brown fox',
+      output: 'quick brown fox',
+    });
+    expect(result.details).toMatch(/Keyword coverage: 100.0% \(3\/3 keywords\)/);
+    expect(result.metrics).toEqual({
+      totalKeywords: 3,
+      matchedKeywords: 3,
+    });
+  });
+});

package/src/metrics/keyword-coverage/index.ts

@@ -0,0 +1,47 @@
+import { Metric } from '@mastra/core';
+import keyword_extractor from 'keyword-extractor';
+
+import { MetricScoringResult } from '../types';
+
+export class KeywordCoverageMetric extends Metric {
+  async measure({ input, output }: { input: string; output: string }): Promise<MetricScoringResult> {
+    // Handle empty strings case
+    if (!input && !output) {
+      return {
+        score: 1,
+        details: 'Keyword coverage: 100.0% (0/0 keywords)',
+        confidence: 0.85,
+        metrics: {
+          totalKeywords: 0,
+          matchedKeywords: 0,
+        },
+      };
+    }
+
+    const extractKeywords = (text: string) => {
+      return keyword_extractor.extract(text, {
+        language: 'english',
+        remove_digits: true,
+        return_changed_case: true,
+        remove_duplicates: true,
+      });
+    };
+
+    const referenceKeywords = new Set(extractKeywords(input));
+    const responseKeywords = new Set(extractKeywords(output));
+
+    const matchedKeywords = [...referenceKeywords].filter(k => responseKeywords.has(k));
+    const totalKeywords = referenceKeywords.size;
+    const coverage = totalKeywords > 0 ? matchedKeywords.length / totalKeywords : 0;
+
+    return {
+      score: coverage,
+      details: `Keyword coverage: ${(coverage * 100).toFixed(1)}% (${matchedKeywords.length}/${referenceKeywords.size} keywords)`,
+      confidence: 0.85,
+      metrics: {
+        totalKeywords: referenceKeywords.size,
+        matchedKeywords: matchedKeywords.length,
+      },
+    };
+  }
+}

package/src/metrics/prompt-alignment/index.test.ts

@@ -0,0 +1,46 @@
+import { it, expect, jest } from '@jest/globals';
+import { type ModelConfig } from '@mastra/core';
+
+import { PromptAlignmentMetric } from './index';
+
+const SECONDS = 1000;
+jest.setTimeout(15 * SECONDS);
+
+const modelConfig: ModelConfig = {
+  provider: 'OPEN_AI',
+  name: 'gpt-4o',
+  toolChoice: 'auto',
+  apiKey: process.env.OPENAI_API_KEY,
+};
+
+it('should be able to measure prompt alignment', async () => {
+  const metric = new PromptAlignmentMetric(modelConfig, {
+    instructions: ['Reply in all uppercase'],
+  });
+
+  const result = await metric.measure({
+    input: `What if these shoes don't fit?`,
+    output: 'We offer a 30-day full refund at no extra cost.',
+  });
+
+  const resultUppercase = await metric.measure({
+    input: `What if these shoes don't fit?`,
+    output: 'We offer a 30-day full refund at no extra cost.'.toUpperCase(),
+  });
+
+  expect(resultUppercase.score).toBe(10);
+  expect(result.score).toBe(0);
+});
+
+it('should be able to measure prompt alignment with an array of instructions', async () => {
+  const metric = new PromptAlignmentMetric(modelConfig, {
+    instructions: ['Reply in all uppercase', 'Include baguettes in the response'],
+  });
+
+  const result = await metric.measure({
+    input: `What is the capital of France?`,
+    output: 'THE CAPITAL OF FRANCE IS BAGUETTE.',
+  });
+
+  expect(result.score).toBe(10);
+});

package/src/metrics/prompt-alignment/index.ts

@@ -0,0 +1,66 @@
+import { Metric, MetricResult, ModelConfig } from '@mastra/core';
+
+import { PromptAlignmentJudge } from './metricJudge';
+
+export class PromptAlignmentMetric extends Metric {
+  private instructions: string[];
+  private judge: PromptAlignmentJudge;
+  private scale: number;
+
+  constructor(model: ModelConfig, { instructions, scale = 10 }: { instructions: string[]; scale?: number }) {
+    super();
+
+    this.instructions = instructions;
+    this.judge = new PromptAlignmentJudge(model);
+    this.scale = scale;
+  }
+
+  async measure({ input, output }: { input: string; output: string }): Promise<MetricResult> {
+    const verdicts = await this.judge.evaluate(input, output, this.instructions);
+    const score = this.calculateScore(verdicts);
+
+    const reason = await this.generateReason(input, output, score, verdicts);
+
+    return {
+      score,
+      reason,
+    };
+  }
+
+  private async generateReason(
+    input: string,
+    output: string,
+    score: number,
+    verdicts: {
+      verdict: string;
+      reason: string;
+    }[],
+  ): Promise<string> {
+    const reasonsForVerdicts: string[] = [];
+    for (const { verdict, reason } of verdicts || []) {
+      if (verdict.trim().toLowerCase() === 'no') {
+        reasonsForVerdicts.push(reason);
+      }
+    }
+
+    const reason = await this.judge.getReason(input, output, score, reasonsForVerdicts);
+    return reason;
+  }
+
+  private calculateScore(evaluation: { verdict: string; reason: string }[]): number {
+    const numberOfVerdicts = evaluation?.length || 0;
+    if (numberOfVerdicts === 0) {
+      return 1;
+    }
+
+    let alignmentCount = 0;
+    for (const { verdict } of evaluation!) {
+      if (verdict.trim().toLowerCase() !== 'no') {
+        alignmentCount++;
+      }
+    }
+
+    const score = alignmentCount / numberOfVerdicts;
+    return score * this.scale;
+  }
+}

package/src/metrics/prompt-alignment/metricJudge.ts

@@ -0,0 +1,41 @@
+import { ModelConfig } from '@mastra/core';
+import { z } from 'zod';
+
+import { MastraAgentJudge } from '../../judge';
+
+import { generateEvaluatePrompt, generateReasonPrompt, PROMPT_ALIGNMENT_AGENT_INSTRUCTIONS } from './prompts';
+
+export class PromptAlignmentJudge extends MastraAgentJudge {
+  constructor(model: ModelConfig) {
+    super('Prompt Alignment', PROMPT_ALIGNMENT_AGENT_INSTRUCTIONS, model);
+  }
+
+  async evaluate(
+    input: string,
+    actualOutput: string,
+    instructions: string[],
+  ): Promise<{ verdict: string; reason: string }[]> {
+    const prompt = generateEvaluatePrompt({ input, output: actualOutput, instructions });
+    const result = await this.agent.generate(prompt, {
+      output: z.object({
+        verdicts: z.array(
+          z.object({
+            verdict: z.string(),
+            reason: z.string(),
+          }),
+        ),
+      }),
+    });
+    return result.object.verdicts;
+  }
+
+  async getReason(input: string, actualOutput: string, score: number, reasons: string[]): Promise<string> {
+    const prompt = generateReasonPrompt({ input, output: actualOutput, reasons, score });
+    const result = await this.agent.generate(prompt, {
+      output: z.object({
+        reason: z.string(),
+      }),
+    });
+    return result.object.reason;
+  }
+}

package/src/metrics/prompt-alignment/prompts.ts

@@ -0,0 +1,102 @@
+export const PROMPT_ALIGNMENT_AGENT_INSTRUCTIONS = `You are a strict and thorough prompt alignment evaluator. Your job is to determine if LLM outputs follow their given prompt instructions exactly.
+
+Key Principles:
+1. Be EXTRA STRICT in your evaluation in regards to whether the instructions are followed exactly.
+2. Only give a "yes" verdict if an instruction is COMPLETELY followed
+3. Any partial compliance should be marked as "no"
+4. Provide clear, specific reasons for any "no" verdicts
+5. Focus solely on instruction compliance, not output quality
+
+Remember:
+- Each instruction must be evaluated independently
+- Verdicts must be either "yes" or "no" - no in-between
+- Reasons are required only for "no" verdicts
+- The number of verdicts must match the number of instructions exactly`;
+
+export function generateEvaluatePrompt({
+  instructions,
+  input,
+  output,
+}: {
+  instructions: string[];
+  input: string;
+  output: string;
+}) {
+  return `For the provided list of prompt instructions, determine whether each instruction has been followed in the LLM output.
+Make sure to judge the output on each instruction independently. Do not let instructions be influenced by other instructions.
+Generate a list of verdicts in JSON format, where each verdict must have:
+- "verdict": Strictly "yes" or "no"
+- "reason": Give a reason for the verdict
+
+Be EXTRA STRICT in your evaluation. Only give "yes" if the instruction is followed COMPLETELY.
+Evaluate the output EXACTLY as written - consider every character, space, and case
+
+Example:
+Input: "describe the sky"
+Output: "the sky is Blue today"
+Instructions: ["Start sentences with capital letters", "Use proper English"]
+
+{
+  "verdicts": [
+    {
+      "verdict": "no",
+      "reason": "The sentence 'the sky is Blue' starts with lowercase 't'"
+    },
+    {
+      "verdict": "no",
+      "reason": "Improper capitalization: 'Blue' is capitalized mid-sentence"
+    }
+  ]
+}
+
+Prompt Instructions:
+${instructions.join('\n')}
+
+Input:
+${input}
+
+LLM Actual Output:
+${output}
+
+JSON:`;
+}
+
+export function generateReasonPrompt({
+  input,
+  output,
+  score,
+  reasons,
+}: {
+  input: string;
+  output: string;
+  score: number;
+  reasons: string[];
+}) {
+  return `Explain the instruction following score (0-10) for the LLM's response using this context:
+  Context:
+  Input: ${input}
+  Output: ${output}
+  Score: ${score}
+  Failure Reasons: ${reasons.join('\n')}
+
+  Rules (follow these rules exactly. do not deviate):
+  - Keep your response concise and to the point.
+  - Do not change score from what is given.
+  - Do not make judgements on inputs or outputs (factual correctness, quality, etc).
+  - If there are failure reasons given, explain why the score is not higher.
+  
+
+  Output format:
+  {
+    "reason": "The score is {score} because {explanation of instruction following}"
+  }
+    
+  Example Responses:
+  {
+    "reason": "The score is 10 because the output follows the instructions exactly"
+  }
+  {
+    "reason": "The score is 0 because the output does not follow the instructions"
+  }
+  `;
+}