@mastra/evals 0.0.1-alpha.0

package/src/metrics/completeness/index.test.ts

@@ -0,0 +1,96 @@
+import { CompletenessMetric } from './index';
+
+describe('CompletenessMetric', () => {
+  let metric: CompletenessMetric;
+
+  beforeEach(() => {
+    metric = new CompletenessMetric();
+  });
+
+  describe('basic functionality', () => {
+    it('should return high score for identical text', async () => {
+      const text = 'The quick brown fox jumps over the lazy dog';
+      const result = await metric.measure({ input: text, output: text });
+
+      expect(result.score).toBeCloseTo(1.0);
+      expect(result.confidence).toBe(0.8);
+    });
+
+    it('should return lower score for simplified text missing elements', async () => {
+      const original = 'The quick brown fox jumps over the lazy dog';
+      const simplified = 'The fox jumps over the dog';
+      const result = await metric.measure({ input: original, output: simplified });
+
+      expect(result.score).toBeLessThan(1.0);
+      expect(result.score).toBeGreaterThan(0.5);
+      expect(result.metrics?.missingElements).toContain('brown');
+      expect(result.metrics?.missingElements).toContain('lazy');
+    });
+
+    it('should handle completely different texts', async () => {
+      const original = 'The weather is sunny today';
+      const simplified = 'I like to eat pizza';
+      const result = await metric.measure({ input: original, output: simplified });
+
+      expect(result.score).toBeLessThan(0.3);
+      const { input, output } = result.metrics?.elementCounts as { input: number; output: number };
+      expect(input).toBeGreaterThan(0);
+      expect(output).toBeGreaterThan(0);
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should handle both empty strings', async () => {
+      const result = await metric.measure({ input: '', output: '' });
+      expect(result.score).toBe(1);
+      const { input, output } = result.metrics?.elementCounts as { input: number; output: number };
+      expect(input).toBe(0);
+      expect(output).toBe(0);
+    });
+
+    it('should handle empty original string', async () => {
+      const result = await metric.measure({ input: '', output: 'some text' });
+      expect(result.score).toBe(0);
+    });
+
+    it('should handle whitespace-only strings', async () => {
+      const result = await metric.measure({ input: '   \n  ', output: '  \n  ' });
+      expect(result.score).toBe(1);
+      const { input, output } = result.metrics?.elementCounts as { input: number; output: number };
+      expect(input).toBe(0);
+      expect(output).toBe(0);
+    });
+
+    it('should handle null and undefined inputs', async () => {
+      // @ts-expect-error Testing invalid input
+      await expect(metric.measure({ input: null, output: '' })).rejects.toThrow();
+      // @ts-expect-error Testing invalid input
+      await expect(metric.measure({ input: '', output: undefined })).rejects.toThrow();
+    });
+  });
+
+  describe('special cases', () => {
+    it('should handle lists and enumerations', async () => {
+      const result = await metric.measure({ input: 'apples, oranges, and bananas', output: 'apples and bananas' });
+      expect(result.score).toBeLessThan(0.8);
+      expect(result.metrics?.missingElements).toContain('oranges');
+    });
+
+    it('should handle repeated elements', async () => {
+      const result = await metric.measure({ input: 'cat cat cat cats', output: 'cat cats' });
+      expect(result.score).toBeGreaterThan(0.7);
+    });
+
+    it('should handle long and multi-paragraph text', async () => {
+      const original = `First paragraph about AI.
+        Second paragraph about ML.
+        Third paragraph about DL.`;
+      const simplified = `First para about AI.
+        Second para about ML.`;
+      const result = await metric.measure({ input: original, output: simplified });
+
+      expect(result.score).toBeGreaterThan(0.5);
+      expect(result.metrics?.missingElements).toBeDefined();
+    });
+  });
+});

package/src/metrics/completeness/index.ts

@@ -0,0 +1,112 @@
+import { Metric } from '@mastra/core';
+import nlp from 'compromise';
+
+import { MetricScoringResult } from '../types';
+
+export class CompletenessMetric extends Metric {
+  async measure({ input, output }: { input: string; output: string }): Promise<MetricScoringResult> {
+    // Handle null/undefined inputs
+    if (input === null || input === undefined || output === null || output === undefined) {
+      throw new Error('Inputs cannot be null or undefined');
+    }
+
+    // Trim both inputs
+    input = input.trim();
+    output = output.trim();
+
+    const inputDoc = nlp(input);
+    const outputDoc = nlp(output);
+
+    // Extract and log elements
+    const inputElements = this.extractElements(inputDoc);
+    const outputElements = this.extractElements(outputDoc);
+    // Maybe we need a more sophisticated matching approach
+    const coverage = this.calculateCoverage(inputElements, outputElements);
+
+    return {
+      score: coverage,
+      details: `Completeness score: ${(coverage * 100).toFixed(1)}%`,
+      confidence: 0.8,
+      metrics: {
+        inputElements,
+        outputElements,
+        missingElements: inputElements.filter(e => !outputElements.includes(e)),
+        elementCounts: {
+          input: inputElements.length,
+          output: outputElements.length,
+        },
+      },
+    };
+  }
+
+  private extractElements(doc: any): string[] {
+    // Get more specific elements and ensure they're arrays
+    const nouns = doc.nouns().out('array') || [];
+    const verbs = doc.verbs().toInfinitive().out('array') || [];
+    const topics = doc.topics().out('array') || [];
+    const terms = doc.terms().out('array') || [];
+
+    // Helper function to clean and split terms
+    const cleanAndSplitTerm = (term: string): string[] => {
+      // First normalize the string
+      const normalized = this.normalizeString(term);
+
+      // Split on word boundaries and filter out empty strings
+      return normalized
+        .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase
+        .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces
+        .trim()
+        .split(/\s+/)
+        .filter(word => word.length > 0);
+    };
+
+    // Process all elements
+    const processedTerms = [
+      ...nouns.flatMap(cleanAndSplitTerm),
+      ...verbs.flatMap(cleanAndSplitTerm),
+      ...topics.flatMap(cleanAndSplitTerm),
+      ...terms.flatMap(cleanAndSplitTerm),
+    ];
+
+    // Remove duplicates
+    return [...new Set(processedTerms)];
+  }
+
+  private normalizeString(str: string): string {
+    // Remove diacritics and convert to lowercase
+    return str
+      .normalize('NFD')
+      .replace(/[\u0300-\u036f]/g, '')
+      .toLowerCase();
+  }
+
+  private calculateCoverage(original: string[], simplified: string[]): number {
+    if (original.length === 0) {
+      return simplified.length === 0 ? 1 : 0;
+    }
+
+    // Exact matching for short words (3 chars or less), substring matching for longer words
+    const covered = original.filter(element =>
+      simplified.some(s => {
+        const elem = this.normalizeString(element);
+        const simp = this.normalizeString(s);
+
+        // For short words (3 chars or less), require exact match
+        if (elem.length <= 3) {
+          return elem === simp;
+        }
+
+        // For longer words, require substantial overlap (more than 60% of the longer word)
+        const longer = elem.length > simp.length ? elem : simp;
+        const shorter = elem.length > simp.length ? simp : elem;
+
+        if (longer.includes(shorter)) {
+          return shorter.length / longer.length > 0.6;
+        }
+
+        return false;
+      }),
+    );
+    return covered.length / original.length;
+  }
+}

package/src/metrics/content-similarity/index.test.ts

@@ -0,0 +1,107 @@
+import { it, expect } from '@jest/globals';
+
+import { ContentSimilarityMetric } from './index';
+
+describe('ContentSimilarityMetric', () => {
+  const metric = new ContentSimilarityMetric();
+
+  it('should return perfect similarity for identical strings', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: 'The quick brown fox',
+    });
+    expect(result.score).toBe(1);
+    expect(result.confidence).toBe(0.9);
+  });
+
+  it('should handle case differences with default options', async () => {
+    const result = await metric.measure({
+      input: 'The Quick Brown Fox',
+      output: 'the quick brown fox',
+    });
+    expect(result.score).toBe(1);
+  });
+
+  it('should handle whitespace differences with default options', async () => {
+    const result = await metric.measure({
+      input: 'The   quick\nbrown    fox',
+      output: 'The quick brown fox',
+    });
+    expect(result.score).toBe(1);
+  });
+
+  it('should be case sensitive when ignoreCase is false', async () => {
+    const caseSensitiveMetric = new ContentSimilarityMetric({ ignoreCase: false });
+    const result = await caseSensitiveMetric.measure({
+      input: 'The Quick Brown FOX',
+      output: 'the quick brown fox',
+    });
+    expect(result.score).toBeLessThan(0.8);
+  });
+
+  it('should preserve whitespace differences when ignoreWhitespace is true', async () => {
+    const whitespaceMetric = new ContentSimilarityMetric({
+      ignoreCase: true,
+      ignoreWhitespace: true,
+    });
+    const result = await whitespaceMetric.measure({
+      input: 'The\tquick  brown\n\nfox',
+      output: 'The quick brown fox',
+    });
+    expect(result.score).toBe(1);
+  });
+
+  it('should handle both case and whitespace sensitivity', async () => {
+    const sensitiveMetric = new ContentSimilarityMetric({
+      ignoreCase: false,
+      ignoreWhitespace: true,
+    });
+    const result = await sensitiveMetric.measure({
+      input: 'The\tQuick  Brown\n\nFOX',
+      output: 'the quick brown fox',
+    });
+    expect(result.score).toBeLessThan(0.8);
+  });
+
+  it('should handle partial similarity', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox jumps over the lazy dog',
+      output: 'The quick brown fox runs past the lazy dog',
+    });
+    expect(result.score).toBeGreaterThan(0.7);
+    expect(result.score).toBeLessThan(0.8);
+  });
+
+  it('should handle completely different strings', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: 'Lorem ipsum dolor sit amet',
+    });
+    expect(result.score).toBeLessThan(0.3);
+  });
+
+  it('should handle empty strings', async () => {
+    const result = await metric.measure({
+      input: '',
+      output: '',
+    });
+    expect(result.score).toBe(1);
+  });
+
+  it('should handle one empty string', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: '',
+    });
+    expect(result.score).toBe(0);
+  });
+
+  it('should include similarity details in result', async () => {
+    const result = await metric.measure({
+      input: 'The quick brown fox',
+      output: 'The quick brown fox',
+    });
+    expect(result.details).toBe('Content similarity: 100.0%');
+    expect(result.metrics).toEqual({ similarity: 1 });
+  });
+});

package/src/metrics/content-similarity/index.ts

@@ -0,0 +1,41 @@
+import { Metric } from '@mastra/core';
+import stringSimilarity from 'string-similarity';
+
+import { MetricOptions, MetricScoringResult } from '../types';
+
+export class ContentSimilarityMetric extends Metric {
+  private options: MetricOptions;
+
+  constructor(options: MetricOptions = {}) {
+    super();
+    this.options = {
+      ignoreCase: true,
+      ignoreWhitespace: true,
+      ...options,
+    };
+  }
+
+  async measure({ input, output }: { input: string; output: string }): Promise<MetricScoringResult> {
+    let processedInput = input;
+    let processedOutput = output;
+
+    if (this.options.ignoreCase) {
+      processedInput = processedInput.toLowerCase();
+      processedOutput = processedOutput.toLowerCase();
+    }
+
+    if (this.options.ignoreWhitespace) {
+      processedInput = processedInput.replace(/\s+/g, ' ').trim();
+      processedOutput = processedOutput.replace(/\s+/g, ' ').trim();
+    }
+
+    const similarity = stringSimilarity.compareTwoStrings(processedInput, processedOutput);
+
+    return {
+      score: similarity,
+      details: `Content similarity: ${(similarity * 100).toFixed(1)}%`,
+      confidence: 0.9,
+      metrics: { similarity },
+    };
+  }
+}

package/src/metrics/context-position/index.test.ts

@@ -0,0 +1,292 @@
+import { it, expect, jest } from '@jest/globals';
+import { type ModelConfig } from '@mastra/core';
+
+import { ContextPositionMetric } from './index';
+
+type TestCase = {
+  input: string;
+  output: string;
+  context: string[];
+  expectedResult: {
+    score: number;
+    reason: string;
+  };
+};
+
+const testCases: TestCase[] = [
+  {
+    // Perfect ordering with all relevant pieces [1,1,1]
+    input: 'What is the Sun?',
+    output: 'The Sun is a star that produces its own light.',
+    context: [
+      'The Sun is a star at the center of our solar system.',
+      'Stars like the Sun produce their own light through nuclear fusion.',
+      'The Sun provides light and heat to Earth.',
+    ],
+    expectedResult: {
+      score: 10,
+      reason: 'All context pieces are relevant and optimally ordered, with the most important definition first.',
+    },
+  },
+  {
+    // Mixed relevance case [1,0,1]
+    input: 'What is photosynthesis?',
+    output: 'Photosynthesis is how plants make food using sunlight.',
+    context: [
+      'Plants use sunlight to create food through photosynthesis.',
+      'Chlorophyll gives plants their green color.',
+      'Plants need water and CO2 for photosynthesis.',
+    ],
+    expectedResult: {
+      score: 7.27,
+      reason:
+        'First and third pieces are relevant, with an irrelevant piece in between, demonstrating proper handling of mixed relevance.',
+    },
+  },
+  {
+    // Domain knowledge relevance [1,1,1]
+    input: 'How does a car engine work?',
+    output: 'A car engine converts gasoline into mechanical energy through combustion.',
+    context: [
+      'Car engines burn gasoline for power.',
+      'Pistons move up and down in the engine.',
+      'Spark plugs create electrical sparks.',
+    ],
+    expectedResult: {
+      score: 10,
+      reason: 'All pieces contribute domain knowledge: direct explanation, mechanical process, and enabling mechanism.',
+    },
+  },
+  {
+    // Mixed relevance with good ordering [1,1,0]
+    input: 'What is JavaScript?',
+    output: 'JavaScript is a programming language used for web development.',
+    context: [
+      'JavaScript is a popular programming language.',
+      'Web browsers can run JavaScript code.',
+      'Computers use binary code.',
+    ],
+    expectedResult: {
+      score: 8.18,
+      reason: 'Two relevant pieces at the start, followed by an irrelevant piece.',
+    },
+  },
+  {
+    // Single relevant at start [1,0,0]
+    input: 'What is precipitation?',
+    output: 'Precipitation is water falling from clouds as rain or snow.',
+    context: ['Precipitation is water falling from clouds.', 'The Earth is round.', 'Plants are green.'],
+    expectedResult: {
+      score: 5.45,
+      reason: 'Single relevant piece at the start, followed by irrelevant pieces.',
+    },
+  },
+  {
+    // Single relevant in middle [0,1,0]
+    input: 'What are clouds?',
+    output: 'Clouds are visible masses of water droplets in the sky.',
+    context: ['The sky is blue.', 'Clouds are made of water droplets.', 'Birds have feathers.'],
+    expectedResult: {
+      score: 2.73,
+      reason: 'Single relevant piece in middle position, with irrelevant pieces before and after.',
+    },
+  },
+  {
+    // Single relevant at end [0,0,1]
+    input: 'What is the moon?',
+    output: "The Moon is Earth's natural satellite.",
+    context: ['Stars twinkle at night.', 'The sky appears blue.', 'The Moon orbits around Earth.'],
+    expectedResult: {
+      score: 1.82,
+      reason: 'Single relevant piece at the end, with irrelevant pieces before it.',
+    },
+  },
+  {
+    // Empty context (edge case)
+    input: 'What is gravity?',
+    output: 'Gravity is a force that attracts objects together.',
+    context: [],
+    expectedResult: {
+      score: 0,
+      reason: 'No context provided to evaluate.',
+    },
+  },
+  {
+    // All irrelevant [0,0,0]
+    input: 'What is electricity?',
+    output: 'Electricity is the flow of electrical charge.',
+    context: ['The weather is sunny today.', 'Birds can fly.', 'Trees grow tall.'],
+    expectedResult: {
+      score: 0,
+      reason: 'No relevant context pieces found.',
+    },
+  },
+  {
+    // Complex interdependent context [1,1,1]
+    input: 'How do plants grow?',
+    output: 'Plants grow through photosynthesis, using water and nutrients from soil.',
+    context: [
+      'Plants need nutrients from soil to grow.',
+      'Photosynthesis converts sunlight to energy.',
+      'Roots absorb water from the soil.',
+    ],
+    expectedResult: {
+      score: 10,
+      reason: 'All pieces are relevant and work together to explain the concept, with logical ordering.',
+    },
+  },
+  {
+    // Single relevant piece [1]
+    input: 'What is DNA?',
+    output: 'DNA contains genetic information.',
+    context: ['DNA stores genetic information in cells.'],
+    expectedResult: {
+      score: 10,
+      reason: 'Single relevant piece in optimal first position.',
+    },
+  },
+  {
+    // Two relevant at end [0,1,1]
+    input: 'What is a volcano?',
+    output: 'A volcano is a mountain that erupts hot lava.',
+    context: ['Mountains can be found worldwide.', 'Volcanoes erupt molten rock.', 'Lava flows from volcanic vents.'],
+    expectedResult: {
+      score: 4.55,
+      reason: 'Two relevant pieces appear at the end, after an irrelevant general statement.',
+    },
+  },
+];
+
+const SECONDS = 10000;
+jest.setTimeout(15 * SECONDS);
+
+const modelConfig: ModelConfig = {
+  provider: 'OPEN_AI',
+  name: 'gpt-4o',
+  toolChoice: 'auto',
+  apiKey: process.env.OPENAI_API_KEY,
+};
+
+describe('ContextPositionMetric', () => {
+  const metric = new ContextPositionMetric(modelConfig);
+
+  it('should handle perfect ordering with all relevant pieces', async () => {
+    const testCase = testCases[0]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle mixed relevance case', async () => {
+    const testCase = testCases[1]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle domain knowledge relevance', async () => {
+    const testCase = testCases[2]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle mixed relevance with good ordering', async () => {
+    const testCase = testCases[3]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle single relevant piece at start', async () => {
+    const testCase = testCases[4]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle single relevant piece in middle', async () => {
+    const testCase = testCases[5]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle single relevant piece at end', async () => {
+    const testCase = testCases[6]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle empty context', async () => {
+    const testCase = testCases[7]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle all irrelevant context', async () => {
+    const testCase = testCases[8]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle complex interdependent context', async () => {
+    const testCase = testCases[9]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle single piece context', async () => {
+    const testCase = testCases[10]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+
+  it('should handle two relevant pieces at end', async () => {
+    const testCase = testCases[11]!;
+    const result = await metric.measure({
+      input: testCase.input,
+      output: testCase.output,
+      context: testCase.context,
+    });
+    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+  });
+});