@mastra/evals 0.1.0-alpha.17 → 0.1.0-alpha.21

package/src/metrics/llm/summarization/index.test.ts

@@ -1,5 +1,5 @@
-import { describe, it, expect, jest } from '@jest/globals';
 import { type ModelConfig } from '@mastra/core';
+import { describe, it, expect } from 'vitest';
 
 import { TestCase } from '../utils';
 
@@ -166,7 +166,6 @@ const testCases: TestCase[] = [
 ];
 
 const SECONDS = 10000;
-jest.setTimeout(15 * SECONDS);
 
 const modelConfig: ModelConfig = {
   provider: 'OPEN_AI',
@@ -175,96 +174,102 @@ const modelConfig: ModelConfig = {
   apiKey: process.env.OPENAI_API_KEY,
 };
 
-describe('SummarizationMetric', () => {
-  const metric = new SummarizationMetric(modelConfig);
+describe(
+  'SummarizationMetric',
+  () => {
+    const metric = new SummarizationMetric(modelConfig);
 
-  it('should handle perfect summarization', async () => {
-    const testCase = testCases[0]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle perfect summarization', async () => {
+      const testCase = testCases[0]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle mixed accuracy with contradictions', async () => {
-    const testCase = testCases[1]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle mixed accuracy with contradictions', async () => {
+      const testCase = testCases[1]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle missing key information', async () => {
-    const testCase = testCases[2]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle missing key information', async () => {
+      const testCase = testCases[2]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle empty output', async () => {
-    const testCase = testCases[3]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBe(testCase.expectedResult.score);
-  });
+    it('should handle empty output', async () => {
+      const testCase = testCases[3]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBe(testCase.expectedResult.score);
+    });
 
-  it('should handle speculative additions', async () => {
-    const testCase = testCases[4]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle speculative additions', async () => {
+      const testCase = testCases[4]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle incorrect emphasis', async () => {
-    const testCase = testCases[5]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle incorrect emphasis', async () => {
+      const testCase = testCases[5]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle technical accuracy with missing context', async () => {
-    const testCase = testCases[6]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle technical accuracy with missing context', async () => {
+      const testCase = testCases[6]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle numerical approximation', async () => {
-    const testCase = testCases[7]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle numerical approximation', async () => {
+      const testCase = testCases[7]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle mixed tenses', async () => {
-    const testCase = testCases[8]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle mixed tenses', async () => {
+      const testCase = testCases[8]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle subjective interpretation', async () => {
-    const testCase = testCases[9]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle subjective interpretation', async () => {
+      const testCase = testCases[9]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle high alignment with low coverage', async () => {
-    const testCase = testCases[10]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle high alignment with low coverage', async () => {
+      const testCase = testCases[10]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle low alignment with high coverage', async () => {
-    const testCase = testCases[11]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle low alignment with high coverage', async () => {
+      const testCase = testCases[11]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle single word summary', async () => {
-    const testCase = testCases[12]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle single word summary', async () => {
+      const testCase = testCases[12]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle repetitive summary', async () => {
-    const testCase = testCases[13]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
+    it('should handle repetitive summary', async () => {
+      const testCase = testCases[13]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
 
-  it('should handle overly verbose summary', async () => {
-    const testCase = testCases[14]!;
-    const result = await metric.measure(testCase.input, testCase.output);
-    expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
-  });
-});
+    it('should handle overly verbose summary', async () => {
+      const testCase = testCases[14]!;
+      const result = await metric.measure(testCase.input, testCase.output);
+      expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
+    });
+  },
+  {
+    timeout: 15 * SECONDS,
+  },
+);

package/src/metrics/llm/toxicity/index.test.ts

@@ -1,5 +1,5 @@
-import { describe, it, expect, jest } from '@jest/globals';
 import { type ModelConfig } from '@mastra/core';
+import { describe, it, expect } from 'vitest';
 
 import { TestCase } from '../utils';
 
@@ -35,7 +35,6 @@ const testCases: TestCase[] = [
 ];
 
 const SECONDS = 10000;
-jest.setTimeout(15 * SECONDS);
 
 const modelConfig: ModelConfig = {
   provider: 'OPEN_AI',
@@ -44,24 +43,30 @@ const modelConfig: ModelConfig = {
   apiKey: process.env.OPENAI_API_KEY,
 };
 
-describe('ToxicityMetric', () => {
-  const metric = new ToxicityMetric(modelConfig);
+describe(
+  'ToxicityMetric',
+  () => {
+    const metric = new ToxicityMetric(modelConfig);
 
-  it('should be able to measure a prompt that is toxic', async () => {
-    const result = await metric.measure(testCases[0].input, testCases[0].output);
+    it('should be able to measure a prompt that is toxic', async () => {
+      const result = await metric.measure(testCases[0].input, testCases[0].output);
 
-    expect(result.score).toBeCloseTo(testCases[0].expectedResult.score, 1);
-  });
+      expect(result.score).toBeCloseTo(testCases[0].expectedResult.score, 1);
+    });
 
-  it('should be able to measure a prompt that is not toxic', async () => {
-    const result = await metric.measure(testCases[1].input, testCases[1].output);
+    it('should be able to measure a prompt that is not toxic', async () => {
+      const result = await metric.measure(testCases[1].input, testCases[1].output);
 
-    expect(result.score).toBeCloseTo(testCases[1].expectedResult.score, 1);
-  });
+      expect(result.score).toBeCloseTo(testCases[1].expectedResult.score, 1);
+    });
 
-  it('should be able to measure a prompt that is midly toxic', async () => {
-    const result = await metric.measure(testCases[2].input, testCases[2].output);
+    it('should be able to measure a prompt that is midly toxic', async () => {
+      const result = await metric.measure(testCases[2].input, testCases[2].output);
 
-    expect(result.score).toBeCloseTo(testCases[2].expectedResult.score, 1);
-  });
-});
+      expect(result.score).toBeCloseTo(testCases[2].expectedResult.score, 1);
+    });
+  },
+  {
+    timeout: 15 * SECONDS,
+  },
+);

package/src/metrics/nlp/completeness/index.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeEach } from '@jest/globals';
+import { describe, it, expect, beforeEach } from 'vitest';
 
 import { CompletenessMetric } from './index';
 

package/src/metrics/nlp/content-similarity/index.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect } from '@jest/globals';
+import { describe, it, expect } from 'vitest';
 
 import { ContentSimilarityMetric } from './index';
 

package/src/metrics/nlp/keyword-coverage/index.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect } from '@jest/globals';
+import { describe, it, expect } from 'vitest';
 
 import { KeywordCoverageMetric } from './index';
 

package/src/metrics/nlp/textual-difference/index.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect } from '@jest/globals';
+import { describe, it, expect } from 'vitest';
 
 import { TextualDifferenceMetric } from './index';
 

package/src/metrics/nlp/tone/index.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect } from '@jest/globals';
+import { describe, it, expect } from 'vitest';
 
 import { ToneConsistencyMetric } from './index';
 

package/vitest.config.ts

@@ -0,0 +1,9 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    environment: 'node',
+    include: ['src/**/*.test.ts'],
+    exclude: ['**/node_modules/**', '**/dist/**'],
+  },
+});

package/jest.config.ts

@@ -1,21 +0,0 @@
-import { config } from 'dotenv';
-
-config();
-
-export default {
-  maxWorkers: 1,
-  preset: 'ts-jest',
-  extensionsToTreatAsEsm: ['.ts'],
-  moduleNameMapper: {
-    '^(\\.{1,2}/.*)\\.js$': '$1',
-  },
-  transform: {
-    '^.+\\.tsx?$': [
-      'ts-jest',
-      {
-        useESM: true,
-        isolatedModules: true,
-      },
-    ],
-  },
-};