@mastra/evals 0.1.0-alpha.30 → 0.1.0-alpha.33

package/src/metrics/llm/bias/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { type MastraLLMBase } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';
@@ -11,8 +11,8 @@ import {
 } from './prompts';
 
 export class BiasJudge extends MastraAgentJudge {
-  constructor(model: ModelConfig) {
-    super('Bias', BIAS_AGENT_INSTRUCTIONS, model);
+  constructor(llm: MastraLLMBase) {
+    super('Bias', BIAS_AGENT_INSTRUCTIONS, llm);
   }
 
   async evaluate(input: string, actualOutput: string): Promise<{ verdict: string; reason: string }[]> {

package/src/metrics/llm/context-position/index.test.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { OpenAI } from '@mastra/core/llm/openai';
 import { describe, it, expect } from 'vitest';
 
 import { TestCaseWithContext } from '../utils';
@@ -151,96 +151,94 @@ const testCases: TestCaseWithContext[] = [
 
 const SECONDS = 10000;
 
-const modelConfig: ModelConfig = {
-  provider: 'OPEN_AI',
+const llm = new OpenAI({
   name: 'gpt-4o',
-  toolChoice: 'auto',
   apiKey: process.env.OPENAI_API_KEY,
-};
+});
 
 describe(
   'ContextPositionMetric',
   () => {
     it('should handle perfect ordering with all relevant pieces', async () => {
       const testCase = testCases[0]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle mixed relevance case', async () => {
       const testCase = testCases[1]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle domain knowledge relevance', async () => {
       const testCase = testCases[2]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle mixed relevance with good ordering', async () => {
       const testCase = testCases[3]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle single relevant piece at start', async () => {
       const testCase = testCases[4]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle single relevant piece in middle', async () => {
       const testCase = testCases[5]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle single relevant piece at end', async () => {
       const testCase = testCases[6]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle empty context', async () => {
       const testCase = testCases[7]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle all irrelevant context', async () => {
       const testCase = testCases[8]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle complex interdependent context', async () => {
       const testCase = testCases[9]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle single piece context', async () => {
       const testCase = testCases[10]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle two relevant pieces at end', async () => {
       const testCase = testCases[11]!;
-      const metric = new ContextPositionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPositionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });

package/src/metrics/llm/context-position/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type ModelConfig } from '@mastra/core';
+import { Metric } from '@mastra/core/eval';
+import { type MastraLLMBase } from '@mastra/core/llm';
 
 import { type MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
@@ -15,11 +16,12 @@ export class ContextPositionMetric extends Metric {
   private scale: number;
   private context: string[];
 
-  constructor(model: ModelConfig, { scale = 1, context }: ContextPositionMetricOptions) {
+  constructor(llm: MastraLLMBase, { scale = 1, context }: ContextPositionMetricOptions) {
     super();
-    this.judge = new ContextPositionJudge(model);
-    this.scale = scale;
+
     this.context = context;
+    this.judge = new ContextPositionJudge(llm);
+    this.scale = scale;
   }
 
   async measure(input: string, output: string): Promise<MetricResultWithReason> {

package/src/metrics/llm/context-position/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { type MastraLLMBase } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';
@@ -6,8 +6,8 @@ import { MastraAgentJudge } from '../../judge';
 import { CONTEXT_POSITION_AGENT_INSTRUCTIONS, generateEvaluatePrompt, generateReasonPrompt } from './prompts';
 
 export class ContextPositionJudge extends MastraAgentJudge {
-  constructor(model: ModelConfig) {
-    super('Context Position', CONTEXT_POSITION_AGENT_INSTRUCTIONS, model);
+  constructor(llm: MastraLLMBase) {
+    super('Context Position', CONTEXT_POSITION_AGENT_INSTRUCTIONS, llm);
   }
 
   async evaluate(

package/src/metrics/llm/context-precision/index.test.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { OpenAI } from '@mastra/core/llm/openai';
 import { describe, it, expect } from 'vitest';
 
 import { TestCaseWithContext } from '../utils';
@@ -128,82 +128,80 @@ const testCases: TestCaseWithContext[] = [
 
 const SECONDS = 10000;
 
-const modelConfig: ModelConfig = {
-  provider: 'OPEN_AI',
+const llm = new OpenAI({
   name: 'gpt-4o',
-  toolChoice: 'auto',
   apiKey: process.env.OPENAI_API_KEY,
-};
+});
 
 describe(
   'ContextPrecisionMetric',
   () => {
     it('should measure perfect context precision with all relevant items', async () => {
       const testCase = testCases[0]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should measure high precision with irrelevant item at end', async () => {
       const testCase = testCases[1]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should measure precision with two relevant items after irrelevant start', async () => {
       const testCase = testCases[2]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should measure precision with alternating relevant items', async () => {
       const testCase = testCases[3]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should measure precision with single relevant item at start', async () => {
       const testCase = testCases[4]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle completely irrelevant context', async () => {
       const testCase = testCases[5]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle single relevant context perfectly', async () => {
       const testCase = testCases[6]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should measure precision with single relevant item at end', async () => {
       const testCase = testCases[7]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle empty context', async () => {
       const testCase = testCases[8]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should handle single irrelevant context', async () => {
       const testCase = testCases[9]!;
-      const metric = new ContextPrecisionMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextPrecisionMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });

package/src/metrics/llm/context-precision/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type ModelConfig } from '@mastra/core';
+import { Metric } from '@mastra/core/eval';
+import { type MastraLLMBase } from '@mastra/core/llm';
 
 import { type MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
@@ -15,11 +16,12 @@ export class ContextPrecisionMetric extends Metric {
   private scale: number;
   private context: string[];
 
-  constructor(model: ModelConfig, { scale = 1, context }: ContextPrecisionMetricOptions) {
+  constructor(llm: MastraLLMBase, { scale = 1, context }: ContextPrecisionMetricOptions) {
     super();
-    this.judge = new ContextPrecisionJudge(model);
-    this.scale = scale;
+
     this.context = context;
+    this.judge = new ContextPrecisionJudge(llm);
+    this.scale = scale;
   }
 
   async measure(input: string, output: string): Promise<MetricResultWithReason> {

package/src/metrics/llm/context-precision/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { type MastraLLMBase } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';
@@ -7,8 +7,8 @@ import './prompts';
 import { CONTEXT_PRECISION_AGENT_INSTRUCTIONS, generateEvaluatePrompt, generateReasonPrompt } from './prompts';
 
 export class ContextPrecisionJudge extends MastraAgentJudge {
-  constructor(model: ModelConfig) {
-    super('Context Precision', CONTEXT_PRECISION_AGENT_INSTRUCTIONS, model);
+  constructor(llm: MastraLLMBase) {
+    super('Context Precision', CONTEXT_PRECISION_AGENT_INSTRUCTIONS, llm);
   }
 
   async evaluate(

package/src/metrics/llm/context-relevancy/index.test.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { OpenAI } from '@mastra/core/llm/openai';
 import { describe, it, expect } from 'vitest';
 
 import { isCloserTo } from '../utils';
@@ -56,33 +56,31 @@ const testCases: TestCaseWithContext[] = [
 
 const SECONDS = 10000;
 
-const modelConfig: ModelConfig = {
-  provider: 'OPEN_AI',
+const llm = new OpenAI({
   name: 'gpt-4o',
-  toolChoice: 'auto',
   apiKey: process.env.OPENAI_API_KEY,
-};
+});
 
 describe(
-  'ContextPrecisionMetric',
+  'ContextRelevancyMetric',
   () => {
     it('should measure perfect context relevancy with all relevant items', async () => {
       const testCase = testCases[0]!;
-      const metric = new ContextRelevancyMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextRelevancyMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });
 
     it('should measure mixed relevancy where only some contexts are relevant', async () => {
       const testCase = testCases[1]!;
-      const metric = new ContextRelevancyMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextRelevancyMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(isCloserTo(result.score, testCase.expectedResult.score, 0)).toBe(true);
     });
 
     it('should measure no relevancy where contexts are completely unrelated', async () => {
       const testCase = testCases[2]!;
-      const metric = new ContextRelevancyMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextRelevancyMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });

package/src/metrics/llm/context-relevancy/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type ModelConfig } from '@mastra/core';
+import { Metric } from '@mastra/core/eval';
+import { type MastraLLMBase } from '@mastra/core/llm';
 
 import { type MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
@@ -15,11 +16,12 @@ export class ContextRelevancyMetric extends Metric {
   private scale: number;
   private context: string[];
 
-  constructor(model: ModelConfig, { scale = 1, context }: ContextRelevancyOptions) {
+  constructor(llm: MastraLLMBase, { scale = 1, context }: ContextRelevancyOptions) {
     super();
-    this.judge = new ContextRelevancyJudge(model);
-    this.scale = scale;
+
     this.context = context;
+    this.judge = new ContextRelevancyJudge(llm);
+    this.scale = scale;
   }
 
   async measure(input: string, output: string): Promise<MetricResultWithReason> {

package/src/metrics/llm/context-relevancy/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { type MastraLLMBase } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';
@@ -6,8 +6,8 @@ import { MastraAgentJudge } from '../../judge';
 import { CONTEXT_RELEVANCY_AGENT_INSTRUCTIONS, generateEvaluatePrompt, generateReasonPrompt } from './prompts';
 
 export class ContextRelevancyJudge extends MastraAgentJudge {
-  constructor(model: ModelConfig) {
-    super('Context Relevancy', CONTEXT_RELEVANCY_AGENT_INSTRUCTIONS, model);
+  constructor(llm: MastraLLMBase) {
+    super('Context Relevancy', CONTEXT_RELEVANCY_AGENT_INSTRUCTIONS, llm);
   }
 
   async evaluate(

package/src/metrics/llm/contextual-recall/index.test.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { OpenAI } from '@mastra/core/llm/openai';
 import { describe, it, expect } from 'vitest';
 
 import { isCloserTo } from '../utils';
@@ -52,26 +52,24 @@ const testCases: TestCaseWithContext[] = [
 
 const SECONDS = 10000;
 
-const modelConfig: ModelConfig = {
-  provider: 'OPEN_AI',
+const llm = new OpenAI({
   name: 'gpt-4o',
-  toolChoice: 'auto',
   apiKey: process.env.OPENAI_API_KEY,
-};
+});
 
 describe(
   'ContextualRecallMetric',
   () => {
     it('should succeed when context is relevant', async () => {
       const testCase = testCases[0]!;
-      const metric = new ContextualRecallMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextualRecallMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 2);
     });
 
     it('should be mixed', async () => {
       const testCase = testCases[1]!;
-      const metric = new ContextualRecallMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextualRecallMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(isCloserTo(result.score, testCase.expectedResult.score, 1)).toBe(true);
@@ -80,7 +78,7 @@ describe(
 
     it('should be none', async () => {
       const testCase = testCases[2]!;
-      const metric = new ContextualRecallMetric(modelConfig, { context: testCase.context });
+      const metric = new ContextualRecallMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
     });

package/src/metrics/llm/contextual-recall/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type ModelConfig } from '@mastra/core';
+import { Metric } from '@mastra/core/eval';
+import { type MastraLLMBase } from '@mastra/core/llm';
 
 import { type MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
@@ -15,11 +16,12 @@ export class ContextualRecallMetric extends Metric {
   private scale: number;
   private context: string[];
 
-  constructor(model: ModelConfig, { scale = 1, context }: ContextualRecallMetricOptions) {
+  constructor(llm: MastraLLMBase, { scale = 1, context }: ContextualRecallMetricOptions) {
     super();
-    this.judge = new ContextualRecallJudge(model);
-    this.scale = scale;
+
     this.context = context;
+    this.judge = new ContextualRecallJudge(llm);
+    this.scale = scale;
   }
 
   async measure(input: string, output: string): Promise<MetricResultWithReason> {

package/src/metrics/llm/contextual-recall/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { type MastraLLMBase } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';
@@ -6,8 +6,8 @@ import { MastraAgentJudge } from '../../judge';
 import { CONTEXT_RECALL_AGENT_INSTRUCTIONS, generateEvaluatePrompt, generateReasonPrompt } from './prompts';
 
 export class ContextualRecallJudge extends MastraAgentJudge {
-  constructor(model: ModelConfig) {
-    super('Contextual Recall', CONTEXT_RECALL_AGENT_INSTRUCTIONS, model);
+  constructor(llm: MastraLLMBase) {
+    super('Contextual Recall', CONTEXT_RECALL_AGENT_INSTRUCTIONS, llm);
   }
 
   async evaluate(

package/src/metrics/llm/faithfulness/index.test.ts

@@ -1,4 +1,4 @@
-import { type ModelConfig } from '@mastra/core';
+import { OpenAI } from '@mastra/core/llm/openai';
 import { describe, it, expect } from 'vitest';
 
 import { TestCaseWithContext } from '../utils';
@@ -147,19 +147,17 @@ const testCases: TestCaseWithContext[] = [
 
 const SECONDS = 10000;
 
-const modelConfig: ModelConfig = {
-  provider: 'OPEN_AI',
+const llm = new OpenAI({
   name: 'gpt-4o',
-  toolChoice: 'auto',
   apiKey: process.env.OPENAI_API_KEY,
-};
+});
 
 describe(
   'FaithfulnessMetric',
   () => {
     it('should handle perfect faithfulness', async () => {
       const testCase = testCases[0]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
@@ -167,7 +165,7 @@ describe(
 
     it('should handle mixed faithfulness with contradictions', async () => {
       const testCase = testCases[1]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
@@ -175,7 +173,7 @@ describe(
 
     it('should handle claims with speculative language', async () => {
       const testCase = testCases[2]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
@@ -183,7 +181,7 @@ describe(
 
     it('should handle empty output', async () => {
       const testCase = testCases[3]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBe(testCase.expectedResult.score);
@@ -191,7 +189,7 @@ describe(
 
     it('should handle empty context', async () => {
       const testCase = testCases[4]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBe(testCase.expectedResult.score);
@@ -199,7 +197,7 @@ describe(
 
     it('should handle subjective claims', async () => {
       const testCase = testCases[5]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBe(testCase.expectedResult.score);
@@ -207,7 +205,7 @@ describe(
 
     it('should handle claims with speculative language appropriately', async () => {
       const testCase = testCases[6]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
@@ -215,7 +213,7 @@ describe(
 
     it('should handle compound statements correctly', async () => {
       const testCase = testCases[7]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
@@ -223,7 +221,7 @@ describe(
 
     it('should handle precise numerical claims', async () => {
       const testCase = testCases[8]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBe(testCase.expectedResult.score);
@@ -231,7 +229,7 @@ describe(
 
     it('should handle partially supported claims', async () => {
       const testCase = testCases[9]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
@@ -239,7 +237,7 @@ describe(
 
     it('should handle mixed factual and speculative claims', async () => {
       const testCase = testCases[10]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);
@@ -247,7 +245,7 @@ describe(
 
     it('should handle implicit information appropriately', async () => {
       const testCase = testCases[11]!;
-      const metric = new FaithfulnessMetric(modelConfig, { context: testCase.context });
+      const metric = new FaithfulnessMetric(llm, { context: testCase.context });
       const result = await metric.measure(testCase.input, testCase.output);
 
       expect(result.score).toBeCloseTo(testCase.expectedResult.score, 1);

package/src/metrics/llm/faithfulness/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type ModelConfig } from '@mastra/core';
+import { Metric } from '@mastra/core/eval';
+import { type MastraLLMBase } from '@mastra/core/llm';
 
 import { type MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
@@ -15,11 +16,12 @@ export class FaithfulnessMetric extends Metric {
   private scale: number;
   private context: string[];
 
-  constructor(model: ModelConfig, { scale = 1, context }: FaithfulnessMetricOptions) {
+  constructor(llm: MastraLLMBase, { scale = 1, context }: FaithfulnessMetricOptions) {
     super();
-    this.scale = scale;
+
     this.context = context;
-    this.judge = new FaithfulnessJudge(model);
+    this.judge = new FaithfulnessJudge(llm);
+    this.scale = scale;
   }
 
   async measure(input: string, output: string): Promise<MetricResultWithReason> {