@mastra/evals 0.1.0 → 0.1.1

package/src/metrics/llm/bias/index.ts

@@ -1,7 +1,7 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
-import { type MetricResultWithReason } from '../types';
+import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
 
 import { BiasJudge } from './metricJudge';
@@ -24,10 +24,10 @@ export class BiasMetric extends Metric {
   async measure(input: string, output: string): Promise<MetricResultWithReason> {
     const verdicts = await this.judge.evaluate(input, output);
     const score = this.calculateScore(verdicts);
-    const reason = await this.judge.getReason(
+    const reason = await this.judge.getReason({
       score,
-      verdicts.filter(Boolean).map(v => v.reason),
-    );
+      biases: verdicts.filter(Boolean).map(v => v.reason),
+    });
 
     return {
       score,

package/src/metrics/llm/bias/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';
@@ -40,8 +40,8 @@ export class BiasJudge extends MastraAgentJudge {
     return result.object.verdicts;
   }
 
-  async getReason(score: number, biases: string[]): Promise<string> {
-    const prompt = generateReasonPrompt({ score, biases });
+  async getReason(args: { score: number; biases: string[] }): Promise<string> {
+    const prompt = generateReasonPrompt(args);
     const result = await this.agent.generate(prompt, {
       output: z.object({
         reason: z.string(),

package/src/metrics/llm/context-position/index.test.ts

@@ -1,7 +1,8 @@
 import { createOpenAI } from '@ai-sdk/openai';
 import { describe, it, expect } from 'vitest';
 
-import { isCloserTo, TestCaseWithContext } from '../utils';
+import type { TestCaseWithContext } from '../utils';
+import { isCloserTo } from '../utils';
 
 import { ContextPositionMetric } from './index';
 

package/src/metrics/llm/context-position/index.ts

@@ -1,7 +1,7 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
-import { type MetricResultWithReason } from '../types';
+import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
 
 import { ContextPositionJudge } from './metricJudge';
@@ -27,7 +27,7 @@ export class ContextPositionMetric extends Metric {
   async measure(input: string, output: string): Promise<MetricResultWithReason> {
     const verdicts = await this.judge.evaluate(input, output, this.context);
     const score = this.calculateScore(verdicts);
-    const reason = await this.judge.getReason(input, output, score, this.scale, verdicts);
+    const reason = await this.judge.getReason({ input, output, score, scale: this.scale, verdicts });
 
     return {
       score,

package/src/metrics/llm/context-position/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';
@@ -34,17 +34,17 @@ export class ContextPositionJudge extends MastraAgentJudge {
     return result.object.verdicts;
   }
 
-  async getReason(
-    input: string,
-    actualOutput: string,
-    score: number,
-    scale: number,
+  async getReason(args: {
+    input: string;
+    output: string;
+    score: number;
+    scale: number;
     verdicts: {
       verdict: string;
       reason: string;
-    }[],
-  ): Promise<string> {
-    const prompt = generateReasonPrompt({ input, output: actualOutput, verdicts, score, scale });
+    }[];
+  }): Promise<string> {
+    const prompt = generateReasonPrompt(args);
     const result = await this.agent.generate(prompt, {
       output: z.object({
         reason: z.string(),

package/src/metrics/llm/context-precision/index.test.ts

@@ -1,7 +1,7 @@
 import { createOpenAI } from '@ai-sdk/openai';
 import { describe, it, expect } from 'vitest';
 
-import { TestCaseWithContext } from '../utils';
+import type { TestCaseWithContext } from '../utils';
 
 import { ContextPrecisionMetric } from './index';
 

package/src/metrics/llm/context-precision/index.ts

@@ -1,7 +1,7 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
-import { type MetricResultWithReason } from '../types';
+import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
 
 import { ContextPrecisionJudge } from './metricJudge';
@@ -27,7 +27,7 @@ export class ContextPrecisionMetric extends Metric {
   async measure(input: string, output: string): Promise<MetricResultWithReason> {
     const verdicts = await this.judge.evaluate(input, output, this.context);
     const score = this.calculateScore(verdicts);
-    const reason = await this.judge.getReason(input, output, score, this.scale, verdicts);
+    const reason = await this.judge.getReason({ input, output, score, scale: this.scale, verdicts });
 
     return {
       score,

package/src/metrics/llm/context-precision/metricJudge.ts

@@ -1,9 +1,8 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';
 
-import './prompts';
 import { CONTEXT_PRECISION_AGENT_INSTRUCTIONS, generateEvaluatePrompt, generateReasonPrompt } from './prompts';
 
 export class ContextPrecisionJudge extends MastraAgentJudge {
@@ -35,17 +34,17 @@ export class ContextPrecisionJudge extends MastraAgentJudge {
     return result.object.verdicts;
   }
 
-  async getReason(
-    input: string,
-    actualOutput: string,
-    score: number,
-    scale: number,
+  async getReason(args: {
+    input: string;
+    output: string;
+    score: number;
+    scale: number;
     verdicts: {
       verdict: string;
       reason: string;
-    }[],
-  ): Promise<string> {
-    const prompt = generateReasonPrompt({ input, output: actualOutput, verdicts, score, scale });
+    }[];
+  }): Promise<string> {
+    const prompt = generateReasonPrompt(args);
     const result = await this.agent.generate(prompt, {
       output: z.object({
         reason: z.string(),

package/src/metrics/llm/context-relevancy/index.test.ts

@@ -1,8 +1,8 @@
 import { createOpenAI } from '@ai-sdk/openai';
 import { describe, it, expect } from 'vitest';
 
+import type { TestCaseWithContext } from '../utils';
 import { isCloserTo } from '../utils';
-import { TestCaseWithContext } from '../utils';
 
 import { ContextRelevancyMetric } from './index';
 

package/src/metrics/llm/context-relevancy/index.ts

@@ -1,7 +1,7 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
-import { type MetricResultWithReason } from '../types';
+import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
 
 import { ContextRelevancyJudge } from './metricJudge';

package/src/metrics/llm/context-relevancy/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';

package/src/metrics/llm/contextual-recall/index.test.ts

@@ -1,8 +1,8 @@
 import { createOpenAI } from '@ai-sdk/openai';
 import { describe, it, expect } from 'vitest';
 
+import type { TestCaseWithContext } from '../utils';
 import { isCloserTo } from '../utils';
-import { TestCaseWithContext } from '../utils';
 
 import { ContextualRecallMetric } from './index';
 

package/src/metrics/llm/contextual-recall/index.ts

@@ -1,7 +1,7 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
-import { type MetricResultWithReason } from '../types';
+import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
 
 import { ContextualRecallJudge } from './metricJudge';

package/src/metrics/llm/contextual-recall/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';

package/src/metrics/llm/faithfulness/index.test.ts

@@ -1,7 +1,7 @@
 import { openai } from '@ai-sdk/openai';
 import { describe, it, expect } from 'vitest';
 
-import { TestCaseWithContext } from '../utils';
+import type { TestCaseWithContext } from '../utils';
 
 import { FaithfulnessMetric } from './index';
 

package/src/metrics/llm/faithfulness/index.ts

@@ -1,7 +1,7 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
-import { type MetricResultWithReason } from '../types';
+import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
 
 import { FaithfulnessJudge } from './metricJudge';

package/src/metrics/llm/faithfulness/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';

package/src/metrics/llm/hallucination/index.test.ts

@@ -1,7 +1,7 @@
 import { openai } from '@ai-sdk/openai';
 import { describe, it, expect } from 'vitest';
 
-import { TestCaseWithContext } from '../utils';
+import type { TestCaseWithContext } from '../utils';
 
 import { HallucinationMetric } from './index';
 

package/src/metrics/llm/hallucination/index.ts

@@ -1,7 +1,7 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
-import { type MetricResultWithReason } from '../types';
+import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
 
 import { HallucinationJudge } from './metricJudge';

package/src/metrics/llm/hallucination/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';

package/src/metrics/llm/index.ts

@@ -2,6 +2,7 @@ export { AnswerRelevancyMetric } from './answer-relevancy';
 export { ContextPositionMetric } from './context-position';
 export { ContextPrecisionMetric } from './context-precision';
 export { FaithfulnessMetric } from './faithfulness';
+export { HallucinationMetric } from './hallucination';
 export { PromptAlignmentMetric } from './prompt-alignment';
 export { ToxicityMetric } from './toxicity';
 export { ContextRelevancyMetric } from './context-relevancy';

package/src/metrics/llm/prompt-alignment/index.test.ts

@@ -1,7 +1,7 @@
 import { createOpenAI } from '@ai-sdk/openai';
 import { describe, it, expect } from 'vitest';
 
-import { TestCase } from '../utils';
+import type { TestCase } from '../utils';
 
 import { PromptAlignmentMetric } from './index';
 

package/src/metrics/llm/prompt-alignment/index.ts

@@ -1,5 +1,5 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
 import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';

package/src/metrics/llm/prompt-alignment/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';

package/src/metrics/llm/summarization/index.test.ts

@@ -1,7 +1,8 @@
 import { createOpenAI } from '@ai-sdk/openai';
 import { describe, it, expect, vi } from 'vitest';
 
-import { isCloserTo, TestCase } from '../utils';
+import type { TestCase } from '../utils';
+import { isCloserTo } from '../utils';
 
 import { SummarizationMetric } from './index';
 

package/src/metrics/llm/summarization/index.ts

@@ -1,7 +1,7 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
-import { type MetricResultWithReason } from '../types';
+import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
 
 import { SummarizationJudge } from './metricJudge';

package/src/metrics/llm/summarization/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';

package/src/metrics/llm/toxicity/index.test.ts

@@ -1,7 +1,7 @@
 import { openai } from '@ai-sdk/openai';
 import { describe, it, expect } from 'vitest';
 
-import { TestCase } from '../utils';
+import type { TestCase } from '../utils';
 
 import { ToxicityMetric } from './index';
 

package/src/metrics/llm/toxicity/index.ts

@@ -1,7 +1,7 @@
 import { Metric } from '@mastra/core/eval';
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 
-import { type MetricResultWithReason } from '../types';
+import type { MetricResultWithReason } from '../types';
 import { roundToTwoDecimals } from '../utils';
 
 import { ToxicityJudge } from './metricJudge';

package/src/metrics/llm/toxicity/metricJudge.ts

@@ -1,4 +1,4 @@
-import { type LanguageModel } from '@mastra/core/llm';
+import type { LanguageModel } from '@mastra/core/llm';
 import { z } from 'zod';
 
 import { MastraAgentJudge } from '../../judge';
@@ -26,8 +26,8 @@ export class ToxicityJudge extends MastraAgentJudge {
     return result.object.verdicts;
   }
 
-  async getReason({ score, toxics }: { score: number; toxics: string[] }): Promise<string> {
-    const prompt = getReasonPrompt({ score, toxics });
+  async getReason(args: { score: number; toxics: string[] }): Promise<string> {
+    const prompt = getReasonPrompt(args);
     const result = await this.agent.generate(prompt, {
       output: z.object({
         reason: z.string(),

package/src/metrics/llm/types.ts

@@ -1,4 +1,4 @@
-import { type MetricResult } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
 
 export interface MetricResultWithReason extends MetricResult {
   info: {

package/src/metrics/nlp/completeness/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type MetricResult } from '@mastra/core/eval';
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
 import nlp from 'compromise';
 
 interface CompletenessMetricResult extends MetricResult {

package/src/metrics/nlp/content-similarity/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type MetricResult } from '@mastra/core/eval';
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
 import stringSimilarity from 'string-similarity';
 
 interface ContentSimilarityResult extends MetricResult {

package/src/metrics/nlp/keyword-coverage/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type MetricResult } from '@mastra/core/eval';
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
 import keyword_extractor from 'keyword-extractor';
 
 interface KeywordCoverageResult extends MetricResult {

package/src/metrics/nlp/textual-difference/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type MetricResult } from '@mastra/core/eval';
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
 import { SequenceMatcher } from 'difflib';
 
 interface TextualDifferenceResult extends MetricResult {

package/src/metrics/nlp/tone/index.ts

@@ -1,4 +1,5 @@
-import { Metric, type MetricResult } from '@mastra/core/eval';
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
 import Sentiment from 'sentiment';
 
 interface ToneConsitencyResult extends MetricResult {