@mastra/evals 1.0.1 → 1.1.0

package/dist/scorers/prebuilt/index.js

@@ -1,4 +1,4 @@
-import { getAssistantMessageFromRunOutput, getUserMessageFromRunInput, roundToTwoDecimals, extractToolCalls, getCombinedSystemPrompt, getTextContentFromMastraDBMessage } from '../../chunk-6EA6D7JG.js';
+import { getAssistantMessageFromRunOutput, getUserMessageFromRunInput, roundToTwoDecimals, extractToolCalls, getCombinedSystemPrompt, getTextContentFromMastraDBMessage } from '../../chunk-OEOE7ZHN.js';
 import { createScorer } from '@mastra/core/evals';
 import { z } from 'zod';
 import nlp from 'compromise';
@@ -1114,10 +1114,16 @@ function createHallucinationScorer({
     outputSchema: z.object({
       verdicts: z.array(z.object({ statement: z.string(), verdict: z.string(), reason: z.string() }))
     }),
-    createPrompt: ({ results }) => {
+    createPrompt: async ({ run, results }) => {
+      let context;
+      if (options?.getContext) {
+        context = await options.getContext({ run, results, step: "analyze" });
+      } else {
+        context = options?.context ?? [];
+      }
       const prompt = createHallucinationAnalyzePrompt({
         claims: results.preprocessStepResult.claims,
-        context: options?.context || []
+        context
       });
       return prompt;
     }
@@ -1131,11 +1137,17 @@ function createHallucinationScorer({
     return roundToTwoDecimals(score);
   }).generateReason({
     description: "Reason about the results",
-    createPrompt: ({ run, results, score }) => {
+    createPrompt: async ({ run, results, score }) => {
+      let context;
+      if (options?.getContext) {
+        context = await options.getContext({ run, results, score, step: "generateReason" });
+      } else {
+        context = options?.context ?? [];
+      }
       const prompt = createHallucinationReasonPrompt({
         input: getUserMessageFromRunInput(run.input) ?? "",
         output: getAssistantMessageFromRunOutput(run.output) ?? "",
-        context: options?.context || [],
+        context,
         score,
         scale: options?.scale || 1,
         verdicts: results.analyzeStepResult?.verdicts || []