@mastra/evals 1.2.1 → 1.2.2

package/dist/scorers/prebuilt/index.js

@@ -1,4 +1,4 @@
-import { getAssistantMessageFromRunOutput, getUserMessageFromRunInput, roundToTwoDecimals, extractToolCalls, getCombinedSystemPrompt, getTextContentFromMastraDBMessage, compareTrajectories, checkTrajectoryEfficiency, checkTrajectoryBlacklist, analyzeToolFailures } from '../../chunk-X4MKZ735.js';
+import { getAssistantMessageFromRunOutput, getUserMessageFromRunInput, roundToTwoDecimals, extractToolCalls, getCombinedSystemPrompt, getTextContentFromMastraDBMessage, compareTrajectories, checkTrajectoryEfficiency, checkTrajectoryBlacklist, analyzeToolFailures, isScorerRunInputForAgent, isScorerRunOutputForAgent } from '../../chunk-ZRHCSFKL.js';
 import { createScorer } from '@mastra/core/evals';
 import { z } from 'zod';
 import nlp from 'compromise';
@@ -689,6 +689,10 @@ Example Responses:
 }
 
 // src/scorers/llm/faithfulness/index.ts
+var getToolInvocationContext = (output) => {
+  if (!Array.isArray(output)) return [];
+  return output.filter((message) => message?.role === "assistant").flatMap((message) => message?.content?.toolInvocations ?? []).filter((toolCall) => toolCall.state === "result").map((toolCall) => JSON.stringify(toolCall.result));
+};
 function createFaithfulnessScorer({
   model,
   options
@@ -715,10 +719,7 @@ function createFaithfulnessScorer({
     description: "Score the relevance of the statements to the input",
     outputSchema: z.object({ verdicts: z.array(z.object({ verdict: z.string(), reason: z.string() })) }),
     createPrompt: ({ results, run }) => {
-      const assistantMessage = run.output.find(({ role }) => role === "assistant");
-      const context = options?.context ?? assistantMessage?.content?.toolInvocations?.map(
-        (toolCall) => toolCall.state === "result" ? JSON.stringify(toolCall.result) : ""
-      ) ?? [];
+      const context = options?.context ?? getToolInvocationContext(run.output);
       const prompt = createFaithfulnessAnalyzePrompt({
         claims: results.preprocessStepResult?.claims || [],
         context
@@ -736,11 +737,10 @@ function createFaithfulnessScorer({
   }).generateReason({
     description: "Reason about the results",
     createPrompt: ({ run, results, score }) => {
-      const assistantMessage = run.output.find(({ role }) => role === "assistant");
       const prompt = createFaithfulnessReasonPrompt({
         input: getUserMessageFromRunInput(run.input) ?? "",
         output: getAssistantMessageFromRunOutput(run.output) ?? "",
-        context: assistantMessage?.content?.toolInvocations?.map((toolCall) => JSON.stringify(toolCall)) || [],
+        context: options?.context ?? getToolInvocationContext(run.output),
         score,
         scale: options?.scale || 1,
         verdicts: results.analyzeStepResult?.verdicts || []
@@ -1627,6 +1627,16 @@ var DEFAULT_PENALTIES = {
   MAX_MISSING_CONTEXT_PENALTY: 0.5
   // Maximum 50% penalty for missing context
 };
+var getContext = ({
+  input,
+  output,
+  options
+}) => {
+  if (options.contextExtractor && isScorerRunInputForAgent(input) && isScorerRunOutputForAgent(output)) {
+    return options.contextExtractor(input, output);
+  }
+  return options.context ?? [];
+};
 function createContextRelevanceScorerLLM({
   model,
   options
@@ -1652,7 +1662,7 @@ function createContextRelevanceScorerLLM({
     createPrompt: ({ run }) => {
       const userQuery = getUserMessageFromRunInput(run.input) ?? "";
       const agentResponse = getAssistantMessageFromRunOutput(run.output) ?? "";
-      const context = options.contextExtractor ? options.contextExtractor(run.input, run.output) : options.context;
+      const context = getContext({ input: run.input, output: run.output, options });
       if (context.length === 0) {
         return createAnalyzePrompt3({
           userQuery,
@@ -1668,7 +1678,7 @@ function createContextRelevanceScorerLLM({
     }
   }).generateScore(({ results, run }) => {
     const evaluations = results.analyzeStepResult?.evaluations || [];
-    const context = options.contextExtractor ? options.contextExtractor(run.input, run.output) : options.context;
+    const context = getContext({ input: run.input, output: run.output, options });
     if (context.length === 0) {
       return 1 * (options.scale || 1);
     }
@@ -1704,7 +1714,7 @@ function createContextRelevanceScorerLLM({
     description: "Generate human-readable explanation of context relevance evaluation",
     createPrompt: ({ run, results, score }) => {
       const userQuery = getUserMessageFromRunInput(run.input) ?? "";
-      const context = options.contextExtractor ? options.contextExtractor(run.input, run.output) : options.context;
+      const context = getContext({ input: run.input, output: run.output, options });
       if (context.length === 0) {
         return `No context was available for evaluation. The agent response was generated without any supporting context. Score: ${score}`;
       }
@@ -1851,6 +1861,16 @@ var contextRelevanceOutputSchema = z.object({
     })
   )
 });
+var getContext2 = ({
+  input,
+  output,
+  options
+}) => {
+  if (options.contextExtractor && isScorerRunInputForAgent(input) && isScorerRunOutputForAgent(output)) {
+    return options.contextExtractor(input, output);
+  }
+  return options.context ?? [];
+};
 function createContextPrecisionScorer({
   model,
   options
@@ -1876,7 +1896,7 @@ function createContextPrecisionScorer({
     createPrompt: ({ run }) => {
       const input = getUserMessageFromRunInput(run.input) ?? "";
       const output = getAssistantMessageFromRunOutput(run.output) ?? "";
-      const context = options.contextExtractor ? options.contextExtractor(run.input, run.output) : options.context;
+      const context = getContext2({ input: run.input, output: run.output, options });
       if (context.length === 0) {
         throw new Error("No context available for evaluation");
       }
@@ -1914,7 +1934,7 @@ function createContextPrecisionScorer({
     createPrompt: ({ run, results, score }) => {
       const input = getUserMessageFromRunInput(run.input) ?? "";
       const output = getAssistantMessageFromRunOutput(run.output) ?? "";
-      const context = options.contextExtractor ? options.contextExtractor(run.input, run.output) : options.context;
+      const context = getContext2({ input: run.input, output: run.output, options });
       return createContextPrecisionReasonPrompt({
         input,
         output,
@@ -2550,8 +2570,8 @@ function createPromptAlignmentScorerLLM({
       if (evaluationMode === "system" && !systemPrompt) {
         throw new Error("System prompt is required for system prompt alignment scoring");
       }
-      if (evaluationMode === "both" && (!userPrompt || !systemPrompt)) {
-        throw new Error("Both user and system prompts are required for combined alignment scoring");
+      if (evaluationMode === "both" && !userPrompt && !systemPrompt) {
+        throw new Error("A user or system prompt is required for combined alignment scoring");
       }
       if (!agentResponse) {
         throw new Error("Agent response is required for prompt alignment scoring");