@mastra/evals 1.7.0 → 1.8.0

package/dist/scorers/prebuilt/index.js

@@ -1,4 +1,4 @@
-import { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, extractAgentResponseMessages, extractToolCalls, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isScorerRunInputForAgent, isScorerRunOutputForAgent, roundToTwoDecimals } from "../utils.js";
+import { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, extractAgentResponseMessages, extractToolCalls, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getConversationHistoryFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isScorerRunInputForAgent, isScorerRunOutputForAgent, roundToTwoDecimals } from "../utils.js";
 import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "../../checks-64AonnEK.js";
 import { createScorer } from "@mastra/core/evals";
 import nlp from "compromise";
@@ -2538,19 +2538,30 @@ Evaluation Guidelines:
 - Be objective and focus on alignment rather than response quality
 
 Score each dimension from 0.0 (completely misaligned) to 1.0 (perfectly aligned).`;
-function createAnalyzePrompt$2({ userPrompt, systemPrompt, agentResponse, evaluationMode }) {
-	let promptContext = "";
+function createConversationHistoryContext(conversationHistory) {
+	if (!conversationHistory) return "";
+	return `Conversation History (prior turns, for context only):
+${conversationHistory}
+
+`;
+}
+const CONVERSATION_HISTORY_GUIDANCE = `
+Note on conversation history:
+- The conversation history is provided only to resolve the meaning of the current prompt. Use it to interpret short or referential prompts (e.g. "A", "yes", "the second one") against what was previously asked or offered.
+- Evaluate only the agent response to the current prompt. Do not score earlier turns, and do not credit or penalise the agent for anything it said in them.`;
+function createAnalyzePrompt$2({ userPrompt, systemPrompt, agentResponse, evaluationMode, conversationHistory }) {
+	let promptContext = createConversationHistoryContext(conversationHistory);
 	let evaluationTarget = "";
 	if (evaluationMode === "user") {
-		promptContext = `User Prompt:
+		promptContext += `User Prompt:
 ${userPrompt}`;
 		evaluationTarget = "the user's prompt";
 	} else if (evaluationMode === "system") {
-		promptContext = `System Prompt:
+		promptContext += `System Prompt:
 ${systemPrompt}`;
 		evaluationTarget = "the system's behavioral guidelines and constraints";
 	} else {
-		promptContext = `User Prompt:
+		promptContext += `User Prompt:
 ${userPrompt}
 
 System Prompt:
@@ -2563,6 +2574,7 @@ ${promptContext}
 
 Agent Response:
 ${agentResponse}
+${conversationHistory ? CONVERSATION_HISTORY_GUIDANCE : ""}
 
 Evaluate the following aspects:
 
@@ -2685,10 +2697,10 @@ Agent Response: "def factorial(n):
   "overallAssessment": "The response perfectly aligns with the prompt, providing a correct Python factorial function with the requested error handling for negative numbers"
 }`;
 }
-function createReasonPrompt$1({ userPrompt, systemPrompt, score, scale, analysis, evaluationMode }) {
+function createReasonPrompt$1({ userPrompt, systemPrompt, score, scale, analysis, evaluationMode, conversationHistory }) {
 	const fulfilledCount = analysis.requirementsFulfillment.requirements.filter((r) => r.isFulfilled).length;
 	const totalRequirements = analysis.requirementsFulfillment.requirements.length;
-	const promptContext = evaluationMode === "system" ? `System Prompt:\n${systemPrompt}` : evaluationMode === "user" ? `User Prompt:\n${userPrompt}` : `User Prompt:\n${userPrompt}\n\nSystem Prompt:\n${systemPrompt}`;
+	const promptContext = createConversationHistoryContext(conversationHistory) + (evaluationMode === "system" ? `System Prompt:\n${systemPrompt}` : evaluationMode === "user" ? `User Prompt:\n${userPrompt}` : `User Prompt:\n${userPrompt}\n\nSystem Prompt:\n${systemPrompt}`);
 	return `Explain the prompt alignment score based on how well the agent's response addresses the ${evaluationMode === "system" ? "system behavioral guidelines and constraints" : evaluationMode === "user" ? "user's prompt" : "both user's prompt and system guidelines"}.
 
 ${promptContext}
@@ -2835,9 +2847,14 @@ const SCORING_WEIGHTS = {
 		SYSTEM_WEIGHT: .3
 	}
 };
+function normalizeConversationHistoryOptions(includeConversationHistory) {
+	if (!includeConversationHistory) return void 0;
+	return includeConversationHistory === true ? {} : includeConversationHistory;
+}
 function createPromptAlignmentScorerLLM({ model, options }) {
 	const scale = options?.scale || 1;
 	const evaluationMode = options?.evaluationMode || "both";
+	const historyOptions = normalizeConversationHistoryOptions(options?.includeConversationHistory);
 	return createScorer({
 		id: "prompt-alignment-scorer",
 		name: "Prompt Alignment (LLM)",
@@ -2861,7 +2878,8 @@ function createPromptAlignmentScorerLLM({ model, options }) {
 				userPrompt,
 				systemPrompt,
 				agentResponse,
-				evaluationMode
+				evaluationMode,
+				conversationHistory: historyOptions && getConversationHistoryFromRunInput(run.input, historyOptions)
 			});
 		}
 	}).generateScore(({ results }) => {
@@ -2897,7 +2915,8 @@ function createPromptAlignmentScorerLLM({ model, options }) {
 				score,
 				scale,
 				analysis,
-				evaluationMode
+				evaluationMode,
+				conversationHistory: historyOptions && getConversationHistoryFromRunInput(run.input, historyOptions)
 			});
 		}
 	});