@mastra/evals 1.10.8-alpha.0 → 1.10.8-alpha.2

package/dist/scorers/prebuilt/index.js

@@ -1,4 +1,4 @@
-import { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, extractAgentResponseMessages, extractToolCalls, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getConversationHistoryFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isScorerRunInputForAgent, isScorerRunOutputForAgent, roundToTwoDecimals } from "../utils.js";
+import { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, extractAgentResponseMessages, extractToolCalls, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getConversationHistoryFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isScorerRunInputForAgent, isScorerRunOutputForAgent, roundToTwoDecimals, toToolLevelSteps } from "../utils.js";
 import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "../../checks-DxbqoaYq.js";
 import { createScorer } from "@mastra/core/evals";
 import nlp from "compromise";
@@ -3809,14 +3809,13 @@ function formatStepDetails(step) {
 			if (step.toolResult !== void 0) parts.push(`result: ${JSON.stringify(step.toolResult)}`);
 			return parts.length > 0 ? ` (${parts.join(", ")})` : "";
 		}
-		case "model_generation": return step.modelId ? ` (model: ${step.modelId})` : "";
 		case "workflow_step": return step.output !== void 0 ? ` (output: ${JSON.stringify(step.output)})` : "";
 		default: return "";
 	}
 }
 function formatTrajectory(trajectory, indent = 0) {
 	const prefix = "  ".repeat(indent);
-	return trajectory.steps.map((step, i) => {
+	return toToolLevelSteps(trajectory.steps).map((step, i) => {
 		let line = `${prefix}${i + 1}. [${step.stepType}] ${step.name}${formatStepDetails(step)}`;
 		if (step.children && step.children.length > 0) line += `\n${formatTrajectory({ steps: step.children }, indent + 1)}`;
 		return line;
@@ -4218,16 +4217,11 @@ function createToneScorer(config = {}) {
 //#region src/scorers/code/tool-call-accuracy/index.ts
 function checkToolOrder(actualTools, expectedOrder, strictMode = false) {
 	if (strictMode) return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);
-	const expectedIndices = [];
+	let searchFrom = 0;
 	for (const expectedTool of expectedOrder) {
-		const index = actualTools.indexOf(expectedTool);
+		const index = actualTools.indexOf(expectedTool, searchFrom);
 		if (index === -1) return false;
-		expectedIndices.push(index);
-	}
-	for (let i = 1; i < expectedIndices.length; i++) {
-		const currentIndex = expectedIndices[i];
-		const prevIndex = expectedIndices[i - 1];
-		if (currentIndex !== void 0 && prevIndex !== void 0 && currentIndex <= prevIndex) return false;
+		searchFrom = index + 1;
 	}
 	return true;
 }
@@ -4351,7 +4345,7 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
 			actualTrajectory,
 			expectedTrajectory: void 0,
 			comparison: void 0,
-			actualStepNames: actualTrajectory.steps.map((s) => s.name),
+			actualStepNames: toToolLevelSteps(actualTrajectory.steps).map((s) => s.name),
 			expectedStepNames: [],
 			error: "No expected trajectory provided (pass via options or dataset item expectedTrajectory)"
 		};
@@ -4366,7 +4360,7 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
 			actualTrajectory,
 			expectedTrajectory: { steps: resolvedExpectedSteps },
 			comparison,
-			actualStepNames: actualTrajectory.steps.map((s) => s.name),
+			actualStepNames: toToolLevelSteps(actualTrajectory.steps).map((s) => s.name),
 			expectedStepNames: resolvedExpectedSteps.map((s) => s.name)
 		};
 	}).generateScore(({ results }) => {
@@ -4389,12 +4383,13 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 	blacklist: .1
 }) {
 	const results = [];
-	const matchedIndices = /* @__PURE__ */ new Set();
+	const matchedSteps = /* @__PURE__ */ new Set();
+	const toolLevelSteps = toToolLevelSteps(actualSteps);
+	const containerNames = new Set(actualSteps.filter((s) => s.stepType === "model_generation" || s.stepType === "processor_run").map((s) => s.name));
 	for (const expectedStep of expectedSteps) {
 		if (!expectedStep.children) continue;
-		const matchIndex = actualSteps.findIndex((s, i) => !matchedIndices.has(i) && s.name === expectedStep.name && (!expectedStep.stepType || s.stepType === expectedStep.stepType));
-		const actualStep = matchIndex >= 0 ? actualSteps[matchIndex] : void 0;
-		if (matchIndex >= 0) matchedIndices.add(matchIndex);
+		const actualStep = (expectedStep.stepType === "model_generation" || expectedStep.stepType === "processor_run" || !expectedStep.stepType && containerNames.has(expectedStep.name) ? actualSteps : toolLevelSteps).find((s) => !matchedSteps.has(s) && s.name === expectedStep.name && (!expectedStep.stepType || s.stepType === expectedStep.stepType));
+		if (actualStep) matchedSteps.add(actualStep);
 		if (!actualStep) {
 			const expectedStepCount = expectedStep.children.steps?.length ?? 0;
 			results.push({