@mastra/evals 1.10.1 → 1.10.2-alpha.1

package/dist/scorers/prebuilt/index.js

@@ -1,5 +1,5 @@
 import { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, extractAgentResponseMessages, extractToolCalls, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getConversationHistoryFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isScorerRunInputForAgent, isScorerRunOutputForAgent, roundToTwoDecimals } from "../utils.js";
-import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "../../checks-64AonnEK.js";
+import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "../../checks-DX9-p5w0.js";
 import { createScorer } from "@mastra/core/evals";
 import nlp from "compromise";
 import keyword_extractor from "keyword-extractor";
@@ -4290,7 +4290,6 @@ function trajectoryStepToExpectedStep(step) {
 * suitable for comparison.
 */
 function expectationToExpectedSteps(expectation) {
-	if (!expectation.steps || expectation.steps.length === 0) return void 0;
 	return expectation.steps;
 }
 /**
@@ -4329,7 +4328,7 @@ function expectationToExpectedSteps(expectation) {
 function createTrajectoryAccuracyScorerCode(options = {}) {
 	const { expectedTrajectory: staticExpectedTrajectory, comparisonOptions = {} } = options;
 	const { ordering = "relaxed", allowRepeatedSteps = true } = comparisonOptions;
-	const staticExpectedSteps = staticExpectedTrajectory ? Array.isArray(staticExpectedTrajectory) && staticExpectedTrajectory.length > 0 && !("steps" in staticExpectedTrajectory[0] || false) ? staticExpectedTrajectory : "steps" in staticExpectedTrajectory ? staticExpectedTrajectory.steps.map(trajectoryStepToExpectedStep) : void 0 : void 0;
+	const staticExpectedSteps = Array.isArray(staticExpectedTrajectory) ? staticExpectedTrajectory : staticExpectedTrajectory?.steps.map(trajectoryStepToExpectedStep);
 	const getDescription = () => {
 		if (staticExpectedSteps) return `Evaluates whether the trajectory matches the expected path: [${staticExpectedSteps.map((s) => s.name).join(" → ")}] (${ordering} ordering)`;
 		return `Evaluates trajectory accuracy against expected trajectory from dataset items (${ordering} ordering)`;
@@ -4346,7 +4345,7 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
 			const expectation = run.expectedTrajectory;
 			resolvedExpectedSteps = expectationToExpectedSteps(expectation);
 		}
-		if (!resolvedExpectedSteps || resolvedExpectedSteps.length === 0) return {
+		if (!resolvedExpectedSteps) return {
 			actualTrajectory,
 			expectedTrajectory: void 0,
 			comparison: void 0,
@@ -4374,6 +4373,9 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
 		return preprocessResult.comparison.score;
 	});
 }
+function hasBlacklistViolation(results) {
+	return results.some((result) => result.blacklist?.score === 0 || hasBlacklistViolation(result.nested ?? []));
+}
 /**
 * Evaluates nested expectations: for each expected step with a `children` config,
 * finds the matching actual step and recursively evaluates its children.
@@ -4391,7 +4393,7 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 		const matchIndex = actualSteps.findIndex((s, i) => !matchedIndices.has(i) && s.name === expectedStep.name && (!expectedStep.stepType || s.stepType === expectedStep.stepType));
 		const actualStep = matchIndex >= 0 ? actualSteps[matchIndex] : void 0;
 		if (matchIndex >= 0) matchedIndices.add(matchIndex);
-		if (!actualStep?.children || actualStep.children.length === 0) {
+		if (!actualStep) {
 			const expectedStepCount = expectedStep.children.steps?.length ?? 0;
 			results.push({
 				stepName: expectedStep.name,
@@ -4410,12 +4412,12 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 			continue;
 		}
 		const childTrajectory = {
-			steps: actualStep.children,
+			steps: actualStep.children ?? [],
 			totalDurationMs: actualStep.durationMs
 		};
 		const childConfig = expectedStep.children;
 		let accuracy;
-		if (childConfig.steps && childConfig.steps.length > 0) accuracy = compareTrajectories(childTrajectory, { steps: childConfig.steps }, {
+		if (childConfig.steps) accuracy = compareTrajectories(childTrajectory, { steps: childConfig.steps }, {
 			ordering: childConfig.ordering ?? "relaxed",
 			allowRepeatedSteps: childConfig.allowRepeatedSteps ?? true
 		});
@@ -4430,7 +4432,7 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 			blacklistedSequences: childConfig.blacklistedSequences
 		}) : void 0;
 		const toolFailures = analyzeToolFailures(childTrajectory, { maxRetriesPerTool: childConfig.maxRetriesPerTool ?? 2 });
-		const nested = childConfig.steps ? evaluateNestedExpectations(childConfig.steps, actualStep.children, weights) : [];
+		const nested = childConfig.steps ? evaluateNestedExpectations(childConfig.steps, childTrajectory.steps, weights) : [];
 		const scores = [];
 		if (accuracy) scores.push({
 			weight: weights.accuracy,
@@ -4469,7 +4471,7 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 		}
 		let finalScore = levelScore;
 		if (nested.length > 0) {
-			if (nested.some((r) => r.blacklist && r.blacklist.score === 0)) {
+			if (hasBlacklistViolation(nested)) {
 				results.push({
 					stepName: expectedStep.name,
 					score: 0,
@@ -4551,7 +4553,7 @@ function createTrajectoryScorerCode(options = {}) {
 		};
 		if (itemExpectation.steps !== void 0) config.steps = itemExpectation.steps;
 		let accuracy;
-		if (config.steps && config.steps.length > 0) accuracy = compareTrajectories(actualTrajectory, { steps: config.steps }, {
+		if (config.steps) accuracy = compareTrajectories(actualTrajectory, { steps: config.steps }, {
 			ordering: config.ordering ?? "relaxed",
 			allowRepeatedSteps: config.allowRepeatedSteps ?? true
 		});
@@ -4602,7 +4604,7 @@ function createTrajectoryScorerCode(options = {}) {
 			levelScore = totalWeight > 0 ? scores.reduce((sum, s) => sum + s.weight / totalWeight * s.value, 0) : 1;
 		}
 		if (nested && nested.length > 0) {
-			if (nested.some((r) => r.blacklist && r.blacklist.score === 0)) return 0;
+			if (hasBlacklistViolation(nested)) return 0;
 			const nestedAvg = nested.reduce((sum, r) => sum + r.score, 0) / nested.length;
 			levelScore = .7 * levelScore + .3 * nestedAvg;
 		}
@@ -4618,8 +4620,8 @@ function createTrajectoryScorerCode(options = {}) {
 			parts.push(`Blacklist violation: ${violations.join(". ")}.`);
 			return parts.join("\n");
 		}
-		if (nested && nested.some((r) => r.blacklist && r.blacklist.score === 0)) {
-			const violating = nested.filter((r) => r.blacklist && r.blacklist.score === 0).map((r) => r.stepName);
+		if (nested && hasBlacklistViolation(nested)) {
+			const violating = nested.filter((r) => hasBlacklistViolation([r])).map((r) => r.stepName);
 			parts.push(`Nested blacklist violation in: ${violating.join(", ")}.`);
 			return parts.join("\n");
 		}