@mastra/evals 1.10.1 → 1.10.2-alpha.0

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-evals
 description: Documentation for @mastra/evals. Use when working with @mastra/evals APIs, configuration, or implementation.
 metadata:
   package: "@mastra/evals"
-  version: "1.10.1"
+  version: "1.10.2-alpha.0"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.10.1",
+  "version": "1.10.2-alpha.0",
   "package": "@mastra/evals",
   "exports": {},
   "modules": {}

package/dist/scorers/code/trajectory/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/trajectory/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,2BAA2B,EAC3B,qBAAqB,EAEtB,MAAM,oBAAoB,CAAC;AAQ5B,OAAO,KAAK,EACV,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EAC1B,MAAM,aAAa,CAAC;AAErB,UAAU,mCAAmC;IAC3C;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,CAAC;IACjD,kCAAkC;IAClC,iBAAiB,CAAC,EAAE,2BAA2B,CAAC;CACjD;AA4BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,kCAAkC,CAAC,OAAO,GAAE,mCAAwC;;;;;;;;;;;QAoEtE,KAAK;;;;;gDAclC;AAID;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,yCAAyC;IACzC,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,wCAAwC;IACxC,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,2CAA2C;IAC3C,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,gDAAgD;IAChD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AAyJF;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAClC,kGAAkG;IAClG,KAAK,EAAE,MAAM,CAAC;IACd,wFAAwF;IACxF,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,mDAAmD;IACnD,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,uDAAuD;IACvD,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,6BAA6B;IAC7B,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,wDAAwD;IACxD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACrC,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,2BAA2B;IAC1C;;;OAGG;IACH,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC;;;;OAIG;IACH,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,GAAE,2BAAgC;;;;;;;6FA0NnF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/trajectory/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,2BAA2B,EAC3B,qBAAqB,EAEtB,MAAM,oBAAoB,CAAC;AAQ5B,OAAO,KAAK,EACV,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EAC1B,MAAM,aAAa,CAAC;AAErB,UAAU,mCAAmC;IAC3C;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,CAAC;IACjD,kCAAkC;IAClC,iBAAiB,CAAC,EAAE,2BAA2B,CAAC;CACjD;AA2BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,kCAAkC,CAAC,OAAO,GAAE,mCAAwC;;;;;;;;;;;QA+DtE,KAAK;;;;;gDAclC;AAID;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,yCAAyC;IACzC,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,wCAAwC;IACxC,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,2CAA2C;IAC3C,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,gDAAgD;IAChD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AA+JF;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAClC,kGAAkG;IAClG,KAAK,EAAE,MAAM,CAAC;IACd,wFAAwF;IACxF,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,mDAAmD;IACnD,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,uDAAuD;IACvD,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,6BAA6B;IAC7B,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,wDAAwD;IACxD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACrC,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,2BAA2B;IAC1C;;;OAGG;IACH,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC;;;;OAIG;IACH,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,GAAE,2BAAgC;;;;;;;6FA0NnF"}

package/dist/scorers/prebuilt/index.cjs

@@ -4295,7 +4295,6 @@ function trajectoryStepToExpectedStep(step) {
 * suitable for comparison.
 */
 function expectationToExpectedSteps(expectation) {
-	if (!expectation.steps || expectation.steps.length === 0) return void 0;
 	return expectation.steps;
 }
 /**
@@ -4334,7 +4333,7 @@ function expectationToExpectedSteps(expectation) {
 function createTrajectoryAccuracyScorerCode(options = {}) {
 	const { expectedTrajectory: staticExpectedTrajectory, comparisonOptions = {} } = options;
 	const { ordering = "relaxed", allowRepeatedSteps = true } = comparisonOptions;
-	const staticExpectedSteps = staticExpectedTrajectory ? Array.isArray(staticExpectedTrajectory) && staticExpectedTrajectory.length > 0 && !("steps" in staticExpectedTrajectory[0] || false) ? staticExpectedTrajectory : "steps" in staticExpectedTrajectory ? staticExpectedTrajectory.steps.map(trajectoryStepToExpectedStep) : void 0 : void 0;
+	const staticExpectedSteps = Array.isArray(staticExpectedTrajectory) ? staticExpectedTrajectory : staticExpectedTrajectory?.steps.map(trajectoryStepToExpectedStep);
 	const getDescription = () => {
 		if (staticExpectedSteps) return `Evaluates whether the trajectory matches the expected path: [${staticExpectedSteps.map((s) => s.name).join(" → ")}] (${ordering} ordering)`;
 		return `Evaluates trajectory accuracy against expected trajectory from dataset items (${ordering} ordering)`;
@@ -4351,7 +4350,7 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
 			const expectation = run.expectedTrajectory;
 			resolvedExpectedSteps = expectationToExpectedSteps(expectation);
 		}
-		if (!resolvedExpectedSteps || resolvedExpectedSteps.length === 0) return {
+		if (!resolvedExpectedSteps) return {
 			actualTrajectory,
 			expectedTrajectory: void 0,
 			comparison: void 0,
@@ -4379,6 +4378,9 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
 		return preprocessResult.comparison.score;
 	});
 }
+function hasBlacklistViolation(results) {
+	return results.some((result) => result.blacklist?.score === 0 || hasBlacklistViolation(result.nested ?? []));
+}
 /**
 * Evaluates nested expectations: for each expected step with a `children` config,
 * finds the matching actual step and recursively evaluates its children.
@@ -4396,7 +4398,7 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 		const matchIndex = actualSteps.findIndex((s, i) => !matchedIndices.has(i) && s.name === expectedStep.name && (!expectedStep.stepType || s.stepType === expectedStep.stepType));
 		const actualStep = matchIndex >= 0 ? actualSteps[matchIndex] : void 0;
 		if (matchIndex >= 0) matchedIndices.add(matchIndex);
-		if (!actualStep?.children || actualStep.children.length === 0) {
+		if (!actualStep) {
 			const expectedStepCount = expectedStep.children.steps?.length ?? 0;
 			results.push({
 				stepName: expectedStep.name,
@@ -4415,12 +4417,12 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 			continue;
 		}
 		const childTrajectory = {
-			steps: actualStep.children,
+			steps: actualStep.children ?? [],
 			totalDurationMs: actualStep.durationMs
 		};
 		const childConfig = expectedStep.children;
 		let accuracy;
-		if (childConfig.steps && childConfig.steps.length > 0) accuracy = require_scorers_utils.compareTrajectories(childTrajectory, { steps: childConfig.steps }, {
+		if (childConfig.steps) accuracy = require_scorers_utils.compareTrajectories(childTrajectory, { steps: childConfig.steps }, {
 			ordering: childConfig.ordering ?? "relaxed",
 			allowRepeatedSteps: childConfig.allowRepeatedSteps ?? true
 		});
@@ -4435,7 +4437,7 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 			blacklistedSequences: childConfig.blacklistedSequences
 		}) : void 0;
 		const toolFailures = require_scorers_utils.analyzeToolFailures(childTrajectory, { maxRetriesPerTool: childConfig.maxRetriesPerTool ?? 2 });
-		const nested = childConfig.steps ? evaluateNestedExpectations(childConfig.steps, actualStep.children, weights) : [];
+		const nested = childConfig.steps ? evaluateNestedExpectations(childConfig.steps, childTrajectory.steps, weights) : [];
 		const scores = [];
 		if (accuracy) scores.push({
 			weight: weights.accuracy,
@@ -4474,7 +4476,7 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 		}
 		let finalScore = levelScore;
 		if (nested.length > 0) {
-			if (nested.some((r) => r.blacklist && r.blacklist.score === 0)) {
+			if (hasBlacklistViolation(nested)) {
 				results.push({
 					stepName: expectedStep.name,
 					score: 0,
@@ -4556,7 +4558,7 @@ function createTrajectoryScorerCode(options = {}) {
 		};
 		if (itemExpectation.steps !== void 0) config.steps = itemExpectation.steps;
 		let accuracy;
-		if (config.steps && config.steps.length > 0) accuracy = require_scorers_utils.compareTrajectories(actualTrajectory, { steps: config.steps }, {
+		if (config.steps) accuracy = require_scorers_utils.compareTrajectories(actualTrajectory, { steps: config.steps }, {
 			ordering: config.ordering ?? "relaxed",
 			allowRepeatedSteps: config.allowRepeatedSteps ?? true
 		});
@@ -4607,7 +4609,7 @@ function createTrajectoryScorerCode(options = {}) {
 			levelScore = totalWeight > 0 ? scores.reduce((sum, s) => sum + s.weight / totalWeight * s.value, 0) : 1;
 		}
 		if (nested && nested.length > 0) {
-			if (nested.some((r) => r.blacklist && r.blacklist.score === 0)) return 0;
+			if (hasBlacklistViolation(nested)) return 0;
 			const nestedAvg = nested.reduce((sum, r) => sum + r.score, 0) / nested.length;
 			levelScore = .7 * levelScore + .3 * nestedAvg;
 		}
@@ -4623,8 +4625,8 @@ function createTrajectoryScorerCode(options = {}) {
 			parts.push(`Blacklist violation: ${violations.join(". ")}.`);
 			return parts.join("\n");
 		}
-		if (nested && nested.some((r) => r.blacklist && r.blacklist.score === 0)) {
-			const violating = nested.filter((r) => r.blacklist && r.blacklist.score === 0).map((r) => r.stepName);
+		if (nested && hasBlacklistViolation(nested)) {
+			const violating = nested.filter((r) => hasBlacklistViolation([r])).map((r) => r.stepName);
 			parts.push(`Nested blacklist violation in: ${violating.join(", ")}.`);
 			return parts.join("\n");
 		}