@mastra/evals 1.10.2 → 1.10.3

package/dist/docs/references/reference-evals-trajectory-accuracy.md

@@ -221,7 +221,7 @@ const scorer = createTrajectoryAccuracyScorerCode({
 const scorer = createTrajectoryAccuracyScorerCode()
 
 await runEvals({
-  target: myAgent,
+  target: mastra.getAgent('myAgent'),
   scorers: { trajectory: [scorer] },
   data: [
     {
@@ -245,16 +245,20 @@ await runEvals({
 
 ### Evaluation modes
 
-The code-based scorer operates in two modes based on `strictOrder`:
+The code-based scorer operates in one of three modes based on `ordering`:
 
-#### Strict mode (`strictOrder: true`)
+#### Strict mode (`ordering: 'strict'`)
 
-Requires an exact match. The actual steps must match the expected steps in the same order with no extra or missing steps. Returns `1.0` for an exact match and `0.0` otherwise.
+Only an exact match (the same steps in the same order, with nothing extra or missing) scores `1.0`. Anything else gets partial credit for the expected steps that matched in position, with a penalty deducted for each extra step. For example, the two expected steps followed by one extra step scores `0.75`.
 
-#### Relaxed mode (`strictOrder: false`, default)
+#### Relaxed mode (`ordering: 'relaxed'`, default)
 
 Allows extra steps. Expected steps must appear in the correct relative order. The score is calculated based on how many expected steps were matched, with optional penalties for extra or repeated steps.
 
+#### Unordered mode (`ordering: 'unordered'`)
+
+Only checks that each expected step is present. Order is ignored, and extra steps are reported in the result but not penalized.
+
 ## Code-based scoring details
 
 - **Continuous scores**: Returns values between 0.0 and 1.0 in relaxed mode; binary (0 or 1) in strict mode
@@ -303,16 +307,16 @@ const scorer = createTrajectoryAccuracyScorerCode({
       { stepType: 'tool_call', name: 'fetch-tool' },
     ],
   },
-  comparisonOptions: { strictOrder: true },
+  comparisonOptions: { ordering: 'strict' },
 })
 
 const result = await runEvals({
-  target: myAgent,
+  target: mastra.getAgent('myAgent'),
   scorers: { trajectory: [scorer] },
   data: [{ input: 'Get my data' }],
 })
 
-console.log(result.scores.trajectory['trajectory-accuracy']) // 1.0
+console.log(result.scores.trajectory['code-trajectory-accuracy-scorer']) // 1.0
 ```
 
 ### Agent trajectory with relaxed ordering
@@ -327,7 +331,7 @@ const scorer = createTrajectoryAccuracyScorerCode({
       { stepType: 'tool_call', name: 'summarize-tool' },
     ],
   },
-  comparisonOptions: { strictOrder: false },
+  comparisonOptions: { ordering: 'relaxed' },
 })
 
 // Agent called search-tool → log-tool → summarize-tool
@@ -354,12 +358,12 @@ const scorer = createTrajectoryAccuracyScorerCode({
 })
 
 const result = await runEvals({
-  target: myWorkflow,
+  target: mastra.getWorkflow('myWorkflow'),
   scorers: { trajectory: [scorer] },
   data: [{ input: { data: 'test' } }],
 })
 
-console.log(result.scores.trajectory['trajectory-accuracy'])
+console.log(result.scores.trajectory['code-trajectory-accuracy-scorer'])
 ```
 
 ### Comparing step data
@@ -517,7 +521,7 @@ const scorer = createTrajectoryScorerCode({
 })
 
 const result = await runEvals({
-  target: myAgent,
+  target: mastra.getAgent('myAgent'),
   scorers: { trajectory: [scorer] },
   data: [
     {
@@ -583,7 +587,7 @@ const trajectoryScorer = createTrajectoryAccuracyScorerCode({
 })
 
 const result = await runEvals({
-  target: myAgent,
+  target: mastra.getAgent('myAgent'),
   scorers: {
     agent: [qualityScorer], // receives raw MastraDBMessage[] output
     trajectory: [trajectoryScorer], // receives pre-extracted Trajectory
@@ -592,7 +596,7 @@ const result = await runEvals({
 })
 
 // result.scores.agent['quality'] — agent-level score
-// result.scores.trajectory['trajectory-accuracy'] — trajectory score
+// result.scores.trajectory['code-trajectory-accuracy-scorer'] — trajectory score
 ```
 
 ### Workflow trajectory evaluation
@@ -612,7 +616,7 @@ const workflowTrajectoryScorer = createTrajectoryAccuracyScorerCode({
 })
 
 const result = await runEvals({
-  target: myWorkflow,
+  target: mastra.getWorkflow('myWorkflow'),
   scorers: {
     workflow: [outputScorer], // receives workflow output
     trajectory: [workflowTrajectoryScorer], // receives pre-extracted Trajectory from step results
@@ -621,7 +625,7 @@ const result = await runEvals({
 })
 
 // result.scores.workflow['output-quality'] — workflow-level score
-// result.scores.trajectory['trajectory-accuracy'] — trajectory score
+// result.scores.trajectory['code-trajectory-accuracy-scorer'] — trajectory score
 ```
 
 ## Related

package/dist/scorers/llm/faithfulness/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/faithfulness/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAQzF,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAYD,wBAAgB,wBAAwB,CAAC,EACvC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,yBAAyB,CAAC;CACrC;;;;;;;6FAgEA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/faithfulness/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAQzF,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAYD,wBAAgB,wBAAwB,CAAC,EACvC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,yBAAyB,CAAC;CACrC;;;;;;;6FAkEA"}

package/dist/scorers/llm/hallucination/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/hallucination/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAIjE,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAQzF,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,0BAA0B,CAAC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACrC,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,aAAa,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,GAAG,gBAAgB,CAAC;CACpC;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,gBAAgB,KAAK,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AAEtF,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,YAAY,CAAC;CAC3B;AAED,wBAAgB,yBAAyB,CAAC,EACxC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,0BAA0B,CAAC;CACtC;;;;;;;;6FA8EA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/hallucination/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAIjE,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAQzF,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,0BAA0B,CAAC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACrC,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,aAAa,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,GAAG,gBAAgB,CAAC;CACpC;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,gBAAgB,KAAK,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AAEtF,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,YAAY,CAAC;CAC3B;AAED,wBAAgB,yBAAyB,CAAC,EACxC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,0BAA0B,CAAC;CACtC;;;;;;;;6FAgFA"}

package/dist/scorers/prebuilt/index.cjs

@@ -431,7 +431,7 @@ const analyzeOutputSchema$7 = {
 				"type": "object",
 				"properties": {
 					"groundTruthUnit": { "type": "string" },
-					"outputUnit": { "anyOf": [{ "type": "string" }, { "type": "null" }] },
+					"outputUnit": { "type": ["string", "null"] },
 					"matchType": {
 						"type": "string",
 						"enum": [
@@ -762,10 +762,11 @@ function createFaithfulnessScorer({ model, options }) {
 			});
 		}
 	}).generateScore(({ results }) => {
-		const totalClaims = results.analyzeStepResult.verdicts.length;
-		const supportedClaims = results.analyzeStepResult.verdicts.filter((v) => v.verdict === "yes").length;
+		const verdicts = results.analyzeStepResult.verdicts;
+		const totalClaims = results.preprocessStepResult?.claims?.length || verdicts.length;
+		const supportedClaims = verdicts.filter((v) => v.verdict.toLowerCase().trim() === "yes").length;
 		if (totalClaims === 0) return 0;
-		return require_scorers_utils.roundToTwoDecimals(supportedClaims / totalClaims * (options?.scale || 1));
+		return require_scorers_utils.roundToTwoDecimals(Math.min(1, supportedClaims / totalClaims) * (options?.scale || 1));
 	}).generateReason({
 		description: "Reason about the results",
 		createPrompt: ({ run, results, score }) => {
@@ -1189,10 +1190,11 @@ function createHallucinationScorer({ model, options }) {
 			});
 		}
 	}).generateScore(({ results }) => {
-		const totalStatements = results.analyzeStepResult.verdicts.length;
-		const contradictedStatements = results.analyzeStepResult.verdicts.filter((v) => v.verdict === "yes").length;
+		const verdicts = results.analyzeStepResult.verdicts;
+		const totalStatements = results.preprocessStepResult?.claims?.length || verdicts.length;
+		const contradictedStatements = verdicts.filter((v) => v.verdict.toLowerCase().trim() === "yes").length;
 		if (totalStatements === 0) return 0;
-		return require_scorers_utils.roundToTwoDecimals(contradictedStatements / totalStatements * (options?.scale || 1));
+		return require_scorers_utils.roundToTwoDecimals(Math.min(1, contradictedStatements / totalStatements) * (options?.scale || 1));
 	}).generateReason({
 		description: "Reason about the results",
 		createPrompt: async ({ run, results, score }) => {