@mastra/evals 1.10.1 → 1.10.2-alpha.1

package/dist/{checks-DGTgg-nW.cjs → checks-CGRgwpDb.cjs}

@@ -323,8 +323,11 @@ function usedNoTools() {
 	});
 }
 /**
-* Scores 1 if none of the tool invocations resulted in an error state.
-* Checks for tool invocations with state other than 'result' (i.e., missing results).
+* Scores 1 if none of the tool invocations resulted in an error.
+*
+* Treats every failure representation a native run can persist as an error: a
+* thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a
+* result the executor flagged with `isError`, and a result carrying an `error` field.
 *
 * @example
 * ```ts
@@ -340,7 +343,7 @@ function noToolErrors() {
 		type: "agent"
 	}).preprocess(async ({ run }) => {
 		const invocations = extractRawInvocations(run.output);
-		const errorCount = invocations.filter((inv) => inv.state === "call" || inv.result && inv.result.error).length;
+		const errorCount = invocations.filter(isErrorInvocation).length;
 		return {
 			errorCount,
 			totalCalls: invocations.length,
@@ -350,13 +353,21 @@ function noToolErrors() {
 		return results.preprocessStepResult?.passed ? 1 : 0;
 	});
 }
+/**
+* True when an invocation represents a tool failure rather than a success.
+*
+* A natively thrown tool call is persisted as `state: 'output-error'` with
+* `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.
+* `isError` is set alongside `state: 'result'` when the executor reported a failure
+* even though the result carries no `error` field; core's `extractTrajectory`
+* likewise defines success as `state === 'result' && isError !== true`.
+*/
+function isErrorInvocation(inv) {
+	return inv.state === "call" || inv.state === "output-error" || inv.isError === true || Boolean(inv.result?.error);
+}
 function extractRawInvocations(output) {
 	const invocations = [];
-	for (const message of output) {
-		const legacy = message?.content?.toolInvocations;
-		const fromParts = legacy ? void 0 : (message?.content)?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
-		for (const inv of legacy ?? fromParts ?? []) if (inv) invocations.push(inv);
-	}
+	for (const message of output) for (const inv of require_scorers_utils.mergeToolInvocations(message)) if (inv) invocations.push(inv);
 	return invocations;
 }
 /**
@@ -476,4 +487,4 @@ Object.defineProperty(exports, "usedNoTools", {
 	}
 });
 
-//# sourceMappingURL=checks-DGTgg-nW.cjs.map
+//# sourceMappingURL=checks-CGRgwpDb.cjs.map

package/dist/checks-CGRgwpDb.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"checks-CGRgwpDb.cjs","names":["getTextContentFromMastraDBMessage","stringSimilarity","extractToolCalls","mergeToolInvocations"],"sources":["../src/scorers/code/checks/index.ts"],"sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error.\n *\n * Treats every failure representation a native run can persist as an error: a\n * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a\n * result the executor flagged with `isError`, and a result carrying an `error` field.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(isErrorInvocation).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\n/**\n * True when an invocation represents a tool failure rather than a success.\n *\n * A natively thrown tool call is persisted as `state: 'output-error'` with\n * `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.\n * `isError` is set alongside `state: 'result'` when the executor reported a failure\n * even though the result carries no `error` field; core's `extractTrajectory`\n * likewise defines success as `state === 'result' && isError !== true`.\n */\nfunction isErrorInvocation(inv: any): boolean {\n  return inv.state === 'call' || inv.state === 'output-error' || inv.isError === true || Boolean(inv.result?.error);\n}\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    // Merge both storage forms so a call present only in `content.parts` — which is\n    // where thrown calls live — is never hidden by a legacy `toolInvocations` array.\n    for (const inv of mergeToolInvocations(message)) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,8BAA8B,SAAS;EACpD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,OAAO,OAAO,SAAS,MAAM;EAAE;CAC1D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,QAAQ,IAAI;CACnD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,wCAAwC,SAAS;EAC9D,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,UAAU,CAAC,OAAO,SAAS,MAAM;EAAE;CAC9D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,WAAW,IAAI;CACtD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,OAAO,UAAkB,UAA2B,CAAC,GAAG;CACtE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,SAAS;EAClD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,SAAS,WAAW;EAAO;CACtD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;AAgBA,SAAgB,QAAQ,SAAiB,UAA0B,CAAC,GAAG;CACrE,MAAM,EAAE,QAAQ,UAAU;CAC1B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,oCAAoC;EACjD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,SAAS,IAAI,OAChB,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EAEV,MAAM,WADQ,QAAQ,IAAI,OAAO,IAAI,QAAQ,OAAO,IAAI,QAAQ,KAAK,IAAI,QAAA,CACnD,KAAK,MAAM;EACjC,OAAO;GAAE;GAAQ,SAAS,QAAQ,SAAS;GAAG;EAAQ;CACxD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,aAAa,MAAM,cAAc;CACzC,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gCAAgC,SAAS;EACtD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,MAAM,QAAQC,kBAAAA,QAAiB,kBAAkB,QAAQ,MAAM;EAC/D,OAAO;GAAE;GAAQ;GAAQ;GAAO;EAAU;CAC5C,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,QAAQ,QAAQ,sBAAsB,SAAS;EACrD,MAAM,IAAI,QAAQ,sBAAsB;EACxC,OAAO,MAAM,KAAA,IAAa,SAAS,IAAI,IAAI,IAAK;CAClD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,QAAQ,MAAM;CACtB,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS,cAAc,QAAQ,IAAI,aAAa,MAAM,UAAU;EAC7F,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUC,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU,eAAe;GAAO,aAAa;GAAO,QAAQ,SAAS;EAAM;CACtF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,WAAW,UAAkB;CAC3C,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS;EACtC,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU;GAAO,QAAQ,UAAU;EAAE;CAChD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,UAAU,eAAyB;CACjD,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,cAAc,KAAK,KAAK,EAAE;EACnE,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAE7C,IAAI,aAAa;EACjB,KAAK,MAAM,QAAQ,OACjB,IAAI,aAAa,cAAc,UAAU,SAAS,cAAc,aAC9D;EAIJ,OAAO;GAAE,aAAa;GAAO;GAAe,QAD7B,eAAe,cAAc;EACO;CACrD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,aAAa,KAAa;CACxC,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,IAAI;EAC7C,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ;GAAK,QAAQ,MAAM,UAAU;EAAI;CACjE,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,cAAc;CAC5B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ,QAAQ,MAAM,WAAW;EAAE;CAC3D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;;;;;AAeA,SAAgB,eAAe;CAC7B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,cAAc,sBAAsB,IAAI,MAAM;EACpD,MAAM,aAAa,YAAY,OAAO,iBAAiB,CAAC,CAAC;EACzD,OAAO;GAAE;GAAY,YAAY,YAAY;GAAQ,QAAQ,eAAe;EAAE;CAChF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAaA,SAAS,kBAAkB,KAAmB;CAC5C,OAAO,IAAI,UAAU,UAAU,IAAI,UAAU,kBAAkB,IAAI,YAAY,QAAQ,QAAQ,IAAI,QAAQ,KAAK;AAClH;AAEA,SAAS,sBAAsB,QAAgD;CAC7E,MAAM,cAAqB,CAAC;CAC5B,KAAK,MAAM,WAAW,QAGpB,KAAK,MAAM,OAAOC,sBAAAA,qBAAqB,OAAO,GAC5C,IAAI,KAAK,YAAY,KAAK,GAAG;CAGjC,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAa,SAAS;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/dist/{checks-64AonnEK.js → checks-DX9-p5w0.js}

@@ -1,4 +1,4 @@
-import { extractToolCalls, getTextContentFromMastraDBMessage } from "./scorers/utils.js";
+import { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from "./scorers/utils.js";
 import { createScorer } from "@mastra/core/evals";
 import stringSimilarity from "string-similarity";
 //#region src/scorers/code/checks/index.ts
@@ -300,8 +300,11 @@ function usedNoTools() {
 	});
 }
 /**
-* Scores 1 if none of the tool invocations resulted in an error state.
-* Checks for tool invocations with state other than 'result' (i.e., missing results).
+* Scores 1 if none of the tool invocations resulted in an error.
+*
+* Treats every failure representation a native run can persist as an error: a
+* thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a
+* result the executor flagged with `isError`, and a result carrying an `error` field.
 *
 * @example
 * ```ts
@@ -317,7 +320,7 @@ function noToolErrors() {
 		type: "agent"
 	}).preprocess(async ({ run }) => {
 		const invocations = extractRawInvocations(run.output);
-		const errorCount = invocations.filter((inv) => inv.state === "call" || inv.result && inv.result.error).length;
+		const errorCount = invocations.filter(isErrorInvocation).length;
 		return {
 			errorCount,
 			totalCalls: invocations.length,
@@ -327,13 +330,21 @@ function noToolErrors() {
 		return results.preprocessStepResult?.passed ? 1 : 0;
 	});
 }
+/**
+* True when an invocation represents a tool failure rather than a success.
+*
+* A natively thrown tool call is persisted as `state: 'output-error'` with
+* `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.
+* `isError` is set alongside `state: 'result'` when the executor reported a failure
+* even though the result carries no `error` field; core's `extractTrajectory`
+* likewise defines success as `state === 'result' && isError !== true`.
+*/
+function isErrorInvocation(inv) {
+	return inv.state === "call" || inv.state === "output-error" || inv.isError === true || Boolean(inv.result?.error);
+}
 function extractRawInvocations(output) {
 	const invocations = [];
-	for (const message of output) {
-		const legacy = message?.content?.toolInvocations;
-		const fromParts = legacy ? void 0 : (message?.content)?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
-		for (const inv of legacy ?? fromParts ?? []) if (inv) invocations.push(inv);
-	}
+	for (const message of output) for (const inv of mergeToolInvocations(message)) if (inv) invocations.push(inv);
 	return invocations;
 }
 /**
@@ -376,4 +387,4 @@ const checks = {
 //#endregion
 export { excludes as a, maxToolCalls as c, toolOrder as d, usedNoTools as f, equals as i, noToolErrors as l, checks as n, includes as o, didNotCall as r, matches as s, calledTool as t, similarity as u };
 
-//# sourceMappingURL=checks-64AonnEK.js.map
+//# sourceMappingURL=checks-DX9-p5w0.js.map

package/dist/checks-DX9-p5w0.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checks-DX9-p5w0.js","names":[],"sources":["../src/scorers/code/checks/index.ts"],"sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error.\n *\n * Treats every failure representation a native run can persist as an error: a\n * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a\n * result the executor flagged with `isError`, and a result carrying an `error` field.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(isErrorInvocation).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\n/**\n * True when an invocation represents a tool failure rather than a success.\n *\n * A natively thrown tool call is persisted as `state: 'output-error'` with\n * `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.\n * `isError` is set alongside `state: 'result'` when the executor reported a failure\n * even though the result carries no `error` field; core's `extractTrajectory`\n * likewise defines success as `state === 'result' && isError !== true`.\n */\nfunction isErrorInvocation(inv: any): boolean {\n  return inv.state === 'call' || inv.state === 'output-error' || inv.isError === true || Boolean(inv.result?.error);\n}\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    // Merge both storage forms so a call present only in `content.parts` — which is\n    // where thrown calls live — is never hidden by a legacy `toolInvocations` array.\n    for (const inv of mergeToolInvocations(message)) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"],"mappings":";;;;;;;;;;;;;AAoBA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,8BAA8B,SAAS;EACpD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,OAAO,OAAO,SAAS,MAAM;EAAE;CAC1D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,QAAQ,IAAI;CACnD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,wCAAwC,SAAS;EAC9D,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,UAAU,CAAC,OAAO,SAAS,MAAM;EAAE;CAC9D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,WAAW,IAAI;CACtD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,OAAO,UAAkB,UAA2B,CAAC,GAAG;CACtE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,SAAS;EAClD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,SAAS,WAAW;EAAO;CACtD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;AAgBA,SAAgB,QAAQ,SAAiB,UAA0B,CAAC,GAAG;CACrE,MAAM,EAAE,QAAQ,UAAU;CAC1B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,oCAAoC;EACjD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,SAAS,IAAI,OAChB,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EAEV,MAAM,WADQ,QAAQ,IAAI,OAAO,IAAI,QAAQ,OAAO,IAAI,QAAQ,KAAK,IAAI,QAAA,CACnD,KAAK,MAAM;EACjC,OAAO;GAAE;GAAQ,SAAS,QAAQ,SAAS;GAAG;EAAQ;CACxD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,aAAa,MAAM,cAAc;CACzC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gCAAgC,SAAS;EACtD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,MAAM,QAAQ,iBAAiB,kBAAkB,QAAQ,MAAM;EAC/D,OAAO;GAAE;GAAQ;GAAQ;GAAO;EAAU;CAC5C,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,QAAQ,QAAQ,sBAAsB,SAAS;EACrD,MAAM,IAAI,QAAQ,sBAAsB;EACxC,OAAO,MAAM,KAAA,IAAa,SAAS,IAAI,IAAI,IAAK;CAClD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,QAAQ,MAAM;CACtB,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS,cAAc,QAAQ,IAAI,aAAa,MAAM,UAAU;EAC7F,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU,eAAe;GAAO,aAAa;GAAO,QAAQ,SAAS;EAAM;CACtF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,WAAW,UAAkB;CAC3C,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS;EACtC,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU;GAAO,QAAQ,UAAU;EAAE;CAChD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,UAAU,eAAyB;CACjD,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,cAAc,KAAK,KAAK,EAAE;EACnE,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAE7C,IAAI,aAAa;EACjB,KAAK,MAAM,QAAQ,OACjB,IAAI,aAAa,cAAc,UAAU,SAAS,cAAc,aAC9D;EAIJ,OAAO;GAAE,aAAa;GAAO;GAAe,QAD7B,eAAe,cAAc;EACO;CACrD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,aAAa,KAAa;CACxC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,IAAI;EAC7C,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ;GAAK,QAAQ,MAAM,UAAU;EAAI;CACjE,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,cAAc;CAC5B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ,QAAQ,MAAM,WAAW;EAAE;CAC3D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;;;;;AAeA,SAAgB,eAAe;CAC7B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,cAAc,sBAAsB,IAAI,MAAM;EACpD,MAAM,aAAa,YAAY,OAAO,iBAAiB,CAAC,CAAC;EACzD,OAAO;GAAE;GAAY,YAAY,YAAY;GAAQ,QAAQ,eAAe;EAAE;CAChF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAaA,SAAS,kBAAkB,KAAmB;CAC5C,OAAO,IAAI,UAAU,UAAU,IAAI,UAAU,kBAAkB,IAAI,YAAY,QAAQ,QAAQ,IAAI,QAAQ,KAAK;AAClH;AAEA,SAAS,sBAAsB,QAAgD;CAC7E,MAAM,cAAqB,CAAC;CAC5B,KAAK,MAAM,WAAW,QAGpB,KAAK,MAAM,OAAO,qBAAqB,OAAO,GAC5C,IAAI,KAAK,YAAY,KAAK,GAAG;CAGjC,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAa,SAAS;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/dist/checks.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_checks = require("./checks-DGTgg-nW.cjs");
+const require_checks = require("./checks-CGRgwpDb.cjs");
 exports.calledTool = require_checks.calledTool;
 exports.checks = require_checks.checks;
 exports.didNotCall = require_checks.didNotCall;

package/dist/checks.js

@@ -1,2 +1,2 @@
-import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "./checks-64AonnEK.js";
+import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "./checks-DX9-p5w0.js";
 export { calledTool, checks, didNotCall, equals, excludes, includes, matches, maxToolCalls, noToolErrors, similarity, toolOrder, usedNoTools };

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-evals
 description: Documentation for @mastra/evals. Use when working with @mastra/evals APIs, configuration, or implementation.
 metadata:
   package: "@mastra/evals"
-  version: "1.10.1"
+  version: "1.10.2-alpha.1"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.10.1",
+  "version": "1.10.2-alpha.1",
   "package": "@mastra/evals",
   "exports": {},
   "modules": {}

package/dist/scorers/code/checks/index.d.ts

@@ -160,8 +160,11 @@ export declare function usedNoTools(): import("@mastra/core/evals").MastraScorer
     passed: boolean;
 }> & Record<"generateScoreStepResult", number>>;
 /**
- * Scores 1 if none of the tool invocations resulted in an error state.
- * Checks for tool invocations with state other than 'result' (i.e., missing results).
+ * Scores 1 if none of the tool invocations resulted in an error.
+ *
+ * Treats every failure representation a native run can persist as an error: a
+ * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a
+ * result the executor flagged with `isError`, and a result carrying an `error` field.
  *
  * @example
  * ```ts

package/dist/scorers/code/checks/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/checks/index.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBrE;AAED,MAAM,WAAW,cAAc;IAC7B,uGAAuG;IACvG,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB;;;;gDAoBpE;AAED,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDA0B3E;AAID,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDAgB3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM;;;;gDAe1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,aAAa,EAAE,MAAM,EAAE;;;;gDAsBhD;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM;;;;gDAcvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW;;;gDAc1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY;;;;gDAe3B;AAsBD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,eAAO,MAAM,MAAM;;;;;;;;;;;;CAYlB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/checks/index.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBrE;AAED,MAAM,WAAW,cAAc;IAC7B,uGAAuG;IACvG,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB;;;;gDAoBpE;AAED,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDA0B3E;AAID,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDAgB3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM;;;;gDAe1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,aAAa,EAAE,MAAM,EAAE;;;;gDAsBhD;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM;;;;gDAcvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW;;;gDAc1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,YAAY;;;;gDAe3B;AA+BD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,eAAO,MAAM,MAAM;;;;;;;;;;;;CAYlB,CAAC"}

package/dist/scorers/code/trajectory/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/trajectory/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,2BAA2B,EAC3B,qBAAqB,EAEtB,MAAM,oBAAoB,CAAC;AAQ5B,OAAO,KAAK,EACV,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EAC1B,MAAM,aAAa,CAAC;AAErB,UAAU,mCAAmC;IAC3C;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,CAAC;IACjD,kCAAkC;IAClC,iBAAiB,CAAC,EAAE,2BAA2B,CAAC;CACjD;AA4BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,kCAAkC,CAAC,OAAO,GAAE,mCAAwC;;;;;;;;;;;QAoEtE,KAAK;;;;;gDAclC;AAID;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,yCAAyC;IACzC,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,wCAAwC;IACxC,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,2CAA2C;IAC3C,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,gDAAgD;IAChD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AAyJF;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAClC,kGAAkG;IAClG,KAAK,EAAE,MAAM,CAAC;IACd,wFAAwF;IACxF,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,mDAAmD;IACnD,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,uDAAuD;IACvD,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,6BAA6B;IAC7B,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,wDAAwD;IACxD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACrC,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,2BAA2B;IAC1C;;;OAGG;IACH,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC;;;;OAIG;IACH,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,GAAE,2BAAgC;;;;;;;6FA0NnF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/trajectory/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,2BAA2B,EAC3B,qBAAqB,EAEtB,MAAM,oBAAoB,CAAC;AAQ5B,OAAO,KAAK,EACV,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EAC1B,MAAM,aAAa,CAAC;AAErB,UAAU,mCAAmC;IAC3C;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,CAAC;IACjD,kCAAkC;IAClC,iBAAiB,CAAC,EAAE,2BAA2B,CAAC;CACjD;AA2BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,kCAAkC,CAAC,OAAO,GAAE,mCAAwC;;;;;;;;;;;QA+DtE,KAAK;;;;;gDAclC;AAID;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,yCAAyC;IACzC,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,wCAAwC;IACxC,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,2CAA2C;IAC3C,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,gDAAgD;IAChD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AA+JF;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAClC,kGAAkG;IAClG,KAAK,EAAE,MAAM,CAAC;IACd,wFAAwF;IACxF,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,mDAAmD;IACnD,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,uDAAuD;IACvD,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,6BAA6B;IAC7B,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,wDAAwD;IACxD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACrC,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,2BAA2B;IAC1C;;;OAGG;IACH,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC;;;;OAIG;IACH,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,GAAE,2BAAgC;;;;;;;6FA0NnF"}

package/dist/scorers/prebuilt/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_checks = require("../../checks-DGTgg-nW.cjs");
+const require_checks = require("../../checks-CGRgwpDb.cjs");
 const require_scorers_utils = require("../utils.cjs");
 let _mastra_core_evals = require("@mastra/core/evals");
 let compromise = require("compromise");
@@ -4295,7 +4295,6 @@ function trajectoryStepToExpectedStep(step) {
 * suitable for comparison.
 */
 function expectationToExpectedSteps(expectation) {
-	if (!expectation.steps || expectation.steps.length === 0) return void 0;
 	return expectation.steps;
 }
 /**
@@ -4334,7 +4333,7 @@ function expectationToExpectedSteps(expectation) {
 function createTrajectoryAccuracyScorerCode(options = {}) {
 	const { expectedTrajectory: staticExpectedTrajectory, comparisonOptions = {} } = options;
 	const { ordering = "relaxed", allowRepeatedSteps = true } = comparisonOptions;
-	const staticExpectedSteps = staticExpectedTrajectory ? Array.isArray(staticExpectedTrajectory) && staticExpectedTrajectory.length > 0 && !("steps" in staticExpectedTrajectory[0] || false) ? staticExpectedTrajectory : "steps" in staticExpectedTrajectory ? staticExpectedTrajectory.steps.map(trajectoryStepToExpectedStep) : void 0 : void 0;
+	const staticExpectedSteps = Array.isArray(staticExpectedTrajectory) ? staticExpectedTrajectory : staticExpectedTrajectory?.steps.map(trajectoryStepToExpectedStep);
 	const getDescription = () => {
 		if (staticExpectedSteps) return `Evaluates whether the trajectory matches the expected path: [${staticExpectedSteps.map((s) => s.name).join(" → ")}] (${ordering} ordering)`;
 		return `Evaluates trajectory accuracy against expected trajectory from dataset items (${ordering} ordering)`;
@@ -4351,7 +4350,7 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
 			const expectation = run.expectedTrajectory;
 			resolvedExpectedSteps = expectationToExpectedSteps(expectation);
 		}
-		if (!resolvedExpectedSteps || resolvedExpectedSteps.length === 0) return {
+		if (!resolvedExpectedSteps) return {
 			actualTrajectory,
 			expectedTrajectory: void 0,
 			comparison: void 0,
@@ -4379,6 +4378,9 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
 		return preprocessResult.comparison.score;
 	});
 }
+function hasBlacklistViolation(results) {
+	return results.some((result) => result.blacklist?.score === 0 || hasBlacklistViolation(result.nested ?? []));
+}
 /**
 * Evaluates nested expectations: for each expected step with a `children` config,
 * finds the matching actual step and recursively evaluates its children.
@@ -4396,7 +4398,7 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 		const matchIndex = actualSteps.findIndex((s, i) => !matchedIndices.has(i) && s.name === expectedStep.name && (!expectedStep.stepType || s.stepType === expectedStep.stepType));
 		const actualStep = matchIndex >= 0 ? actualSteps[matchIndex] : void 0;
 		if (matchIndex >= 0) matchedIndices.add(matchIndex);
-		if (!actualStep?.children || actualStep.children.length === 0) {
+		if (!actualStep) {
 			const expectedStepCount = expectedStep.children.steps?.length ?? 0;
 			results.push({
 				stepName: expectedStep.name,
@@ -4415,12 +4417,12 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 			continue;
 		}
 		const childTrajectory = {
-			steps: actualStep.children,
+			steps: actualStep.children ?? [],
 			totalDurationMs: actualStep.durationMs
 		};
 		const childConfig = expectedStep.children;
 		let accuracy;
-		if (childConfig.steps && childConfig.steps.length > 0) accuracy = require_scorers_utils.compareTrajectories(childTrajectory, { steps: childConfig.steps }, {
+		if (childConfig.steps) accuracy = require_scorers_utils.compareTrajectories(childTrajectory, { steps: childConfig.steps }, {
 			ordering: childConfig.ordering ?? "relaxed",
 			allowRepeatedSteps: childConfig.allowRepeatedSteps ?? true
 		});
@@ -4435,7 +4437,7 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 			blacklistedSequences: childConfig.blacklistedSequences
 		}) : void 0;
 		const toolFailures = require_scorers_utils.analyzeToolFailures(childTrajectory, { maxRetriesPerTool: childConfig.maxRetriesPerTool ?? 2 });
-		const nested = childConfig.steps ? evaluateNestedExpectations(childConfig.steps, actualStep.children, weights) : [];
+		const nested = childConfig.steps ? evaluateNestedExpectations(childConfig.steps, childTrajectory.steps, weights) : [];
 		const scores = [];
 		if (accuracy) scores.push({
 			weight: weights.accuracy,
@@ -4474,7 +4476,7 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
 		}
 		let finalScore = levelScore;
 		if (nested.length > 0) {
-			if (nested.some((r) => r.blacklist && r.blacklist.score === 0)) {
+			if (hasBlacklistViolation(nested)) {
 				results.push({
 					stepName: expectedStep.name,
 					score: 0,
@@ -4556,7 +4558,7 @@ function createTrajectoryScorerCode(options = {}) {
 		};
 		if (itemExpectation.steps !== void 0) config.steps = itemExpectation.steps;
 		let accuracy;
-		if (config.steps && config.steps.length > 0) accuracy = require_scorers_utils.compareTrajectories(actualTrajectory, { steps: config.steps }, {
+		if (config.steps) accuracy = require_scorers_utils.compareTrajectories(actualTrajectory, { steps: config.steps }, {
 			ordering: config.ordering ?? "relaxed",
 			allowRepeatedSteps: config.allowRepeatedSteps ?? true
 		});
@@ -4607,7 +4609,7 @@ function createTrajectoryScorerCode(options = {}) {
 			levelScore = totalWeight > 0 ? scores.reduce((sum, s) => sum + s.weight / totalWeight * s.value, 0) : 1;
 		}
 		if (nested && nested.length > 0) {
-			if (nested.some((r) => r.blacklist && r.blacklist.score === 0)) return 0;
+			if (hasBlacklistViolation(nested)) return 0;
 			const nestedAvg = nested.reduce((sum, r) => sum + r.score, 0) / nested.length;
 			levelScore = .7 * levelScore + .3 * nestedAvg;
 		}
@@ -4623,8 +4625,8 @@ function createTrajectoryScorerCode(options = {}) {
 			parts.push(`Blacklist violation: ${violations.join(". ")}.`);
 			return parts.join("\n");
 		}
-		if (nested && nested.some((r) => r.blacklist && r.blacklist.score === 0)) {
-			const violating = nested.filter((r) => r.blacklist && r.blacklist.score === 0).map((r) => r.stepName);
+		if (nested && hasBlacklistViolation(nested)) {
+			const violating = nested.filter((r) => hasBlacklistViolation([r])).map((r) => r.stepName);
 			parts.push(`Nested blacklist violation in: ${violating.join(", ")}.`);
 			return parts.join("\n");
 		}