@mastra/evals 1.10.2-alpha.0 → 1.10.2-alpha.1

package/dist/{checks-DGTgg-nW.cjs → checks-CGRgwpDb.cjs}

@@ -323,8 +323,11 @@ function usedNoTools() {
 	});
 }
 /**
-* Scores 1 if none of the tool invocations resulted in an error state.
-* Checks for tool invocations with state other than 'result' (i.e., missing results).
+* Scores 1 if none of the tool invocations resulted in an error.
+*
+* Treats every failure representation a native run can persist as an error: a
+* thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a
+* result the executor flagged with `isError`, and a result carrying an `error` field.
 *
 * @example
 * ```ts
@@ -340,7 +343,7 @@ function noToolErrors() {
 		type: "agent"
 	}).preprocess(async ({ run }) => {
 		const invocations = extractRawInvocations(run.output);
-		const errorCount = invocations.filter((inv) => inv.state === "call" || inv.result && inv.result.error).length;
+		const errorCount = invocations.filter(isErrorInvocation).length;
 		return {
 			errorCount,
 			totalCalls: invocations.length,
@@ -350,13 +353,21 @@ function noToolErrors() {
 		return results.preprocessStepResult?.passed ? 1 : 0;
 	});
 }
+/**
+* True when an invocation represents a tool failure rather than a success.
+*
+* A natively thrown tool call is persisted as `state: 'output-error'` with
+* `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.
+* `isError` is set alongside `state: 'result'` when the executor reported a failure
+* even though the result carries no `error` field; core's `extractTrajectory`
+* likewise defines success as `state === 'result' && isError !== true`.
+*/
+function isErrorInvocation(inv) {
+	return inv.state === "call" || inv.state === "output-error" || inv.isError === true || Boolean(inv.result?.error);
+}
 function extractRawInvocations(output) {
 	const invocations = [];
-	for (const message of output) {
-		const legacy = message?.content?.toolInvocations;
-		const fromParts = legacy ? void 0 : (message?.content)?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
-		for (const inv of legacy ?? fromParts ?? []) if (inv) invocations.push(inv);
-	}
+	for (const message of output) for (const inv of require_scorers_utils.mergeToolInvocations(message)) if (inv) invocations.push(inv);
 	return invocations;
 }
 /**
@@ -476,4 +487,4 @@ Object.defineProperty(exports, "usedNoTools", {
 	}
 });
 
-//# sourceMappingURL=checks-DGTgg-nW.cjs.map
+//# sourceMappingURL=checks-CGRgwpDb.cjs.map

package/dist/checks-CGRgwpDb.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"checks-CGRgwpDb.cjs","names":["getTextContentFromMastraDBMessage","stringSimilarity","extractToolCalls","mergeToolInvocations"],"sources":["../src/scorers/code/checks/index.ts"],"sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error.\n *\n * Treats every failure representation a native run can persist as an error: a\n * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a\n * result the executor flagged with `isError`, and a result carrying an `error` field.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(isErrorInvocation).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\n/**\n * True when an invocation represents a tool failure rather than a success.\n *\n * A natively thrown tool call is persisted as `state: 'output-error'` with\n * `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.\n * `isError` is set alongside `state: 'result'` when the executor reported a failure\n * even though the result carries no `error` field; core's `extractTrajectory`\n * likewise defines success as `state === 'result' && isError !== true`.\n */\nfunction isErrorInvocation(inv: any): boolean {\n  return inv.state === 'call' || inv.state === 'output-error' || inv.isError === true || Boolean(inv.result?.error);\n}\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    // Merge both storage forms so a call present only in `content.parts` — which is\n    // where thrown calls live — is never hidden by a legacy `toolInvocations` array.\n    for (const inv of mergeToolInvocations(message)) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,8BAA8B,SAAS;EACpD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,OAAO,OAAO,SAAS,MAAM;EAAE;CAC1D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,QAAQ,IAAI;CACnD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,wCAAwC,SAAS;EAC9D,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,UAAU,CAAC,OAAO,SAAS,MAAM;EAAE;CAC9D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,WAAW,IAAI;CACtD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,OAAO,UAAkB,UAA2B,CAAC,GAAG;CACtE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,SAAS;EAClD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,SAAS,WAAW;EAAO;CACtD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;AAgBA,SAAgB,QAAQ,SAAiB,UAA0B,CAAC,GAAG;CACrE,MAAM,EAAE,QAAQ,UAAU;CAC1B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,oCAAoC;EACjD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,SAAS,IAAI,OAChB,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EAEV,MAAM,WADQ,QAAQ,IAAI,OAAO,IAAI,QAAQ,OAAO,IAAI,QAAQ,KAAK,IAAI,QAAA,CACnD,KAAK,MAAM;EACjC,OAAO;GAAE;GAAQ,SAAS,QAAQ,SAAS;GAAG;EAAQ;CACxD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,aAAa,MAAM,cAAc;CACzC,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gCAAgC,SAAS;EACtD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,MAAM,QAAQC,kBAAAA,QAAiB,kBAAkB,QAAQ,MAAM;EAC/D,OAAO;GAAE;GAAQ;GAAQ;GAAO;EAAU;CAC5C,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,QAAQ,QAAQ,sBAAsB,SAAS;EACrD,MAAM,IAAI,QAAQ,sBAAsB;EACxC,OAAO,MAAM,KAAA,IAAa,SAAS,IAAI,IAAI,IAAK;CAClD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,QAAQ,MAAM;CACtB,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS,cAAc,QAAQ,IAAI,aAAa,MAAM,UAAU;EAC7F,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUC,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU,eAAe;GAAO,aAAa;GAAO,QAAQ,SAAS;EAAM;CACtF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,WAAW,UAAkB;CAC3C,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS;EACtC,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU;GAAO,QAAQ,UAAU;EAAE;CAChD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,UAAU,eAAyB;CACjD,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,cAAc,KAAK,KAAK,EAAE;EACnE,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAE7C,IAAI,aAAa;EACjB,KAAK,MAAM,QAAQ,OACjB,IAAI,aAAa,cAAc,UAAU,SAAS,cAAc,aAC9D;EAIJ,OAAO;GAAE,aAAa;GAAO;GAAe,QAD7B,eAAe,cAAc;EACO;CACrD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,aAAa,KAAa;CACxC,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,IAAI;EAC7C,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ;GAAK,QAAQ,MAAM,UAAU;EAAI;CACjE,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,cAAc;CAC5B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ,QAAQ,MAAM,WAAW;EAAE;CAC3D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;;;;;AAeA,SAAgB,eAAe;CAC7B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,cAAc,sBAAsB,IAAI,MAAM;EACpD,MAAM,aAAa,YAAY,OAAO,iBAAiB,CAAC,CAAC;EACzD,OAAO;GAAE;GAAY,YAAY,YAAY;GAAQ,QAAQ,eAAe;EAAE;CAChF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAaA,SAAS,kBAAkB,KAAmB;CAC5C,OAAO,IAAI,UAAU,UAAU,IAAI,UAAU,kBAAkB,IAAI,YAAY,QAAQ,QAAQ,IAAI,QAAQ,KAAK;AAClH;AAEA,SAAS,sBAAsB,QAAgD;CAC7E,MAAM,cAAqB,CAAC;CAC5B,KAAK,MAAM,WAAW,QAGpB,KAAK,MAAM,OAAOC,sBAAAA,qBAAqB,OAAO,GAC5C,IAAI,KAAK,YAAY,KAAK,GAAG;CAGjC,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAa,SAAS;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/dist/{checks-64AonnEK.js → checks-DX9-p5w0.js}

@@ -1,4 +1,4 @@
-import { extractToolCalls, getTextContentFromMastraDBMessage } from "./scorers/utils.js";
+import { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from "./scorers/utils.js";
 import { createScorer } from "@mastra/core/evals";
 import stringSimilarity from "string-similarity";
 //#region src/scorers/code/checks/index.ts
@@ -300,8 +300,11 @@ function usedNoTools() {
 	});
 }
 /**
-* Scores 1 if none of the tool invocations resulted in an error state.
-* Checks for tool invocations with state other than 'result' (i.e., missing results).
+* Scores 1 if none of the tool invocations resulted in an error.
+*
+* Treats every failure representation a native run can persist as an error: a
+* thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a
+* result the executor flagged with `isError`, and a result carrying an `error` field.
 *
 * @example
 * ```ts
@@ -317,7 +320,7 @@ function noToolErrors() {
 		type: "agent"
 	}).preprocess(async ({ run }) => {
 		const invocations = extractRawInvocations(run.output);
-		const errorCount = invocations.filter((inv) => inv.state === "call" || inv.result && inv.result.error).length;
+		const errorCount = invocations.filter(isErrorInvocation).length;
 		return {
 			errorCount,
 			totalCalls: invocations.length,
@@ -327,13 +330,21 @@ function noToolErrors() {
 		return results.preprocessStepResult?.passed ? 1 : 0;
 	});
 }
+/**
+* True when an invocation represents a tool failure rather than a success.
+*
+* A natively thrown tool call is persisted as `state: 'output-error'` with
+* `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.
+* `isError` is set alongside `state: 'result'` when the executor reported a failure
+* even though the result carries no `error` field; core's `extractTrajectory`
+* likewise defines success as `state === 'result' && isError !== true`.
+*/
+function isErrorInvocation(inv) {
+	return inv.state === "call" || inv.state === "output-error" || inv.isError === true || Boolean(inv.result?.error);
+}
 function extractRawInvocations(output) {
 	const invocations = [];
-	for (const message of output) {
-		const legacy = message?.content?.toolInvocations;
-		const fromParts = legacy ? void 0 : (message?.content)?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
-		for (const inv of legacy ?? fromParts ?? []) if (inv) invocations.push(inv);
-	}
+	for (const message of output) for (const inv of mergeToolInvocations(message)) if (inv) invocations.push(inv);
 	return invocations;
 }
 /**
@@ -376,4 +387,4 @@ const checks = {
 //#endregion
 export { excludes as a, maxToolCalls as c, toolOrder as d, usedNoTools as f, equals as i, noToolErrors as l, checks as n, includes as o, didNotCall as r, matches as s, calledTool as t, similarity as u };
 
-//# sourceMappingURL=checks-64AonnEK.js.map
+//# sourceMappingURL=checks-DX9-p5w0.js.map

package/dist/checks-DX9-p5w0.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checks-DX9-p5w0.js","names":[],"sources":["../src/scorers/code/checks/index.ts"],"sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error.\n *\n * Treats every failure representation a native run can persist as an error: a\n * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a\n * result the executor flagged with `isError`, and a result carrying an `error` field.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(isErrorInvocation).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\n/**\n * True when an invocation represents a tool failure rather than a success.\n *\n * A natively thrown tool call is persisted as `state: 'output-error'` with\n * `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.\n * `isError` is set alongside `state: 'result'` when the executor reported a failure\n * even though the result carries no `error` field; core's `extractTrajectory`\n * likewise defines success as `state === 'result' && isError !== true`.\n */\nfunction isErrorInvocation(inv: any): boolean {\n  return inv.state === 'call' || inv.state === 'output-error' || inv.isError === true || Boolean(inv.result?.error);\n}\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    // Merge both storage forms so a call present only in `content.parts` — which is\n    // where thrown calls live — is never hidden by a legacy `toolInvocations` array.\n    for (const inv of mergeToolInvocations(message)) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"],"mappings":";;;;;;;;;;;;;AAoBA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,8BAA8B,SAAS;EACpD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,OAAO,OAAO,SAAS,MAAM;EAAE;CAC1D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,QAAQ,IAAI;CACnD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,wCAAwC,SAAS;EAC9D,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,UAAU,CAAC,OAAO,SAAS,MAAM;EAAE;CAC9D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,WAAW,IAAI;CACtD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,OAAO,UAAkB,UAA2B,CAAC,GAAG;CACtE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,SAAS;EAClD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,SAAS,WAAW;EAAO;CACtD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;AAgBA,SAAgB,QAAQ,SAAiB,UAA0B,CAAC,GAAG;CACrE,MAAM,EAAE,QAAQ,UAAU;CAC1B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,oCAAoC;EACjD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,SAAS,IAAI,OAChB,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EAEV,MAAM,WADQ,QAAQ,IAAI,OAAO,IAAI,QAAQ,OAAO,IAAI,QAAQ,KAAK,IAAI,QAAA,CACnD,KAAK,MAAM;EACjC,OAAO;GAAE;GAAQ,SAAS,QAAQ,SAAS;GAAG;EAAQ;CACxD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,aAAa,MAAM,cAAc;CACzC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gCAAgC,SAAS;EACtD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,MAAM,QAAQ,iBAAiB,kBAAkB,QAAQ,MAAM;EAC/D,OAAO;GAAE;GAAQ;GAAQ;GAAO;EAAU;CAC5C,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,QAAQ,QAAQ,sBAAsB,SAAS;EACrD,MAAM,IAAI,QAAQ,sBAAsB;EACxC,OAAO,MAAM,KAAA,IAAa,SAAS,IAAI,IAAI,IAAK;CAClD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,QAAQ,MAAM;CACtB,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS,cAAc,QAAQ,IAAI,aAAa,MAAM,UAAU;EAC7F,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU,eAAe;GAAO,aAAa;GAAO,QAAQ,SAAS;EAAM;CACtF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,WAAW,UAAkB;CAC3C,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS;EACtC,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU;GAAO,QAAQ,UAAU;EAAE;CAChD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,UAAU,eAAyB;CACjD,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,cAAc,KAAK,KAAK,EAAE;EACnE,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAE7C,IAAI,aAAa;EACjB,KAAK,MAAM,QAAQ,OACjB,IAAI,aAAa,cAAc,UAAU,SAAS,cAAc,aAC9D;EAIJ,OAAO;GAAE,aAAa;GAAO;GAAe,QAD7B,eAAe,cAAc;EACO;CACrD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,aAAa,KAAa;CACxC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,IAAI;EAC7C,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ;GAAK,QAAQ,MAAM,UAAU;EAAI;CACjE,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,cAAc;CAC5B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ,QAAQ,MAAM,WAAW;EAAE;CAC3D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;;;;;AAeA,SAAgB,eAAe;CAC7B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,cAAc,sBAAsB,IAAI,MAAM;EACpD,MAAM,aAAa,YAAY,OAAO,iBAAiB,CAAC,CAAC;EACzD,OAAO;GAAE;GAAY,YAAY,YAAY;GAAQ,QAAQ,eAAe;EAAE;CAChF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAaA,SAAS,kBAAkB,KAAmB;CAC5C,OAAO,IAAI,UAAU,UAAU,IAAI,UAAU,kBAAkB,IAAI,YAAY,QAAQ,QAAQ,IAAI,QAAQ,KAAK;AAClH;AAEA,SAAS,sBAAsB,QAAgD;CAC7E,MAAM,cAAqB,CAAC;CAC5B,KAAK,MAAM,WAAW,QAGpB,KAAK,MAAM,OAAO,qBAAqB,OAAO,GAC5C,IAAI,KAAK,YAAY,KAAK,GAAG;CAGjC,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAa,SAAS;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/dist/checks.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_checks = require("./checks-DGTgg-nW.cjs");
+const require_checks = require("./checks-CGRgwpDb.cjs");
 exports.calledTool = require_checks.calledTool;
 exports.checks = require_checks.checks;
 exports.didNotCall = require_checks.didNotCall;

package/dist/checks.js

@@ -1,2 +1,2 @@
-import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "./checks-64AonnEK.js";
+import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "./checks-DX9-p5w0.js";
 export { calledTool, checks, didNotCall, equals, excludes, includes, matches, maxToolCalls, noToolErrors, similarity, toolOrder, usedNoTools };

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-evals
 description: Documentation for @mastra/evals. Use when working with @mastra/evals APIs, configuration, or implementation.
 metadata:
   package: "@mastra/evals"
-  version: "1.10.2-alpha.0"
+  version: "1.10.2-alpha.1"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.10.2-alpha.0",
+  "version": "1.10.2-alpha.1",
   "package": "@mastra/evals",
   "exports": {},
   "modules": {}

package/dist/scorers/code/checks/index.d.ts

@@ -160,8 +160,11 @@ export declare function usedNoTools(): import("@mastra/core/evals").MastraScorer
     passed: boolean;
 }> & Record<"generateScoreStepResult", number>>;
 /**
- * Scores 1 if none of the tool invocations resulted in an error state.
- * Checks for tool invocations with state other than 'result' (i.e., missing results).
+ * Scores 1 if none of the tool invocations resulted in an error.
+ *
+ * Treats every failure representation a native run can persist as an error: a
+ * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a
+ * result the executor flagged with `isError`, and a result carrying an `error` field.
  *
  * @example
  * ```ts

package/dist/scorers/code/checks/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/checks/index.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBrE;AAED,MAAM,WAAW,cAAc;IAC7B,uGAAuG;IACvG,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB;;;;gDAoBpE;AAED,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDA0B3E;AAID,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDAgB3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM;;;;gDAe1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,aAAa,EAAE,MAAM,EAAE;;;;gDAsBhD;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM;;;;gDAcvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW;;;gDAc1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY;;;;gDAe3B;AAsBD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,eAAO,MAAM,MAAM;;;;;;;;;;;;CAYlB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/checks/index.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBrE;AAED,MAAM,WAAW,cAAc;IAC7B,uGAAuG;IACvG,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB;;;;gDAoBpE;AAED,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDA0B3E;AAID,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDAgB3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM;;;;gDAe1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,aAAa,EAAE,MAAM,EAAE;;;;gDAsBhD;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM;;;;gDAcvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW;;;gDAc1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,YAAY;;;;gDAe3B;AA+BD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,eAAO,MAAM,MAAM;;;;;;;;;;;;CAYlB,CAAC"}

package/dist/scorers/prebuilt/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_checks = require("../../checks-DGTgg-nW.cjs");
+const require_checks = require("../../checks-CGRgwpDb.cjs");
 const require_scorers_utils = require("../utils.cjs");
 let _mastra_core_evals = require("@mastra/core/evals");
 let compromise = require("compromise");

package/dist/scorers/prebuilt/index.js

@@ -1,5 +1,5 @@
 import { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, extractAgentResponseMessages, extractToolCalls, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getConversationHistoryFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isScorerRunInputForAgent, isScorerRunOutputForAgent, roundToTwoDecimals } from "../utils.js";
-import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "../../checks-64AonnEK.js";
+import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "../../checks-DX9-p5w0.js";
 import { createScorer } from "@mastra/core/evals";
 import nlp from "compromise";
 import keyword_extractor from "keyword-extractor";

package/dist/scorers/utils.cjs

@@ -352,8 +352,10 @@ const getReasoningFromRunOutput = (output) => {
 * @param options.toolCallId - Unique identifier for the tool call
 * @param options.toolName - Name of the tool being called
 * @param options.args - Arguments passed to the tool
-* @param options.result - Result returned by the tool
+* @param options.result - Result returned by the tool (absent for a thrown call)
 * @param options.state - State of the invocation (default: 'result')
+* @param options.errorText - Error text recorded when a tool threw (`state: 'output-error'`)
+* @param options.isError - Set alongside `state: 'result'` when the executor reported a failure
 * @returns A tool invocation object
 *
 * @example
@@ -364,15 +366,26 @@ const getReasoningFromRunOutput = (output) => {
 *   args: { location: 'London' },
 *   result: { temperature: 20, condition: 'sunny' },
 * });
+*
+* // A tool that threw
+* const thrown = createToolInvocation({
+*   toolCallId: 'call-456',
+*   toolName: 'saveTool',
+*   args: { value: 'x' },
+*   state: 'output-error',
+*   errorText: 'Save failed',
+* });
 * ```
 */
-const createToolInvocation = ({ toolCallId, toolName, args, result, state = "result" }) => {
+const createToolInvocation = ({ toolCallId, toolName, args, result, state = "result", errorText, isError }) => {
 	return {
 		toolCallId,
 		toolName,
 		args,
 		result,
-		state
+		state,
+		...errorText !== void 0 && { errorText },
+		...isError !== void 0 && { isError }
 	};
 };
 /**
@@ -406,15 +419,34 @@ const createToolInvocation = ({ toolCallId, toolName, args, result, state = "res
 *     state: 'result',
 *   }],
 * });
+*
+* // With native V2 parts — the only form that can carry a thrown call
+* const messageWithThrownTool = createTestMessage({
+*   content: 'That failed.',
+*   role: 'assistant',
+*   parts: [
+*     { type: 'text', text: 'That failed.' },
+*     {
+*       type: 'tool-invocation',
+*       toolInvocation: createToolInvocation({
+*         toolCallId: 'call-2',
+*         toolName: 'saveTool',
+*         args: { value: 'x' },
+*         state: 'output-error',
+*         errorText: 'Save failed',
+*       }),
+*     },
+*   ],
+* });
 * ```
 */
-function createTestMessage({ content, role, id = "test-message", toolInvocations = [] }) {
+function createTestMessage({ content, role, id = "test-message", toolInvocations = [], parts }) {
 	return {
 		id,
 		role,
 		content: {
 			format: 2,
-			parts: [{
+			parts: parts ?? [{
 				type: "text",
 				text: content
 			}],
@@ -503,10 +535,56 @@ const createTrajectoryTestRun = ({ inputMessages = [], trajectory, rememberedMes
 	};
 };
 /**
+* Merges the two places a message can store tool invocations into one list.
+*
+* A native message may carry `content.parts` tool-invocation entries, the legacy
+* `content.toolInvocations` array, or both. Neither form is a superset of the
+* other: thrown calls are recorded in parts as `state: 'output-error'` and can be
+* missing from the legacy array, while older persisted messages may only carry the
+* legacy array. Reading one form exclusively therefore hides real calls.
+*
+* Parts win when the same `toolCallId` appears in both, since they hold the real
+* terminal state. Legacy-only calls are retained before their next shared anchor so
+* compatible call order survives instead of being appended.
+*
+* Mirrors `extractTrajectory` in `@mastra/core` (`packages/core/src/evals/types.ts`)
+* so tool-use checks and trajectory scorers agree on what a run actually called.
+* Keep the two in step.
+*
+* @param message - A single message from the scorer run output
+* @returns The merged tool invocations for that message
+*/
+function mergeToolInvocations(message) {
+	const legacy = message?.content?.toolInvocations;
+	const fromParts = message?.content?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation).filter(Boolean) ?? [];
+	const partCallIds = new Set(fromParts.map((invocation) => invocation.toolCallId).filter(Boolean));
+	const legacyInvocations = legacy ?? [];
+	const legacyPositions = new Map(legacyInvocations.map((invocation, index) => [invocation?.toolCallId, index]));
+	const toolInvocations = [];
+	let legacyIndex = 0;
+	for (const invocation of fromParts) {
+		const sharedIndex = invocation.toolCallId ? legacyPositions.get(invocation.toolCallId) : void 0;
+		if (sharedIndex !== void 0 && sharedIndex >= legacyIndex) {
+			for (; legacyIndex < sharedIndex; legacyIndex++) {
+				const previous = legacyInvocations[legacyIndex];
+				if (previous && !partCallIds.has(previous.toolCallId)) toolInvocations.push(previous);
+			}
+			legacyIndex++;
+		}
+		toolInvocations.push(invocation);
+	}
+	for (; legacyIndex < legacyInvocations.length; legacyIndex++) {
+		const invocation = legacyInvocations[legacyIndex];
+		if (invocation && !partCallIds.has(invocation.toolCallId)) toolInvocations.push(invocation);
+	}
+	return toolInvocations;
+}
+/**
 * Extracts all tool calls from a scorer run output.
 *
-* Iterates through all messages and their tool invocations to collect
-* information about tools that were called (with state 'result' or 'call').
+* Iterates through all messages and their tool invocations to collect information
+* about tools that were called. A call counts regardless of outcome, so the states
+* `'result'`, `'call'`, and `'output-error'` (a tool that threw) are all included.
 *
 * @param output - The scorer run output (array of MastraDBMessage)
 * @returns An object containing tool names and detailed tool call info
@@ -528,13 +606,11 @@ function extractToolCalls(output) {
 	const toolCallInfos = [];
 	for (let messageIndex = 0; messageIndex < output.length; messageIndex++) {
 		const message = output[messageIndex];
-		const legacy = message?.content?.toolInvocations;
-		const fromParts = legacy ? void 0 : message?.content?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
-		const toolInvocations = legacy ?? fromParts;
-		if (!toolInvocations?.length) continue;
+		const toolInvocations = mergeToolInvocations(message);
+		if (!toolInvocations.length) continue;
 		for (let invocationIndex = 0; invocationIndex < toolInvocations.length; invocationIndex++) {
 			const invocation = toolInvocations[invocationIndex];
-			if (invocation && invocation.toolName && (invocation.state === "result" || invocation.state === "call")) {
+			if (invocation && invocation.toolName && (invocation.state === "result" || invocation.state === "call" || invocation.state === "output-error")) {
 				toolCalls.push(invocation.toolName);
 				toolCallInfos.push({
 					toolName: invocation.toolName,
@@ -616,10 +692,8 @@ const extractAgentResponseMessages = (runOutput) => {
 function extractToolResults(output) {
 	const results = [];
 	for (const message of output) {
-		const legacy = message?.content?.toolInvocations;
-		const fromParts = legacy ? void 0 : message?.content?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
-		const toolInvocations = legacy ?? fromParts;
-		if (!toolInvocations?.length) continue;
+		const toolInvocations = mergeToolInvocations(message);
+		if (!toolInvocations.length) continue;
 		for (const invocation of toolInvocations) if (invocation.state === "result" && invocation.result !== void 0) results.push({
 			toolName: invocation.toolName,
 			toolCallId: invocation.toolCallId || "",
@@ -1030,6 +1104,7 @@ exports.getUserMessageFromRunInput = getUserMessageFromRunInput;
 exports.isCloserTo = isCloserTo;
 exports.isScorerRunInputForAgent = isScorerRunInputForAgent;
 exports.isScorerRunOutputForAgent = isScorerRunOutputForAgent;
+exports.mergeToolInvocations = mergeToolInvocations;
 exports.roundToTwoDecimals = roundToTwoDecimals;
 
 //# sourceMappingURL=utils.cjs.map