@mastra/evals 1.10.4-alpha.0 → 1.10.5-alpha.0

package/dist/{checks-CGRgwpDb.cjs → checks-B7c-zPoM.cjs}

@@ -139,7 +139,9 @@ function matches(pattern, options = {}) {
 		type: "agent"
 	}).preprocess(async ({ run }) => {
 		const output = run.output.filter((m) => m.role === "assistant").map((m) => require_scorers_utils.getTextContentFromMastraDBMessage(m)).join("");
-		const matched = (exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern).test(output);
+		const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;
+		regex.lastIndex = 0;
+		const matched = regex.test(output);
 		return {
 			output,
 			pattern: pattern.toString(),
@@ -487,4 +489,4 @@ Object.defineProperty(exports, "usedNoTools", {
 	}
 });
 
-//# sourceMappingURL=checks-CGRgwpDb.cjs.map
+//# sourceMappingURL=checks-B7c-zPoM.cjs.map

package/dist/checks-B7c-zPoM.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"checks-B7c-zPoM.cjs","names":["getTextContentFromMastraDBMessage","stringSimilarity","extractToolCalls","mergeToolInvocations"],"sources":["../src/scorers/code/checks/index.ts"],"sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      // g/y regexes keep lastIndex between calls, and the scorer is reused across items\n      regex.lastIndex = 0;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error.\n *\n * Treats every failure representation a native run can persist as an error: a\n * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a\n * result the executor flagged with `isError`, and a result carrying an `error` field.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(isErrorInvocation).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\n/**\n * True when an invocation represents a tool failure rather than a success.\n *\n * A natively thrown tool call is persisted as `state: 'output-error'` with\n * `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.\n * `isError` is set alongside `state: 'result'` when the executor reported a failure\n * even though the result carries no `error` field; core's `extractTrajectory`\n * likewise defines success as `state === 'result' && isError !== true`.\n */\nfunction isErrorInvocation(inv: any): boolean {\n  return inv.state === 'call' || inv.state === 'output-error' || inv.isError === true || Boolean(inv.result?.error);\n}\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    // Merge both storage forms so a call present only in `content.parts` — which is\n    // where thrown calls live — is never hidden by a legacy `toolInvocations` array.\n    for (const inv of mergeToolInvocations(message)) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,8BAA8B,SAAS;EACpD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,OAAO,OAAO,SAAS,MAAM;EAAE;CAC1D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,QAAQ,IAAI;CACnD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,wCAAwC,SAAS;EAC9D,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,UAAU,CAAC,OAAO,SAAS,MAAM;EAAE;CAC9D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,WAAW,IAAI;CACtD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,OAAO,UAAkB,UAA2B,CAAC,GAAG;CACtE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,SAAS;EAClD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,SAAS,WAAW;EAAO;CACtD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;AAgBA,SAAgB,QAAQ,SAAiB,UAA0B,CAAC,GAAG;CACrE,MAAM,EAAE,QAAQ,UAAU;CAC1B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,oCAAoC;EACjD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,SAAS,IAAI,OAChB,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,MAAM,QAAQ,QAAQ,IAAI,OAAO,IAAI,QAAQ,OAAO,IAAI,QAAQ,KAAK,IAAI;EAEzE,MAAM,YAAY;EAClB,MAAM,UAAU,MAAM,KAAK,MAAM;EACjC,OAAO;GAAE;GAAQ,SAAS,QAAQ,SAAS;GAAG;EAAQ;CACxD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,aAAa,MAAM,cAAc;CACzC,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gCAAgC,SAAS;EACtD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,MAAM,QAAQC,kBAAAA,QAAiB,kBAAkB,QAAQ,MAAM;EAC/D,OAAO;GAAE;GAAQ;GAAQ;GAAO;EAAU;CAC5C,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,QAAQ,QAAQ,sBAAsB,SAAS;EACrD,MAAM,IAAI,QAAQ,sBAAsB;EACxC,OAAO,MAAM,KAAA,IAAa,SAAS,IAAI,IAAI,IAAK;CAClD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,QAAQ,MAAM;CACtB,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS,cAAc,QAAQ,IAAI,aAAa,MAAM,UAAU;EAC7F,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUC,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU,eAAe;GAAO,aAAa;GAAO,QAAQ,SAAS;EAAM;CACtF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,WAAW,UAAkB;CAC3C,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS;EACtC,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU;GAAO,QAAQ,UAAU;EAAE;CAChD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,UAAU,eAAyB;CACjD,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,cAAc,KAAK,KAAK,EAAE;EACnE,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAE7C,IAAI,aAAa;EACjB,KAAK,MAAM,QAAQ,OACjB,IAAI,aAAa,cAAc,UAAU,SAAS,cAAc,aAC9D;EAIJ,OAAO;GAAE,aAAa;GAAO;GAAe,QAD7B,eAAe,cAAc;EACO;CACrD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,aAAa,KAAa;CACxC,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,IAAI;EAC7C,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ;GAAK,QAAQ,MAAM,UAAU;EAAI;CACjE,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,cAAc;CAC5B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ,QAAQ,MAAM,WAAW;EAAE;CAC3D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;;;;;AAeA,SAAgB,eAAe;CAC7B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,cAAc,sBAAsB,IAAI,MAAM;EACpD,MAAM,aAAa,YAAY,OAAO,iBAAiB,CAAC,CAAC;EACzD,OAAO;GAAE;GAAY,YAAY,YAAY;GAAQ,QAAQ,eAAe;EAAE;CAChF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAaA,SAAS,kBAAkB,KAAmB;CAC5C,OAAO,IAAI,UAAU,UAAU,IAAI,UAAU,kBAAkB,IAAI,YAAY,QAAQ,QAAQ,IAAI,QAAQ,KAAK;AAClH;AAEA,SAAS,sBAAsB,QAAgD;CAC7E,MAAM,cAAqB,CAAC;CAC5B,KAAK,MAAM,WAAW,QAGpB,KAAK,MAAM,OAAOC,sBAAAA,qBAAqB,OAAO,GAC5C,IAAI,KAAK,YAAY,KAAK,GAAG;CAGjC,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAa,SAAS;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/dist/{checks-DX9-p5w0.js → checks-DxbqoaYq.js}

@@ -116,7 +116,9 @@ function matches(pattern, options = {}) {
 		type: "agent"
 	}).preprocess(async ({ run }) => {
 		const output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join("");
-		const matched = (exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern).test(output);
+		const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;
+		regex.lastIndex = 0;
+		const matched = regex.test(output);
 		return {
 			output,
 			pattern: pattern.toString(),
@@ -387,4 +389,4 @@ const checks = {
 //#endregion
 export { excludes as a, maxToolCalls as c, toolOrder as d, usedNoTools as f, equals as i, noToolErrors as l, checks as n, includes as o, didNotCall as r, matches as s, calledTool as t, similarity as u };
 
-//# sourceMappingURL=checks-DX9-p5w0.js.map
+//# sourceMappingURL=checks-DxbqoaYq.js.map

package/dist/checks-DxbqoaYq.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checks-DxbqoaYq.js","names":[],"sources":["../src/scorers/code/checks/index.ts"],"sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      // g/y regexes keep lastIndex between calls, and the scorer is reused across items\n      regex.lastIndex = 0;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error.\n *\n * Treats every failure representation a native run can persist as an error: a\n * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a\n * result the executor flagged with `isError`, and a result carrying an `error` field.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(isErrorInvocation).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\n/**\n * True when an invocation represents a tool failure rather than a success.\n *\n * A natively thrown tool call is persisted as `state: 'output-error'` with\n * `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.\n * `isError` is set alongside `state: 'result'` when the executor reported a failure\n * even though the result carries no `error` field; core's `extractTrajectory`\n * likewise defines success as `state === 'result' && isError !== true`.\n */\nfunction isErrorInvocation(inv: any): boolean {\n  return inv.state === 'call' || inv.state === 'output-error' || inv.isError === true || Boolean(inv.result?.error);\n}\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    // Merge both storage forms so a call present only in `content.parts` — which is\n    // where thrown calls live — is never hidden by a legacy `toolInvocations` array.\n    for (const inv of mergeToolInvocations(message)) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"],"mappings":";;;;;;;;;;;;;AAoBA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,8BAA8B,SAAS;EACpD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,OAAO,OAAO,SAAS,MAAM;EAAE;CAC1D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,QAAQ,IAAI;CACnD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,wCAAwC,SAAS;EAC9D,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,UAAU,CAAC,OAAO,SAAS,MAAM;EAAE;CAC9D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,WAAW,IAAI;CACtD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,OAAO,UAAkB,UAA2B,CAAC,GAAG;CACtE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,SAAS;EAClD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,SAAS,WAAW;EAAO;CACtD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;AAgBA,SAAgB,QAAQ,SAAiB,UAA0B,CAAC,GAAG;CACrE,MAAM,EAAE,QAAQ,UAAU;CAC1B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,oCAAoC;EACjD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,SAAS,IAAI,OAChB,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,MAAM,QAAQ,QAAQ,IAAI,OAAO,IAAI,QAAQ,OAAO,IAAI,QAAQ,KAAK,IAAI;EAEzE,MAAM,YAAY;EAClB,MAAM,UAAU,MAAM,KAAK,MAAM;EACjC,OAAO;GAAE;GAAQ,SAAS,QAAQ,SAAS;GAAG;EAAQ;CACxD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,aAAa,MAAM,cAAc;CACzC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gCAAgC,SAAS;EACtD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,MAAM,QAAQ,iBAAiB,kBAAkB,QAAQ,MAAM;EAC/D,OAAO;GAAE;GAAQ;GAAQ;GAAO;EAAU;CAC5C,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,QAAQ,QAAQ,sBAAsB,SAAS;EACrD,MAAM,IAAI,QAAQ,sBAAsB;EACxC,OAAO,MAAM,KAAA,IAAa,SAAS,IAAI,IAAI,IAAK;CAClD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,QAAQ,MAAM;CACtB,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS,cAAc,QAAQ,IAAI,aAAa,MAAM,UAAU;EAC7F,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU,eAAe;GAAO,aAAa;GAAO,QAAQ,SAAS;EAAM;CACtF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,WAAW,UAAkB;CAC3C,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS;EACtC,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU;GAAO,QAAQ,UAAU;EAAE;CAChD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,UAAU,eAAyB;CACjD,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,cAAc,KAAK,KAAK,EAAE;EACnE,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAE7C,IAAI,aAAa;EACjB,KAAK,MAAM,QAAQ,OACjB,IAAI,aAAa,cAAc,UAAU,SAAS,cAAc,aAC9D;EAIJ,OAAO;GAAE,aAAa;GAAO;GAAe,QAD7B,eAAe,cAAc;EACO;CACrD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,aAAa,KAAa;CACxC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,IAAI;EAC7C,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ;GAAK,QAAQ,MAAM,UAAU;EAAI;CACjE,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,cAAc;CAC5B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ,QAAQ,MAAM,WAAW;EAAE;CAC3D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;;;;;AAeA,SAAgB,eAAe;CAC7B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,cAAc,sBAAsB,IAAI,MAAM;EACpD,MAAM,aAAa,YAAY,OAAO,iBAAiB,CAAC,CAAC;EACzD,OAAO;GAAE;GAAY,YAAY,YAAY;GAAQ,QAAQ,eAAe;EAAE;CAChF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAaA,SAAS,kBAAkB,KAAmB;CAC5C,OAAO,IAAI,UAAU,UAAU,IAAI,UAAU,kBAAkB,IAAI,YAAY,QAAQ,QAAQ,IAAI,QAAQ,KAAK;AAClH;AAEA,SAAS,sBAAsB,QAAgD;CAC7E,MAAM,cAAqB,CAAC;CAC5B,KAAK,MAAM,WAAW,QAGpB,KAAK,MAAM,OAAO,qBAAqB,OAAO,GAC5C,IAAI,KAAK,YAAY,KAAK,GAAG;CAGjC,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAa,SAAS;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/dist/checks.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_checks = require("./checks-CGRgwpDb.cjs");
+const require_checks = require("./checks-B7c-zPoM.cjs");
 exports.calledTool = require_checks.calledTool;
 exports.checks = require_checks.checks;
 exports.didNotCall = require_checks.didNotCall;

package/dist/checks.js

@@ -1,2 +1,2 @@
-import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "./checks-DX9-p5w0.js";
+import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "./checks-DxbqoaYq.js";
 export { calledTool, checks, didNotCall, equals, excludes, includes, matches, maxToolCalls, noToolErrors, similarity, toolOrder, usedNoTools };

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-evals
 description: Documentation for @mastra/evals. Use when working with @mastra/evals APIs, configuration, or implementation.
 metadata:
   package: "@mastra/evals"
-  version: "1.10.4-alpha.0"
+  version: "1.10.5-alpha.0"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.10.4-alpha.0",
+  "version": "1.10.5-alpha.0",
   "package": "@mastra/evals",
   "exports": {},
   "modules": {}

package/dist/docs/references/docs-evals-vitest-integration.md

@@ -77,7 +77,7 @@ test.for([
 })
 ```
 
-Each item passes or fails independently, so a single regression shows up as one failing test instead of a lowered aggregate pass rate.
+Each item passes or fails independently. A single regression shows up as one failing test instead of a lowered aggregate pass rate.
 
 ## Asserting on results with matchers
 

package/dist/scorers/code/checks/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/checks/index.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBrE;AAED,MAAM,WAAW,cAAc;IAC7B,uGAAuG;IACvG,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB;;;;gDAoBpE;AAED,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDA0B3E;AAID,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDAgB3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM;;;;gDAe1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,aAAa,EAAE,MAAM,EAAE;;;;gDAsBhD;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM;;;;gDAcvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW;;;gDAc1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,YAAY;;;;gDAe3B;AA+BD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,eAAO,MAAM,MAAM;;;;;;;;;;;;CAYlB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/checks/index.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBrE;AAED,MAAM,WAAW,cAAc;IAC7B,uGAAuG;IACvG,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB;;;;gDAsBpE;AAED,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDA0B3E;AAID,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDAgB3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM;;;;gDAe1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,aAAa,EAAE,MAAM,EAAE;;;;gDAsBhD;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM;;;;gDAcvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW;;;gDAc1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,YAAY;;;;gDAe3B;AA+BD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,eAAO,MAAM,MAAM;;;;;;;;;;;;CAYlB,CAAC"}

package/dist/scorers/prebuilt/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_checks = require("../../checks-CGRgwpDb.cjs");
+const require_checks = require("../../checks-B7c-zPoM.cjs");
 const require_scorers_utils = require("../utils.cjs");
 let _mastra_core_evals = require("@mastra/core/evals");
 let compromise = require("compromise");

package/dist/scorers/prebuilt/index.js

@@ -1,5 +1,5 @@
 import { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, extractAgentResponseMessages, extractToolCalls, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getConversationHistoryFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isScorerRunInputForAgent, isScorerRunOutputForAgent, roundToTwoDecimals } from "../utils.js";
-import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "../../checks-DX9-p5w0.js";
+import { a as excludes, c as maxToolCalls, d as toolOrder, f as usedNoTools, i as equals, l as noToolErrors, n as checks, o as includes, r as didNotCall, s as matches, t as calledTool, u as similarity } from "../../checks-DxbqoaYq.js";
 import { createScorer } from "@mastra/core/evals";
 import nlp from "compromise";
 import keyword_extractor from "keyword-extractor";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/evals",
-  "version": "1.10.4-alpha.0",
+  "version": "1.10.5-alpha.0",
   "description": "",
   "type": "module",
   "files": [
@@ -113,12 +113,12 @@
     "typescript": "^7.0.2",
     "vitest": "4.1.11",
     "zod": "^4.6.4",
-    "@internal/lint": "0.0.137",
-    "@internal/ai-sdk-v5": "0.0.84",
-    "@internal/llm-recorder": "0.0.73",
-    "@internal/types-builder": "0.0.112",
-    "@mastra/core": "1.72.0-alpha.7",
-    "@internal/test-utils": "0.0.73"
+    "@internal/ai-sdk-v5": "0.0.85",
+    "@internal/test-utils": "0.0.74",
+    "@internal/types-builder": "0.0.113",
+    "@internal/lint": "0.0.138",
+    "@internal/llm-recorder": "0.0.74",
+    "@mastra/core": "1.73.0-alpha.1"
   },
   "engines": {
     "node": ">=22.13.0"

package/dist/checks-CGRgwpDb.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"checks-CGRgwpDb.cjs","names":["getTextContentFromMastraDBMessage","stringSimilarity","extractToolCalls","mergeToolInvocations"],"sources":["../src/scorers/code/checks/index.ts"],"sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error.\n *\n * Treats every failure representation a native run can persist as an error: a\n * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a\n * result the executor flagged with `isError`, and a result carrying an `error` field.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(isErrorInvocation).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\n/**\n * True when an invocation represents a tool failure rather than a success.\n *\n * A natively thrown tool call is persisted as `state: 'output-error'` with\n * `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.\n * `isError` is set alongside `state: 'result'` when the executor reported a failure\n * even though the result carries no `error` field; core's `extractTrajectory`\n * likewise defines success as `state === 'result' && isError !== true`.\n */\nfunction isErrorInvocation(inv: any): boolean {\n  return inv.state === 'call' || inv.state === 'output-error' || inv.isError === true || Boolean(inv.result?.error);\n}\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    // Merge both storage forms so a call present only in `content.parts` — which is\n    // where thrown calls live — is never hidden by a legacy `toolInvocations` array.\n    for (const inv of mergeToolInvocations(message)) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,8BAA8B,SAAS;EACpD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,OAAO,OAAO,SAAS,MAAM;EAAE;CAC1D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,QAAQ,IAAI;CACnD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,wCAAwC,SAAS;EAC9D,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,UAAU,CAAC,OAAO,SAAS,MAAM;EAAE;CAC9D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,WAAW,IAAI;CACtD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,OAAO,UAAkB,UAA2B,CAAC,GAAG;CACtE,MAAM,EAAE,aAAa,SAAS;CAC9B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,SAAS;EAClD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,SAAS,WAAW;EAAO;CACtD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;AAgBA,SAAgB,QAAQ,SAAiB,UAA0B,CAAC,GAAG;CACrE,MAAM,EAAE,QAAQ,UAAU;CAC1B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,oCAAoC;EACjD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,SAAS,IAAI,OAChB,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EAEV,MAAM,WADQ,QAAQ,IAAI,OAAO,IAAI,QAAQ,OAAO,IAAI,QAAQ,KAAK,IAAI,QAAA,CACnD,KAAK,MAAM;EACjC,OAAO;GAAE;GAAQ,SAAS,QAAQ,SAAS;GAAG;EAAQ;CACxD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,aAAa,MAAM,cAAc;CACzC,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gCAAgC,SAAS;EACtD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAKA,sBAAAA,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,MAAM,QAAQC,kBAAAA,QAAiB,kBAAkB,QAAQ,MAAM;EAC/D,OAAO;GAAE;GAAQ;GAAQ;GAAO;EAAU;CAC5C,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,QAAQ,QAAQ,sBAAsB,SAAS;EACrD,MAAM,IAAI,QAAQ,sBAAsB;EACxC,OAAO,MAAM,KAAA,IAAa,SAAS,IAAI,IAAI,IAAK;CAClD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,QAAQ,MAAM;CACtB,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS,cAAc,QAAQ,IAAI,aAAa,MAAM,UAAU;EAC7F,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUC,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU,eAAe;GAAO,aAAa;GAAO,QAAQ,SAAS;EAAM;CACtF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,WAAW,UAAkB;CAC3C,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS;EACtC,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU;GAAO,QAAQ,UAAU;EAAE;CAChD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,UAAU,eAAyB;CACjD,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,cAAc,KAAK,KAAK,EAAE;EACnE,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAE7C,IAAI,aAAa;EACjB,KAAK,MAAM,QAAQ,OACjB,IAAI,aAAa,cAAc,UAAU,SAAS,cAAc,aAC9D;EAIJ,OAAO;GAAE,aAAa;GAAO;GAAe,QAD7B,eAAe,cAAc;EACO;CACrD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,aAAa,KAAa;CACxC,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,IAAI;EAC7C,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ;GAAK,QAAQ,MAAM,UAAU;EAAI;CACjE,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,cAAc;CAC5B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAUA,sBAAAA,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ,QAAQ,MAAM,WAAW;EAAE;CAC3D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;;;;;AAeA,SAAgB,eAAe;CAC7B,QAAA,GAAA,mBAAA,aAAA,CAAoB;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,cAAc,sBAAsB,IAAI,MAAM;EACpD,MAAM,aAAa,YAAY,OAAO,iBAAiB,CAAC,CAAC;EACzD,OAAO;GAAE;GAAY,YAAY,YAAY;GAAQ,QAAQ,eAAe;EAAE;CAChF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAaA,SAAS,kBAAkB,KAAmB;CAC5C,OAAO,IAAI,UAAU,UAAU,IAAI,UAAU,kBAAkB,IAAI,YAAY,QAAQ,QAAQ,IAAI,QAAQ,KAAK;AAClH;AAEA,SAAS,sBAAsB,QAAgD;CAC7E,MAAM,cAAqB,CAAC;CAC5B,KAAK,MAAM,WAAW,QAGpB,KAAK,MAAM,OAAOC,sBAAAA,qBAAqB,OAAO,GAC5C,IAAI,KAAK,YAAY,KAAK,GAAG;CAGjC,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAa,SAAS;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}

package/dist/checks-DX9-p5w0.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"checks-DX9-p5w0.js","names":[],"sources":["../src/scorers/code/checks/index.ts"],"sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage, mergeToolInvocations } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error.\n *\n * Treats every failure representation a native run can persist as an error: a\n * thrown call (`state: 'output-error'`), an incomplete call (`state: 'call'`), a\n * result the executor flagged with `isError`, and a result carrying an `error` field.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(isErrorInvocation).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\n/**\n * True when an invocation represents a tool failure rather than a success.\n *\n * A natively thrown tool call is persisted as `state: 'output-error'` with\n * `errorText`, so keying only on `state: 'call'` or `result.error` lets it pass.\n * `isError` is set alongside `state: 'result'` when the executor reported a failure\n * even though the result carries no `error` field; core's `extractTrajectory`\n * likewise defines success as `state === 'result' && isError !== true`.\n */\nfunction isErrorInvocation(inv: any): boolean {\n  return inv.state === 'call' || inv.state === 'output-error' || inv.isError === true || Boolean(inv.result?.error);\n}\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    // Merge both storage forms so a call present only in `content.parts` — which is\n    // where thrown calls live — is never hidden by a legacy `toolInvocations` array.\n    for (const inv of mergeToolInvocations(message)) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"],"mappings":";;;;;;;;;;;;;AAoBA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,8BAA8B,SAAS;EACpD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,OAAO,OAAO,SAAS,MAAM;EAAE;CAC1D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,QAAQ,IAAI;CACnD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,SAAS,UAAkB,UAA2B,CAAC,GAAG;CACxE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,wCAAwC,SAAS;EAC9D,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,UAAU,CAAC,OAAO,SAAS,MAAM;EAAE;CAC9D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,WAAW,IAAI;CACtD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,OAAO,UAAkB,UAA2B,CAAC,GAAG;CACtE,MAAM,EAAE,aAAa,SAAS;CAC9B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,SAAS;EAClD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EACV,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,OAAO;GAAE;GAAQ;GAAQ,SAAS,WAAW;EAAO;CACtD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;AAgBA,SAAgB,QAAQ,SAAiB,UAA0B,CAAC,GAAG;CACrE,MAAM,EAAE,QAAQ,UAAU;CAC1B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,oCAAoC;EACjD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,SAAS,IAAI,OAChB,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,EAAE;EAEV,MAAM,WADQ,QAAQ,IAAI,OAAO,IAAI,QAAQ,OAAO,IAAI,QAAQ,KAAK,IAAI,QAAA,CACnD,KAAK,MAAM;EACjC,OAAO;GAAE;GAAQ,SAAS,QAAQ,SAAS;GAAG;EAAQ;CACxD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,UAAU,IAAI;CACrD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,aAAa,MAAM,cAAc;CACzC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gCAAgC,SAAS;EACtD,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,IAAI,SAAS,IAAI,OACd,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CACnC,KAAI,MAAK,kCAAkC,CAAC,CAAC,CAAC,CAC9C,KAAK,GAAG;EACX,IAAI,SAAS;EACb,IAAI,YAAY;GACd,SAAS,OAAO,YAAY;GAC5B,SAAS,OAAO,YAAY;EAC9B;EACA,MAAM,QAAQ,iBAAiB,kBAAkB,QAAQ,MAAM;EAC/D,OAAO;GAAE;GAAQ;GAAQ;GAAO;EAAU;CAC5C,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,QAAQ,QAAQ,sBAAsB,SAAS;EACrD,MAAM,IAAI,QAAQ,sBAAsB;EACxC,OAAO,MAAM,KAAA,IAAa,SAAS,IAAI,IAAI,IAAK;CAClD,CAAC;AACL;;;;;;;;;;;AAmBA,SAAgB,WAAW,UAAkB,UAA6B,CAAC,GAAG;CAC5E,MAAM,EAAE,QAAQ,MAAM;CACtB,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS,cAAc,QAAQ,IAAI,aAAa,MAAM,UAAU;EAC7F,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU,eAAe;GAAO,aAAa;GAAO,QAAQ,SAAS;EAAM;CACtF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,WAAW,UAAkB;CAC3C,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,gBAAgB,SAAS;EACtC,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,MAAM,QAAQ,MAAM,QAAO,MAAK,MAAM,QAAQ,CAAC,CAAC;EAChD,OAAO;GAAE;GAAU;GAAO,QAAQ,UAAU;EAAE;CAChD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,UAAU,eAAyB;CACjD,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,cAAc,KAAK,KAAK,EAAE;EACnE,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAE7C,IAAI,aAAa;EACjB,KAAK,MAAM,QAAQ,OACjB,IAAI,aAAa,cAAc,UAAU,SAAS,cAAc,aAC9D;EAIJ,OAAO;GAAE,aAAa;GAAO;GAAe,QAD7B,eAAe,cAAc;EACO;CACrD,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,aAAa,KAAa;CACxC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa,4BAA4B,IAAI;EAC7C,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ;GAAK,QAAQ,MAAM,UAAU;EAAI;CACjE,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAWA,SAAgB,cAAc;CAC5B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,EAAE,UAAU,iBAAiB,IAAI,MAAM;EAC7C,OAAO;GAAE,OAAO,MAAM;GAAQ,QAAQ,MAAM,WAAW;EAAE;CAC3D,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;;;;;AAeA,SAAgB,eAAe;CAC7B,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,cAAc,sBAAsB,IAAI,MAAM;EACpD,MAAM,aAAa,YAAY,OAAO,iBAAiB,CAAC,CAAC;EACzD,OAAO;GAAE;GAAY,YAAY,YAAY;GAAQ,QAAQ,eAAe;EAAE;CAChF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,OAAO,QAAQ,sBAAsB,SAAS,IAAI;CACpD,CAAC;AACL;;;;;;;;;;AAaA,SAAS,kBAAkB,KAAmB;CAC5C,OAAO,IAAI,UAAU,UAAU,IAAI,UAAU,kBAAkB,IAAI,YAAY,QAAQ,QAAQ,IAAI,QAAQ,KAAK;AAClH;AAEA,SAAS,sBAAsB,QAAgD;CAC7E,MAAM,cAAqB,CAAC;CAC5B,KAAK,MAAM,WAAW,QAGpB,KAAK,MAAM,OAAO,qBAAqB,OAAO,GAC5C,IAAI,KAAK,YAAY,KAAK,GAAG;CAGjC,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAa,SAAS;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF"}