@mastra/evals 1.4.0 → 1.5.0-alpha.1

package/CHANGELOG.md

@@ -1,5 +1,62 @@
 # @mastra/evals
 
+## 1.5.0-alpha.1
+
+### Patch Changes
+
+- **Gates and verdict for `runEvals`** ([#18394](https://github.com/mastra-ai/mastra/pull/18394))
+
+  New optional `gates` field accepts scorers that must score 1.0 for the run to pass. Scorers can now use a `{ scorer, threshold }` form to set pass/fail thresholds. `threshold` accepts a number (minimum) or `{ min, max }` for range-based checks (e.g. hallucination where high = bad). The result includes `verdict`, `gateResults`, and `thresholdResults`. Fully backward compatible.
+
+  ```ts
+  import { runEvals } from '@mastra/core/evals';
+  import { checks } from '@mastra/evals/checks';
+
+  const result = await runEvals({
+    data: [{ input: 'What is the weather?' }],
+    target: weatherAgent,
+    gates: [checks.calledTool('get_weather')],
+    scorers: [
+      { scorer: faithfulnessScorer, threshold: 0.7 },
+      { scorer: hallucinationScorer, threshold: { max: 0.3 } },
+    ],
+  });
+
+  result.verdict; // 'passed' | 'scored' | 'failed'
+  ```
+
+- Updated dependencies [[`86623c1`](https://github.com/mastra-ai/mastra/commit/86623c1adf7d22de32cc916dda17f4155184db36), [`7c9dd77`](https://github.com/mastra-ai/mastra/commit/7c9dd77bd18cb8dc72797e25f1a0fbdc71a11347), [`9990965`](https://github.com/mastra-ai/mastra/commit/999096571635a83b42ef40841fd7028cfa630779), [`c0ffa3c`](https://github.com/mastra-ai/mastra/commit/c0ffa3c897ccd326de880df734740a7f0681a18f), [`0504bf5`](https://github.com/mastra-ai/mastra/commit/0504bf5e8cffc571a4b343326178de371e6f859b), [`5afe423`](https://github.com/mastra-ai/mastra/commit/5afe423e4badf040f1b0d4525183a856fcb8146e), [`86623c1`](https://github.com/mastra-ai/mastra/commit/86623c1adf7d22de32cc916dda17f4155184db36), [`8c9f1c0`](https://github.com/mastra-ai/mastra/commit/8c9f1c0361d89066f9bcd14a2f69e761b01766c8)]:
+  - @mastra/core@1.47.0-alpha.2
+
+## 1.5.0-alpha.0
+
+### Minor Changes
+
+- Added `checks`, a new namespace of micro-scorers for common eval assertions. ([#18392](https://github.com/mastra-ai/mastra/pull/18392))
+
+  **What changed**
+  - Added text checks: `includes`, `excludes`, `equals`, `matches`, and `similarity`.
+  - Added tool checks: `calledTool`, `didNotCall`, `toolOrder`, `maxToolCalls`, `usedNoTools`, and `noToolErrors`.
+  - You can now import checks from `@mastra/evals/checks`.
+
+  **Example**
+
+  ```ts
+  import { checks } from '@mastra/evals/checks';
+
+  const scorers = [
+    checks.includes('sunny'),
+    checks.calledTool('get_weather'),
+    checks.toolOrder(['search', 'summarize']),
+    checks.noToolErrors(),
+  ];
+  ```
+
+### Patch Changes
+
+- Updated dependencies [[`7f9ae70`](https://github.com/mastra-ai/mastra/commit/7f9ae70826b047e5a66218f9e92f20e54a2d791f), [`1505c07`](https://github.com/mastra-ai/mastra/commit/1505c07603f6346bae12aa82f140e8b88ffea9ab), [`e940f09`](https://github.com/mastra-ai/mastra/commit/e940f099ef5d18b403e6f2b4937e086a4da857b1)]:
+  - @mastra/core@1.46.1-alpha.1
+
 ## 1.4.0
 
 ### Minor Changes

package/dist/checks.cjs

@@ -0,0 +1,56 @@
+'use strict';
+
+var chunkNXVFY4CK_cjs = require('./chunk-NXVFY4CK.cjs');
+
+
+
+Object.defineProperty(exports, "calledTool", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.calledTool; }
+});
+Object.defineProperty(exports, "checks", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.checks; }
+});
+Object.defineProperty(exports, "didNotCall", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.didNotCall; }
+});
+Object.defineProperty(exports, "equals", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.equals; }
+});
+Object.defineProperty(exports, "excludes", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.excludes; }
+});
+Object.defineProperty(exports, "includes", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.includes; }
+});
+Object.defineProperty(exports, "matches", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.matches; }
+});
+Object.defineProperty(exports, "maxToolCalls", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.maxToolCalls; }
+});
+Object.defineProperty(exports, "noToolErrors", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.noToolErrors; }
+});
+Object.defineProperty(exports, "similarity", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.similarity; }
+});
+Object.defineProperty(exports, "toolOrder", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.toolOrder; }
+});
+Object.defineProperty(exports, "usedNoTools", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.usedNoTools; }
+});
+//# sourceMappingURL=checks.cjs.map
+//# sourceMappingURL=checks.cjs.map

package/dist/checks.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"checks.cjs"}

package/dist/checks.d.ts

@@ -0,0 +1,3 @@
+export { checks } from './scorers/code/checks/index.js';
+export * from './scorers/code/checks/index.js';
+//# sourceMappingURL=checks.d.ts.map

package/dist/checks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checks.d.ts","sourceRoot":"","sources":["../src/checks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,cAAc,uBAAuB,CAAC"}

package/dist/checks.js

@@ -0,0 +1,3 @@
+export { calledTool, checks, didNotCall, equals, excludes, includes, matches, maxToolCalls, noToolErrors, similarity, toolOrder, usedNoTools } from './chunk-QRAONHNV.js';
+//# sourceMappingURL=checks.js.map
+//# sourceMappingURL=checks.js.map

package/dist/checks.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"checks.js"}

package/dist/chunk-NXVFY4CK.cjs

@@ -0,0 +1,233 @@
+'use strict';
+
+var chunkUNQXHPOD_cjs = require('./chunk-UNQXHPOD.cjs');
+var evals = require('@mastra/core/evals');
+var stringSimilarity = require('string-similarity');
+
+function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
+
+var stringSimilarity__default = /*#__PURE__*/_interopDefault(stringSimilarity);
+
+function includes(expected, options = {}) {
+  const { ignoreCase = true } = options;
+  return evals.createScorer({
+    id: "check-includes",
+    name: "Includes Check",
+    description: `Checks if output includes "${expected}"`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    let output = run.output.filter((m) => m.role === "assistant").map((m) => chunkUNQXHPOD_cjs.getTextContentFromMastraDBMessage(m)).join(" ");
+    let target = expected;
+    if (ignoreCase) {
+      output = output.toLowerCase();
+      target = target.toLowerCase();
+    }
+    return { output, target, found: output.includes(target) };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.found ? 1 : 0;
+  });
+}
+function excludes(unwanted, options = {}) {
+  const { ignoreCase = true } = options;
+  return evals.createScorer({
+    id: "check-excludes",
+    name: "Excludes Check",
+    description: `Checks that output does not include "${unwanted}"`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    let output = run.output.filter((m) => m.role === "assistant").map((m) => chunkUNQXHPOD_cjs.getTextContentFromMastraDBMessage(m)).join(" ");
+    let target = unwanted;
+    if (ignoreCase) {
+      output = output.toLowerCase();
+      target = target.toLowerCase();
+    }
+    return { output, target, excluded: !output.includes(target) };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.excluded ? 1 : 0;
+  });
+}
+function equals(expected, options = {}) {
+  const { ignoreCase = true } = options;
+  return evals.createScorer({
+    id: "check-equals",
+    name: "Equals Check",
+    description: `Checks if output equals "${expected}"`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    let output = run.output.filter((m) => m.role === "assistant").map((m) => chunkUNQXHPOD_cjs.getTextContentFromMastraDBMessage(m)).join("");
+    let target = expected;
+    if (ignoreCase) {
+      output = output.toLowerCase();
+      target = target.toLowerCase();
+    }
+    return { output, target, isEqual: output === target };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.isEqual ? 1 : 0;
+  });
+}
+function matches(pattern, options = {}) {
+  const { exact = false } = options;
+  return evals.createScorer({
+    id: "check-matches",
+    name: "Matches Check",
+    description: `Checks if output matches pattern ${pattern}`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const output = run.output.filter((m) => m.role === "assistant").map((m) => chunkUNQXHPOD_cjs.getTextContentFromMastraDBMessage(m)).join("");
+    const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;
+    const matched = regex.test(output);
+    return { output, pattern: pattern.toString(), matched };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.matched ? 1 : 0;
+  });
+}
+function similarity(expected, options = {}) {
+  const { ignoreCase = true, threshold } = options;
+  return evals.createScorer({
+    id: "check-similarity",
+    name: "Similarity Check",
+    description: `Checks string similarity to "${expected}"`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    let output = run.output.filter((m) => m.role === "assistant").map((m) => chunkUNQXHPOD_cjs.getTextContentFromMastraDBMessage(m)).join(" ");
+    let target = expected;
+    if (ignoreCase) {
+      output = output.toLowerCase();
+      target = target.toLowerCase();
+    }
+    const score = stringSimilarity__default.default.compareTwoStrings(output, target);
+    return { output, target, score, threshold };
+  }).generateScore(({ results }) => {
+    const score = results.preprocessStepResult?.score ?? 0;
+    const t = results.preprocessStepResult?.threshold;
+    return t !== void 0 ? score >= t ? 1 : 0 : score;
+  });
+}
+function calledTool(toolName, options = {}) {
+  const { times = 1 } = options;
+  return evals.createScorer({
+    id: "check-called-tool",
+    name: "Called Tool Check",
+    description: `Checks that "${toolName}" was called${times > 1 ? ` at least ${times} times` : ""}`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = chunkUNQXHPOD_cjs.extractToolCalls(run.output);
+    const count = tools.filter((t) => t === toolName).length;
+    return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function didNotCall(toolName) {
+  return evals.createScorer({
+    id: "check-did-not-call",
+    name: "Did Not Call Check",
+    description: `Checks that "${toolName}" was NOT called`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = chunkUNQXHPOD_cjs.extractToolCalls(run.output);
+    const count = tools.filter((t) => t === toolName).length;
+    return { toolName, count, passed: count === 0 };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function toolOrder(expectedOrder) {
+  return evals.createScorer({
+    id: "check-tool-order",
+    name: "Tool Order Check",
+    description: `Checks tool call order: [${expectedOrder.join(" \u2192 ")}]`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = chunkUNQXHPOD_cjs.extractToolCalls(run.output);
+    let orderIndex = 0;
+    for (const tool of tools) {
+      if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {
+        orderIndex++;
+      }
+    }
+    const passed = orderIndex === expectedOrder.length;
+    return { actualTools: tools, expectedOrder, passed };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function maxToolCalls(max) {
+  return evals.createScorer({
+    id: "check-max-tool-calls",
+    name: "Max Tool Calls Check",
+    description: `Checks that no more than ${max} tool calls were made`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = chunkUNQXHPOD_cjs.extractToolCalls(run.output);
+    return { count: tools.length, max, passed: tools.length <= max };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function usedNoTools() {
+  return evals.createScorer({
+    id: "check-used-no-tools",
+    name: "Used No Tools Check",
+    description: "Checks that no tools were called",
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = chunkUNQXHPOD_cjs.extractToolCalls(run.output);
+    return { count: tools.length, passed: tools.length === 0 };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function noToolErrors() {
+  return evals.createScorer({
+    id: "check-no-tool-errors",
+    name: "No Tool Errors Check",
+    description: "Checks that no tool calls resulted in errors",
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const invocations = extractRawInvocations(run.output);
+    const errorCount = invocations.filter((inv) => inv.state === "call" || inv.result && inv.result.error).length;
+    return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function extractRawInvocations(output) {
+  const invocations = [];
+  for (const message of output) {
+    const legacy = message?.content?.toolInvocations;
+    const fromParts = legacy ? void 0 : message?.content?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
+    for (const inv of legacy ?? fromParts ?? []) {
+      if (inv) invocations.push(inv);
+    }
+  }
+  return invocations;
+}
+var checks = {
+  includes,
+  excludes,
+  equals,
+  matches,
+  similarity,
+  calledTool,
+  didNotCall,
+  toolOrder,
+  maxToolCalls,
+  usedNoTools,
+  noToolErrors
+};
+
+exports.calledTool = calledTool;
+exports.checks = checks;
+exports.didNotCall = didNotCall;
+exports.equals = equals;
+exports.excludes = excludes;
+exports.includes = includes;
+exports.matches = matches;
+exports.maxToolCalls = maxToolCalls;
+exports.noToolErrors = noToolErrors;
+exports.similarity = similarity;
+exports.toolOrder = toolOrder;
+exports.usedNoTools = usedNoTools;
+//# sourceMappingURL=chunk-NXVFY4CK.cjs.map
+//# sourceMappingURL=chunk-NXVFY4CK.cjs.map

package/dist/chunk-NXVFY4CK.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/scorers/code/checks/index.ts"],"names":["createScorer","getTextContentFromMastraDBMessage","stringSimilarity","extractToolCalls"],"mappings":";;;;;;;;;;AAoBO,SAAS,QAAA,CAAS,QAAA,EAAkB,OAAA,GAA2B,EAAC,EAAG;AACxE,EAAA,MAAM,EAAE,UAAA,GAAa,IAAA,EAAK,GAAI,OAAA;AAC9B,EAAA,OAAOA,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,gBAAA;AAAA,IACJ,IAAA,EAAM,gBAAA;AAAA,IACN,WAAA,EAAa,8BAA8B,QAAQ,CAAA,CAAA,CAAA;AAAA,IACnD,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,SAAS,GAAA,CAAI,MAAA,CACd,MAAA,CAAO,CAAA,CAAA,KAAK,EAAE,IAAA,KAAS,WAAW,CAAA,CAClC,GAAA,CAAI,OAAKC,mDAAA,CAAkC,CAAC,CAAC,CAAA,CAC7C,KAAK,GAAG,CAAA;AACX,IAAA,IAAI,MAAA,GAAS,QAAA;AACb,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,MAAA,GAAS,OAAO,WAAA,EAAY;AAC5B,MAAA,MAAA,GAAS,OAAO,WAAA,EAAY;AAAA,IAC9B;AACA,IAAA,OAAO,EAAE,MAAA,EAAQ,MAAA,EAAQ,OAAO,MAAA,CAAO,QAAA,CAAS,MAAM,CAAA,EAAE;AAAA,EAC1D,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,KAAA,GAAQ,CAAA,GAAI,CAAA;AAAA,EACnD,CAAC,CAAA;AACL;AAWO,SAAS,QAAA,CAAS,QAAA,EAAkB,OAAA,GAA2B,EAAC,EAAG;AACxE,EAAA,MAAM,EAAE,UAAA,GAAa,IAAA,EAAK,GAAI,OAAA;AAC9B,EAAA,OAAOD,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,gBAAA;AAAA,IACJ,IAAA,EAAM,gBAAA;AAAA,IACN,WAAA,EAAa,wCAAwC,QAAQ,CAAA,CAAA,CAAA;AAAA,IAC7D,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,SAAS,GAAA,CAAI,MAAA,CACd,MAAA,CAAO,CAAA,CAAA,KAAK,EAAE,IAAA,KAAS,WAAW,CAAA,CAClC,GAAA,CAAI,OAAKC,mDAAA,CAAkC,CAAC,CAAC,CAAA,CAC7C,KAAK,GAAG,CAAA;AACX,IAAA,IAAI,MAAA,GAAS,QAAA;AACb,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,MAAA,GAAS,OAAO,WAAA,EAAY;AAC5B,MAAA,MAAA,GAAS,OAAO,WAAA,EAAY;AAAA,IAC9B;AACA,IAAA,OAAO,EAAE,QAAQ,MAAA,EAAQ,QAAA,EAAU,CAAC,MAAA,CAAO,QAAA,CAAS,MAAM,CAAA,EAAE;AAAA,EAC9D,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,QAAA,GAAW,CAAA,GAAI,CAAA;AAAA,EACtD,CAAC,CAAA;AACL;AAWO,SAAS,MAAA,CAAO,QAAA,EAAkB,OAAA,GAA2B,EAAC,EAAG;AACtE,EAAA,MAAM,EAAE,UAAA,GAAa,IAAA,EAAK,GAAI,OAAA;AAC9B,EAAA,OAAOD,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,cAAA;AAAA,IACJ,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EAAa,4BAA4B,QAAQ,CAAA,CAAA,CAAA;AAAA,IACjD,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,SAAS,GAAA,CAAI,MAAA,CACd,MAAA,CAAO,CAAA,CAAA,KAAK,EAAE,IAAA,KAAS,WAAW,CAAA,CAClC,GAAA,CAAI,OAAKC,mDAAA,CAAkC,CAAC,CAAC,CAAA,CAC7C,KAAK,EAAE,CAAA;AACV,IAAA,IAAI,MAAA,GAAS,QAAA;AACb,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,MAAA,GAAS,OAAO,WAAA,EAAY;AAC5B,MAAA,MAAA,GAAS,OAAO,WAAA,EAAY;AAAA,IAC9B;AACA,IAAA,OAAO,EAAE,MAAA,EAAQ,MAAA,EAAQ,OAAA,EAAS,WAAW,MAAA,EAAO;AAAA,EACtD,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,OAAA,GAAU,CAAA,GAAI,CAAA;AAAA,EACrD,CAAC,CAAA;AACL;AAgBO,SAAS,OAAA,CAAQ,OAAA,EAAiB,OAAA,GAA0B,EAAC,EAAG;AACrE,EAAA,MAAM,EAAE,KAAA,GAAQ,KAAA,EAAM,GAAI,OAAA;AAC1B,EAAA,OAAOD,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,eAAA;AAAA,IACJ,IAAA,EAAM,eAAA;AAAA,IACN,WAAA,EAAa,oCAAoC,OAAO,CAAA,CAAA;AAAA,IACxD,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAS,GAAA,CAAI,MAAA,CAChB,MAAA,CAAO,CAAA,CAAA,KAAK,EAAE,IAAA,KAAS,WAAW,CAAA,CAClC,GAAA,CAAI,OAAKC,mDAAA,CAAkC,CAAC,CAAC,CAAA,CAC7C,KAAK,EAAE,CAAA;AACV,IAAA,MAAM,KAAA,GAAQ,KAAA,GAAQ,IAAI,MAAA,CAAO,CAAA,CAAA,EAAI,QAAQ,MAAM,CAAA,CAAA,CAAA,EAAK,OAAA,CAAQ,KAAK,CAAA,GAAI,OAAA;AACzE,IAAA,MAAM,OAAA,GAAU,KAAA,CAAM,IAAA,CAAK,MAAM,CAAA;AACjC,IAAA,OAAO,EAAE,MAAA,EAAQ,OAAA,EAAS,OAAA,CAAQ,QAAA,IAAY,OAAA,EAAQ;AAAA,EACxD,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,OAAA,GAAU,CAAA,GAAI,CAAA;AAAA,EACrD,CAAC,CAAA;AACL;AAmBO,SAAS,UAAA,CAAW,QAAA,EAAkB,OAAA,GAA6B,EAAC,EAAG;AAC5E,EAAA,MAAM,EAAE,UAAA,GAAa,IAAA,EAAM,SAAA,EAAU,GAAI,OAAA;AACzC,EAAA,OAAOD,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,kBAAA;AAAA,IACJ,IAAA,EAAM,kBAAA;AAAA,IACN,WAAA,EAAa,gCAAgC,QAAQ,CAAA,CAAA,CAAA;AAAA,IACrD,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,SAAS,GAAA,CAAI,MAAA,CACd,MAAA,CAAO,CAAA,CAAA,KAAK,EAAE,IAAA,KAAS,WAAW,CAAA,CAClC,GAAA,CAAI,OAAKC,mDAAA,CAAkC,CAAC,CAAC,CAAA,CAC7C,KAAK,GAAG,CAAA;AACX,IAAA,IAAI,MAAA,GAAS,QAAA;AACb,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,MAAA,GAAS,OAAO,WAAA,EAAY;AAC5B,MAAA,MAAA,GAAS,OAAO,WAAA,EAAY;AAAA,IAC9B;AACA,IAAA,MAAM,KAAA,GAAQC,iCAAA,CAAiB,iBAAA,CAAkB,MAAA,EAAQ,MAAM,CAAA;AAC/D,IAAA,OAAO,EAAE,MAAA,EAAQ,MAAA,EAAQ,KAAA,EAAO,SAAA,EAAU;AAAA,EAC5C,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,oBAAA,EAAsB,KAAA,IAAS,CAAA;AACrD,IAAA,MAAM,CAAA,GAAI,QAAQ,oBAAA,EAAsB,SAAA;AACxC,IAAA,OAAO,CAAA,KAAM,MAAA,GAAa,KAAA,IAAS,CAAA,GAAI,IAAI,CAAA,GAAK,KAAA;AAAA,EAClD,CAAC,CAAA;AACL;AAmBO,SAAS,UAAA,CAAW,QAAA,EAAkB,OAAA,GAA6B,EAAC,EAAG;AAC5E,EAAA,MAAM,EAAE,KAAA,GAAQ,CAAA,EAAE,GAAI,OAAA;AACtB,EAAA,OAAOF,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,mBAAA;AAAA,IACJ,IAAA,EAAM,mBAAA;AAAA,IACN,WAAA,EAAa,gBAAgB,QAAQ,CAAA,YAAA,EAAe,QAAQ,CAAA,GAAI,CAAA,UAAA,EAAa,KAAK,CAAA,MAAA,CAAA,GAAW,EAAE,CAAA,CAAA;AAAA,IAC/F,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,EAAE,KAAA,EAAM,GAAIG,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAC7C,IAAA,MAAM,QAAQ,KAAA,CAAM,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,KAAM,QAAQ,CAAA,CAAE,MAAA;AAChD,IAAA,OAAO,EAAE,UAAU,aAAA,EAAe,KAAA,EAAO,aAAa,KAAA,EAAO,MAAA,EAAQ,SAAS,KAAA,EAAM;AAAA,EACtF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,MAAA,GAAS,CAAA,GAAI,CAAA;AAAA,EACpD,CAAC,CAAA;AACL;AAWO,SAAS,WAAW,QAAA,EAAkB;AAC3C,EAAA,OAAOH,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,oBAAA;AAAA,IACJ,IAAA,EAAM,oBAAA;AAAA,IACN,WAAA,EAAa,gBAAgB,QAAQ,CAAA,gBAAA,CAAA;AAAA,IACrC,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,EAAE,KAAA,EAAM,GAAIG,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAC7C,IAAA,MAAM,QAAQ,KAAA,CAAM,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,KAAM,QAAQ,CAAA,CAAE,MAAA;AAChD,IAAA,OAAO,EAAE,QAAA,EAAU,KAAA,EAAO,MAAA,EAAQ,UAAU,CAAA,EAAE;AAAA,EAChD,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,MAAA,GAAS,CAAA,GAAI,CAAA;AAAA,EACpD,CAAC,CAAA;AACL;AAWO,SAAS,UAAU,aAAA,EAAyB;AACjD,EAAA,OAAOH,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,kBAAA;AAAA,IACJ,IAAA,EAAM,kBAAA;AAAA,IACN,WAAA,EAAa,CAAA,yBAAA,EAA4B,aAAA,CAAc,IAAA,CAAK,UAAK,CAAC,CAAA,CAAA,CAAA;AAAA,IAClE,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,EAAE,KAAA,EAAM,GAAIG,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAE7C,IAAA,IAAI,UAAA,GAAa,CAAA;AACjB,IAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,MAAA,IAAI,aAAa,aAAA,CAAc,MAAA,IAAU,IAAA,KAAS,aAAA,CAAc,UAAU,CAAA,EAAG;AAC3E,QAAA,UAAA,EAAA;AAAA,MACF;AAAA,IACF;AACA,IAAA,MAAM,MAAA,GAAS,eAAe,aAAA,CAAc,MAAA;AAC5C,IAAA,OAAO,EAAE,WAAA,EAAa,KAAA,EAAO,aAAA,EAAe,MAAA,EAAO;AAAA,EACrD,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,MAAA,GAAS,CAAA,GAAI,CAAA;AAAA,EACpD,CAAC,CAAA;AACL;AAWO,SAAS,aAAa,GAAA,EAAa;AACxC,EAAA,OAAOH,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,sBAAA;AAAA,IACJ,IAAA,EAAM,sBAAA;AAAA,IACN,WAAA,EAAa,4BAA4B,GAAG,CAAA,qBAAA,CAAA;AAAA,IAC5C,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,EAAE,KAAA,EAAM,GAAIG,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAC7C,IAAA,OAAO,EAAE,OAAO,KAAA,CAAM,MAAA,EAAQ,KAAK,MAAA,EAAQ,KAAA,CAAM,UAAU,GAAA,EAAI;AAAA,EACjE,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,MAAA,GAAS,CAAA,GAAI,CAAA;AAAA,EACpD,CAAC,CAAA;AACL;AAWO,SAAS,WAAA,GAAc;AAC5B,EAAA,OAAOH,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,qBAAA;AAAA,IACJ,IAAA,EAAM,qBAAA;AAAA,IACN,WAAA,EAAa,kCAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,EAAE,KAAA,EAAM,GAAIG,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAC7C,IAAA,OAAO,EAAE,KAAA,EAAO,KAAA,CAAM,QAAQ,MAAA,EAAQ,KAAA,CAAM,WAAW,CAAA,EAAE;AAAA,EAC3D,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,MAAA,GAAS,CAAA,GAAI,CAAA;AAAA,EACpD,CAAC,CAAA;AACL;AAYO,SAAS,YAAA,GAAe;AAC7B,EAAA,OAAOH,kBAAA,CAAa;AAAA,IAClB,EAAA,EAAI,sBAAA;AAAA,IACJ,IAAA,EAAM,sBAAA;AAAA,IACN,WAAA,EAAa,8CAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,WAAA,GAAc,qBAAA,CAAsB,GAAA,CAAI,MAAM,CAAA;AACpD,IAAA,MAAM,UAAA,GAAa,WAAA,CAAY,MAAA,CAAO,CAAA,GAAA,KAAO,GAAA,CAAI,KAAA,KAAU,MAAA,IAAW,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,MAAA,CAAO,KAAM,CAAA,CAAE,MAAA;AACvG,IAAA,OAAO,EAAE,UAAA,EAAY,UAAA,EAAY,YAAY,MAAA,EAAQ,MAAA,EAAQ,eAAe,CAAA,EAAE;AAAA,EAChF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,OAAA,CAAQ,oBAAA,EAAsB,MAAA,GAAS,CAAA,GAAI,CAAA;AAAA,EACpD,CAAC,CAAA;AACL;AAIA,SAAS,sBAAsB,MAAA,EAAgD;AAC7E,EAAA,MAAM,cAAqB,EAAC;AAC5B,EAAA,KAAA,MAAW,WAAW,MAAA,EAAQ;AAC5B,IAAA,MAAM,MAAA,GAAS,SAAS,OAAA,EAAS,eAAA;AACjC,IAAA,MAAM,YAAY,MAAA,GACd,MAAA,GACC,OAAA,EAAS,OAAA,EAAiB,OACvB,MAAA,CAAO,CAAC,CAAA,KAAW,CAAA,CAAE,SAAS,iBAAiB,CAAA,CAChD,IAAI,CAAC,CAAA,KAAW,EAAE,cAAc,CAAA;AACvC,IAAA,KAAA,MAAW,GAAA,IAAO,MAAA,IAAU,SAAA,IAAa,EAAC,EAAG;AAC3C,MAAA,IAAI,GAAA,EAAK,WAAA,CAAY,IAAA,CAAK,GAAG,CAAA;AAAA,IAC/B;AAAA,EACF;AACA,EAAA,OAAO,WAAA;AACT;AA4BO,IAAM,MAAA,GAAS;AAAA,EACpB,QAAA;AAAA,EACA,QAAA;AAAA,EACA,MAAA;AAAA,EACA,OAAA;AAAA,EACA,UAAA;AAAA,EACA,UAAA;AAAA,EACA,UAAA;AAAA,EACA,SAAA;AAAA,EACA,YAAA;AAAA,EACA,WAAA;AAAA,EACA;AACF","file":"chunk-NXVFY4CK.cjs","sourcesContent":["import { createScorer } from '@mastra/core/evals';\nimport stringSimilarity from 'string-similarity';\nimport { extractToolCalls, getTextContentFromMastraDBMessage } from '../../utils';\n\n// ─── Output Text Checks ───────────────────────────────────────────────────────\n\nexport interface IncludesOptions {\n  /** Case-insensitive match (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.includes('sunny');\n * ```\n */\nexport function includes(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-includes',\n    name: 'Includes Check',\n    description: `Checks if output includes \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, found: output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.found ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.excludes('error');\n * ```\n */\nexport function excludes(unwanted: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-excludes',\n    name: 'Excludes Check',\n    description: `Checks that output does not include \"${unwanted}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = unwanted;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, excluded: !output.includes(target) };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.excluded ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the output text exactly equals the expected string (after optional normalization).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.equals('Hello, world!');\n * ```\n */\nexport function equals(expected: string, options: IncludesOptions = {}) {\n  const { ignoreCase = true } = options;\n  return createScorer({\n    id: 'check-equals',\n    name: 'Equals Check',\n    description: `Checks if output equals \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      return { output, target, isEqual: output === target };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.isEqual ? 1 : 0;\n    });\n}\n\nexport interface MatchesOptions {\n  /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */\n  exact?: boolean;\n}\n\n/**\n * Scores 1 if the output matches the given regular expression, 0 otherwise.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.matches(/\\d{1,3}°[FC]/);\n * ```\n */\nexport function matches(pattern: RegExp, options: MatchesOptions = {}) {\n  const { exact = false } = options;\n  return createScorer({\n    id: 'check-matches',\n    name: 'Matches Check',\n    description: `Checks if output matches pattern ${pattern}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join('');\n      const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;\n      const matched = regex.test(output);\n      return { output, pattern: pattern.toString(), matched };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.matched ? 1 : 0;\n    });\n}\n\nexport interface SimilarityOptions {\n  /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */\n  threshold?: number;\n  /** Case-insensitive comparison (default: true) */\n  ignoreCase?: boolean;\n}\n\n/**\n * Returns the string similarity score (0-1) between the output and an expected string.\n * Useful for fuzzy matching when exact equality is too strict.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.similarity('Sunny, 72°F');\n * ```\n */\nexport function similarity(expected: string, options: SimilarityOptions = {}) {\n  const { ignoreCase = true, threshold } = options;\n  return createScorer({\n    id: 'check-similarity',\n    name: 'Similarity Check',\n    description: `Checks string similarity to \"${expected}\"`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let output = run.output\n        .filter(m => m.role === 'assistant')\n        .map(m => getTextContentFromMastraDBMessage(m))\n        .join(' ');\n      let target = expected;\n      if (ignoreCase) {\n        output = output.toLowerCase();\n        target = target.toLowerCase();\n      }\n      const score = stringSimilarity.compareTwoStrings(output, target);\n      return { output, target, score, threshold };\n    })\n    .generateScore(({ results }) => {\n      const score = results.preprocessStepResult?.score ?? 0;\n      const t = results.preprocessStepResult?.threshold;\n      return t !== undefined ? (score >= t ? 1 : 0) : score;\n    });\n}\n\n// ─── Tool Call Checks ─────────────────────────────────────────────────────────\n\nexport interface CalledToolOptions {\n  /** Minimum number of times the tool must be called. Default: 1 */\n  times?: number;\n}\n\n/**\n * Scores 1 if the agent called the specified tool (at least `times` times).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.calledTool('get_weather');\n * const twice = checks.calledTool('search', { times: 2 });\n * ```\n */\nexport function calledTool(toolName: string, options: CalledToolOptions = {}) {\n  const { times = 1 } = options;\n  return createScorer({\n    id: 'check-called-tool',\n    name: 'Called Tool Check',\n    description: `Checks that \"${toolName}\" was called${times > 1 ? ` at least ${times} times` : ''}`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent did NOT call the specified tool.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.didNotCall('delete_user');\n * ```\n */\nexport function didNotCall(toolName: string) {\n  return createScorer({\n    id: 'check-did-not-call',\n    name: 'Did Not Call Check',\n    description: `Checks that \"${toolName}\" was NOT called`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      const count = tools.filter(t => t === toolName).length;\n      return { toolName, count, passed: count === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);\n * ```\n */\nexport function toolOrder(expectedOrder: string[]) {\n  return createScorer({\n    id: 'check-tool-order',\n    name: 'Tool Order Check',\n    description: `Checks tool call order: [${expectedOrder.join(' → ')}]`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      // Check that expectedOrder appears as a subsequence of tools\n      let orderIndex = 0;\n      for (const tool of tools) {\n        if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {\n          orderIndex++;\n        }\n      }\n      const passed = orderIndex === expectedOrder.length;\n      return { actualTools: tools, expectedOrder, passed };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent used no more than `max` tool calls.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.maxToolCalls(5);\n * ```\n */\nexport function maxToolCalls(max: number) {\n  return createScorer({\n    id: 'check-max-tool-calls',\n    name: 'Max Tool Calls Check',\n    description: `Checks that no more than ${max} tool calls were made`,\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, max, passed: tools.length <= max };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if the agent made no tool calls at all.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.usedNoTools();\n * ```\n */\nexport function usedNoTools() {\n  return createScorer({\n    id: 'check-used-no-tools',\n    name: 'Used No Tools Check',\n    description: 'Checks that no tools were called',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const { tools } = extractToolCalls(run.output);\n      return { count: tools.length, passed: tools.length === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n/**\n * Scores 1 if none of the tool invocations resulted in an error state.\n * Checks for tool invocations with state other than 'result' (i.e., missing results).\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n * const scorer = checks.noToolErrors();\n * ```\n */\nexport function noToolErrors() {\n  return createScorer({\n    id: 'check-no-tool-errors',\n    name: 'No Tool Errors Check',\n    description: 'Checks that no tool calls resulted in errors',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const invocations = extractRawInvocations(run.output);\n      const errorCount = invocations.filter(inv => inv.state === 'call' || (inv.result && inv.result.error)).length;\n      return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.passed ? 1 : 0;\n    });\n}\n\n// ─── Internal helpers ──────────────────────────────────────────────────────────\n\nfunction extractRawInvocations(output: Parameters<typeof extractToolCalls>[0]) {\n  const invocations: any[] = [];\n  for (const message of output) {\n    const legacy = message?.content?.toolInvocations;\n    const fromParts = legacy\n      ? undefined\n      : (message?.content as any)?.parts\n          ?.filter((p: any) => p.type === 'tool-invocation')\n          .map((p: any) => p.toolInvocation);\n    for (const inv of legacy ?? fromParts ?? []) {\n      if (inv) invocations.push(inv);\n    }\n  }\n  return invocations;\n}\n\n// ─── Convenience namespace ────────────────────────────────────────────────────\n\n/**\n * Quick Checks — composable micro-scorers for common assertions.\n *\n * These are zero-LLM, zero-ceremony scorers that plug into the existing\n * `scorers: [...]` array anywhere scorers are used. Internally they are\n * standard `createScorer()` instances with the same observability, storage,\n * and pipeline integration as any other scorer.\n *\n * @example\n * ```ts\n * import { checks } from '@mastra/evals';\n *\n * await runEvals({\n *   data: [...],\n *   target: myAgent,\n *   scorers: [\n *     checks.includes('sunny'),\n *     checks.calledTool('get_weather'),\n *     checks.toolOrder(['search', 'summarize']),\n *     checks.noToolErrors(),\n *   ],\n * });\n * ```\n */\nexport const checks = {\n  includes,\n  excludes,\n  equals,\n  matches,\n  similarity,\n  calledTool,\n  didNotCall,\n  toolOrder,\n  maxToolCalls,\n  usedNoTools,\n  noToolErrors,\n};\n"]}

package/dist/chunk-QRAONHNV.js

@@ -0,0 +1,216 @@
+import { getTextContentFromMastraDBMessage, extractToolCalls } from './chunk-BE5F2OUQ.js';
+import { createScorer } from '@mastra/core/evals';
+import stringSimilarity from 'string-similarity';
+
+function includes(expected, options = {}) {
+  const { ignoreCase = true } = options;
+  return createScorer({
+    id: "check-includes",
+    name: "Includes Check",
+    description: `Checks if output includes "${expected}"`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    let output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join(" ");
+    let target = expected;
+    if (ignoreCase) {
+      output = output.toLowerCase();
+      target = target.toLowerCase();
+    }
+    return { output, target, found: output.includes(target) };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.found ? 1 : 0;
+  });
+}
+function excludes(unwanted, options = {}) {
+  const { ignoreCase = true } = options;
+  return createScorer({
+    id: "check-excludes",
+    name: "Excludes Check",
+    description: `Checks that output does not include "${unwanted}"`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    let output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join(" ");
+    let target = unwanted;
+    if (ignoreCase) {
+      output = output.toLowerCase();
+      target = target.toLowerCase();
+    }
+    return { output, target, excluded: !output.includes(target) };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.excluded ? 1 : 0;
+  });
+}
+function equals(expected, options = {}) {
+  const { ignoreCase = true } = options;
+  return createScorer({
+    id: "check-equals",
+    name: "Equals Check",
+    description: `Checks if output equals "${expected}"`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    let output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join("");
+    let target = expected;
+    if (ignoreCase) {
+      output = output.toLowerCase();
+      target = target.toLowerCase();
+    }
+    return { output, target, isEqual: output === target };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.isEqual ? 1 : 0;
+  });
+}
+function matches(pattern, options = {}) {
+  const { exact = false } = options;
+  return createScorer({
+    id: "check-matches",
+    name: "Matches Check",
+    description: `Checks if output matches pattern ${pattern}`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join("");
+    const regex = exact ? new RegExp(`^${pattern.source}$`, pattern.flags) : pattern;
+    const matched = regex.test(output);
+    return { output, pattern: pattern.toString(), matched };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.matched ? 1 : 0;
+  });
+}
+function similarity(expected, options = {}) {
+  const { ignoreCase = true, threshold } = options;
+  return createScorer({
+    id: "check-similarity",
+    name: "Similarity Check",
+    description: `Checks string similarity to "${expected}"`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    let output = run.output.filter((m) => m.role === "assistant").map((m) => getTextContentFromMastraDBMessage(m)).join(" ");
+    let target = expected;
+    if (ignoreCase) {
+      output = output.toLowerCase();
+      target = target.toLowerCase();
+    }
+    const score = stringSimilarity.compareTwoStrings(output, target);
+    return { output, target, score, threshold };
+  }).generateScore(({ results }) => {
+    const score = results.preprocessStepResult?.score ?? 0;
+    const t = results.preprocessStepResult?.threshold;
+    return t !== void 0 ? score >= t ? 1 : 0 : score;
+  });
+}
+function calledTool(toolName, options = {}) {
+  const { times = 1 } = options;
+  return createScorer({
+    id: "check-called-tool",
+    name: "Called Tool Check",
+    description: `Checks that "${toolName}" was called${times > 1 ? ` at least ${times} times` : ""}`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = extractToolCalls(run.output);
+    const count = tools.filter((t) => t === toolName).length;
+    return { toolName, expectedTimes: times, actualCount: count, passed: count >= times };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function didNotCall(toolName) {
+  return createScorer({
+    id: "check-did-not-call",
+    name: "Did Not Call Check",
+    description: `Checks that "${toolName}" was NOT called`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = extractToolCalls(run.output);
+    const count = tools.filter((t) => t === toolName).length;
+    return { toolName, count, passed: count === 0 };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function toolOrder(expectedOrder) {
+  return createScorer({
+    id: "check-tool-order",
+    name: "Tool Order Check",
+    description: `Checks tool call order: [${expectedOrder.join(" \u2192 ")}]`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = extractToolCalls(run.output);
+    let orderIndex = 0;
+    for (const tool of tools) {
+      if (orderIndex < expectedOrder.length && tool === expectedOrder[orderIndex]) {
+        orderIndex++;
+      }
+    }
+    const passed = orderIndex === expectedOrder.length;
+    return { actualTools: tools, expectedOrder, passed };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function maxToolCalls(max) {
+  return createScorer({
+    id: "check-max-tool-calls",
+    name: "Max Tool Calls Check",
+    description: `Checks that no more than ${max} tool calls were made`,
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = extractToolCalls(run.output);
+    return { count: tools.length, max, passed: tools.length <= max };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function usedNoTools() {
+  return createScorer({
+    id: "check-used-no-tools",
+    name: "Used No Tools Check",
+    description: "Checks that no tools were called",
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const { tools } = extractToolCalls(run.output);
+    return { count: tools.length, passed: tools.length === 0 };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function noToolErrors() {
+  return createScorer({
+    id: "check-no-tool-errors",
+    name: "No Tool Errors Check",
+    description: "Checks that no tool calls resulted in errors",
+    type: "agent"
+  }).preprocess(async ({ run }) => {
+    const invocations = extractRawInvocations(run.output);
+    const errorCount = invocations.filter((inv) => inv.state === "call" || inv.result && inv.result.error).length;
+    return { errorCount, totalCalls: invocations.length, passed: errorCount === 0 };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.passed ? 1 : 0;
+  });
+}
+function extractRawInvocations(output) {
+  const invocations = [];
+  for (const message of output) {
+    const legacy = message?.content?.toolInvocations;
+    const fromParts = legacy ? void 0 : message?.content?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
+    for (const inv of legacy ?? fromParts ?? []) {
+      if (inv) invocations.push(inv);
+    }
+  }
+  return invocations;
+}
+var checks = {
+  includes,
+  excludes,
+  equals,
+  matches,
+  similarity,
+  calledTool,
+  didNotCall,
+  toolOrder,
+  maxToolCalls,
+  usedNoTools,
+  noToolErrors
+};
+
+export { calledTool, checks, didNotCall, equals, excludes, includes, matches, maxToolCalls, noToolErrors, similarity, toolOrder, usedNoTools };
+//# sourceMappingURL=chunk-QRAONHNV.js.map
+//# sourceMappingURL=chunk-QRAONHNV.js.map