@mastra/evals 1.4.0-alpha.0 → 1.5.0-alpha.0

package/dist/scorers/code/checks/index.d.ts

@@ -0,0 +1,214 @@
+export interface IncludesOptions {
+    /** Case-insensitive match (default: true) */
+    ignoreCase?: boolean;
+}
+/**
+ * Scores 1 if the agent's output text contains the expected substring, 0 otherwise.
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.includes('sunny');
+ * ```
+ */
+export declare function includes(expected: string, options?: IncludesOptions): import("@mastra/core/evals").MastraScorer<"check-includes", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    output: string;
+    target: string;
+    found: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+/**
+ * Scores 1 if the agent's output text does NOT contain the substring, 0 otherwise.
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.excludes('error');
+ * ```
+ */
+export declare function excludes(unwanted: string, options?: IncludesOptions): import("@mastra/core/evals").MastraScorer<"check-excludes", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    output: string;
+    target: string;
+    excluded: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+/**
+ * Scores 1 if the output text exactly equals the expected string (after optional normalization).
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.equals('Hello, world!');
+ * ```
+ */
+export declare function equals(expected: string, options?: IncludesOptions): import("@mastra/core/evals").MastraScorer<"check-equals", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    output: string;
+    target: string;
+    isEqual: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+export interface MatchesOptions {
+    /** If true, the output must match the pattern exactly (anchored). Default: false (substring match). */
+    exact?: boolean;
+}
+/**
+ * Scores 1 if the output matches the given regular expression, 0 otherwise.
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.matches(/\d{1,3}°[FC]/);
+ * ```
+ */
+export declare function matches(pattern: RegExp, options?: MatchesOptions): import("@mastra/core/evals").MastraScorer<"check-matches", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    output: string;
+    pattern: string;
+    matched: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+export interface SimilarityOptions {
+    /** Minimum similarity threshold (0-1) to score 1. Default: 0.7 */
+    threshold?: number;
+    /** Case-insensitive comparison (default: true) */
+    ignoreCase?: boolean;
+}
+/**
+ * Returns the string similarity score (0-1) between the output and an expected string.
+ * Useful for fuzzy matching when exact equality is too strict.
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.similarity('Sunny, 72°F');
+ * ```
+ */
+export declare function similarity(expected: string, options?: SimilarityOptions): import("@mastra/core/evals").MastraScorer<"check-similarity", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    output: string;
+    target: string;
+    score: number;
+    threshold: number | undefined;
+}> & Record<"generateScoreStepResult", number>>;
+export interface CalledToolOptions {
+    /** Minimum number of times the tool must be called. Default: 1 */
+    times?: number;
+}
+/**
+ * Scores 1 if the agent called the specified tool (at least `times` times).
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.calledTool('get_weather');
+ * const twice = checks.calledTool('search', { times: 2 });
+ * ```
+ */
+export declare function calledTool(toolName: string, options?: CalledToolOptions): import("@mastra/core/evals").MastraScorer<"check-called-tool", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    toolName: string;
+    expectedTimes: number;
+    actualCount: number;
+    passed: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+/**
+ * Scores 1 if the agent did NOT call the specified tool.
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.didNotCall('delete_user');
+ * ```
+ */
+export declare function didNotCall(toolName: string): import("@mastra/core/evals").MastraScorer<"check-did-not-call", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    toolName: string;
+    count: number;
+    passed: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+/**
+ * Scores 1 if the tools were called in the specified order (relaxed: allows other calls in between).
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.toolOrder(['search', 'summarize', 'respond']);
+ * ```
+ */
+export declare function toolOrder(expectedOrder: string[]): import("@mastra/core/evals").MastraScorer<"check-tool-order", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    actualTools: string[];
+    expectedOrder: string[];
+    passed: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+/**
+ * Scores 1 if the agent used no more than `max` tool calls.
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.maxToolCalls(5);
+ * ```
+ */
+export declare function maxToolCalls(max: number): import("@mastra/core/evals").MastraScorer<"check-max-tool-calls", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    count: number;
+    max: number;
+    passed: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+/**
+ * Scores 1 if the agent made no tool calls at all.
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.usedNoTools();
+ * ```
+ */
+export declare function usedNoTools(): import("@mastra/core/evals").MastraScorer<"check-used-no-tools", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    count: number;
+    passed: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+/**
+ * Scores 1 if none of the tool invocations resulted in an error state.
+ * Checks for tool invocations with state other than 'result' (i.e., missing results).
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ * const scorer = checks.noToolErrors();
+ * ```
+ */
+export declare function noToolErrors(): import("@mastra/core/evals").MastraScorer<"check-no-tool-errors", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    errorCount: number;
+    totalCalls: number;
+    passed: boolean;
+}> & Record<"generateScoreStepResult", number>>;
+/**
+ * Quick Checks — composable micro-scorers for common assertions.
+ *
+ * These are zero-LLM, zero-ceremony scorers that plug into the existing
+ * `scorers: [...]` array anywhere scorers are used. Internally they are
+ * standard `createScorer()` instances with the same observability, storage,
+ * and pipeline integration as any other scorer.
+ *
+ * @example
+ * ```ts
+ * import { checks } from '@mastra/evals';
+ *
+ * await runEvals({
+ *   data: [...],
+ *   target: myAgent,
+ *   scorers: [
+ *     checks.includes('sunny'),
+ *     checks.calledTool('get_weather'),
+ *     checks.toolOrder(['search', 'summarize']),
+ *     checks.noToolErrors(),
+ *   ],
+ * });
+ * ```
+ */
+export declare const checks: {
+    includes: typeof includes;
+    excludes: typeof excludes;
+    equals: typeof equals;
+    matches: typeof matches;
+    similarity: typeof similarity;
+    calledTool: typeof calledTool;
+    didNotCall: typeof didNotCall;
+    toolOrder: typeof toolOrder;
+    maxToolCalls: typeof maxToolCalls;
+    usedNoTools: typeof usedNoTools;
+    noToolErrors: typeof noToolErrors;
+};
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/code/checks/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/checks/index.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB;;;;gDAuBrE;AAED,MAAM,WAAW,cAAc;IAC7B,uGAAuG;IACvG,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB;;;;gDAoBpE;AAED,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDA0B3E;AAID,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB;;;;;gDAgB3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM;;;;gDAe1C;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,aAAa,EAAE,MAAM,EAAE;;;;gDAsBhD;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM;;;;gDAcvC;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW;;;gDAc1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY;;;;gDAe3B;AAsBD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,eAAO,MAAM,MAAM;;;;;;;;;;;;CAYlB,CAAC"}

package/dist/scorers/code/index.d.ts

@@ -5,4 +5,5 @@ export * from './content-similarity/index.js';
 export * from './tone/index.js';
 export * from './tool-call-accuracy/index.js';
 export * from './trajectory/index.js';
+export * from './checks/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/scorers/code/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scorers/code/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,QAAQ,CAAC;AACvB,cAAc,sBAAsB,CAAC;AACrC,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scorers/code/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,QAAQ,CAAC;AACvB,cAAc,sBAAsB,CAAC;AACrC,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC"}

package/dist/scorers/prebuilt/index.cjs

@@ -1,5 +1,6 @@
 'use strict';
 
+var chunkNXVFY4CK_cjs = require('../../chunk-NXVFY4CK.cjs');
 var chunkUNQXHPOD_cjs = require('../../chunk-UNQXHPOD.cjs');
 var evals = require('@mastra/core/evals');
 var nlp = require('compromise');
@@ -3675,9 +3676,9 @@ function calculateRatio(input, output) {
   if (input.length === 0 || output.length === 0) {
     return 0;
   }
-  const matches = longestCommonSubsequence(input, output);
+  const matches2 = longestCommonSubsequence(input, output);
   const total = input.length + output.length;
-  return total > 0 ? 2 * matches / total : 0;
+  return total > 0 ? 2 * matches2 / total : 0;
 }
 function longestCommonSubsequence(str1, str2) {
   const m = str1.length;
@@ -3728,18 +3729,18 @@ function countChanges(input, output) {
   return changes;
 }
 function findCommonWords(arr1, arr2) {
-  let matches = 0;
+  let matches2 = 0;
   const used = /* @__PURE__ */ new Set();
   for (let i = 0; i < arr1.length; i++) {
     for (let j = 0; j < arr2.length; j++) {
       if (!used.has(j) && arr1[i] === arr2[j]) {
-        matches++;
+        matches2++;
         used.add(j);
         break;
       }
     }
   }
-  return matches;
+  return matches2;
 }
 function createTextualDifferenceScorer() {
   return evals.createScorer({
@@ -3841,11 +3842,11 @@ function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace } = { igno
       processedOutput
     };
   }).generateScore(({ results }) => {
-    const similarity = stringSimilarity__default.default.compareTwoStrings(
+    const similarity2 = stringSimilarity__default.default.compareTwoStrings(
       results.preprocessStepResult?.processedInput,
       results.preprocessStepResult?.processedOutput
     );
-    return similarity;
+    return similarity2;
   });
 }
 function createToneScorer(config = {}) {
@@ -4302,6 +4303,54 @@ function createTrajectoryScorerCode(options = {}) {
   });
 }
 
+Object.defineProperty(exports, "calledTool", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.calledTool; }
+});
+Object.defineProperty(exports, "checks", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.checks; }
+});
+Object.defineProperty(exports, "didNotCall", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.didNotCall; }
+});
+Object.defineProperty(exports, "equals", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.equals; }
+});
+Object.defineProperty(exports, "excludes", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.excludes; }
+});
+Object.defineProperty(exports, "includes", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.includes; }
+});
+Object.defineProperty(exports, "matches", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.matches; }
+});
+Object.defineProperty(exports, "maxToolCalls", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.maxToolCalls; }
+});
+Object.defineProperty(exports, "noToolErrors", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.noToolErrors; }
+});
+Object.defineProperty(exports, "similarity", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.similarity; }
+});
+Object.defineProperty(exports, "toolOrder", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.toolOrder; }
+});
+Object.defineProperty(exports, "usedNoTools", {
+  enumerable: true,
+  get: function () { return chunkNXVFY4CK_cjs.usedNoTools; }
+});
 exports.ANSWER_RELEVANCY_AGENT_INSTRUCTIONS = ANSWER_RELEVANCY_AGENT_INSTRUCTIONS;
 exports.ANSWER_SIMILARITY_DEFAULT_OPTIONS = ANSWER_SIMILARITY_DEFAULT_OPTIONS;
 exports.ANSWER_SIMILARITY_INSTRUCTIONS = ANSWER_SIMILARITY_INSTRUCTIONS;