@mastra/evals 0.13.5 → 0.13.6

package/dist/index.cjs

@@ -41,7 +41,7 @@ var getCurrentTestInfo = async () => {
     };
   }
   try {
-    const vitest = await import('./dist-QNM75ISG.cjs');
+    const vitest = await import('./dist-ALHZKHK6.cjs');
     if (typeof vitest !== "undefined" && vitest.expect?.getState) {
       const state = vitest.expect.getState();
       return {

package/dist/index.js

@@ -39,7 +39,7 @@ var getCurrentTestInfo = async () => {
     };
   }
   try {
-    const vitest = await import('./dist-KXHZV6E4.js');
+    const vitest = await import('./dist-HPW4UI62.js');
     if (typeof vitest !== "undefined" && vitest.expect?.getState) {
       const state = vitest.expect.getState();
       return {

package/dist/scorers/code/index.cjs

@@ -1,6 +1,6 @@
 'use strict';
 
-var chunkEKSPLMYP_cjs = require('../../chunk-EKSPLMYP.cjs');
+var chunkQKR2PMLZ_cjs = require('../../chunk-QKR2PMLZ.cjs');
 var scores = require('@mastra/core/scores');
 var nlp = require('compromise');
 var difflib = require('difflib');
@@ -287,7 +287,7 @@ function createToolCallAccuracyScorerCode(options) {
     if (isInputInvalid || isOutputInvalid) {
       throw new Error("Input and output messages cannot be null or empty");
     }
-    const { tools: actualTools, toolCallInfos } = chunkEKSPLMYP_cjs.extractToolCalls(run.output);
+    const { tools: actualTools, toolCallInfos } = chunkQKR2PMLZ_cjs.extractToolCalls(run.output);
     const correctToolCalled = expectedTool ? strictMode ? actualTools.length === 1 && actualTools[0] === expectedTool : actualTools.includes(expectedTool) : false;
     return {
       expectedTool,

package/dist/scorers/code/index.js

@@ -1,4 +1,4 @@
-import { extractToolCalls } from '../../chunk-4LRZVFXR.js';
+import { extractToolCalls } from '../../chunk-KHEXN75Q.js';
 import { createScorer } from '@mastra/core/scores';
 import nlp from 'compromise';
 import { SequenceMatcher } from 'difflib';

package/dist/scorers/llm/answer-similarity/index.d.ts

@@ -0,0 +1,34 @@
+import type { MastraLanguageModel } from '@mastra/core/agent';
+import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
+export interface AnswerSimilarityOptions {
+    requireGroundTruth?: boolean;
+    semanticThreshold?: number;
+    exactMatchBonus?: number;
+    missingPenalty?: number;
+    contradictionPenalty?: number;
+    extraInfoPenalty?: number;
+    scale?: number;
+}
+export declare const ANSWER_SIMILARITY_DEFAULT_OPTIONS: Required<AnswerSimilarityOptions>;
+export declare const ANSWER_SIMILARITY_INSTRUCTIONS = "\nYou are a precise answer similarity evaluator for CI/CD testing. Your role is to compare agent outputs against ground truth answers to ensure consistency and accuracy in automated testing.\n\nKey Principles:\n1. Focus on semantic equivalence, not just string matching\n2. Recognize that different phrasings can convey the same information\n3. Identify missing critical information from the ground truth\n4. Detect contradictions between output and ground truth\n5. Provide actionable feedback for improving answer accuracy\n6. Be strict but fair - partial credit for partial matches\n";
+export declare function createAnswerSimilarityScorer({ model, options, }: {
+    model: MastraLanguageModel;
+    options?: AnswerSimilarityOptions;
+}): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    outputUnits: string[];
+    groundTruthUnits: string[];
+}> & Record<"analyzeStepResult", {
+    matches: {
+        groundTruthUnit: string;
+        outputUnit: string | null;
+        matchType: "exact" | "semantic" | "partial" | "missing";
+        explanation: string;
+    }[];
+    extraInOutput: string[];
+    contradictions: {
+        groundTruthUnit: string;
+        outputUnit: string;
+        explanation: string;
+    }[];
+}> & Record<"generateScoreStepResult", number> & Record<"generateReasonStepResult", string>>;
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/llm/answer-similarity/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-similarity/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAM3F,MAAM,WAAW,uBAAuB;IACtC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,eAAO,MAAM,iCAAiC,EAAE,QAAQ,CAAC,uBAAuB,CAQ/E,CAAC;AAEF,eAAO,MAAM,8BAA8B,+kBAU1C,CAAC;AA0BF,wBAAgB,4BAA4B,CAAC,EAC3C,KAAK,EACL,OAA2C,GAC5C,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC;;;;;;;;;;;;;;;;6FA4HA"}

package/dist/scorers/llm/answer-similarity/prompts.d.ts

@@ -0,0 +1,29 @@
+export declare const createExtractPrompt: ({ output, groundTruth }: {
+    output: string;
+    groundTruth: string;
+}) => string;
+export declare const createAnalyzePrompt: ({ outputUnits, groundTruthUnits, }: {
+    outputUnits: string[];
+    groundTruthUnits: string[];
+}) => string;
+export declare const createReasonPrompt: ({ output, groundTruth, score, analysis, scale, }: {
+    output: string;
+    groundTruth: string;
+    score: number;
+    analysis: {
+        matches: Array<{
+            groundTruthUnit: string;
+            outputUnit: string | null;
+            matchType: string;
+            explanation: string;
+        }>;
+        extraInOutput: string[];
+        contradictions: Array<{
+            outputUnit: string;
+            groundTruthUnit: string;
+            explanation: string;
+        }>;
+    };
+    scale: number;
+}) => string;
+//# sourceMappingURL=prompts.d.ts.map

package/dist/scorers/llm/answer-similarity/prompts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-similarity/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,mBAAmB,GAAI,yBAAyB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,WAiCnG,CAAC;AAEF,eAAO,MAAM,mBAAmB,GAAI,oCAGjC;IACD,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,WAgDA,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAI,kDAMhC;IACD,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE;QACR,OAAO,EAAE,KAAK,CAAC;YACb,eAAe,EAAE,MAAM,CAAC;YACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;YAC1B,SAAS,EAAE,MAAM,CAAC;YAClB,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC,CAAC;QACH,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,cAAc,EAAE,KAAK,CAAC;YACpB,UAAU,EAAE,MAAM,CAAC;YACnB,eAAe,EAAE,MAAM,CAAC;YACxB,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC,CAAC;KACJ,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;CACf,WAuBA,CAAC"}