@mastra/evals 0.14.3 → 0.14.4-alpha.0

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @mastra/evals
 
+## 0.14.4-alpha.0
+
+### Patch Changes
+
+- Remove difflib ([#9931](https://github.com/mastra-ai/mastra/pull/9931))
+
+- Updated dependencies [[`0a0aa87`](https://github.com/mastra-ai/mastra/commit/0a0aa87910dd9234ae11e8146fbf54d71f0b1516), [`56bbbd0`](https://github.com/mastra-ai/mastra/commit/56bbbd0eb4510455ff03d6bf2827fdbd307938be), [`d576fd8`](https://github.com/mastra-ai/mastra/commit/d576fd8f2a3c61bcf2f7d5d2bdfb496d442c46c2), [`5e48406`](https://github.com/mastra-ai/mastra/commit/5e48406766fa94e2b7c56b70fb7d6068cf7f3989), [`7fcce62`](https://github.com/mastra-ai/mastra/commit/7fcce62880c3525fbf752d59c0ac2c478cffe024), [`16a324f`](https://github.com/mastra-ai/mastra/commit/16a324f8c30a07d0d899bc2e4e7998c6b40a4cb6), [`26aee16`](https://github.com/mastra-ai/mastra/commit/26aee160149e7acb84a533bf45631aaed6dd7077), [`b063a81`](https://github.com/mastra-ai/mastra/commit/b063a8144176915a766ea15888e1e8a06a020776), [`5ff9462`](https://github.com/mastra-ai/mastra/commit/5ff9462691c80a6841b014bcc68f6a85c3fd3fbf)]:
+  - @mastra/core@0.24.1-alpha.0
+
 ## 0.14.3
 
 ### Patch Changes

package/dist/chunk-44PMY5ES.js

@@ -0,0 +1,78 @@
+// src/ratio.ts
+function calculateRatio(input, output) {
+  if (input === output) {
+    return 1;
+  }
+  if (input.length === 0 || output.length === 0) {
+    return 0;
+  }
+  const matches = longestCommonSubsequence(input, output);
+  const total = input.length + output.length;
+  return total > 0 ? 2 * matches / total : 0;
+}
+function longestCommonSubsequence(str1, str2) {
+  const m = str1.length;
+  const n = str2.length;
+  const dp = [];
+  for (let i = 0; i <= m; i++) {
+    dp[i] = [];
+    for (let j = 0; j <= n; j++) {
+      dp[i][j] = 0;
+    }
+  }
+  for (let i = 1; i <= m; i++) {
+    for (let j = 1; j <= n; j++) {
+      if (str1[i - 1] === str2[j - 1]) {
+        dp[i][j] = dp[i - 1][j - 1] + 1;
+      } else {
+        dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]);
+      }
+    }
+  }
+  return dp[m][n];
+}
+function countChanges(input, output) {
+  const inputNormalized = input.replace(/\s+/g, " ").trim();
+  const outputNormalized = output.replace(/\s+/g, " ").trim();
+  if (inputNormalized === outputNormalized) {
+    if (input !== output) {
+      const inputWords2 = input.split(/\s+/).filter((w) => w.length > 0);
+      const outputWords2 = output.split(/\s+/).filter((w) => w.length > 0);
+      return Math.abs(inputWords2.length - outputWords2.length) || 1;
+    }
+    return 0;
+  }
+  const inputWords = inputNormalized.split(/\s+/).filter((w) => w.length > 0);
+  const outputWords = outputNormalized.split(/\s+/).filter((w) => w.length > 0);
+  if (inputWords.length === 0 && outputWords.length === 0) {
+    return 0;
+  }
+  if (inputWords.length === 0) {
+    return outputWords.length;
+  }
+  if (outputWords.length === 0) {
+    return inputWords.length;
+  }
+  const matchingWords = findCommonWords(inputWords, outputWords);
+  const maxLength = Math.max(inputWords.length, outputWords.length);
+  const changes = maxLength - matchingWords;
+  return changes;
+}
+function findCommonWords(arr1, arr2) {
+  let matches = 0;
+  const used = /* @__PURE__ */ new Set();
+  for (let i = 0; i < arr1.length; i++) {
+    for (let j = 0; j < arr2.length; j++) {
+      if (!used.has(j) && arr1[i] === arr2[j]) {
+        matches++;
+        used.add(j);
+        break;
+      }
+    }
+  }
+  return matches;
+}
+
+export { calculateRatio, countChanges };
+//# sourceMappingURL=chunk-44PMY5ES.js.map
+//# sourceMappingURL=chunk-44PMY5ES.js.map

package/dist/chunk-44PMY5ES.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/ratio.ts"],"names":["inputWords","outputWords"],"mappings":";AAKO,SAAS,cAAA,CAAe,OAAe,MAAA,EAAwB;AACpE,EAAA,IAAI,UAAU,MAAA,EAAQ;AACpB,IAAA,OAAO,CAAA;AAAA,EACT;AACA,EAAA,IAAI,KAAA,CAAM,MAAA,KAAW,CAAA,IAAK,MAAA,CAAO,WAAW,CAAA,EAAG;AAC7C,IAAA,OAAO,CAAA;AAAA,EACT;AAGA,EAAA,MAAM,OAAA,GAAU,wBAAA,CAAyB,KAAA,EAAO,MAAM,CAAA;AACtD,EAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,MAAA,GAAS,MAAA,CAAO,MAAA;AAEpC,EAAA,OAAO,KAAA,GAAQ,CAAA,GAAK,CAAA,GAAM,OAAA,GAAW,KAAA,GAAQ,CAAA;AAC/C;AAKA,SAAS,wBAAA,CAAyB,MAAc,IAAA,EAAsB;AACpE,EAAA,MAAM,IAAI,IAAA,CAAK,MAAA;AACf,EAAA,MAAM,IAAI,IAAA,CAAK,MAAA;AACf,EAAA,MAAM,KAAiB,EAAC;AAGxB,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,IAAK,CAAA,EAAG,CAAA,EAAA,EAAK;AAC3B,IAAA,EAAA,CAAG,CAAC,IAAI,EAAC;AACT,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,IAAK,CAAA,EAAG,CAAA,EAAA,EAAK;AAC3B,MAAA,EAAA,CAAG,CAAC,CAAA,CAAG,CAAC,CAAA,GAAI,CAAA;AAAA,IACd;AAAA,EACF;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,IAAK,CAAA,EAAG,CAAA,EAAA,EAAK;AAC3B,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,IAAK,CAAA,EAAG,CAAA,EAAA,EAAK;AAC3B,MAAA,IAAI,KAAK,CAAA,GAAI,CAAC,MAAM,IAAA,CAAK,CAAA,GAAI,CAAC,CAAA,EAAG;AAC/B,QAAA,EAAA,CAAG,CAAC,CAAA,CAAG,CAAC,CAAA,GAAK,EAAA,CAAG,IAAI,CAAC,CAAA,CAAG,CAAA,GAAI,CAAC,CAAA,GAAK,CAAA;AAAA,MACpC,CAAA,MAAO;AACL,QAAA,EAAA,CAAG,CAAC,CAAA,CAAG,CAAC,CAAA,GAAK,IAAA,CAAK,IAAI,EAAA,CAAG,CAAA,GAAI,CAAC,CAAA,CAAG,CAAC,CAAA,EAAI,EAAA,CAAG,CAAC,CAAA,CAAG,CAAA,GAAI,CAAC,CAAE,CAAA;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAA,CAAG,CAAC,CAAA,CAAG,CAAC,CAAA;AACjB;AAOO,SAAS,YAAA,CAAa,OAAe,MAAA,EAAwB;AAElE,EAAA,MAAM,kBAAkB,KAAA,CAAM,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AACxD,EAAA,MAAM,mBAAmB,MAAA,CAAO,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAG1D,EAAA,IAAI,oBAAoB,gBAAA,EAAkB;AAExC,IAAA,IAAI,UAAU,MAAA,EAAQ;AAEpB,MAAA,MAAMA,WAAAA,GAAa,MAAM,KAAA,CAAM,KAAK,EAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,MAAA,GAAS,CAAC,CAAA;AAC9D,MAAA,MAAMC,YAAAA,GAAc,OAAO,KAAA,CAAM,KAAK,EAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,MAAA,GAAS,CAAC,CAAA;AAChE,MAAA,OAAO,KAAK,GAAA,CAAID,WAAAA,CAAW,MAAA,GAASC,YAAAA,CAAY,MAAM,CAAA,IAAK,CAAA;AAAA,IAC7D;AACA,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,MAAM,UAAA,GAAa,gBAAgB,KAAA,CAAM,KAAK,EAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,MAAA,GAAS,CAAC,CAAA;AACxE,EAAA,MAAM,WAAA,GAAc,iBAAiB,KAAA,CAAM,KAAK,EAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,MAAA,GAAS,CAAC,CAAA;AAE1E,EAAA,IAAI,UAAA,CAAW,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,WAAW,CAAA,EAAG;AACvD,IAAA,OAAO,CAAA;AAAA,EACT;AACA,EAAA,IAAI,UAAA,CAAW,WAAW,CAAA,EAAG;AAC3B,IAAA,OAAO,WAAA,CAAY,MAAA;AAAA,EACrB;AACA,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,UAAA,CAAW,MAAA;AAAA,EACpB;AAIA,EAAA,MAAM,aAAA,GAAgB,eAAA,CAAgB,UAAA,EAAY,WAAW,CAAA;AAC7D,EAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,MAAA,EAAQ,YAAY,MAAM,CAAA;AAChE,EAAA,MAAM,UAAU,SAAA,GAAY,aAAA;AAE5B,EAAA,OAAO,OAAA;AACT;AAKA,SAAS,eAAA,CAAgB,MAAgB,IAAA,EAAwB;AAC/D,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,MAAM,IAAA,uBAAW,GAAA,EAAY;AAE7B,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,IAAA,CAAK,QAAQ,CAAA,EAAA,EAAK;AACpC,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,IAAA,CAAK,QAAQ,CAAA,EAAA,EAAK;AACpC,MAAA,IAAI,CAAC,IAAA,CAAK,GAAA,CAAI,CAAC,CAAA,IAAK,KAAK,CAAC,CAAA,KAAM,IAAA,CAAK,CAAC,CAAA,EAAG;AACvC,QAAA,OAAA,EAAA;AACA,QAAA,IAAA,CAAK,IAAI,CAAC,CAAA;AACV,QAAA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO,OAAA;AACT","file":"chunk-44PMY5ES.js","sourcesContent":["/**\n * Calculates similarity ratio similar to SequenceMatcher.ratio()\n * Uses longest common subsequence (LCS) approach\n * Ratio = 2.0 * matches / total\n */\nexport function calculateRatio(input: string, output: string): number {\n  if (input === output) {\n    return 1.0;\n  }\n  if (input.length === 0 || output.length === 0) {\n    return 0.0;\n  }\n\n  // Use character-level LCS for more accurate matching (similar to SequenceMatcher)\n  const matches = longestCommonSubsequence(input, output);\n  const total = input.length + output.length;\n\n  return total > 0 ? (2.0 * matches) / total : 0.0;\n}\n\n/**\n * Finds the length of the longest common subsequence between two strings\n */\nfunction longestCommonSubsequence(str1: string, str2: string): number {\n  const m = str1.length;\n  const n = str2.length;\n  const dp: number[][] = [];\n\n  // Initialize DP table\n  for (let i = 0; i <= m; i++) {\n    dp[i] = [];\n    for (let j = 0; j <= n; j++) {\n      dp[i]![j] = 0;\n    }\n  }\n\n  for (let i = 1; i <= m; i++) {\n    for (let j = 1; j <= n; j++) {\n      if (str1[i - 1] === str2[j - 1]) {\n        dp[i]![j]! = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j]! = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  return dp[m]![n]!;\n}\n\n/**\n * Counts the number of differences between two strings\n * Uses opcodes-like approach: counts insertions, deletions, and replacements\n * For whitespace differences, preserves the original strings before word splitting\n */\nexport function countChanges(input: string, output: string): number {\n  // Normalize whitespace for comparison but preserve original for change detection\n  const inputNormalized = input.replace(/\\s+/g, ' ').trim();\n  const outputNormalized = output.replace(/\\s+/g, ' ').trim();\n\n  // If normalized strings are identical, check if there are whitespace differences\n  if (inputNormalized === outputNormalized) {\n    // If original strings differ only in whitespace, count that as a change\n    if (input !== output) {\n      // Count whitespace differences\n      const inputWords = input.split(/\\s+/).filter(w => w.length > 0);\n      const outputWords = output.split(/\\s+/).filter(w => w.length > 0);\n      return Math.abs(inputWords.length - outputWords.length) || 1;\n    }\n    return 0;\n  }\n\n  const inputWords = inputNormalized.split(/\\s+/).filter(w => w.length > 0);\n  const outputWords = outputNormalized.split(/\\s+/).filter(w => w.length > 0);\n\n  if (inputWords.length === 0 && outputWords.length === 0) {\n    return 0;\n  }\n  if (inputWords.length === 0) {\n    return outputWords.length;\n  }\n  if (outputWords.length === 0) {\n    return inputWords.length;\n  }\n\n  // Use LCS approach: changes = total - 2 * matches\n  // But for word-level, we want to count replacements as single changes\n  const matchingWords = findCommonWords(inputWords, outputWords);\n  const maxLength = Math.max(inputWords.length, outputWords.length);\n  const changes = maxLength - matchingWords;\n\n  return changes;\n}\n\n/**\n * Finds the number of common words between two arrays using a greedy matching approach\n */\nfunction findCommonWords(arr1: string[], arr2: string[]): number {\n  let matches = 0;\n  const used = new Set<number>();\n\n  for (let i = 0; i < arr1.length; i++) {\n    for (let j = 0; j < arr2.length; j++) {\n      if (!used.has(j) && arr1[i] === arr2[j]) {\n        matches++;\n        used.add(j);\n        break;\n      }\n    }\n  }\n\n  return matches;\n}\n"]}

package/dist/chunk-PWGOG6ML.cjs

@@ -0,0 +1,81 @@
+'use strict';
+
+// src/ratio.ts
+function calculateRatio(input, output) {
+  if (input === output) {
+    return 1;
+  }
+  if (input.length === 0 || output.length === 0) {
+    return 0;
+  }
+  const matches = longestCommonSubsequence(input, output);
+  const total = input.length + output.length;
+  return total > 0 ? 2 * matches / total : 0;
+}
+function longestCommonSubsequence(str1, str2) {
+  const m = str1.length;
+  const n = str2.length;
+  const dp = [];
+  for (let i = 0; i <= m; i++) {
+    dp[i] = [];
+    for (let j = 0; j <= n; j++) {
+      dp[i][j] = 0;
+    }
+  }
+  for (let i = 1; i <= m; i++) {
+    for (let j = 1; j <= n; j++) {
+      if (str1[i - 1] === str2[j - 1]) {
+        dp[i][j] = dp[i - 1][j - 1] + 1;
+      } else {
+        dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]);
+      }
+    }
+  }
+  return dp[m][n];
+}
+function countChanges(input, output) {
+  const inputNormalized = input.replace(/\s+/g, " ").trim();
+  const outputNormalized = output.replace(/\s+/g, " ").trim();
+  if (inputNormalized === outputNormalized) {
+    if (input !== output) {
+      const inputWords2 = input.split(/\s+/).filter((w) => w.length > 0);
+      const outputWords2 = output.split(/\s+/).filter((w) => w.length > 0);
+      return Math.abs(inputWords2.length - outputWords2.length) || 1;
+    }
+    return 0;
+  }
+  const inputWords = inputNormalized.split(/\s+/).filter((w) => w.length > 0);
+  const outputWords = outputNormalized.split(/\s+/).filter((w) => w.length > 0);
+  if (inputWords.length === 0 && outputWords.length === 0) {
+    return 0;
+  }
+  if (inputWords.length === 0) {
+    return outputWords.length;
+  }
+  if (outputWords.length === 0) {
+    return inputWords.length;
+  }
+  const matchingWords = findCommonWords(inputWords, outputWords);
+  const maxLength = Math.max(inputWords.length, outputWords.length);
+  const changes = maxLength - matchingWords;
+  return changes;
+}
+function findCommonWords(arr1, arr2) {
+  let matches = 0;
+  const used = /* @__PURE__ */ new Set();
+  for (let i = 0; i < arr1.length; i++) {
+    for (let j = 0; j < arr2.length; j++) {
+      if (!used.has(j) && arr1[i] === arr2[j]) {
+        matches++;
+        used.add(j);
+        break;
+      }
+    }
+  }
+  return matches;
+}
+
+exports.calculateRatio = calculateRatio;
+exports.countChanges = countChanges;
+//# sourceMappingURL=chunk-PWGOG6ML.cjs.map
+//# sourceMappingURL=chunk-PWGOG6ML.cjs.map

package/dist/chunk-PWGOG6ML.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/ratio.ts"],"names":["inputWords","outputWords"],"mappings":";;;AAKO,SAAS,cAAA,CAAe,OAAe,MAAA,EAAwB;AACpE,EAAA,IAAI,UAAU,MAAA,EAAQ;AACpB,IAAA,OAAO,CAAA;AAAA,EACT;AACA,EAAA,IAAI,KAAA,CAAM,MAAA,KAAW,CAAA,IAAK,MAAA,CAAO,WAAW,CAAA,EAAG;AAC7C,IAAA,OAAO,CAAA;AAAA,EACT;AAGA,EAAA,MAAM,OAAA,GAAU,wBAAA,CAAyB,KAAA,EAAO,MAAM,CAAA;AACtD,EAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,MAAA,GAAS,MAAA,CAAO,MAAA;AAEpC,EAAA,OAAO,KAAA,GAAQ,CAAA,GAAK,CAAA,GAAM,OAAA,GAAW,KAAA,GAAQ,CAAA;AAC/C;AAKA,SAAS,wBAAA,CAAyB,MAAc,IAAA,EAAsB;AACpE,EAAA,MAAM,IAAI,IAAA,CAAK,MAAA;AACf,EAAA,MAAM,IAAI,IAAA,CAAK,MAAA;AACf,EAAA,MAAM,KAAiB,EAAC;AAGxB,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,IAAK,CAAA,EAAG,CAAA,EAAA,EAAK;AAC3B,IAAA,EAAA,CAAG,CAAC,IAAI,EAAC;AACT,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,IAAK,CAAA,EAAG,CAAA,EAAA,EAAK;AAC3B,MAAA,EAAA,CAAG,CAAC,CAAA,CAAG,CAAC,CAAA,GAAI,CAAA;AAAA,IACd;AAAA,EACF;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,IAAK,CAAA,EAAG,CAAA,EAAA,EAAK;AAC3B,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,IAAK,CAAA,EAAG,CAAA,EAAA,EAAK;AAC3B,MAAA,IAAI,KAAK,CAAA,GAAI,CAAC,MAAM,IAAA,CAAK,CAAA,GAAI,CAAC,CAAA,EAAG;AAC/B,QAAA,EAAA,CAAG,CAAC,CAAA,CAAG,CAAC,CAAA,GAAK,EAAA,CAAG,IAAI,CAAC,CAAA,CAAG,CAAA,GAAI,CAAC,CAAA,GAAK,CAAA;AAAA,MACpC,CAAA,MAAO;AACL,QAAA,EAAA,CAAG,CAAC,CAAA,CAAG,CAAC,CAAA,GAAK,IAAA,CAAK,IAAI,EAAA,CAAG,CAAA,GAAI,CAAC,CAAA,CAAG,CAAC,CAAA,EAAI,EAAA,CAAG,CAAC,CAAA,CAAG,CAAA,GAAI,CAAC,CAAE,CAAA;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAA,CAAG,CAAC,CAAA,CAAG,CAAC,CAAA;AACjB;AAOO,SAAS,YAAA,CAAa,OAAe,MAAA,EAAwB;AAElE,EAAA,MAAM,kBAAkB,KAAA,CAAM,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AACxD,EAAA,MAAM,mBAAmB,MAAA,CAAO,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAG1D,EAAA,IAAI,oBAAoB,gBAAA,EAAkB;AAExC,IAAA,IAAI,UAAU,MAAA,EAAQ;AAEpB,MAAA,MAAMA,WAAAA,GAAa,MAAM,KAAA,CAAM,KAAK,EAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,MAAA,GAAS,CAAC,CAAA;AAC9D,MAAA,MAAMC,YAAAA,GAAc,OAAO,KAAA,CAAM,KAAK,EAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,MAAA,GAAS,CAAC,CAAA;AAChE,MAAA,OAAO,KAAK,GAAA,CAAID,WAAAA,CAAW,MAAA,GAASC,YAAAA,CAAY,MAAM,CAAA,IAAK,CAAA;AAAA,IAC7D;AACA,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,MAAM,UAAA,GAAa,gBAAgB,KAAA,CAAM,KAAK,EAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,MAAA,GAAS,CAAC,CAAA;AACxE,EAAA,MAAM,WAAA,GAAc,iBAAiB,KAAA,CAAM,KAAK,EAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,MAAA,GAAS,CAAC,CAAA;AAE1E,EAAA,IAAI,UAAA,CAAW,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,WAAW,CAAA,EAAG;AACvD,IAAA,OAAO,CAAA;AAAA,EACT;AACA,EAAA,IAAI,UAAA,CAAW,WAAW,CAAA,EAAG;AAC3B,IAAA,OAAO,WAAA,CAAY,MAAA;AAAA,EACrB;AACA,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,UAAA,CAAW,MAAA;AAAA,EACpB;AAIA,EAAA,MAAM,aAAA,GAAgB,eAAA,CAAgB,UAAA,EAAY,WAAW,CAAA;AAC7D,EAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,UAAA,CAAW,MAAA,EAAQ,YAAY,MAAM,CAAA;AAChE,EAAA,MAAM,UAAU,SAAA,GAAY,aAAA;AAE5B,EAAA,OAAO,OAAA;AACT;AAKA,SAAS,eAAA,CAAgB,MAAgB,IAAA,EAAwB;AAC/D,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,MAAM,IAAA,uBAAW,GAAA,EAAY;AAE7B,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,IAAA,CAAK,QAAQ,CAAA,EAAA,EAAK;AACpC,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,IAAA,CAAK,QAAQ,CAAA,EAAA,EAAK;AACpC,MAAA,IAAI,CAAC,IAAA,CAAK,GAAA,CAAI,CAAC,CAAA,IAAK,KAAK,CAAC,CAAA,KAAM,IAAA,CAAK,CAAC,CAAA,EAAG;AACvC,QAAA,OAAA,EAAA;AACA,QAAA,IAAA,CAAK,IAAI,CAAC,CAAA;AACV,QAAA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO,OAAA;AACT","file":"chunk-PWGOG6ML.cjs","sourcesContent":["/**\n * Calculates similarity ratio similar to SequenceMatcher.ratio()\n * Uses longest common subsequence (LCS) approach\n * Ratio = 2.0 * matches / total\n */\nexport function calculateRatio(input: string, output: string): number {\n  if (input === output) {\n    return 1.0;\n  }\n  if (input.length === 0 || output.length === 0) {\n    return 0.0;\n  }\n\n  // Use character-level LCS for more accurate matching (similar to SequenceMatcher)\n  const matches = longestCommonSubsequence(input, output);\n  const total = input.length + output.length;\n\n  return total > 0 ? (2.0 * matches) / total : 0.0;\n}\n\n/**\n * Finds the length of the longest common subsequence between two strings\n */\nfunction longestCommonSubsequence(str1: string, str2: string): number {\n  const m = str1.length;\n  const n = str2.length;\n  const dp: number[][] = [];\n\n  // Initialize DP table\n  for (let i = 0; i <= m; i++) {\n    dp[i] = [];\n    for (let j = 0; j <= n; j++) {\n      dp[i]![j] = 0;\n    }\n  }\n\n  for (let i = 1; i <= m; i++) {\n    for (let j = 1; j <= n; j++) {\n      if (str1[i - 1] === str2[j - 1]) {\n        dp[i]![j]! = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j]! = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  return dp[m]![n]!;\n}\n\n/**\n * Counts the number of differences between two strings\n * Uses opcodes-like approach: counts insertions, deletions, and replacements\n * For whitespace differences, preserves the original strings before word splitting\n */\nexport function countChanges(input: string, output: string): number {\n  // Normalize whitespace for comparison but preserve original for change detection\n  const inputNormalized = input.replace(/\\s+/g, ' ').trim();\n  const outputNormalized = output.replace(/\\s+/g, ' ').trim();\n\n  // If normalized strings are identical, check if there are whitespace differences\n  if (inputNormalized === outputNormalized) {\n    // If original strings differ only in whitespace, count that as a change\n    if (input !== output) {\n      // Count whitespace differences\n      const inputWords = input.split(/\\s+/).filter(w => w.length > 0);\n      const outputWords = output.split(/\\s+/).filter(w => w.length > 0);\n      return Math.abs(inputWords.length - outputWords.length) || 1;\n    }\n    return 0;\n  }\n\n  const inputWords = inputNormalized.split(/\\s+/).filter(w => w.length > 0);\n  const outputWords = outputNormalized.split(/\\s+/).filter(w => w.length > 0);\n\n  if (inputWords.length === 0 && outputWords.length === 0) {\n    return 0;\n  }\n  if (inputWords.length === 0) {\n    return outputWords.length;\n  }\n  if (outputWords.length === 0) {\n    return inputWords.length;\n  }\n\n  // Use LCS approach: changes = total - 2 * matches\n  // But for word-level, we want to count replacements as single changes\n  const matchingWords = findCommonWords(inputWords, outputWords);\n  const maxLength = Math.max(inputWords.length, outputWords.length);\n  const changes = maxLength - matchingWords;\n\n  return changes;\n}\n\n/**\n * Finds the number of common words between two arrays using a greedy matching approach\n */\nfunction findCommonWords(arr1: string[], arr2: string[]): number {\n  let matches = 0;\n  const used = new Set<number>();\n\n  for (let i = 0; i < arr1.length; i++) {\n    for (let j = 0; j < arr2.length; j++) {\n      if (!used.has(j) && arr1[i] === arr2[j]) {\n        matches++;\n        used.add(j);\n        break;\n      }\n    }\n  }\n\n  return matches;\n}\n"]}

package/dist/metrics/nlp/index.cjs

@@ -1,9 +1,9 @@
 'use strict';
 
+var chunkPWGOG6ML_cjs = require('../../chunk-PWGOG6ML.cjs');
 var _eval = require('@mastra/core/eval');
 var nlp = require('compromise');
 var stringSimilarity = require('string-similarity');
-var difflib = require('difflib');
 var keyword_extractor = require('keyword-extractor');
 var Sentiment = require('sentiment');
 
@@ -111,10 +111,8 @@ var ContentSimilarityMetric = class extends _eval.Metric {
 };
 var TextualDifferenceMetric = class extends _eval.Metric {
   async measure(input, output) {
-    const matcher = new difflib.SequenceMatcher(null, input, output);
-    const ratio = matcher.ratio();
-    const ops = matcher.getOpcodes();
-    const changes = ops.filter(([op]) => op !== "equal").length;
+    const ratio = chunkPWGOG6ML_cjs.calculateRatio(input, output);
+    const changes = chunkPWGOG6ML_cjs.countChanges(input, output);
     const maxLength = Math.max(input.length, output.length);
     const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
     const confidence = 1 - lengthDiff;

package/dist/metrics/nlp/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/metrics/nlp/completeness/index.ts","../../../src/metrics/nlp/content-similarity/index.ts","../../../src/metrics/nlp/textual-difference/index.ts","../../../src/metrics/nlp/keyword-coverage/index.ts","../../../src/metrics/nlp/tone/index.ts"],"names":["Metric","nlp","stringSimilarity","SequenceMatcher","keyword_extractor","Sentiment"],"mappings":";;;;;;;;;;;;;;;;AAgBO,IAAM,kBAAA,GAAN,cAAiCA,YAAA,CAAO;AAAA,EAC7C,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAmD;AAE9E,IAAA,IAAI,UAAU,IAAA,IAAQ,KAAA,KAAU,UAAa,MAAA,KAAW,IAAA,IAAQ,WAAW,MAAA,EAAW;AACpF,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAGA,IAAA,KAAA,GAAQ,MAAM,IAAA,EAAK;AACnB,IAAA,MAAA,GAAS,OAAO,IAAA,EAAK;AAErB,IAAA,MAAM,QAAA,GAAWC,qBAAI,KAAK,CAAA;AAC1B,IAAA,MAAM,SAAA,GAAYA,qBAAI,MAAM,CAAA;AAG5B,IAAA,MAAM,aAAA,GAAgB,IAAA,CAAK,eAAA,CAAgB,QAAQ,CAAA;AACnD,IAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,eAAA,CAAgB,SAAS,CAAA;AAErD,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,iBAAA,CAAkB,aAAA,EAAe,cAAc,CAAA;AAErE,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,aAAA;AAAA,QACA,cAAA;AAAA,QACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,QACtE,aAAA,EAAe;AAAA,UACb,OAAO,aAAA,CAAc,MAAA;AAAA,UACrB,QAAQ,cAAA,CAAe;AAAA;AACzB;AACF,KACF;AAAA,EACF;AAAA,EAEQ,gBAAgB,GAAA,EAAoB;AAE1C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,IAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,IAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,MAAA,MAAM,UAAA,GAAa,IAAA,CAAK,eAAA,CAAgB,IAAI,CAAA;AAG5C,MAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,IACnC,CAAA;AAGA,IAAA,MAAM,cAAA,GAAiB;AAAA,MACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,KACpC;AAGA,IAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AAAA,EACpC;AAAA,EAEQ,gBAAgB,GAAA,EAAqB;AAE3C,IAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AAAA,EACjB;AAAA,EAEQ,iBAAA,CAAkB,UAAoB,UAAA,EAA8B;AAC1E,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,IACvC;AAGA,IAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,MAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,OAAO,CAAA;AACzC,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,CAAC,CAAA;AAGnC,QAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,UAAA,OAAO,IAAA,KAAS,IAAA;AAAA,QAClB;AAGA,QAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,QAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,UAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,QAC1C;AAEA,QAAA,OAAO,KAAA;AAAA,MACT,CAAC;AAAA,KACH;AACA,IAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AAAA,EACnC;AACF;ACzGO,IAAM,uBAAA,GAAN,cAAsCD,YAAAA,CAAO;AAAA,EAC1C,OAAA;AAAA,EAER,WAAA,CAAY,OAAA,GAAoC,EAAC,EAAG;AAClD,IAAA,KAAA,EAAM;AACN,IAAA,IAAA,CAAK,OAAA,GAAU;AAAA,MACb,UAAA,EAAY,IAAA;AAAA,MACZ,gBAAA,EAAkB,IAAA;AAAA,MAClB,GAAG;AAAA,KACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,IAAI,cAAA,GAAiB,KAAA;AACrB,IAAA,IAAI,eAAA,GAAkB,MAAA;AAEtB,IAAA,IAAI,IAAA,CAAK,QAAQ,UAAA,EAAY;AAC3B,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,IAAA,CAAK,QAAQ,gBAAA,EAAkB;AACjC,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,MAAM,UAAA,GAAaE,iCAAA,CAAiB,iBAAA,CAAkB,cAAA,EAAgB,eAAe,CAAA;AAErF,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,UAAA;AAAA,MACP,IAAA,EAAM,EAAE,UAAA;AAAW,KACrB;AAAA,EACF;AACF;ACnCO,IAAM,uBAAA,GAAN,cAAsCF,YAAAA,CAAO;AAAA,EAClD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,MAAM,OAAA,GAAU,IAAIG,uBAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,KAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,UAAA;AAAA,QACA,KAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA;AACF,KACF;AAAA,EACF;AACF;AC1BO,IAAM,qBAAA,GAAN,cAAoCH,YAAAA,CAAO;AAAA,EAChD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAgD;AAE3E,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,aAAA,EAAe,CAAA;AAAA,UACf,eAAA,EAAiB;AAAA;AACnB,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOI,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AAExD,IAAA,MAAM,eAAA,GAAkB,CAAC,GAAG,iBAAiB,CAAA,CAAE,OAAO,CAAA,CAAA,KAAK,gBAAA,CAAiB,GAAA,CAAI,CAAC,CAAC,CAAA;AAClF,IAAA,MAAM,gBAAgB,iBAAA,CAAkB,IAAA;AACxC,IAAA,MAAM,QAAA,GAAW,aAAA,GAAgB,CAAA,GAAI,eAAA,CAAgB,SAAS,aAAA,GAAgB,CAAA;AAE9E,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,eAAe,iBAAA,CAAkB,IAAA;AAAA,QACjC,iBAAiB,eAAA,CAAgB;AAAA;AACnC,KACF;AAAA,EACF;AACF;AC/BO,IAAM,qBAAA,GAAN,cAAoCJ,YAAAA,CAAO;AAAA,EACxC,SAAA,GAAY,IAAIK,0BAAA,EAAU;AAAA,EAElC,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAA+C;AAC1E,IAAA,MAAM,iBAAA,GAAoB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,KAAK,CAAA;AAEtD,IAAA,IAAI,MAAA,EAAQ;AAEV,MAAA,MAAM,kBAAA,GAAqB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,MAAM,CAAA;AACxD,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,UACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,UACvC,UAAA,EAAY;AAAA;AACd,OACF;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,KAAA,CAAM,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,KAAK,CAAA;AACzD,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,KAAK,SAAA,CAAU,OAAA,CAAQ,CAAC,CAAA,CAAE,WAAW,CAAA;AAC3E,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,YAAA;AAAA,QACA,iBAAA,EAAmB;AAAA;AACrB,KACF;AAAA,EACF;AACF","file":"index.cjs","sourcesContent":["import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport nlp from 'compromise';\n\ninterface CompletenessMetricResult extends MetricResult {\n  info: {\n    inputElements: string[];\n    outputElements: string[];\n    missingElements: string[];\n    elementCounts: {\n      input: number;\n      output: number;\n    };\n  };\n}\n\nexport class CompletenessMetric extends Metric {\n  async measure(input: string, output: string): Promise<CompletenessMetricResult> {\n    // Handle null/undefined inputs\n    if (input === null || input === undefined || output === null || output === undefined) {\n      throw new Error('Inputs cannot be null or undefined');\n    }\n\n    // Trim both inputs\n    input = input.trim();\n    output = output.trim();\n\n    const inputDoc = nlp(input);\n    const outputDoc = nlp(output);\n\n    // Extract and log elements\n    const inputElements = this.extractElements(inputDoc);\n    const outputElements = this.extractElements(outputDoc);\n    // Maybe we need a more sophisticated matching approach\n    const coverage = this.calculateCoverage(inputElements, outputElements);\n\n    return {\n      score: coverage,\n      info: {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      },\n    };\n  }\n\n  private extractElements(doc: any): string[] {\n    // Get more specific elements and ensure they're arrays\n    const nouns = doc.nouns().out('array') || [];\n    const verbs = doc.verbs().toInfinitive().out('array') || [];\n    const topics = doc.topics().out('array') || [];\n    const terms = doc.terms().out('array') || [];\n\n    // Helper function to clean and split terms\n    const cleanAndSplitTerm = (term: string): string[] => {\n      // First normalize the string\n      const normalized = this.normalizeString(term);\n\n      // Split on word boundaries and filter out empty strings\n      return normalized\n        .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n        .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n        .trim()\n        .split(/\\s+/)\n        .filter(word => word.length > 0);\n    };\n\n    // Process all elements\n    const processedTerms = [\n      ...nouns.flatMap(cleanAndSplitTerm),\n      ...verbs.flatMap(cleanAndSplitTerm),\n      ...topics.flatMap(cleanAndSplitTerm),\n      ...terms.flatMap(cleanAndSplitTerm),\n    ];\n\n    // Remove duplicates\n    return [...new Set(processedTerms)];\n  }\n\n  private normalizeString(str: string): string {\n    // Remove diacritics and convert to lowercase\n    return str\n      .normalize('NFD')\n      .replace(/[\\u0300-\\u036f]/g, '')\n      .toLowerCase();\n  }\n\n  private calculateCoverage(original: string[], simplified: string[]): number {\n    if (original.length === 0) {\n      return simplified.length === 0 ? 1 : 0;\n    }\n\n    // Exact matching for short words (3 chars or less), substring matching for longer words\n    const covered = original.filter(element =>\n      simplified.some(s => {\n        const elem = this.normalizeString(element);\n        const simp = this.normalizeString(s);\n\n        // For short words (3 chars or less), require exact match\n        if (elem.length <= 3) {\n          return elem === simp;\n        }\n\n        // For longer words, require substantial overlap (more than 60% of the longer word)\n        const longer = elem.length > simp.length ? elem : simp;\n        const shorter = elem.length > simp.length ? simp : elem;\n\n        if (longer.includes(shorter)) {\n          return shorter.length / longer.length > 0.6;\n        }\n\n        return false;\n      }),\n    );\n    return covered.length / original.length;\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityResult extends MetricResult {\n  info: {\n    similarity: number;\n  };\n}\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport class ContentSimilarityMetric extends Metric {\n  private options: ContentSimilarityOptions;\n\n  constructor(options: ContentSimilarityOptions = {}) {\n    super();\n    this.options = {\n      ignoreCase: true,\n      ignoreWhitespace: true,\n      ...options,\n    };\n  }\n\n  async measure(input: string, output: string): Promise<ContentSimilarityResult> {\n    let processedInput = input;\n    let processedOutput = output;\n\n    if (this.options.ignoreCase) {\n      processedInput = processedInput.toLowerCase();\n      processedOutput = processedOutput.toLowerCase();\n    }\n\n    if (this.options.ignoreWhitespace) {\n      processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n      processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n    }\n\n    const similarity = stringSimilarity.compareTwoStrings(processedInput, processedOutput);\n\n    return {\n      score: similarity,\n      info: { similarity },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport { SequenceMatcher } from 'difflib';\n\ninterface TextualDifferenceResult extends MetricResult {\n  info: {\n    ratio: number;\n    changes: number;\n    lengthDiff: number;\n    confidence: number;\n  };\n}\n\nexport class TextualDifferenceMetric extends Metric {\n  async measure(input: string, output: string): Promise<TextualDifferenceResult> {\n    const matcher = new SequenceMatcher(null, input, output);\n    const ratio = matcher.ratio();\n\n    // Get detailed operations\n    const ops = matcher.getOpcodes();\n    const changes = ops.filter(([op]) => op !== 'equal').length;\n\n    // Calculate confidence based on text length difference\n    const maxLength = Math.max(input.length, output.length);\n    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n    const confidence = 1 - lengthDiff;\n\n    return {\n      score: ratio,\n      info: {\n        confidence,\n        ratio,\n        changes,\n        lengthDiff,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport keyword_extractor from 'keyword-extractor';\n\ninterface KeywordCoverageResult extends MetricResult {\n  info: {\n    totalKeywords: number;\n    matchedKeywords: number;\n  };\n}\n\nexport class KeywordCoverageMetric extends Metric {\n  async measure(input: string, output: string): Promise<KeywordCoverageResult> {\n    // Handle empty strings case\n    if (!input && !output) {\n      return {\n        score: 1,\n        info: {\n          totalKeywords: 0,\n          matchedKeywords: 0,\n        },\n      };\n    }\n\n    const extractKeywords = (text: string) => {\n      return keyword_extractor.extract(text, {\n        language: 'english',\n        remove_digits: true,\n        return_changed_case: true,\n        remove_duplicates: true,\n      });\n    };\n\n    const referenceKeywords = new Set(extractKeywords(input));\n    const responseKeywords = new Set(extractKeywords(output));\n\n    const matchedKeywords = [...referenceKeywords].filter(k => responseKeywords.has(k));\n    const totalKeywords = referenceKeywords.size;\n    const coverage = totalKeywords > 0 ? matchedKeywords.length / totalKeywords : 0;\n\n    return {\n      score: coverage,\n      info: {\n        totalKeywords: referenceKeywords.size,\n        matchedKeywords: matchedKeywords.length,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport Sentiment from 'sentiment';\n\ninterface ToneConsitencyResult extends MetricResult {\n  info:\n    | {\n        responseSentiment: number;\n        referenceSentiment: number;\n        difference: number;\n      }\n    | {\n        avgSentiment: number;\n        sentimentVariance: number;\n      };\n}\n\nexport class ToneConsistencyMetric extends Metric {\n  private sentiment = new Sentiment();\n\n  async measure(input: string, output: string): Promise<ToneConsitencyResult> {\n    const responseSentiment = this.sentiment.analyze(input);\n\n    if (output) {\n      // Compare sentiment with reference\n      const referenceSentiment = this.sentiment.analyze(output);\n      const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n      const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n      return {\n        score: normalizedScore,\n        info: {\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        },\n      };\n    }\n\n    // Evaluate sentiment stability across response\n    const sentences = input.match(/[^.!?]+[.!?]+/g) || [input];\n    const sentiments = sentences.map(s => this.sentiment.analyze(s).comparative);\n    const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n    const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n    const stability = Math.max(0, 1 - variance);\n\n    return {\n      score: stability,\n      info: {\n        avgSentiment,\n        sentimentVariance: variance,\n      },\n    };\n  }\n}\n"]}
+{"version":3,"sources":["../../../src/metrics/nlp/completeness/index.ts","../../../src/metrics/nlp/content-similarity/index.ts","../../../src/metrics/nlp/textual-difference/index.ts","../../../src/metrics/nlp/keyword-coverage/index.ts","../../../src/metrics/nlp/tone/index.ts"],"names":["Metric","nlp","stringSimilarity","calculateRatio","countChanges","keyword_extractor","Sentiment"],"mappings":";;;;;;;;;;;;;;;;AAgBO,IAAM,kBAAA,GAAN,cAAiCA,YAAA,CAAO;AAAA,EAC7C,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAmD;AAE9E,IAAA,IAAI,UAAU,IAAA,IAAQ,KAAA,KAAU,UAAa,MAAA,KAAW,IAAA,IAAQ,WAAW,MAAA,EAAW;AACpF,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAGA,IAAA,KAAA,GAAQ,MAAM,IAAA,EAAK;AACnB,IAAA,MAAA,GAAS,OAAO,IAAA,EAAK;AAErB,IAAA,MAAM,QAAA,GAAWC,qBAAI,KAAK,CAAA;AAC1B,IAAA,MAAM,SAAA,GAAYA,qBAAI,MAAM,CAAA;AAG5B,IAAA,MAAM,aAAA,GAAgB,IAAA,CAAK,eAAA,CAAgB,QAAQ,CAAA;AACnD,IAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,eAAA,CAAgB,SAAS,CAAA;AAErD,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,iBAAA,CAAkB,aAAA,EAAe,cAAc,CAAA;AAErE,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,aAAA;AAAA,QACA,cAAA;AAAA,QACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,QACtE,aAAA,EAAe;AAAA,UACb,OAAO,aAAA,CAAc,MAAA;AAAA,UACrB,QAAQ,cAAA,CAAe;AAAA;AACzB;AACF,KACF;AAAA,EACF;AAAA,EAEQ,gBAAgB,GAAA,EAAoB;AAE1C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,IAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,IAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,MAAA,MAAM,UAAA,GAAa,IAAA,CAAK,eAAA,CAAgB,IAAI,CAAA;AAG5C,MAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,IACnC,CAAA;AAGA,IAAA,MAAM,cAAA,GAAiB;AAAA,MACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,KACpC;AAGA,IAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AAAA,EACpC;AAAA,EAEQ,gBAAgB,GAAA,EAAqB;AAE3C,IAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AAAA,EACjB;AAAA,EAEQ,iBAAA,CAAkB,UAAoB,UAAA,EAA8B;AAC1E,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,IACvC;AAGA,IAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,MAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,OAAO,CAAA;AACzC,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,CAAC,CAAA;AAGnC,QAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,UAAA,OAAO,IAAA,KAAS,IAAA;AAAA,QAClB;AAGA,QAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,QAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,UAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,QAC1C;AAEA,QAAA,OAAO,KAAA;AAAA,MACT,CAAC;AAAA,KACH;AACA,IAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AAAA,EACnC;AACF;ACzGO,IAAM,uBAAA,GAAN,cAAsCD,YAAAA,CAAO;AAAA,EAC1C,OAAA;AAAA,EAER,WAAA,CAAY,OAAA,GAAoC,EAAC,EAAG;AAClD,IAAA,KAAA,EAAM;AACN,IAAA,IAAA,CAAK,OAAA,GAAU;AAAA,MACb,UAAA,EAAY,IAAA;AAAA,MACZ,gBAAA,EAAkB,IAAA;AAAA,MAClB,GAAG;AAAA,KACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,IAAI,cAAA,GAAiB,KAAA;AACrB,IAAA,IAAI,eAAA,GAAkB,MAAA;AAEtB,IAAA,IAAI,IAAA,CAAK,QAAQ,UAAA,EAAY;AAC3B,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,IAAA,CAAK,QAAQ,gBAAA,EAAkB;AACjC,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,MAAM,UAAA,GAAaE,iCAAA,CAAiB,iBAAA,CAAkB,cAAA,EAAgB,eAAe,CAAA;AAErF,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,UAAA;AAAA,MACP,IAAA,EAAM,EAAE,UAAA;AAAW,KACrB;AAAA,EACF;AACF;ACnCO,IAAM,uBAAA,GAAN,cAAsCF,YAAAA,CAAO;AAAA,EAClD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAE7E,IAAA,MAAM,KAAA,GAAQG,gCAAA,CAAe,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,OAAA,GAAUC,8BAAA,CAAa,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,KAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,UAAA;AAAA,QACA,KAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA;AACF,KACF;AAAA,EACF;AACF;ACzBO,IAAM,qBAAA,GAAN,cAAoCJ,YAAAA,CAAO;AAAA,EAChD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAgD;AAE3E,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,aAAA,EAAe,CAAA;AAAA,UACf,eAAA,EAAiB;AAAA;AACnB,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOK,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AAExD,IAAA,MAAM,eAAA,GAAkB,CAAC,GAAG,iBAAiB,CAAA,CAAE,OAAO,CAAA,CAAA,KAAK,gBAAA,CAAiB,GAAA,CAAI,CAAC,CAAC,CAAA;AAClF,IAAA,MAAM,gBAAgB,iBAAA,CAAkB,IAAA;AACxC,IAAA,MAAM,QAAA,GAAW,aAAA,GAAgB,CAAA,GAAI,eAAA,CAAgB,SAAS,aAAA,GAAgB,CAAA;AAE9E,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,eAAe,iBAAA,CAAkB,IAAA;AAAA,QACjC,iBAAiB,eAAA,CAAgB;AAAA;AACnC,KACF;AAAA,EACF;AACF;AC/BO,IAAM,qBAAA,GAAN,cAAoCL,YAAAA,CAAO;AAAA,EACxC,SAAA,GAAY,IAAIM,0BAAA,EAAU;AAAA,EAElC,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAA+C;AAC1E,IAAA,MAAM,iBAAA,GAAoB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,KAAK,CAAA;AAEtD,IAAA,IAAI,MAAA,EAAQ;AAEV,MAAA,MAAM,kBAAA,GAAqB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,MAAM,CAAA;AACxD,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,UACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,UACvC,UAAA,EAAY;AAAA;AACd,OACF;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,KAAA,CAAM,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,KAAK,CAAA;AACzD,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,KAAK,SAAA,CAAU,OAAA,CAAQ,CAAC,CAAA,CAAE,WAAW,CAAA;AAC3E,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,YAAA;AAAA,QACA,iBAAA,EAAmB;AAAA;AACrB,KACF;AAAA,EACF;AACF","file":"index.cjs","sourcesContent":["import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport nlp from 'compromise';\n\ninterface CompletenessMetricResult extends MetricResult {\n  info: {\n    inputElements: string[];\n    outputElements: string[];\n    missingElements: string[];\n    elementCounts: {\n      input: number;\n      output: number;\n    };\n  };\n}\n\nexport class CompletenessMetric extends Metric {\n  async measure(input: string, output: string): Promise<CompletenessMetricResult> {\n    // Handle null/undefined inputs\n    if (input === null || input === undefined || output === null || output === undefined) {\n      throw new Error('Inputs cannot be null or undefined');\n    }\n\n    // Trim both inputs\n    input = input.trim();\n    output = output.trim();\n\n    const inputDoc = nlp(input);\n    const outputDoc = nlp(output);\n\n    // Extract and log elements\n    const inputElements = this.extractElements(inputDoc);\n    const outputElements = this.extractElements(outputDoc);\n    // Maybe we need a more sophisticated matching approach\n    const coverage = this.calculateCoverage(inputElements, outputElements);\n\n    return {\n      score: coverage,\n      info: {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      },\n    };\n  }\n\n  private extractElements(doc: any): string[] {\n    // Get more specific elements and ensure they're arrays\n    const nouns = doc.nouns().out('array') || [];\n    const verbs = doc.verbs().toInfinitive().out('array') || [];\n    const topics = doc.topics().out('array') || [];\n    const terms = doc.terms().out('array') || [];\n\n    // Helper function to clean and split terms\n    const cleanAndSplitTerm = (term: string): string[] => {\n      // First normalize the string\n      const normalized = this.normalizeString(term);\n\n      // Split on word boundaries and filter out empty strings\n      return normalized\n        .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n        .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n        .trim()\n        .split(/\\s+/)\n        .filter(word => word.length > 0);\n    };\n\n    // Process all elements\n    const processedTerms = [\n      ...nouns.flatMap(cleanAndSplitTerm),\n      ...verbs.flatMap(cleanAndSplitTerm),\n      ...topics.flatMap(cleanAndSplitTerm),\n      ...terms.flatMap(cleanAndSplitTerm),\n    ];\n\n    // Remove duplicates\n    return [...new Set(processedTerms)];\n  }\n\n  private normalizeString(str: string): string {\n    // Remove diacritics and convert to lowercase\n    return str\n      .normalize('NFD')\n      .replace(/[\\u0300-\\u036f]/g, '')\n      .toLowerCase();\n  }\n\n  private calculateCoverage(original: string[], simplified: string[]): number {\n    if (original.length === 0) {\n      return simplified.length === 0 ? 1 : 0;\n    }\n\n    // Exact matching for short words (3 chars or less), substring matching for longer words\n    const covered = original.filter(element =>\n      simplified.some(s => {\n        const elem = this.normalizeString(element);\n        const simp = this.normalizeString(s);\n\n        // For short words (3 chars or less), require exact match\n        if (elem.length <= 3) {\n          return elem === simp;\n        }\n\n        // For longer words, require substantial overlap (more than 60% of the longer word)\n        const longer = elem.length > simp.length ? elem : simp;\n        const shorter = elem.length > simp.length ? simp : elem;\n\n        if (longer.includes(shorter)) {\n          return shorter.length / longer.length > 0.6;\n        }\n\n        return false;\n      }),\n    );\n    return covered.length / original.length;\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityResult extends MetricResult {\n  info: {\n    similarity: number;\n  };\n}\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport class ContentSimilarityMetric extends Metric {\n  private options: ContentSimilarityOptions;\n\n  constructor(options: ContentSimilarityOptions = {}) {\n    super();\n    this.options = {\n      ignoreCase: true,\n      ignoreWhitespace: true,\n      ...options,\n    };\n  }\n\n  async measure(input: string, output: string): Promise<ContentSimilarityResult> {\n    let processedInput = input;\n    let processedOutput = output;\n\n    if (this.options.ignoreCase) {\n      processedInput = processedInput.toLowerCase();\n      processedOutput = processedOutput.toLowerCase();\n    }\n\n    if (this.options.ignoreWhitespace) {\n      processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n      processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n    }\n\n    const similarity = stringSimilarity.compareTwoStrings(processedInput, processedOutput);\n\n    return {\n      score: similarity,\n      info: { similarity },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport { calculateRatio, countChanges } from '../../../ratio';\n\ninterface TextualDifferenceResult extends MetricResult {\n  info: {\n    ratio: number;\n    changes: number;\n    lengthDiff: number;\n    confidence: number;\n  };\n}\n\nexport class TextualDifferenceMetric extends Metric {\n  async measure(input: string, output: string): Promise<TextualDifferenceResult> {\n    // Calculate similarity ratio using LCS approach (similar to SequenceMatcher.ratio())\n    const ratio = calculateRatio(input, output);\n\n    // Count changes by comparing words\n    const changes = countChanges(input, output);\n\n    // Calculate confidence based on text length difference\n    const maxLength = Math.max(input.length, output.length);\n    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n    const confidence = 1 - lengthDiff;\n\n    return {\n      score: ratio,\n      info: {\n        confidence,\n        ratio,\n        changes,\n        lengthDiff,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport keyword_extractor from 'keyword-extractor';\n\ninterface KeywordCoverageResult extends MetricResult {\n  info: {\n    totalKeywords: number;\n    matchedKeywords: number;\n  };\n}\n\nexport class KeywordCoverageMetric extends Metric {\n  async measure(input: string, output: string): Promise<KeywordCoverageResult> {\n    // Handle empty strings case\n    if (!input && !output) {\n      return {\n        score: 1,\n        info: {\n          totalKeywords: 0,\n          matchedKeywords: 0,\n        },\n      };\n    }\n\n    const extractKeywords = (text: string) => {\n      return keyword_extractor.extract(text, {\n        language: 'english',\n        remove_digits: true,\n        return_changed_case: true,\n        remove_duplicates: true,\n      });\n    };\n\n    const referenceKeywords = new Set(extractKeywords(input));\n    const responseKeywords = new Set(extractKeywords(output));\n\n    const matchedKeywords = [...referenceKeywords].filter(k => responseKeywords.has(k));\n    const totalKeywords = referenceKeywords.size;\n    const coverage = totalKeywords > 0 ? matchedKeywords.length / totalKeywords : 0;\n\n    return {\n      score: coverage,\n      info: {\n        totalKeywords: referenceKeywords.size,\n        matchedKeywords: matchedKeywords.length,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport Sentiment from 'sentiment';\n\ninterface ToneConsitencyResult extends MetricResult {\n  info:\n    | {\n        responseSentiment: number;\n        referenceSentiment: number;\n        difference: number;\n      }\n    | {\n        avgSentiment: number;\n        sentimentVariance: number;\n      };\n}\n\nexport class ToneConsistencyMetric extends Metric {\n  private sentiment = new Sentiment();\n\n  async measure(input: string, output: string): Promise<ToneConsitencyResult> {\n    const responseSentiment = this.sentiment.analyze(input);\n\n    if (output) {\n      // Compare sentiment with reference\n      const referenceSentiment = this.sentiment.analyze(output);\n      const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n      const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n      return {\n        score: normalizedScore,\n        info: {\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        },\n      };\n    }\n\n    // Evaluate sentiment stability across response\n    const sentences = input.match(/[^.!?]+[.!?]+/g) || [input];\n    const sentiments = sentences.map(s => this.sentiment.analyze(s).comparative);\n    const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n    const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n    const stability = Math.max(0, 1 - variance);\n\n    return {\n      score: stability,\n      info: {\n        avgSentiment,\n        sentimentVariance: variance,\n      },\n    };\n  }\n}\n"]}

package/dist/metrics/nlp/index.js

@@ -1,7 +1,7 @@
+import { calculateRatio, countChanges } from '../../chunk-44PMY5ES.js';
 import { Metric } from '@mastra/core/eval';
 import nlp from 'compromise';
 import stringSimilarity from 'string-similarity';
-import { SequenceMatcher } from 'difflib';
 import keyword_extractor from 'keyword-extractor';
 import Sentiment from 'sentiment';
 
@@ -102,10 +102,8 @@ var ContentSimilarityMetric = class extends Metric {
 };
 var TextualDifferenceMetric = class extends Metric {
   async measure(input, output) {
-    const matcher = new SequenceMatcher(null, input, output);
-    const ratio = matcher.ratio();
-    const ops = matcher.getOpcodes();
-    const changes = ops.filter(([op]) => op !== "equal").length;
+    const ratio = calculateRatio(input, output);
+    const changes = countChanges(input, output);
     const maxLength = Math.max(input.length, output.length);
     const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
     const confidence = 1 - lengthDiff;

package/dist/metrics/nlp/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/metrics/nlp/completeness/index.ts","../../../src/metrics/nlp/content-similarity/index.ts","../../../src/metrics/nlp/textual-difference/index.ts","../../../src/metrics/nlp/keyword-coverage/index.ts","../../../src/metrics/nlp/tone/index.ts"],"names":["Metric"],"mappings":";;;;;;;AAgBO,IAAM,kBAAA,GAAN,cAAiC,MAAA,CAAO;AAAA,EAC7C,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAmD;AAE9E,IAAA,IAAI,UAAU,IAAA,IAAQ,KAAA,KAAU,UAAa,MAAA,KAAW,IAAA,IAAQ,WAAW,MAAA,EAAW;AACpF,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAGA,IAAA,KAAA,GAAQ,MAAM,IAAA,EAAK;AACnB,IAAA,MAAA,GAAS,OAAO,IAAA,EAAK;AAErB,IAAA,MAAM,QAAA,GAAW,IAAI,KAAK,CAAA;AAC1B,IAAA,MAAM,SAAA,GAAY,IAAI,MAAM,CAAA;AAG5B,IAAA,MAAM,aAAA,GAAgB,IAAA,CAAK,eAAA,CAAgB,QAAQ,CAAA;AACnD,IAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,eAAA,CAAgB,SAAS,CAAA;AAErD,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,iBAAA,CAAkB,aAAA,EAAe,cAAc,CAAA;AAErE,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,aAAA;AAAA,QACA,cAAA;AAAA,QACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,QACtE,aAAA,EAAe;AAAA,UACb,OAAO,aAAA,CAAc,MAAA;AAAA,UACrB,QAAQ,cAAA,CAAe;AAAA;AACzB;AACF,KACF;AAAA,EACF;AAAA,EAEQ,gBAAgB,GAAA,EAAoB;AAE1C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,IAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,IAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,MAAA,MAAM,UAAA,GAAa,IAAA,CAAK,eAAA,CAAgB,IAAI,CAAA;AAG5C,MAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,IACnC,CAAA;AAGA,IAAA,MAAM,cAAA,GAAiB;AAAA,MACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,KACpC;AAGA,IAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AAAA,EACpC;AAAA,EAEQ,gBAAgB,GAAA,EAAqB;AAE3C,IAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AAAA,EACjB;AAAA,EAEQ,iBAAA,CAAkB,UAAoB,UAAA,EAA8B;AAC1E,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,IACvC;AAGA,IAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,MAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,OAAO,CAAA;AACzC,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,CAAC,CAAA;AAGnC,QAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,UAAA,OAAO,IAAA,KAAS,IAAA;AAAA,QAClB;AAGA,QAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,QAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,UAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,QAC1C;AAEA,QAAA,OAAO,KAAA;AAAA,MACT,CAAC;AAAA,KACH;AACA,IAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AAAA,EACnC;AACF;ACzGO,IAAM,uBAAA,GAAN,cAAsCA,MAAAA,CAAO;AAAA,EAC1C,OAAA;AAAA,EAER,WAAA,CAAY,OAAA,GAAoC,EAAC,EAAG;AAClD,IAAA,KAAA,EAAM;AACN,IAAA,IAAA,CAAK,OAAA,GAAU;AAAA,MACb,UAAA,EAAY,IAAA;AAAA,MACZ,gBAAA,EAAkB,IAAA;AAAA,MAClB,GAAG;AAAA,KACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,IAAI,cAAA,GAAiB,KAAA;AACrB,IAAA,IAAI,eAAA,GAAkB,MAAA;AAEtB,IAAA,IAAI,IAAA,CAAK,QAAQ,UAAA,EAAY;AAC3B,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,IAAA,CAAK,QAAQ,gBAAA,EAAkB;AACjC,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,MAAM,UAAA,GAAa,gBAAA,CAAiB,iBAAA,CAAkB,cAAA,EAAgB,eAAe,CAAA;AAErF,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,UAAA;AAAA,MACP,IAAA,EAAM,EAAE,UAAA;AAAW,KACrB;AAAA,EACF;AACF;ACnCO,IAAM,uBAAA,GAAN,cAAsCA,MAAAA,CAAO;AAAA,EAClD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,MAAM,OAAA,GAAU,IAAI,eAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,KAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,UAAA;AAAA,QACA,KAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA;AACF,KACF;AAAA,EACF;AACF;AC1BO,IAAM,qBAAA,GAAN,cAAoCA,MAAAA,CAAO;AAAA,EAChD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAgD;AAE3E,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,aAAA,EAAe,CAAA;AAAA,UACf,eAAA,EAAiB;AAAA;AACnB,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AAExD,IAAA,MAAM,eAAA,GAAkB,CAAC,GAAG,iBAAiB,CAAA,CAAE,OAAO,CAAA,CAAA,KAAK,gBAAA,CAAiB,GAAA,CAAI,CAAC,CAAC,CAAA;AAClF,IAAA,MAAM,gBAAgB,iBAAA,CAAkB,IAAA;AACxC,IAAA,MAAM,QAAA,GAAW,aAAA,GAAgB,CAAA,GAAI,eAAA,CAAgB,SAAS,aAAA,GAAgB,CAAA;AAE9E,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,eAAe,iBAAA,CAAkB,IAAA;AAAA,QACjC,iBAAiB,eAAA,CAAgB;AAAA;AACnC,KACF;AAAA,EACF;AACF;AC/BO,IAAM,qBAAA,GAAN,cAAoCA,MAAAA,CAAO;AAAA,EACxC,SAAA,GAAY,IAAI,SAAA,EAAU;AAAA,EAElC,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAA+C;AAC1E,IAAA,MAAM,iBAAA,GAAoB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,KAAK,CAAA;AAEtD,IAAA,IAAI,MAAA,EAAQ;AAEV,MAAA,MAAM,kBAAA,GAAqB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,MAAM,CAAA;AACxD,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,UACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,UACvC,UAAA,EAAY;AAAA;AACd,OACF;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,KAAA,CAAM,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,KAAK,CAAA;AACzD,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,KAAK,SAAA,CAAU,OAAA,CAAQ,CAAC,CAAA,CAAE,WAAW,CAAA;AAC3E,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,YAAA;AAAA,QACA,iBAAA,EAAmB;AAAA;AACrB,KACF;AAAA,EACF;AACF","file":"index.js","sourcesContent":["import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport nlp from 'compromise';\n\ninterface CompletenessMetricResult extends MetricResult {\n  info: {\n    inputElements: string[];\n    outputElements: string[];\n    missingElements: string[];\n    elementCounts: {\n      input: number;\n      output: number;\n    };\n  };\n}\n\nexport class CompletenessMetric extends Metric {\n  async measure(input: string, output: string): Promise<CompletenessMetricResult> {\n    // Handle null/undefined inputs\n    if (input === null || input === undefined || output === null || output === undefined) {\n      throw new Error('Inputs cannot be null or undefined');\n    }\n\n    // Trim both inputs\n    input = input.trim();\n    output = output.trim();\n\n    const inputDoc = nlp(input);\n    const outputDoc = nlp(output);\n\n    // Extract and log elements\n    const inputElements = this.extractElements(inputDoc);\n    const outputElements = this.extractElements(outputDoc);\n    // Maybe we need a more sophisticated matching approach\n    const coverage = this.calculateCoverage(inputElements, outputElements);\n\n    return {\n      score: coverage,\n      info: {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      },\n    };\n  }\n\n  private extractElements(doc: any): string[] {\n    // Get more specific elements and ensure they're arrays\n    const nouns = doc.nouns().out('array') || [];\n    const verbs = doc.verbs().toInfinitive().out('array') || [];\n    const topics = doc.topics().out('array') || [];\n    const terms = doc.terms().out('array') || [];\n\n    // Helper function to clean and split terms\n    const cleanAndSplitTerm = (term: string): string[] => {\n      // First normalize the string\n      const normalized = this.normalizeString(term);\n\n      // Split on word boundaries and filter out empty strings\n      return normalized\n        .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n        .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n        .trim()\n        .split(/\\s+/)\n        .filter(word => word.length > 0);\n    };\n\n    // Process all elements\n    const processedTerms = [\n      ...nouns.flatMap(cleanAndSplitTerm),\n      ...verbs.flatMap(cleanAndSplitTerm),\n      ...topics.flatMap(cleanAndSplitTerm),\n      ...terms.flatMap(cleanAndSplitTerm),\n    ];\n\n    // Remove duplicates\n    return [...new Set(processedTerms)];\n  }\n\n  private normalizeString(str: string): string {\n    // Remove diacritics and convert to lowercase\n    return str\n      .normalize('NFD')\n      .replace(/[\\u0300-\\u036f]/g, '')\n      .toLowerCase();\n  }\n\n  private calculateCoverage(original: string[], simplified: string[]): number {\n    if (original.length === 0) {\n      return simplified.length === 0 ? 1 : 0;\n    }\n\n    // Exact matching for short words (3 chars or less), substring matching for longer words\n    const covered = original.filter(element =>\n      simplified.some(s => {\n        const elem = this.normalizeString(element);\n        const simp = this.normalizeString(s);\n\n        // For short words (3 chars or less), require exact match\n        if (elem.length <= 3) {\n          return elem === simp;\n        }\n\n        // For longer words, require substantial overlap (more than 60% of the longer word)\n        const longer = elem.length > simp.length ? elem : simp;\n        const shorter = elem.length > simp.length ? simp : elem;\n\n        if (longer.includes(shorter)) {\n          return shorter.length / longer.length > 0.6;\n        }\n\n        return false;\n      }),\n    );\n    return covered.length / original.length;\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityResult extends MetricResult {\n  info: {\n    similarity: number;\n  };\n}\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport class ContentSimilarityMetric extends Metric {\n  private options: ContentSimilarityOptions;\n\n  constructor(options: ContentSimilarityOptions = {}) {\n    super();\n    this.options = {\n      ignoreCase: true,\n      ignoreWhitespace: true,\n      ...options,\n    };\n  }\n\n  async measure(input: string, output: string): Promise<ContentSimilarityResult> {\n    let processedInput = input;\n    let processedOutput = output;\n\n    if (this.options.ignoreCase) {\n      processedInput = processedInput.toLowerCase();\n      processedOutput = processedOutput.toLowerCase();\n    }\n\n    if (this.options.ignoreWhitespace) {\n      processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n      processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n    }\n\n    const similarity = stringSimilarity.compareTwoStrings(processedInput, processedOutput);\n\n    return {\n      score: similarity,\n      info: { similarity },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport { SequenceMatcher } from 'difflib';\n\ninterface TextualDifferenceResult extends MetricResult {\n  info: {\n    ratio: number;\n    changes: number;\n    lengthDiff: number;\n    confidence: number;\n  };\n}\n\nexport class TextualDifferenceMetric extends Metric {\n  async measure(input: string, output: string): Promise<TextualDifferenceResult> {\n    const matcher = new SequenceMatcher(null, input, output);\n    const ratio = matcher.ratio();\n\n    // Get detailed operations\n    const ops = matcher.getOpcodes();\n    const changes = ops.filter(([op]) => op !== 'equal').length;\n\n    // Calculate confidence based on text length difference\n    const maxLength = Math.max(input.length, output.length);\n    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n    const confidence = 1 - lengthDiff;\n\n    return {\n      score: ratio,\n      info: {\n        confidence,\n        ratio,\n        changes,\n        lengthDiff,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport keyword_extractor from 'keyword-extractor';\n\ninterface KeywordCoverageResult extends MetricResult {\n  info: {\n    totalKeywords: number;\n    matchedKeywords: number;\n  };\n}\n\nexport class KeywordCoverageMetric extends Metric {\n  async measure(input: string, output: string): Promise<KeywordCoverageResult> {\n    // Handle empty strings case\n    if (!input && !output) {\n      return {\n        score: 1,\n        info: {\n          totalKeywords: 0,\n          matchedKeywords: 0,\n        },\n      };\n    }\n\n    const extractKeywords = (text: string) => {\n      return keyword_extractor.extract(text, {\n        language: 'english',\n        remove_digits: true,\n        return_changed_case: true,\n        remove_duplicates: true,\n      });\n    };\n\n    const referenceKeywords = new Set(extractKeywords(input));\n    const responseKeywords = new Set(extractKeywords(output));\n\n    const matchedKeywords = [...referenceKeywords].filter(k => responseKeywords.has(k));\n    const totalKeywords = referenceKeywords.size;\n    const coverage = totalKeywords > 0 ? matchedKeywords.length / totalKeywords : 0;\n\n    return {\n      score: coverage,\n      info: {\n        totalKeywords: referenceKeywords.size,\n        matchedKeywords: matchedKeywords.length,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport Sentiment from 'sentiment';\n\ninterface ToneConsitencyResult extends MetricResult {\n  info:\n    | {\n        responseSentiment: number;\n        referenceSentiment: number;\n        difference: number;\n      }\n    | {\n        avgSentiment: number;\n        sentimentVariance: number;\n      };\n}\n\nexport class ToneConsistencyMetric extends Metric {\n  private sentiment = new Sentiment();\n\n  async measure(input: string, output: string): Promise<ToneConsitencyResult> {\n    const responseSentiment = this.sentiment.analyze(input);\n\n    if (output) {\n      // Compare sentiment with reference\n      const referenceSentiment = this.sentiment.analyze(output);\n      const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n      const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n      return {\n        score: normalizedScore,\n        info: {\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        },\n      };\n    }\n\n    // Evaluate sentiment stability across response\n    const sentences = input.match(/[^.!?]+[.!?]+/g) || [input];\n    const sentiments = sentences.map(s => this.sentiment.analyze(s).comparative);\n    const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n    const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n    const stability = Math.max(0, 1 - variance);\n\n    return {\n      score: stability,\n      info: {\n        avgSentiment,\n        sentimentVariance: variance,\n      },\n    };\n  }\n}\n"]}
+{"version":3,"sources":["../../../src/metrics/nlp/completeness/index.ts","../../../src/metrics/nlp/content-similarity/index.ts","../../../src/metrics/nlp/textual-difference/index.ts","../../../src/metrics/nlp/keyword-coverage/index.ts","../../../src/metrics/nlp/tone/index.ts"],"names":["Metric"],"mappings":";;;;;;;AAgBO,IAAM,kBAAA,GAAN,cAAiC,MAAA,CAAO;AAAA,EAC7C,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAmD;AAE9E,IAAA,IAAI,UAAU,IAAA,IAAQ,KAAA,KAAU,UAAa,MAAA,KAAW,IAAA,IAAQ,WAAW,MAAA,EAAW;AACpF,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAGA,IAAA,KAAA,GAAQ,MAAM,IAAA,EAAK;AACnB,IAAA,MAAA,GAAS,OAAO,IAAA,EAAK;AAErB,IAAA,MAAM,QAAA,GAAW,IAAI,KAAK,CAAA;AAC1B,IAAA,MAAM,SAAA,GAAY,IAAI,MAAM,CAAA;AAG5B,IAAA,MAAM,aAAA,GAAgB,IAAA,CAAK,eAAA,CAAgB,QAAQ,CAAA;AACnD,IAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,eAAA,CAAgB,SAAS,CAAA;AAErD,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,iBAAA,CAAkB,aAAA,EAAe,cAAc,CAAA;AAErE,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,aAAA;AAAA,QACA,cAAA;AAAA,QACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,QACtE,aAAA,EAAe;AAAA,UACb,OAAO,aAAA,CAAc,MAAA;AAAA,UACrB,QAAQ,cAAA,CAAe;AAAA;AACzB;AACF,KACF;AAAA,EACF;AAAA,EAEQ,gBAAgB,GAAA,EAAoB;AAE1C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,IAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,IAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,MAAA,MAAM,UAAA,GAAa,IAAA,CAAK,eAAA,CAAgB,IAAI,CAAA;AAG5C,MAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,IACnC,CAAA;AAGA,IAAA,MAAM,cAAA,GAAiB;AAAA,MACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,KACpC;AAGA,IAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AAAA,EACpC;AAAA,EAEQ,gBAAgB,GAAA,EAAqB;AAE3C,IAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AAAA,EACjB;AAAA,EAEQ,iBAAA,CAAkB,UAAoB,UAAA,EAA8B;AAC1E,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,IACvC;AAGA,IAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,MAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,OAAO,CAAA;AACzC,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,CAAC,CAAA;AAGnC,QAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,UAAA,OAAO,IAAA,KAAS,IAAA;AAAA,QAClB;AAGA,QAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,QAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,UAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,QAC1C;AAEA,QAAA,OAAO,KAAA;AAAA,MACT,CAAC;AAAA,KACH;AACA,IAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AAAA,EACnC;AACF;ACzGO,IAAM,uBAAA,GAAN,cAAsCA,MAAAA,CAAO;AAAA,EAC1C,OAAA;AAAA,EAER,WAAA,CAAY,OAAA,GAAoC,EAAC,EAAG;AAClD,IAAA,KAAA,EAAM;AACN,IAAA,IAAA,CAAK,OAAA,GAAU;AAAA,MACb,UAAA,EAAY,IAAA;AAAA,MACZ,gBAAA,EAAkB,IAAA;AAAA,MAClB,GAAG;AAAA,KACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,IAAI,cAAA,GAAiB,KAAA;AACrB,IAAA,IAAI,eAAA,GAAkB,MAAA;AAEtB,IAAA,IAAI,IAAA,CAAK,QAAQ,UAAA,EAAY;AAC3B,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,IAAA,CAAK,QAAQ,gBAAA,EAAkB;AACjC,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,MAAM,UAAA,GAAa,gBAAA,CAAiB,iBAAA,CAAkB,cAAA,EAAgB,eAAe,CAAA;AAErF,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,UAAA;AAAA,MACP,IAAA,EAAM,EAAE,UAAA;AAAW,KACrB;AAAA,EACF;AACF;ACnCO,IAAM,uBAAA,GAAN,cAAsCA,MAAAA,CAAO;AAAA,EAClD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAE7E,IAAA,MAAM,KAAA,GAAQ,cAAA,CAAe,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,OAAA,GAAU,YAAA,CAAa,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,KAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,UAAA;AAAA,QACA,KAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA;AACF,KACF;AAAA,EACF;AACF;ACzBO,IAAM,qBAAA,GAAN,cAAoCA,MAAAA,CAAO;AAAA,EAChD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAgD;AAE3E,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,aAAA,EAAe,CAAA;AAAA,UACf,eAAA,EAAiB;AAAA;AACnB,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AAExD,IAAA,MAAM,eAAA,GAAkB,CAAC,GAAG,iBAAiB,CAAA,CAAE,OAAO,CAAA,CAAA,KAAK,gBAAA,CAAiB,GAAA,CAAI,CAAC,CAAC,CAAA;AAClF,IAAA,MAAM,gBAAgB,iBAAA,CAAkB,IAAA;AACxC,IAAA,MAAM,QAAA,GAAW,aAAA,GAAgB,CAAA,GAAI,eAAA,CAAgB,SAAS,aAAA,GAAgB,CAAA;AAE9E,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,eAAe,iBAAA,CAAkB,IAAA;AAAA,QACjC,iBAAiB,eAAA,CAAgB;AAAA;AACnC,KACF;AAAA,EACF;AACF;AC/BO,IAAM,qBAAA,GAAN,cAAoCA,MAAAA,CAAO;AAAA,EACxC,SAAA,GAAY,IAAI,SAAA,EAAU;AAAA,EAElC,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAA+C;AAC1E,IAAA,MAAM,iBAAA,GAAoB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,KAAK,CAAA;AAEtD,IAAA,IAAI,MAAA,EAAQ;AAEV,MAAA,MAAM,kBAAA,GAAqB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,MAAM,CAAA;AACxD,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,UACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,UACvC,UAAA,EAAY;AAAA;AACd,OACF;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,KAAA,CAAM,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,KAAK,CAAA;AACzD,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,KAAK,SAAA,CAAU,OAAA,CAAQ,CAAC,CAAA,CAAE,WAAW,CAAA;AAC3E,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,YAAA;AAAA,QACA,iBAAA,EAAmB;AAAA;AACrB,KACF;AAAA,EACF;AACF","file":"index.js","sourcesContent":["import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport nlp from 'compromise';\n\ninterface CompletenessMetricResult extends MetricResult {\n  info: {\n    inputElements: string[];\n    outputElements: string[];\n    missingElements: string[];\n    elementCounts: {\n      input: number;\n      output: number;\n    };\n  };\n}\n\nexport class CompletenessMetric extends Metric {\n  async measure(input: string, output: string): Promise<CompletenessMetricResult> {\n    // Handle null/undefined inputs\n    if (input === null || input === undefined || output === null || output === undefined) {\n      throw new Error('Inputs cannot be null or undefined');\n    }\n\n    // Trim both inputs\n    input = input.trim();\n    output = output.trim();\n\n    const inputDoc = nlp(input);\n    const outputDoc = nlp(output);\n\n    // Extract and log elements\n    const inputElements = this.extractElements(inputDoc);\n    const outputElements = this.extractElements(outputDoc);\n    // Maybe we need a more sophisticated matching approach\n    const coverage = this.calculateCoverage(inputElements, outputElements);\n\n    return {\n      score: coverage,\n      info: {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      },\n    };\n  }\n\n  private extractElements(doc: any): string[] {\n    // Get more specific elements and ensure they're arrays\n    const nouns = doc.nouns().out('array') || [];\n    const verbs = doc.verbs().toInfinitive().out('array') || [];\n    const topics = doc.topics().out('array') || [];\n    const terms = doc.terms().out('array') || [];\n\n    // Helper function to clean and split terms\n    const cleanAndSplitTerm = (term: string): string[] => {\n      // First normalize the string\n      const normalized = this.normalizeString(term);\n\n      // Split on word boundaries and filter out empty strings\n      return normalized\n        .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n        .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n        .trim()\n        .split(/\\s+/)\n        .filter(word => word.length > 0);\n    };\n\n    // Process all elements\n    const processedTerms = [\n      ...nouns.flatMap(cleanAndSplitTerm),\n      ...verbs.flatMap(cleanAndSplitTerm),\n      ...topics.flatMap(cleanAndSplitTerm),\n      ...terms.flatMap(cleanAndSplitTerm),\n    ];\n\n    // Remove duplicates\n    return [...new Set(processedTerms)];\n  }\n\n  private normalizeString(str: string): string {\n    // Remove diacritics and convert to lowercase\n    return str\n      .normalize('NFD')\n      .replace(/[\\u0300-\\u036f]/g, '')\n      .toLowerCase();\n  }\n\n  private calculateCoverage(original: string[], simplified: string[]): number {\n    if (original.length === 0) {\n      return simplified.length === 0 ? 1 : 0;\n    }\n\n    // Exact matching for short words (3 chars or less), substring matching for longer words\n    const covered = original.filter(element =>\n      simplified.some(s => {\n        const elem = this.normalizeString(element);\n        const simp = this.normalizeString(s);\n\n        // For short words (3 chars or less), require exact match\n        if (elem.length <= 3) {\n          return elem === simp;\n        }\n\n        // For longer words, require substantial overlap (more than 60% of the longer word)\n        const longer = elem.length > simp.length ? elem : simp;\n        const shorter = elem.length > simp.length ? simp : elem;\n\n        if (longer.includes(shorter)) {\n          return shorter.length / longer.length > 0.6;\n        }\n\n        return false;\n      }),\n    );\n    return covered.length / original.length;\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityResult extends MetricResult {\n  info: {\n    similarity: number;\n  };\n}\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport class ContentSimilarityMetric extends Metric {\n  private options: ContentSimilarityOptions;\n\n  constructor(options: ContentSimilarityOptions = {}) {\n    super();\n    this.options = {\n      ignoreCase: true,\n      ignoreWhitespace: true,\n      ...options,\n    };\n  }\n\n  async measure(input: string, output: string): Promise<ContentSimilarityResult> {\n    let processedInput = input;\n    let processedOutput = output;\n\n    if (this.options.ignoreCase) {\n      processedInput = processedInput.toLowerCase();\n      processedOutput = processedOutput.toLowerCase();\n    }\n\n    if (this.options.ignoreWhitespace) {\n      processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n      processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n    }\n\n    const similarity = stringSimilarity.compareTwoStrings(processedInput, processedOutput);\n\n    return {\n      score: similarity,\n      info: { similarity },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport { calculateRatio, countChanges } from '../../../ratio';\n\ninterface TextualDifferenceResult extends MetricResult {\n  info: {\n    ratio: number;\n    changes: number;\n    lengthDiff: number;\n    confidence: number;\n  };\n}\n\nexport class TextualDifferenceMetric extends Metric {\n  async measure(input: string, output: string): Promise<TextualDifferenceResult> {\n    // Calculate similarity ratio using LCS approach (similar to SequenceMatcher.ratio())\n    const ratio = calculateRatio(input, output);\n\n    // Count changes by comparing words\n    const changes = countChanges(input, output);\n\n    // Calculate confidence based on text length difference\n    const maxLength = Math.max(input.length, output.length);\n    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n    const confidence = 1 - lengthDiff;\n\n    return {\n      score: ratio,\n      info: {\n        confidence,\n        ratio,\n        changes,\n        lengthDiff,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport keyword_extractor from 'keyword-extractor';\n\ninterface KeywordCoverageResult extends MetricResult {\n  info: {\n    totalKeywords: number;\n    matchedKeywords: number;\n  };\n}\n\nexport class KeywordCoverageMetric extends Metric {\n  async measure(input: string, output: string): Promise<KeywordCoverageResult> {\n    // Handle empty strings case\n    if (!input && !output) {\n      return {\n        score: 1,\n        info: {\n          totalKeywords: 0,\n          matchedKeywords: 0,\n        },\n      };\n    }\n\n    const extractKeywords = (text: string) => {\n      return keyword_extractor.extract(text, {\n        language: 'english',\n        remove_digits: true,\n        return_changed_case: true,\n        remove_duplicates: true,\n      });\n    };\n\n    const referenceKeywords = new Set(extractKeywords(input));\n    const responseKeywords = new Set(extractKeywords(output));\n\n    const matchedKeywords = [...referenceKeywords].filter(k => responseKeywords.has(k));\n    const totalKeywords = referenceKeywords.size;\n    const coverage = totalKeywords > 0 ? matchedKeywords.length / totalKeywords : 0;\n\n    return {\n      score: coverage,\n      info: {\n        totalKeywords: referenceKeywords.size,\n        matchedKeywords: matchedKeywords.length,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport Sentiment from 'sentiment';\n\ninterface ToneConsitencyResult extends MetricResult {\n  info:\n    | {\n        responseSentiment: number;\n        referenceSentiment: number;\n        difference: number;\n      }\n    | {\n        avgSentiment: number;\n        sentimentVariance: number;\n      };\n}\n\nexport class ToneConsistencyMetric extends Metric {\n  private sentiment = new Sentiment();\n\n  async measure(input: string, output: string): Promise<ToneConsitencyResult> {\n    const responseSentiment = this.sentiment.analyze(input);\n\n    if (output) {\n      // Compare sentiment with reference\n      const referenceSentiment = this.sentiment.analyze(output);\n      const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n      const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n      return {\n        score: normalizedScore,\n        info: {\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        },\n      };\n    }\n\n    // Evaluate sentiment stability across response\n    const sentences = input.match(/[^.!?]+[.!?]+/g) || [input];\n    const sentiments = sentences.map(s => this.sentiment.analyze(s).comparative);\n    const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n    const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n    const stability = Math.max(0, 1 - variance);\n\n    return {\n      score: stability,\n      info: {\n        avgSentiment,\n        sentimentVariance: variance,\n      },\n    };\n  }\n}\n"]}

package/dist/metrics/nlp/textual-difference/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/metrics/nlp/textual-difference/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGtD,UAAU,uBAAwB,SAAQ,YAAY;IACpD,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,qBAAa,uBAAwB,SAAQ,MAAM;IAC3C,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC;CAuB/E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/metrics/nlp/textual-difference/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGtD,UAAU,uBAAwB,SAAQ,YAAY;IACpD,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,qBAAa,uBAAwB,SAAQ,MAAM;IAC3C,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC;CAsB/E"}

package/dist/ratio.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Calculates similarity ratio similar to SequenceMatcher.ratio()
+ * Uses longest common subsequence (LCS) approach
+ * Ratio = 2.0 * matches / total
+ */
+export declare function calculateRatio(input: string, output: string): number;
+/**
+ * Counts the number of differences between two strings
+ * Uses opcodes-like approach: counts insertions, deletions, and replacements
+ * For whitespace differences, preserves the original strings before word splitting
+ */
+export declare function countChanges(input: string, output: string): number;
+//# sourceMappingURL=ratio.d.ts.map

package/dist/ratio.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ratio.d.ts","sourceRoot":"","sources":["../src/ratio.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAapE;AA+BD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAqClE"}

package/dist/scorers/code/index.cjs

@@ -1,9 +1,9 @@
 'use strict';
 
+var chunkPWGOG6ML_cjs = require('../../chunk-PWGOG6ML.cjs');
 var chunkQKR2PMLZ_cjs = require('../../chunk-QKR2PMLZ.cjs');
 var scores = require('@mastra/core/scores');
 var nlp = require('compromise');
-var difflib = require('difflib');
 var keyword_extractor = require('keyword-extractor');
 var stringSimilarity = require('string-similarity');
 var Sentiment = require('sentiment');
@@ -101,10 +101,8 @@ function createTextualDifferenceScorer() {
   }).preprocess(async ({ run }) => {
     const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
     const output = run.output?.map((i) => i.content).join(", ") || "";
-    const matcher = new difflib.SequenceMatcher(null, input, output);
-    const ratio = matcher.ratio();
-    const ops = matcher.getOpcodes();
-    const changes = ops.filter(([op]) => op !== "equal").length;
+    const ratio = chunkPWGOG6ML_cjs.calculateRatio(input, output);
+    const changes = chunkPWGOG6ML_cjs.countChanges(input, output);
     const maxLength = Math.max(input.length, output.length);
     const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
     const confidence = 1 - lengthDiff;

package/dist/scorers/code/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer","nlp","SequenceMatcher","keyword_extractor","stringSimilarity","Sentiment","extractToolCalls"],"mappings":";;;;;;;;;;;;;;;;;AAGA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAOA,mBAAA,CAAa;AAAA,IAClB,IAAA,EAAM,qBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAWC,oBAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAYA,oBAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;AC1HO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOD,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,2FAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAIE,uBAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOF,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,yBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOG,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC5DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOH,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,qGAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAaI,iCAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOJ,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,aAAA;AAAA,IACN,WAAA,EACE,oJAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAIK,0BAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC3CA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOL,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,aAAa,cAAA,EAAe;AAAA,IAC5B,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAIM,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.cjs","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer({\n    name: 'Completeness Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer({\n    name: 'Textual Difference Scorer',\n    description: 'Calculate textual difference between input and output using sequence matching algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const matcher = new SequenceMatcher(null, input, output);\n      const ratio = matcher.ratio();\n\n      // Get detailed operations\n      const ops = matcher.getOpcodes();\n      const changes = ops.filter(([op]) => op !== 'equal').length;\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer({\n    name: 'Keyword Coverage Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer({\n    name: 'Content Similarity Scorer',\n    description: 'Calculates content similarity between input and output messages using string comparison algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer({\n    name: 'Tone Scorer',\n    description:\n      'Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\ninterface ToolCallAccuracyOptions {\n  expectedTool?: string;\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n  if (strictMode) {\n    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n  }\n\n  const expectedIndices: number[] = [];\n  for (const expectedTool of expectedOrder) {\n    const index = actualTools.indexOf(expectedTool);\n    if (index === -1) {\n      return false;\n    }\n    expectedIndices.push(index);\n  }\n\n  for (let i = 1; i < expectedIndices.length; i++) {\n    const currentIndex = expectedIndices[i];\n    const prevIndex = expectedIndices[i - 1];\n    if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction calculateAccuracy({\n  expectedTool,\n  actualTools,\n  strictMode = false,\n  expectedToolOrder,\n}: {\n  expectedTool?: string;\n  actualTools: string[];\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}): number {\n  if (actualTools.length === 0) {\n    return 0;\n  }\n\n  if (expectedToolOrder && expectedToolOrder.length > 0) {\n    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n  }\n\n  if (!expectedTool) {\n    return 0;\n  }\n\n  if (strictMode) {\n    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n  }\n\n  return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n  const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n  if (!expectedTool && !expectedToolOrder) {\n    throw new Error('Either expectedTool or expectedToolOrder must be provided');\n  }\n\n  const getDescription = () => {\n    return expectedToolOrder\n      ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n      : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n  };\n\n  return createScorer({\n    name: 'Tool Call Accuracy Scorer',\n    description: getDescription(),\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;\n      const isOutputInvalid = !run.output || run.output.length === 0;\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Input and output messages cannot be null or empty');\n      }\n\n      const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n      const correctToolCalled = expectedTool\n        ? strictMode\n          ? actualTools.length === 1 && actualTools[0] === expectedTool\n          : actualTools.includes(expectedTool)\n        : false;\n\n      return {\n        expectedTool,\n        actualTools,\n        strictMode,\n        expectedToolOrder,\n        hasToolCalls: actualTools.length > 0,\n        correctToolCalled,\n        toolCallInfos,\n        correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n      };\n    })\n    .generateScore(({ results }) => {\n      const preprocessResult = results.preprocessStepResult;\n      if (!preprocessResult) {\n        return 0;\n      }\n\n      return calculateAccuracy({\n        expectedTool: preprocessResult.expectedTool,\n        actualTools: preprocessResult.actualTools,\n        strictMode: preprocessResult.strictMode,\n        expectedToolOrder: preprocessResult.expectedToolOrder,\n      });\n    });\n}\n"]}
+{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer","nlp","calculateRatio","countChanges","keyword_extractor","stringSimilarity","Sentiment","extractToolCalls"],"mappings":";;;;;;;;;;;;;;;;;AAGA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAOA,mBAAA,CAAa;AAAA,IAClB,IAAA,EAAM,qBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAWC,oBAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAYA,oBAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;AC1HO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOD,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,2FAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAGpF,IAAA,MAAM,KAAA,GAAQE,gCAAA,CAAe,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,OAAA,GAAUC,8BAAA,CAAa,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOH,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,yBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOI,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC5DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOJ,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,qGAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAaK,iCAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOL,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,aAAA;AAAA,IACN,WAAA,EACE,oJAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAIM,0BAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC3CA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAON,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,aAAa,cAAA,EAAe;AAAA,IAC5B,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAIO,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.cjs","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer({\n    name: 'Completeness Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { calculateRatio, countChanges } from '../../../ratio';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer({\n    name: 'Textual Difference Scorer',\n    description: 'Calculate textual difference between input and output using sequence matching algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      // Calculate similarity ratio using LCS approach (similar to SequenceMatcher.ratio())\n      const ratio = calculateRatio(input, output);\n\n      // Count changes by comparing words\n      const changes = countChanges(input, output);\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer({\n    name: 'Keyword Coverage Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer({\n    name: 'Content Similarity Scorer',\n    description: 'Calculates content similarity between input and output messages using string comparison algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer({\n    name: 'Tone Scorer',\n    description:\n      'Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\ninterface ToolCallAccuracyOptions {\n  expectedTool?: string;\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n  if (strictMode) {\n    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n  }\n\n  const expectedIndices: number[] = [];\n  for (const expectedTool of expectedOrder) {\n    const index = actualTools.indexOf(expectedTool);\n    if (index === -1) {\n      return false;\n    }\n    expectedIndices.push(index);\n  }\n\n  for (let i = 1; i < expectedIndices.length; i++) {\n    const currentIndex = expectedIndices[i];\n    const prevIndex = expectedIndices[i - 1];\n    if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction calculateAccuracy({\n  expectedTool,\n  actualTools,\n  strictMode = false,\n  expectedToolOrder,\n}: {\n  expectedTool?: string;\n  actualTools: string[];\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}): number {\n  if (actualTools.length === 0) {\n    return 0;\n  }\n\n  if (expectedToolOrder && expectedToolOrder.length > 0) {\n    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n  }\n\n  if (!expectedTool) {\n    return 0;\n  }\n\n  if (strictMode) {\n    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n  }\n\n  return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n  const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n  if (!expectedTool && !expectedToolOrder) {\n    throw new Error('Either expectedTool or expectedToolOrder must be provided');\n  }\n\n  const getDescription = () => {\n    return expectedToolOrder\n      ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n      : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n  };\n\n  return createScorer({\n    name: 'Tool Call Accuracy Scorer',\n    description: getDescription(),\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;\n      const isOutputInvalid = !run.output || run.output.length === 0;\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Input and output messages cannot be null or empty');\n      }\n\n      const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n      const correctToolCalled = expectedTool\n        ? strictMode\n          ? actualTools.length === 1 && actualTools[0] === expectedTool\n          : actualTools.includes(expectedTool)\n        : false;\n\n      return {\n        expectedTool,\n        actualTools,\n        strictMode,\n        expectedToolOrder,\n        hasToolCalls: actualTools.length > 0,\n        correctToolCalled,\n        toolCallInfos,\n        correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n      };\n    })\n    .generateScore(({ results }) => {\n      const preprocessResult = results.preprocessStepResult;\n      if (!preprocessResult) {\n        return 0;\n      }\n\n      return calculateAccuracy({\n        expectedTool: preprocessResult.expectedTool,\n        actualTools: preprocessResult.actualTools,\n        strictMode: preprocessResult.strictMode,\n        expectedToolOrder: preprocessResult.expectedToolOrder,\n      });\n    });\n}\n"]}

package/dist/scorers/code/index.js

@@ -1,7 +1,7 @@
+import { calculateRatio, countChanges } from '../../chunk-44PMY5ES.js';
 import { extractToolCalls } from '../../chunk-KHEXN75Q.js';
 import { createScorer } from '@mastra/core/scores';
 import nlp from 'compromise';
-import { SequenceMatcher } from 'difflib';
 import keyword_extractor from 'keyword-extractor';
 import stringSimilarity from 'string-similarity';
 import Sentiment from 'sentiment';
@@ -92,10 +92,8 @@ function createTextualDifferenceScorer() {
   }).preprocess(async ({ run }) => {
     const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
     const output = run.output?.map((i) => i.content).join(", ") || "";
-    const matcher = new SequenceMatcher(null, input, output);
-    const ratio = matcher.ratio();
-    const ops = matcher.getOpcodes();
-    const changes = ops.filter(([op]) => op !== "equal").length;
+    const ratio = calculateRatio(input, output);
+    const changes = countChanges(input, output);
     const maxLength = Math.max(input.length, output.length);
     const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
     const confidence = 1 - lengthDiff;

package/dist/scorers/code/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer"],"mappings":";;;;;;;;AAGA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAO,YAAA,CAAa;AAAA,IAClB,IAAA,EAAM,qBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAW,GAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAY,GAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;AC1HO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,2FAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAI,eAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,yBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC5DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,qGAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAa,gBAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,aAAA;AAAA,IACN,WAAA,EACE,oJAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAI,SAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC3CA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,aAAa,cAAA,EAAe;AAAA,IAC5B,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAI,gBAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.js","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer({\n    name: 'Completeness Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer({\n    name: 'Textual Difference Scorer',\n    description: 'Calculate textual difference between input and output using sequence matching algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const matcher = new SequenceMatcher(null, input, output);\n      const ratio = matcher.ratio();\n\n      // Get detailed operations\n      const ops = matcher.getOpcodes();\n      const changes = ops.filter(([op]) => op !== 'equal').length;\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer({\n    name: 'Keyword Coverage Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer({\n    name: 'Content Similarity Scorer',\n    description: 'Calculates content similarity between input and output messages using string comparison algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer({\n    name: 'Tone Scorer',\n    description:\n      'Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\ninterface ToolCallAccuracyOptions {\n  expectedTool?: string;\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n  if (strictMode) {\n    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n  }\n\n  const expectedIndices: number[] = [];\n  for (const expectedTool of expectedOrder) {\n    const index = actualTools.indexOf(expectedTool);\n    if (index === -1) {\n      return false;\n    }\n    expectedIndices.push(index);\n  }\n\n  for (let i = 1; i < expectedIndices.length; i++) {\n    const currentIndex = expectedIndices[i];\n    const prevIndex = expectedIndices[i - 1];\n    if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction calculateAccuracy({\n  expectedTool,\n  actualTools,\n  strictMode = false,\n  expectedToolOrder,\n}: {\n  expectedTool?: string;\n  actualTools: string[];\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}): number {\n  if (actualTools.length === 0) {\n    return 0;\n  }\n\n  if (expectedToolOrder && expectedToolOrder.length > 0) {\n    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n  }\n\n  if (!expectedTool) {\n    return 0;\n  }\n\n  if (strictMode) {\n    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n  }\n\n  return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n  const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n  if (!expectedTool && !expectedToolOrder) {\n    throw new Error('Either expectedTool or expectedToolOrder must be provided');\n  }\n\n  const getDescription = () => {\n    return expectedToolOrder\n      ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n      : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n  };\n\n  return createScorer({\n    name: 'Tool Call Accuracy Scorer',\n    description: getDescription(),\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;\n      const isOutputInvalid = !run.output || run.output.length === 0;\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Input and output messages cannot be null or empty');\n      }\n\n      const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n      const correctToolCalled = expectedTool\n        ? strictMode\n          ? actualTools.length === 1 && actualTools[0] === expectedTool\n          : actualTools.includes(expectedTool)\n        : false;\n\n      return {\n        expectedTool,\n        actualTools,\n        strictMode,\n        expectedToolOrder,\n        hasToolCalls: actualTools.length > 0,\n        correctToolCalled,\n        toolCallInfos,\n        correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n      };\n    })\n    .generateScore(({ results }) => {\n      const preprocessResult = results.preprocessStepResult;\n      if (!preprocessResult) {\n        return 0;\n      }\n\n      return calculateAccuracy({\n        expectedTool: preprocessResult.expectedTool,\n        actualTools: preprocessResult.actualTools,\n        strictMode: preprocessResult.strictMode,\n        expectedToolOrder: preprocessResult.expectedToolOrder,\n      });\n    });\n}\n"]}
+{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer"],"mappings":";;;;;;;;AAGA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAO,YAAA,CAAa;AAAA,IAClB,IAAA,EAAM,qBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAW,GAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAY,GAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;AC1HO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,2FAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAGpF,IAAA,MAAM,KAAA,GAAQ,cAAA,CAAe,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,OAAA,GAAU,YAAA,CAAa,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,yBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC5DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,qGAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAa,gBAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,aAAA;AAAA,IACN,WAAA,EACE,oJAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAI,SAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC3CA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,aAAa,cAAA,EAAe;AAAA,IAC5B,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAI,gBAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.js","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer({\n    name: 'Completeness Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { calculateRatio, countChanges } from '../../../ratio';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer({\n    name: 'Textual Difference Scorer',\n    description: 'Calculate textual difference between input and output using sequence matching algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      // Calculate similarity ratio using LCS approach (similar to SequenceMatcher.ratio())\n      const ratio = calculateRatio(input, output);\n\n      // Count changes by comparing words\n      const changes = countChanges(input, output);\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer({\n    name: 'Keyword Coverage Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer({\n    name: 'Content Similarity Scorer',\n    description: 'Calculates content similarity between input and output messages using string comparison algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer({\n    name: 'Tone Scorer',\n    description:\n      'Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\ninterface ToolCallAccuracyOptions {\n  expectedTool?: string;\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n  if (strictMode) {\n    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n  }\n\n  const expectedIndices: number[] = [];\n  for (const expectedTool of expectedOrder) {\n    const index = actualTools.indexOf(expectedTool);\n    if (index === -1) {\n      return false;\n    }\n    expectedIndices.push(index);\n  }\n\n  for (let i = 1; i < expectedIndices.length; i++) {\n    const currentIndex = expectedIndices[i];\n    const prevIndex = expectedIndices[i - 1];\n    if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction calculateAccuracy({\n  expectedTool,\n  actualTools,\n  strictMode = false,\n  expectedToolOrder,\n}: {\n  expectedTool?: string;\n  actualTools: string[];\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}): number {\n  if (actualTools.length === 0) {\n    return 0;\n  }\n\n  if (expectedToolOrder && expectedToolOrder.length > 0) {\n    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n  }\n\n  if (!expectedTool) {\n    return 0;\n  }\n\n  if (strictMode) {\n    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n  }\n\n  return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n  const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n  if (!expectedTool && !expectedToolOrder) {\n    throw new Error('Either expectedTool or expectedToolOrder must be provided');\n  }\n\n  const getDescription = () => {\n    return expectedToolOrder\n      ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n      : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n  };\n\n  return createScorer({\n    name: 'Tool Call Accuracy Scorer',\n    description: getDescription(),\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;\n      const isOutputInvalid = !run.output || run.output.length === 0;\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Input and output messages cannot be null or empty');\n      }\n\n      const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n      const correctToolCalled = expectedTool\n        ? strictMode\n          ? actualTools.length === 1 && actualTools[0] === expectedTool\n          : actualTools.includes(expectedTool)\n        : false;\n\n      return {\n        expectedTool,\n        actualTools,\n        strictMode,\n        expectedToolOrder,\n        hasToolCalls: actualTools.length > 0,\n        correctToolCalled,\n        toolCallInfos,\n        correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n      };\n    })\n    .generateScore(({ results }) => {\n      const preprocessResult = results.preprocessStepResult;\n      if (!preprocessResult) {\n        return 0;\n      }\n\n      return calculateAccuracy({\n        expectedTool: preprocessResult.expectedTool,\n        actualTools: preprocessResult.actualTools,\n        strictMode: preprocessResult.strictMode,\n        expectedToolOrder: preprocessResult.expectedToolOrder,\n      });\n    });\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/evals",
-  "version": "0.14.3",
+  "version": "0.14.4-alpha.0",
   "description": "",
   "type": "module",
   "files": [
@@ -96,7 +96,6 @@
   "license": "Apache-2.0",
   "dependencies": {
     "compromise": "^14.14.4",
-    "difflib": "^0.2.4",
     "fs-extra": "^11.3.2",
     "keyword-extractor": "^0.0.28",
     "sentiment": "^5.0.2",
@@ -110,7 +109,6 @@
   "devDependencies": {
     "@ai-sdk/openai": "^1.3.24",
     "@microsoft/api-extractor": "^7.52.8",
-    "@types/difflib": "^0.2.7",
     "@types/fs-extra": "^11.0.4",
     "@types/sentiment": "^5.0.4",
     "@types/string-similarity": "^4.0.2",
@@ -121,8 +119,8 @@
     "typescript": "^5.8.3",
     "vitest": "^3.2.4",
     "zod": "^3.25.76",
-    "@mastra/core": "0.24.0",
     "@internal/lint": "0.0.58",
+    "@mastra/core": "0.24.1-alpha.0",
     "@internal/types-builder": "0.0.33"
   },
   "publishConfig": {