@mastra/evals 0.11.0 → 0.12.0-alpha.0

package/dist/scorers/code/index.cjs

@@ -58,198 +58,174 @@ function calculateCoverage({ original, simplified }) {
 function createCompletenessScorer() {
   return scores.createScorer({
     name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
-    extract: async (run) => {
-      const isInputInvalid = !run.input || run.input.some((i) => i.content === null || i.content === void 0);
-      const isOutputInvalid = !run.output || run.output.text === null || run.output.text === void 0;
-      if (isInputInvalid || isOutputInvalid) {
-        throw new Error("Inputs cannot be null or undefined");
-      }
-      const input = run.input?.map((i) => i.content).join(", ") || "";
-      const output = run.output.text;
-      const inputToProcess = input;
-      const outputToProcess = output;
-      const inputDoc = nlp__default.default(inputToProcess.trim());
-      const outputDoc = nlp__default.default(outputToProcess.trim());
-      const inputElements = extractElements(inputDoc);
-      const outputElements = extractElements(outputDoc);
-      return {
-        result: {
-          inputElements,
-          outputElements,
-          missingElements: inputElements.filter((e) => !outputElements.includes(e)),
-          elementCounts: {
-            input: inputElements.length,
-            output: outputElements.length
-          }
-        }
-      };
-    },
-    analyze: async (run) => {
-      const inputElements = run.extractStepResult?.inputElements;
-      const outputElements = run.extractStepResult?.outputElements;
-      return {
-        score: calculateCoverage({
-          original: inputElements,
-          simplified: outputElements
-        })
-      };
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+  }).preprocess(async ({ run }) => {
+    const isInputInvalid = !run.input || run.input.inputMessages.some((i) => i.content === null || i.content === void 0);
+    const isOutputInvalid = !run.output || run.output.some((i) => i.content === null || i.content === void 0);
+    if (isInputInvalid || isOutputInvalid) {
+      throw new Error("Inputs cannot be null or undefined");
     }
+    const input = run.input?.inputMessages.map((i) => i.content).join(", ") || "";
+    const output = run.output?.map(({ content }) => content).join(", ") || "";
+    const inputToProcess = input;
+    const outputToProcess = output;
+    const inputDoc = nlp__default.default(inputToProcess.trim());
+    const outputDoc = nlp__default.default(outputToProcess.trim());
+    const inputElements = extractElements(inputDoc);
+    const outputElements = extractElements(outputDoc);
+    return {
+      inputElements,
+      outputElements,
+      missingElements: inputElements.filter((e) => !outputElements.includes(e)),
+      elementCounts: {
+        input: inputElements.length,
+        output: outputElements.length
+      }
+    };
+  }).generateScore(({ results }) => {
+    const inputElements = results.preprocessStepResult?.inputElements;
+    const outputElements = results.preprocessStepResult?.outputElements;
+    return calculateCoverage({
+      original: inputElements,
+      simplified: outputElements
+    });
   });
 }
 function createTextualDifferenceScorer() {
   return scores.createScorer({
     name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
-    analyze: async (run) => {
-      const input = run.input?.map((i) => i.content).join(", ") || "";
-      const output = run.output.text;
-      const matcher = new difflib.SequenceMatcher(null, input, output);
-      const ratio = matcher.ratio();
-      const ops = matcher.getOpcodes();
-      const changes = ops.filter(([op]) => op !== "equal").length;
-      const maxLength = Math.max(input.length, output.length);
-      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
-      const confidence = 1 - lengthDiff;
-      return {
-        score: ratio,
-        result: {
-          confidence,
-          changes,
-          lengthDiff
-        }
-      };
-    }
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+  }).preprocess(async ({ run }) => {
+    const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
+    const output = run.output?.map((i) => i.content).join(", ") || "";
+    const matcher = new difflib.SequenceMatcher(null, input, output);
+    const ratio = matcher.ratio();
+    const ops = matcher.getOpcodes();
+    const changes = ops.filter(([op]) => op !== "equal").length;
+    const maxLength = Math.max(input.length, output.length);
+    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
+    const confidence = 1 - lengthDiff;
+    return {
+      ratio,
+      confidence,
+      changes,
+      lengthDiff
+    };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.ratio;
   });
 }
 function createKeywordCoverageScorer() {
   return scores.createScorer({
     name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
-    extract: async (run) => {
-      const input = run.input?.map((i) => i.content).join(", ") || "";
-      const output = run.output.text;
-      if (!input && !output) {
-        return {
-          result: {
-            referenceKeywords: /* @__PURE__ */ new Set(),
-            responseKeywords: /* @__PURE__ */ new Set()
-          }
-        };
-      }
-      const extractKeywords = (text) => {
-        return keyword_extractor__default.default.extract(text, {
-          language: "english",
-          remove_digits: true,
-          return_changed_case: true,
-          remove_duplicates: true
-        });
-      };
-      const referenceKeywords = new Set(extractKeywords(input));
-      const responseKeywords = new Set(extractKeywords(output));
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+  }).preprocess(async ({ run }) => {
+    const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
+    const output = run.output?.map((i) => i.content).join(", ") || "";
+    if (!input && !output) {
       return {
         result: {
-          referenceKeywords,
-          responseKeywords
+          referenceKeywords: /* @__PURE__ */ new Set(),
+          responseKeywords: /* @__PURE__ */ new Set()
         }
       };
-    },
-    analyze: async (run) => {
-      if (!run.extractStepResult?.referenceKeywords.size && !run.extractStepResult?.responseKeywords.size) {
-        return {
-          score: 1,
-          result: {
-            totalKeywords: 0,
-            matchedKeywords: 0
-          }
-        };
-      }
-      const matchedKeywords = [...run.extractStepResult?.referenceKeywords].filter(
-        (k) => run.extractStepResult?.responseKeywords.has(k)
-      );
-      const totalKeywords = run.extractStepResult?.referenceKeywords.size;
-      const coverage = totalKeywords > 0 ? matchedKeywords.length / totalKeywords : 0;
+    }
+    const extractKeywords = (text) => {
+      return keyword_extractor__default.default.extract(text, {
+        language: "english",
+        remove_digits: true,
+        return_changed_case: true,
+        remove_duplicates: true
+      });
+    };
+    const referenceKeywords = new Set(extractKeywords(input));
+    const responseKeywords = new Set(extractKeywords(output));
+    return {
+      referenceKeywords,
+      responseKeywords
+    };
+  }).analyze(async ({ results }) => {
+    if (!results.preprocessStepResult?.referenceKeywords?.size && !results.preprocessStepResult?.responseKeywords?.size) {
       return {
-        score: coverage,
-        result: {
-          totalKeywords: run.extractStepResult?.referenceKeywords.size,
-          matchedKeywords: matchedKeywords.length
-        }
+        totalKeywordsLength: 0,
+        matchedKeywordsLength: 0
       };
     }
+    const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(
+      (k) => results.preprocessStepResult?.responseKeywords?.has(k)
+    );
+    return {
+      totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,
+      matchedKeywordsLength: matchedKeywords.length ?? 0
+    };
+  }).generateScore(({ results }) => {
+    if (!results.analyzeStepResult?.totalKeywordsLength) {
+      return 1;
+    }
+    const totalKeywords = results.analyzeStepResult?.totalKeywordsLength;
+    const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength;
+    return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;
   });
 }
 function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace } = { ignoreCase: true, ignoreWhitespace: true }) {
   return scores.createScorer({
     name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
-    extract: async (run) => {
-      let processedInput = run.input?.map((i) => i.content).join(", ") || "";
-      let processedOutput = run.output.text;
-      if (ignoreCase) {
-        processedInput = processedInput.toLowerCase();
-        processedOutput = processedOutput.toLowerCase();
-      }
-      if (ignoreWhitespace) {
-        processedInput = processedInput.replace(/\s+/g, " ").trim();
-        processedOutput = processedOutput.replace(/\s+/g, " ").trim();
-      }
-      return {
-        result: {
-          processedInput,
-          processedOutput
-        }
-      };
-    },
-    analyze: async (run) => {
-      const similarity = stringSimilarity__default.default.compareTwoStrings(
-        run.extractStepResult?.processedInput,
-        run.extractStepResult?.processedOutput
-      );
-      return {
-        score: similarity,
-        result: {
-          similarity
-        }
-      };
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+  }).preprocess(async ({ run }) => {
+    let processedInput = run.input?.inputMessages.map((i) => i.content).join(", ") || "";
+    let processedOutput = run.output.map((i) => i.content).join(", ") || "";
+    if (ignoreCase) {
+      processedInput = processedInput.toLowerCase();
+      processedOutput = processedOutput.toLowerCase();
     }
+    if (ignoreWhitespace) {
+      processedInput = processedInput.replace(/\s+/g, " ").trim();
+      processedOutput = processedOutput.replace(/\s+/g, " ").trim();
+    }
+    return {
+      processedInput,
+      processedOutput
+    };
+  }).generateScore(({ results }) => {
+    const similarity = stringSimilarity__default.default.compareTwoStrings(
+      results.preprocessStepResult?.processedInput,
+      results.preprocessStepResult?.processedOutput
+    );
+    return similarity;
   });
 }
-function createToneScorer() {
+function createToneScorer(config = {}) {
+  const { referenceTone } = config;
   return scores.createScorer({
     name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
-    analyze: async (run) => {
-      const sentiment = new Sentiment__default.default();
-      const input = run.input?.map((i) => i.content).join(", ") || "";
-      const output = run.output.text;
-      const responseSentiment = sentiment.analyze(input);
-      if (output) {
-        const referenceSentiment = sentiment.analyze(output);
-        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);
-        const normalizedScore = Math.max(0, 1 - sentimentDiff);
-        return {
-          score: normalizedScore,
-          result: {
-            responseSentiment: responseSentiment.comparative,
-            referenceSentiment: referenceSentiment.comparative,
-            difference: sentimentDiff
-          }
-        };
-      }
-      const sentences = input.match(/[^.!?]+[.!?]+/g) || [input];
-      const sentiments = sentences.map((s) => sentiment.analyze(s).comparative);
-      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;
-      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;
-      const stability = Math.max(0, 1 - variance);
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+  }).preprocess(async ({ run }) => {
+    const sentiment = new Sentiment__default.default();
+    const agentMessage = run.output?.map((i) => i.content).join(", ") || "";
+    const responseSentiment = sentiment.analyze(agentMessage);
+    if (referenceTone) {
+      const referenceSentiment = sentiment.analyze(referenceTone);
+      const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);
+      const normalizedScore = Math.max(0, 1 - sentimentDiff);
       return {
-        score: stability,
-        result: {
-          avgSentiment,
-          sentimentVariance: variance
-        }
+        score: normalizedScore,
+        responseSentiment: responseSentiment.comparative,
+        referenceSentiment: referenceSentiment.comparative,
+        difference: sentimentDiff
       };
     }
+    const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];
+    const sentiments = sentences.map((s) => sentiment.analyze(s).comparative);
+    const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;
+    const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;
+    const stability = Math.max(0, 1 - variance);
+    return {
+      score: stability,
+      avgSentiment,
+      sentimentVariance: variance
+    };
+  }).generateScore(({ results }) => {
+    return results.preprocessStepResult?.score;
   });
 }
 
@@ -258,3 +234,5 @@ exports.createContentSimilarityScorer = createContentSimilarityScorer;
 exports.createKeywordCoverageScorer = createKeywordCoverageScorer;
 exports.createTextualDifferenceScorer = createTextualDifferenceScorer;
 exports.createToneScorer = createToneScorer;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/scorers/code/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts"],"names":["createScorer","nlp","SequenceMatcher","keyword_extractor","stringSimilarity","Sentiment"],"mappings":";;;;;;;;;;;;;;;;AAIA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAOA,mBAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAWC,oBAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAYA,oBAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;ACzHO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOD,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAIE,uBAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOF,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOG,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC3DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOH,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAaI,iCAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOJ,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAIK,0BAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL","file":"index.cjs","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const matcher = new SequenceMatcher(null, input, output);\n      const ratio = matcher.ratio();\n\n      // Get detailed operations\n      const ops = matcher.getOpcodes();\n      const changes = ops.filter(([op]) => op !== 'equal').length;\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n"]}

package/dist/scorers/code/index.d.ts

@@ -1,5 +1,6 @@
-export { createCompletenessScorer_alias_1 as createCompletenessScorer } from '../../_tsup-dts-rollup.js';
-export { createTextualDifferenceScorer_alias_1 as createTextualDifferenceScorer } from '../../_tsup-dts-rollup.js';
-export { createKeywordCoverageScorer_alias_1 as createKeywordCoverageScorer } from '../../_tsup-dts-rollup.js';
-export { createContentSimilarityScorer_alias_1 as createContentSimilarityScorer } from '../../_tsup-dts-rollup.js';
-export { createToneScorer_alias_1 as createToneScorer } from '../../_tsup-dts-rollup.js';
+export * from './completeness';
+export * from './textual-difference';
+export * from './keyword-coverage';
+export * from './content-similarity';
+export * from './tone';
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/code/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scorers/code/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,QAAQ,CAAC"}