@mastra/evals 0.14.4 → 1.0.0-beta.1

package/dist/scorers/code/index.js

@@ -1,313 +0,0 @@
-import { calculateRatio, countChanges } from '../../chunk-44PMY5ES.js';
-import { extractToolCalls } from '../../chunk-KHEXN75Q.js';
-import { createScorer } from '@mastra/core/scores';
-import nlp from 'compromise';
-import keyword_extractor from 'keyword-extractor';
-import stringSimilarity from 'string-similarity';
-import Sentiment from 'sentiment';
-
-function normalizeString(str) {
-  return str.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase();
-}
-function extractElements(doc) {
-  const nouns = doc.nouns().out("array") || [];
-  const verbs = doc.verbs().toInfinitive().out("array") || [];
-  const topics = doc.topics().out("array") || [];
-  const terms = doc.terms().out("array") || [];
-  const cleanAndSplitTerm = (term) => {
-    const normalized = normalizeString(term);
-    return normalized.replace(/([a-z])([A-Z])/g, "$1 $2").replace(/[^a-z0-9]+/g, " ").trim().split(/\s+/).filter((word) => word.length > 0);
-  };
-  const processedTerms = [
-    ...nouns.flatMap(cleanAndSplitTerm),
-    ...verbs.flatMap(cleanAndSplitTerm),
-    ...topics.flatMap(cleanAndSplitTerm),
-    ...terms.flatMap(cleanAndSplitTerm)
-  ];
-  return [...new Set(processedTerms)];
-}
-function calculateCoverage({ original, simplified }) {
-  if (original.length === 0) {
-    return simplified.length === 0 ? 1 : 0;
-  }
-  const covered = original.filter(
-    (element) => simplified.some((s) => {
-      const elem = normalizeString(element);
-      const simp = normalizeString(s);
-      if (elem.length <= 3) {
-        return elem === simp;
-      }
-      const longer = elem.length > simp.length ? elem : simp;
-      const shorter = elem.length > simp.length ? simp : elem;
-      if (longer.includes(shorter)) {
-        return shorter.length / longer.length > 0.6;
-      }
-      return false;
-    })
-  );
-  return covered.length / original.length;
-}
-function createCompletenessScorer() {
-  return createScorer({
-    name: "Completeness Scorer",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
-    type: "agent"
-  }).preprocess(async ({ run }) => {
-    const isInputInvalid = !run.input || run.input.inputMessages.some((i) => i.content === null || i.content === void 0);
-    const isOutputInvalid = !run.output || run.output.some((i) => i.content === null || i.content === void 0);
-    if (isInputInvalid || isOutputInvalid) {
-      throw new Error("Inputs cannot be null or undefined");
-    }
-    const input = run.input?.inputMessages.map((i) => i.content).join(", ") || "";
-    const output = run.output?.map(({ content }) => content).join(", ") || "";
-    const inputToProcess = input;
-    const outputToProcess = output;
-    const inputDoc = nlp(inputToProcess.trim());
-    const outputDoc = nlp(outputToProcess.trim());
-    const inputElements = extractElements(inputDoc);
-    const outputElements = extractElements(outputDoc);
-    return {
-      inputElements,
-      outputElements,
-      missingElements: inputElements.filter((e) => !outputElements.includes(e)),
-      elementCounts: {
-        input: inputElements.length,
-        output: outputElements.length
-      }
-    };
-  }).generateScore(({ results }) => {
-    const inputElements = results.preprocessStepResult?.inputElements;
-    const outputElements = results.preprocessStepResult?.outputElements;
-    return calculateCoverage({
-      original: inputElements,
-      simplified: outputElements
-    });
-  });
-}
-function createTextualDifferenceScorer() {
-  return createScorer({
-    name: "Textual Difference Scorer",
-    description: "Calculate textual difference between input and output using sequence matching algorithms.",
-    type: "agent"
-  }).preprocess(async ({ run }) => {
-    const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
-    const output = run.output?.map((i) => i.content).join(", ") || "";
-    const ratio = calculateRatio(input, output);
-    const changes = countChanges(input, output);
-    const maxLength = Math.max(input.length, output.length);
-    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
-    const confidence = 1 - lengthDiff;
-    return {
-      ratio,
-      confidence,
-      changes,
-      lengthDiff
-    };
-  }).generateScore(({ results }) => {
-    return results.preprocessStepResult?.ratio;
-  });
-}
-function createKeywordCoverageScorer() {
-  return createScorer({
-    name: "Keyword Coverage Scorer",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
-    type: "agent"
-  }).preprocess(async ({ run }) => {
-    const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
-    const output = run.output?.map((i) => i.content).join(", ") || "";
-    if (!input && !output) {
-      return {
-        result: {
-          referenceKeywords: /* @__PURE__ */ new Set(),
-          responseKeywords: /* @__PURE__ */ new Set()
-        }
-      };
-    }
-    const extractKeywords = (text) => {
-      return keyword_extractor.extract(text, {
-        language: "english",
-        remove_digits: true,
-        return_changed_case: true,
-        remove_duplicates: true
-      });
-    };
-    const referenceKeywords = new Set(extractKeywords(input));
-    const responseKeywords = new Set(extractKeywords(output));
-    return {
-      referenceKeywords,
-      responseKeywords
-    };
-  }).analyze(async ({ results }) => {
-    if (!results.preprocessStepResult?.referenceKeywords?.size && !results.preprocessStepResult?.responseKeywords?.size) {
-      return {
-        totalKeywordsLength: 0,
-        matchedKeywordsLength: 0
-      };
-    }
-    const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(
-      (k) => results.preprocessStepResult?.responseKeywords?.has(k)
-    );
-    return {
-      totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,
-      matchedKeywordsLength: matchedKeywords.length ?? 0
-    };
-  }).generateScore(({ results }) => {
-    if (!results.analyzeStepResult?.totalKeywordsLength) {
-      return 1;
-    }
-    const totalKeywords = results.analyzeStepResult?.totalKeywordsLength;
-    const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength;
-    return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;
-  });
-}
-function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace } = { ignoreCase: true, ignoreWhitespace: true }) {
-  return createScorer({
-    name: "Content Similarity Scorer",
-    description: "Calculates content similarity between input and output messages using string comparison algorithms.",
-    type: "agent"
-  }).preprocess(async ({ run }) => {
-    let processedInput = run.input?.inputMessages.map((i) => i.content).join(", ") || "";
-    let processedOutput = run.output.map((i) => i.content).join(", ") || "";
-    if (ignoreCase) {
-      processedInput = processedInput.toLowerCase();
-      processedOutput = processedOutput.toLowerCase();
-    }
-    if (ignoreWhitespace) {
-      processedInput = processedInput.replace(/\s+/g, " ").trim();
-      processedOutput = processedOutput.replace(/\s+/g, " ").trim();
-    }
-    return {
-      processedInput,
-      processedOutput
-    };
-  }).generateScore(({ results }) => {
-    const similarity = stringSimilarity.compareTwoStrings(
-      results.preprocessStepResult?.processedInput,
-      results.preprocessStepResult?.processedOutput
-    );
-    return similarity;
-  });
-}
-function createToneScorer(config = {}) {
-  const { referenceTone } = config;
-  return createScorer({
-    name: "Tone Scorer",
-    description: "Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.",
-    type: "agent"
-  }).preprocess(async ({ run }) => {
-    const sentiment = new Sentiment();
-    const agentMessage = run.output?.map((i) => i.content).join(", ") || "";
-    const responseSentiment = sentiment.analyze(agentMessage);
-    if (referenceTone) {
-      const referenceSentiment = sentiment.analyze(referenceTone);
-      const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);
-      const normalizedScore = Math.max(0, 1 - sentimentDiff);
-      return {
-        score: normalizedScore,
-        responseSentiment: responseSentiment.comparative,
-        referenceSentiment: referenceSentiment.comparative,
-        difference: sentimentDiff
-      };
-    }
-    const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];
-    const sentiments = sentences.map((s) => sentiment.analyze(s).comparative);
-    const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;
-    const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;
-    const stability = Math.max(0, 1 - variance);
-    return {
-      score: stability,
-      avgSentiment,
-      sentimentVariance: variance
-    };
-  }).generateScore(({ results }) => {
-    return results.preprocessStepResult?.score;
-  });
-}
-function checkToolOrder(actualTools, expectedOrder, strictMode = false) {
-  if (strictMode) {
-    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);
-  }
-  const expectedIndices = [];
-  for (const expectedTool of expectedOrder) {
-    const index = actualTools.indexOf(expectedTool);
-    if (index === -1) {
-      return false;
-    }
-    expectedIndices.push(index);
-  }
-  for (let i = 1; i < expectedIndices.length; i++) {
-    const currentIndex = expectedIndices[i];
-    const prevIndex = expectedIndices[i - 1];
-    if (currentIndex !== void 0 && prevIndex !== void 0 && currentIndex <= prevIndex) {
-      return false;
-    }
-  }
-  return true;
-}
-function calculateAccuracy({
-  expectedTool,
-  actualTools,
-  strictMode = false,
-  expectedToolOrder
-}) {
-  if (actualTools.length === 0) {
-    return 0;
-  }
-  if (expectedToolOrder && expectedToolOrder.length > 0) {
-    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;
-  }
-  if (!expectedTool) {
-    return 0;
-  }
-  if (strictMode) {
-    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;
-  }
-  return actualTools.includes(expectedTool) ? 1 : 0;
-}
-function createToolCallAccuracyScorerCode(options) {
-  const { expectedTool, strictMode = false, expectedToolOrder } = options;
-  if (!expectedTool && !expectedToolOrder) {
-    throw new Error("Either expectedTool or expectedToolOrder must be provided");
-  }
-  const getDescription = () => {
-    return expectedToolOrder ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(", ")}]` : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;
-  };
-  return createScorer({
-    name: "Tool Call Accuracy Scorer",
-    description: getDescription(),
-    type: "agent"
-  }).preprocess(async ({ run }) => {
-    const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;
-    const isOutputInvalid = !run.output || run.output.length === 0;
-    if (isInputInvalid || isOutputInvalid) {
-      throw new Error("Input and output messages cannot be null or empty");
-    }
-    const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);
-    const correctToolCalled = expectedTool ? strictMode ? actualTools.length === 1 && actualTools[0] === expectedTool : actualTools.includes(expectedTool) : false;
-    return {
-      expectedTool,
-      actualTools,
-      strictMode,
-      expectedToolOrder,
-      hasToolCalls: actualTools.length > 0,
-      correctToolCalled,
-      toolCallInfos,
-      correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null
-    };
-  }).generateScore(({ results }) => {
-    const preprocessResult = results.preprocessStepResult;
-    if (!preprocessResult) {
-      return 0;
-    }
-    return calculateAccuracy({
-      expectedTool: preprocessResult.expectedTool,
-      actualTools: preprocessResult.actualTools,
-      strictMode: preprocessResult.strictMode,
-      expectedToolOrder: preprocessResult.expectedToolOrder
-    });
-  });
-}
-
-export { createCompletenessScorer, createContentSimilarityScorer, createKeywordCoverageScorer, createTextualDifferenceScorer, createToneScorer, createToolCallAccuracyScorerCode };
-//# sourceMappingURL=index.js.map
-//# sourceMappingURL=index.js.map

package/dist/scorers/code/index.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer"],"mappings":";;;;;;;;AAGA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAO,YAAA,CAAa;AAAA,IAClB,IAAA,EAAM,qBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAW,GAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAY,GAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;AC1HO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,2FAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAGpF,IAAA,MAAM,KAAA,GAAQ,cAAA,CAAe,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,OAAA,GAAU,YAAA,CAAa,KAAA,EAAO,MAAM,CAAA;AAG1C,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,yBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC5DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,qGAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAa,gBAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,aAAA;AAAA,IACN,WAAA,EACE,oJAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAI,SAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC3CA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,aAAa,cAAA,EAAe;AAAA,IAC5B,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAI,gBAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.js","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer({\n    name: 'Completeness Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { calculateRatio, countChanges } from '../../../ratio';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer({\n    name: 'Textual Difference Scorer',\n    description: 'Calculate textual difference between input and output using sequence matching algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      // Calculate similarity ratio using LCS approach (similar to SequenceMatcher.ratio())\n      const ratio = calculateRatio(input, output);\n\n      // Count changes by comparing words\n      const changes = countChanges(input, output);\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer({\n    name: 'Keyword Coverage Scorer',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer({\n    name: 'Content Similarity Scorer',\n    description: 'Calculates content similarity between input and output messages using string comparison algorithms.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer({\n    name: 'Tone Scorer',\n    description:\n      'Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.',\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\ninterface ToolCallAccuracyOptions {\n  expectedTool?: string;\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n  if (strictMode) {\n    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n  }\n\n  const expectedIndices: number[] = [];\n  for (const expectedTool of expectedOrder) {\n    const index = actualTools.indexOf(expectedTool);\n    if (index === -1) {\n      return false;\n    }\n    expectedIndices.push(index);\n  }\n\n  for (let i = 1; i < expectedIndices.length; i++) {\n    const currentIndex = expectedIndices[i];\n    const prevIndex = expectedIndices[i - 1];\n    if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction calculateAccuracy({\n  expectedTool,\n  actualTools,\n  strictMode = false,\n  expectedToolOrder,\n}: {\n  expectedTool?: string;\n  actualTools: string[];\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}): number {\n  if (actualTools.length === 0) {\n    return 0;\n  }\n\n  if (expectedToolOrder && expectedToolOrder.length > 0) {\n    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n  }\n\n  if (!expectedTool) {\n    return 0;\n  }\n\n  if (strictMode) {\n    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n  }\n\n  return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n  const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n  if (!expectedTool && !expectedToolOrder) {\n    throw new Error('Either expectedTool or expectedToolOrder must be provided');\n  }\n\n  const getDescription = () => {\n    return expectedToolOrder\n      ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n      : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n  };\n\n  return createScorer({\n    name: 'Tool Call Accuracy Scorer',\n    description: getDescription(),\n    type: 'agent',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;\n      const isOutputInvalid = !run.output || run.output.length === 0;\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Input and output messages cannot be null or empty');\n      }\n\n      const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n      const correctToolCalled = expectedTool\n        ? strictMode\n          ? actualTools.length === 1 && actualTools[0] === expectedTool\n          : actualTools.includes(expectedTool)\n        : false;\n\n      return {\n        expectedTool,\n        actualTools,\n        strictMode,\n        expectedToolOrder,\n        hasToolCalls: actualTools.length > 0,\n        correctToolCalled,\n        toolCallInfos,\n        correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n      };\n    })\n    .generateScore(({ results }) => {\n      const preprocessResult = results.preprocessStepResult;\n      if (!preprocessResult) {\n        return 0;\n      }\n\n      return calculateAccuracy({\n        expectedTool: preprocessResult.expectedTool,\n        actualTools: preprocessResult.actualTools,\n        strictMode: preprocessResult.strictMode,\n        expectedToolOrder: preprocessResult.expectedToolOrder,\n      });\n    });\n}\n"]}