npm - @mastra/evals - Versions diffs - 0.13.8-alpha.1 → 0.13.8-alpha.2 - Mend

@mastra/evals 0.13.8-alpha.1 → 0.13.8-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (41) hide show

package/CHANGELOG.md CHANGED Viewed

@@ -1,5 +1,11 @@
 # @mastra/evals
+## 0.13.8-alpha.2
+### Patch Changes
+- Add explicit type 'agent' for built in scorers ([#8132](https://github.com/mastra-ai/mastra/pull/8132))
 ## 0.13.8-alpha.1
 ### Patch Changes

package/dist/scorers/code/completeness/index.d.ts CHANGED Viewed

@@ -1,5 +1,4 @@
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
-export declare function createCompletenessScorer(): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+export declare function createCompletenessScorer(): import("@mastra/core/scores").MastraScorer<"Completeness Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
     inputElements: string[];
     outputElements: string[];
     missingElements: string[];

package/dist/scorers/code/completeness/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/completeness/index.ts"],"names":[],"mappings":"~~AACA~~,~~OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AA0E3F,~~wBAAgB,wBAAwB;;;;;;;;~~gDAkDvC~~"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/completeness/index.ts"],"names":[],"mappings":"AA0EA,wBAAgB,wBAAwB;;;;;;;;gDAmDvC"}

package/dist/scorers/code/content-similarity/index.d.ts CHANGED Viewed

@@ -1,9 +1,8 @@
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
 interface ContentSimilarityOptions {
     ignoreCase?: boolean;
     ignoreWhitespace?: boolean;
 }
-export declare function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace }?: ContentSimilarityOptions): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+export declare function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace }?: ContentSimilarityOptions): import("@mastra/core/scores").MastraScorer<"Content Similarity Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
     processedInput: string;
     processedOutput: string;
 }> & Record<"generateScoreStepResult", number>>;

package/dist/scorers/code/content-similarity/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/content-similarity/index.ts"],"names":[],"mappings":"~~AACA~~,~~OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAG3F,~~UAAU,wBAAwB;IAChC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,wBAAgB,6BAA6B,CAC3C,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAE,wBAAuE;;;gDAkC1G"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/content-similarity/index.ts"],"names":[],"mappings":"AAGA,UAAU,wBAAwB;IAChC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,wBAAgB,6BAA6B,CAC3C,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAE,wBAAuE;;;gDAkC1G"}

package/dist/scorers/code/index.cjs CHANGED Viewed

@@ -58,8 +58,9 @@ function calculateCoverage({ original, simplified }) {
 }
 function createCompletenessScorer() {
   return scores.createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Completeness Scorer",
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const isInputInvalid = !run.input || run.input.inputMessages.some((i) => i.content === null || i.content === void 0);
     const isOutputInvalid = !run.output || run.output.some((i) => i.content === null || i.content === void 0);
@@ -94,8 +95,9 @@ function createCompletenessScorer() {
 }
 function createTextualDifferenceScorer() {
   return scores.createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Textual Difference Scorer",
+    description: "Calculate textual difference between input and output using sequence matching algorithms.",
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
     const output = run.output?.map((i) => i.content).join(", ") || "";
@@ -118,8 +120,9 @@ function createTextualDifferenceScorer() {
 }
 function createKeywordCoverageScorer() {
   return scores.createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Keyword Coverage Scorer",
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
     const output = run.output?.map((i) => i.content).join(", ") || "";
@@ -170,8 +173,9 @@ function createKeywordCoverageScorer() {
 }
 function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace } = { ignoreCase: true, ignoreWhitespace: true }) {
   return scores.createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Content Similarity Scorer",
+    description: "Calculates content similarity between input and output messages using string comparison algorithms.",
+    type: "agent"
   }).preprocess(async ({ run }) => {
     let processedInput = run.input?.inputMessages.map((i) => i.content).join(", ") || "";
     let processedOutput = run.output.map((i) => i.content).join(", ") || "";
@@ -198,8 +202,9 @@ function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace } = { igno
 function createToneScorer(config = {}) {
   const { referenceTone } = config;
   return scores.createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Tone Scorer",
+    description: "Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.",
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const sentiment = new Sentiment__default.default();
     const agentMessage = run.output?.map((i) => i.content).join(", ") || "";
@@ -279,8 +284,9 @@ function createToolCallAccuracyScorerCode(options) {
     return expectedToolOrder ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(", ")}]` : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;
   };
   return scores.createScorer({
-    name: "Tool Call Accuracy",
-    description: getDescription()
+    name: "Tool Call Accuracy Scorer",
+    description: getDescription(),
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;
     const isOutputInvalid = !run.output || run.output.length === 0;

package/dist/scorers/code/index.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer","nlp","SequenceMatcher","keyword_extractor","stringSimilarity","Sentiment","extractToolCalls"],"mappings":";;;;;;;;;;;;;;;;;AAIA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAOA,mBAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAWC,oBAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAYA,oBAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;ACzHO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOD,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAIE,uBAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOF,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOG,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC3DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOH,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAaI,iCAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOJ,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAIK,0BAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;ACzCA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOL,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,oBAAA;AAAA,IACN,aAAa,cAAA;AAAe,GAC7B,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAIM,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.cjs","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n // Remove diacritics and convert to lowercase\n return str\n .normalize('NFD')\n .replace(/[\\u0300-\\u036f]/g, '')\n .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n // Get more specific elements and ensure they're arrays\n const nouns = doc.nouns().out('array') \|\| [];\n const verbs = doc.verbs().toInfinitive().out('array') \|\| [];\n const topics = doc.topics().out('array') \|\| [];\n const terms = doc.terms().out('array') \|\| [];\n\n // Helper function to clean and split terms\n const cleanAndSplitTerm = (term: string): string[] => {\n // First normalize the string\n const normalized = normalizeString(term);\n\n // Split on word boundaries and filter out empty strings\n return normalized\n .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n .trim()\n .split(/\\s+/)\n .filter(word => word.length > 0);\n };\n\n // Process all elements\n const processedTerms = [\n ...nouns.flatMap(cleanAndSplitTerm),\n ...verbs.flatMap(cleanAndSplitTerm),\n ...topics.flatMap(cleanAndSplitTerm),\n ...terms.flatMap(cleanAndSplitTerm),\n ];\n\n // Remove duplicates\n return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n if (original.length === 0) {\n return simplified.length === 0 ? 1 : 0;\n }\n\n // Exact matching for short words (3 chars or less), substring matching for longer words\n const covered = original.filter(element =>\n simplified.some(s => {\n const elem = normalizeString(element);\n const simp = normalizeString(s);\n\n // For short words (3 chars or less), require exact match\n if (elem.length <= 3) {\n return elem === simp;\n }\n\n // For longer words, require substantial overlap (more than 60% of the longer word)\n const longer = elem.length > simp.length ? elem : simp;\n const shorter = elem.length > simp.length ? simp : elem;\n\n if (longer.includes(shorter)) {\n return shorter.length / longer.length > 0.6;\n }\n\n return false;\n }),\n );\n return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n const isInputInvalid =\n !run.input \|\|\n run.input.inputMessages.some((i: { content: string }) => i.content === null \|\| i.content === undefined);\n\n const isOutputInvalid =\n !run.output \|\| run.output.some((i: { content: string }) => i.content === null \|\| i.content === undefined);\n\n if (isInputInvalid \|\| isOutputInvalid) {\n throw new Error('Inputs cannot be null or undefined');\n }\n\n const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map(({ content }: { content: string }) => content).join(', ') \|\| '';\n\n const inputToProcess = input;\n const outputToProcess = output;\n\n const inputDoc = nlp(inputToProcess.trim());\n const outputDoc = nlp(outputToProcess.trim());\n\n // Extract and log elements\n const inputElements = extractElements(inputDoc);\n const outputElements = extractElements(outputDoc);\n\n return {\n inputElements,\n outputElements,\n missingElements: inputElements.filter(e => !outputElements.includes(e)),\n elementCounts: {\n input: inputElements.length,\n output: outputElements.length,\n },\n };\n })\n .generateScore(({ results }) => {\n const inputElements = results.preprocessStepResult?.inputElements;\n const outputElements = results.preprocessStepResult?.outputElements;\n\n return calculateCoverage({\n original: inputElements,\n simplified: outputElements,\n });\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const matcher = new SequenceMatcher(null, input, output);\n const ratio = matcher.ratio();\n\n // Get detailed operations\n const ops = matcher.getOpcodes();\n const changes = ops.filter(([op]) => op !== 'equal').length;\n\n // Calculate confidence based on text length difference\n const maxLength = Math.max(input.length, output.length);\n const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n const confidence = 1 - lengthDiff;\n\n return {\n ratio,\n confidence,\n changes,\n lengthDiff,\n };\n })\n .generateScore(({ results }) => {\n return results.preprocessStepResult?.ratio;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n\n if (!input && !output) {\n return {\n result: {\n referenceKeywords: new Set<string>(),\n responseKeywords: new Set<string>(),\n },\n };\n }\n\n const extractKeywords = (text: string) => {\n return keyword_extractor.extract(text, {\n language: 'english',\n remove_digits: true,\n return_changed_case: true,\n remove_duplicates: true,\n });\n };\n\n const referenceKeywords = new Set(extractKeywords(input));\n const responseKeywords = new Set(extractKeywords(output));\n return {\n referenceKeywords,\n responseKeywords,\n };\n })\n .analyze(async ({ results }) => {\n if (\n !results.preprocessStepResult?.referenceKeywords?.size &&\n !results.preprocessStepResult?.responseKeywords?.size\n ) {\n return {\n totalKeywordsLength: 0,\n matchedKeywordsLength: 0,\n };\n }\n\n const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n results.preprocessStepResult?.responseKeywords?.has(k),\n );\n\n return {\n totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n matchedKeywordsLength: matchedKeywords.length ?? 0,\n };\n })\n .generateScore(({ results }) => {\n if (!results.analyzeStepResult?.totalKeywordsLength) {\n return 1;\n }\n\n const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n ignoreCase?: boolean;\n ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') \|\| '';\n let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') \|\| '';\n\n if (ignoreCase) {\n processedInput = processedInput.toLowerCase();\n processedOutput = processedOutput.toLowerCase();\n }\n\n if (ignoreWhitespace) {\n processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n }\n\n return {\n processedInput,\n processedOutput,\n };\n })\n .generateScore(({ results }) => {\n const similarity = stringSimilarity.compareTwoStrings(\n results.preprocessStepResult?.processedInput,\n results.preprocessStepResult?.processedOutput,\n );\n\n return similarity;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n const { referenceTone } = config;\n\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n const sentiment = new Sentiment();\n const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const responseSentiment = sentiment.analyze(agentMessage);\n\n if (referenceTone) {\n // Compare sentiment with reference\n const referenceSentiment = sentiment.analyze(referenceTone);\n const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n return {\n score: normalizedScore,\n responseSentiment: responseSentiment.comparative,\n referenceSentiment: referenceSentiment.comparative,\n difference: sentimentDiff,\n };\n }\n\n // Evaluate sentiment stability across response\n const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) \|\| [agentMessage];\n const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n const stability = Math.max(0, 1 - variance);\n\n return {\n score: stability,\n avgSentiment,\n sentimentVariance: variance,\n };\n })\n .generateScore(({ results }) => {\n return results.preprocessStepResult?.score;\n });\n}\n","import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\n\ninterface ToolCallAccuracyOptions {\n expectedTool?: string;\n strictMode?: boolean;\n expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n if (strictMode) {\n return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n }\n\n const expectedIndices: number[] = [];\n for (const expectedTool of expectedOrder) {\n const index = actualTools.indexOf(expectedTool);\n if (index === -1) {\n return false;\n }\n expectedIndices.push(index);\n }\n\n for (let i = 1; i < expectedIndices.length; i++) {\n const currentIndex = expectedIndices[i];\n const prevIndex = expectedIndices[i - 1];\n if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n return false;\n }\n }\n\n return true;\n}\n\nfunction calculateAccuracy({\n expectedTool,\n actualTools,\n strictMode = false,\n expectedToolOrder,\n}: {\n expectedTool?: string;\n actualTools: string[];\n strictMode?: boolean;\n expectedToolOrder?: string[];\n}): number {\n if (actualTools.length === 0) {\n return 0;\n }\n\n if (expectedToolOrder && expectedToolOrder.length > 0) {\n return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n }\n\n if (!expectedTool) {\n return 0;\n }\n\n if (strictMode) {\n return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n }\n\n return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n if (!expectedTool && !expectedToolOrder) {\n throw new Error('Either expectedTool or expectedToolOrder must be provided');\n }\n\n const getDescription = () => {\n return expectedToolOrder\n ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n };\n\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Tool Call Accuracy',\n description: getDescription(),\n })\n .preprocess(async ({ run }) => {\n const isInputInvalid = !run.input \|\| !run.input.inputMessages \|\| run.input.inputMessages.length === 0;\n const isOutputInvalid = !run.output \|\| run.output.length === 0;\n\n if (isInputInvalid \|\| isOutputInvalid) {\n throw new Error('Input and output messages cannot be null or empty');\n }\n\n const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n const correctToolCalled = expectedTool\n ? strictMode\n ? actualTools.length === 1 && actualTools[0] === expectedTool\n : actualTools.includes(expectedTool)\n : false;\n\n return {\n expectedTool,\n actualTools,\n strictMode,\n expectedToolOrder,\n hasToolCalls: actualTools.length > 0,\n correctToolCalled,\n toolCallInfos,\n correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n };\n })\n .generateScore(({ results }) => {\n const preprocessResult = results.preprocessStepResult;\n if (!preprocessResult) {\n return 0;\n }\n\n return calculateAccuracy({\n expectedTool: preprocessResult.expectedTool,\n actualTools: preprocessResult.actualTools,\n strictMode: preprocessResult.strictMode,\n expectedToolOrder: preprocessResult.expectedToolOrder,\n });\n });\n}\n"]}
1	+ {"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer","nlp","SequenceMatcher","keyword_extractor","stringSimilarity","Sentiment","extractToolCalls"],"mappings":";;;;;;;;;;;;;;;;;AAGA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAOA,mBAAA,CAAa;AAAA,IAClB,IAAA,EAAM,qBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAWC,oBAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAYA,oBAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;AC1HO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOD,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,2FAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAIE,uBAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOF,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,yBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOG,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC5DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOH,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,qGAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAaI,iCAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOJ,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,aAAA;AAAA,IACN,WAAA,EACE,oJAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAIK,0BAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC3CA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOL,mBAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,aAAa,cAAA,EAAe;AAAA,IAC5B,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAIM,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.cjs","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n // Remove diacritics and convert to lowercase\n return str\n .normalize('NFD')\n .replace(/[\\u0300-\\u036f]/g, '')\n .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n // Get more specific elements and ensure they're arrays\n const nouns = doc.nouns().out('array') \|\| [];\n const verbs = doc.verbs().toInfinitive().out('array') \|\| [];\n const topics = doc.topics().out('array') \|\| [];\n const terms = doc.terms().out('array') \|\| [];\n\n // Helper function to clean and split terms\n const cleanAndSplitTerm = (term: string): string[] => {\n // First normalize the string\n const normalized = normalizeString(term);\n\n // Split on word boundaries and filter out empty strings\n return normalized\n .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n .trim()\n .split(/\\s+/)\n .filter(word => word.length > 0);\n };\n\n // Process all elements\n const processedTerms = [\n ...nouns.flatMap(cleanAndSplitTerm),\n ...verbs.flatMap(cleanAndSplitTerm),\n ...topics.flatMap(cleanAndSplitTerm),\n ...terms.flatMap(cleanAndSplitTerm),\n ];\n\n // Remove duplicates\n return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n if (original.length === 0) {\n return simplified.length === 0 ? 1 : 0;\n }\n\n // Exact matching for short words (3 chars or less), substring matching for longer words\n const covered = original.filter(element =>\n simplified.some(s => {\n const elem = normalizeString(element);\n const simp = normalizeString(s);\n\n // For short words (3 chars or less), require exact match\n if (elem.length <= 3) {\n return elem === simp;\n }\n\n // For longer words, require substantial overlap (more than 60% of the longer word)\n const longer = elem.length > simp.length ? elem : simp;\n const shorter = elem.length > simp.length ? simp : elem;\n\n if (longer.includes(shorter)) {\n return shorter.length / longer.length > 0.6;\n }\n\n return false;\n }),\n );\n return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n return createScorer({\n name: 'Completeness Scorer',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const isInputInvalid =\n !run.input \|\|\n run.input.inputMessages.some((i: { content: string }) => i.content === null \|\| i.content === undefined);\n\n const isOutputInvalid =\n !run.output \|\| run.output.some((i: { content: string }) => i.content === null \|\| i.content === undefined);\n\n if (isInputInvalid \|\| isOutputInvalid) {\n throw new Error('Inputs cannot be null or undefined');\n }\n\n const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map(({ content }: { content: string }) => content).join(', ') \|\| '';\n\n const inputToProcess = input;\n const outputToProcess = output;\n\n const inputDoc = nlp(inputToProcess.trim());\n const outputDoc = nlp(outputToProcess.trim());\n\n // Extract and log elements\n const inputElements = extractElements(inputDoc);\n const outputElements = extractElements(outputDoc);\n\n return {\n inputElements,\n outputElements,\n missingElements: inputElements.filter(e => !outputElements.includes(e)),\n elementCounts: {\n input: inputElements.length,\n output: outputElements.length,\n },\n };\n })\n .generateScore(({ results }) => {\n const inputElements = results.preprocessStepResult?.inputElements;\n const outputElements = results.preprocessStepResult?.outputElements;\n\n return calculateCoverage({\n original: inputElements,\n simplified: outputElements,\n });\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n return createScorer({\n name: 'Textual Difference Scorer',\n description: 'Calculate textual difference between input and output using sequence matching algorithms.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const matcher = new SequenceMatcher(null, input, output);\n const ratio = matcher.ratio();\n\n // Get detailed operations\n const ops = matcher.getOpcodes();\n const changes = ops.filter(([op]) => op !== 'equal').length;\n\n // Calculate confidence based on text length difference\n const maxLength = Math.max(input.length, output.length);\n const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n const confidence = 1 - lengthDiff;\n\n return {\n ratio,\n confidence,\n changes,\n lengthDiff,\n };\n })\n .generateScore(({ results }) => {\n return results.preprocessStepResult?.ratio;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n return createScorer({\n name: 'Keyword Coverage Scorer',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n\n if (!input && !output) {\n return {\n result: {\n referenceKeywords: new Set<string>(),\n responseKeywords: new Set<string>(),\n },\n };\n }\n\n const extractKeywords = (text: string) => {\n return keyword_extractor.extract(text, {\n language: 'english',\n remove_digits: true,\n return_changed_case: true,\n remove_duplicates: true,\n });\n };\n\n const referenceKeywords = new Set(extractKeywords(input));\n const responseKeywords = new Set(extractKeywords(output));\n return {\n referenceKeywords,\n responseKeywords,\n };\n })\n .analyze(async ({ results }) => {\n if (\n !results.preprocessStepResult?.referenceKeywords?.size &&\n !results.preprocessStepResult?.responseKeywords?.size\n ) {\n return {\n totalKeywordsLength: 0,\n matchedKeywordsLength: 0,\n };\n }\n\n const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n results.preprocessStepResult?.responseKeywords?.has(k),\n );\n\n return {\n totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n matchedKeywordsLength: matchedKeywords.length ?? 0,\n };\n })\n .generateScore(({ results }) => {\n if (!results.analyzeStepResult?.totalKeywordsLength) {\n return 1;\n }\n\n const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n ignoreCase?: boolean;\n ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n return createScorer({\n name: 'Content Similarity Scorer',\n description: 'Calculates content similarity between input and output messages using string comparison algorithms.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') \|\| '';\n let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') \|\| '';\n\n if (ignoreCase) {\n processedInput = processedInput.toLowerCase();\n processedOutput = processedOutput.toLowerCase();\n }\n\n if (ignoreWhitespace) {\n processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n }\n\n return {\n processedInput,\n processedOutput,\n };\n })\n .generateScore(({ results }) => {\n const similarity = stringSimilarity.compareTwoStrings(\n results.preprocessStepResult?.processedInput,\n results.preprocessStepResult?.processedOutput,\n );\n\n return similarity;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n const { referenceTone } = config;\n\n return createScorer({\n name: 'Tone Scorer',\n description:\n 'Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const sentiment = new Sentiment();\n const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const responseSentiment = sentiment.analyze(agentMessage);\n\n if (referenceTone) {\n // Compare sentiment with reference\n const referenceSentiment = sentiment.analyze(referenceTone);\n const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n return {\n score: normalizedScore,\n responseSentiment: responseSentiment.comparative,\n referenceSentiment: referenceSentiment.comparative,\n difference: sentimentDiff,\n };\n }\n\n // Evaluate sentiment stability across response\n const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) \|\| [agentMessage];\n const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n const stability = Math.max(0, 1 - variance);\n\n return {\n score: stability,\n avgSentiment,\n sentimentVariance: variance,\n };\n })\n .generateScore(({ results }) => {\n return results.preprocessStepResult?.score;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\ninterface ToolCallAccuracyOptions {\n expectedTool?: string;\n strictMode?: boolean;\n expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n if (strictMode) {\n return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n }\n\n const expectedIndices: number[] = [];\n for (const expectedTool of expectedOrder) {\n const index = actualTools.indexOf(expectedTool);\n if (index === -1) {\n return false;\n }\n expectedIndices.push(index);\n }\n\n for (let i = 1; i < expectedIndices.length; i++) {\n const currentIndex = expectedIndices[i];\n const prevIndex = expectedIndices[i - 1];\n if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n return false;\n }\n }\n\n return true;\n}\n\nfunction calculateAccuracy({\n expectedTool,\n actualTools,\n strictMode = false,\n expectedToolOrder,\n}: {\n expectedTool?: string;\n actualTools: string[];\n strictMode?: boolean;\n expectedToolOrder?: string[];\n}): number {\n if (actualTools.length === 0) {\n return 0;\n }\n\n if (expectedToolOrder && expectedToolOrder.length > 0) {\n return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n }\n\n if (!expectedTool) {\n return 0;\n }\n\n if (strictMode) {\n return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n }\n\n return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n if (!expectedTool && !expectedToolOrder) {\n throw new Error('Either expectedTool or expectedToolOrder must be provided');\n }\n\n const getDescription = () => {\n return expectedToolOrder\n ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n };\n\n return createScorer({\n name: 'Tool Call Accuracy Scorer',\n description: getDescription(),\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const isInputInvalid = !run.input \|\| !run.input.inputMessages \|\| run.input.inputMessages.length === 0;\n const isOutputInvalid = !run.output \|\| run.output.length === 0;\n\n if (isInputInvalid \|\| isOutputInvalid) {\n throw new Error('Input and output messages cannot be null or empty');\n }\n\n const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n const correctToolCalled = expectedTool\n ? strictMode\n ? actualTools.length === 1 && actualTools[0] === expectedTool\n : actualTools.includes(expectedTool)\n : false;\n\n return {\n expectedTool,\n actualTools,\n strictMode,\n expectedToolOrder,\n hasToolCalls: actualTools.length > 0,\n correctToolCalled,\n toolCallInfos,\n correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n };\n })\n .generateScore(({ results }) => {\n const preprocessResult = results.preprocessStepResult;\n if (!preprocessResult) {\n return 0;\n }\n\n return calculateAccuracy({\n expectedTool: preprocessResult.expectedTool,\n actualTools: preprocessResult.actualTools,\n strictMode: preprocessResult.strictMode,\n expectedToolOrder: preprocessResult.expectedToolOrder,\n });\n });\n}\n"]}

package/dist/scorers/code/index.js CHANGED Viewed

@@ -49,8 +49,9 @@ function calculateCoverage({ original, simplified }) {
 }
 function createCompletenessScorer() {
   return createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Completeness Scorer",
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const isInputInvalid = !run.input || run.input.inputMessages.some((i) => i.content === null || i.content === void 0);
     const isOutputInvalid = !run.output || run.output.some((i) => i.content === null || i.content === void 0);
@@ -85,8 +86,9 @@ function createCompletenessScorer() {
 }
 function createTextualDifferenceScorer() {
   return createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Textual Difference Scorer",
+    description: "Calculate textual difference between input and output using sequence matching algorithms.",
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
     const output = run.output?.map((i) => i.content).join(", ") || "";
@@ -109,8 +111,9 @@ function createTextualDifferenceScorer() {
 }
 function createKeywordCoverageScorer() {
   return createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Keyword Coverage Scorer",
+    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.',
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const input = run.input?.inputMessages?.map((i) => i.content).join(", ") || "";
     const output = run.output?.map((i) => i.content).join(", ") || "";
@@ -161,8 +164,9 @@ function createKeywordCoverageScorer() {
 }
 function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace } = { ignoreCase: true, ignoreWhitespace: true }) {
   return createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Content Similarity Scorer",
+    description: "Calculates content similarity between input and output messages using string comparison algorithms.",
+    type: "agent"
   }).preprocess(async ({ run }) => {
     let processedInput = run.input?.inputMessages.map((i) => i.content).join(", ") || "";
     let processedOutput = run.output.map((i) => i.content).join(", ") || "";
@@ -189,8 +193,9 @@ function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace } = { igno
 function createToneScorer(config = {}) {
   const { referenceTone } = config;
   return createScorer({
-    name: "Completeness",
-    description: 'Leverage the nlp method from "compromise" to extract elements from the input and output and calculate the coverage.'
+    name: "Tone Scorer",
+    description: "Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.",
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const sentiment = new Sentiment();
     const agentMessage = run.output?.map((i) => i.content).join(", ") || "";
@@ -270,8 +275,9 @@ function createToolCallAccuracyScorerCode(options) {
     return expectedToolOrder ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(", ")}]` : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;
   };
   return createScorer({
-    name: "Tool Call Accuracy",
-    description: getDescription()
+    name: "Tool Call Accuracy Scorer",
+    description: getDescription(),
+    type: "agent"
   }).preprocess(async ({ run }) => {
     const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;
     const isOutputInvalid = !run.output || run.output.length === 0;

package/dist/scorers/code/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer"],"mappings":";;;;;;;;AAIA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAO,YAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAW,GAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAY,GAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;ACzHO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAI,eAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC3DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAa,gBAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAI,SAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;ACzCA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,oBAAA;AAAA,IACN,aAAa,cAAA;AAAe,GAC7B,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAI,gBAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.js","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n // Remove diacritics and convert to lowercase\n return str\n .normalize('NFD')\n .replace(/[\\u0300-\\u036f]/g, '')\n .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n // Get more specific elements and ensure they're arrays\n const nouns = doc.nouns().out('array') \|\| [];\n const verbs = doc.verbs().toInfinitive().out('array') \|\| [];\n const topics = doc.topics().out('array') \|\| [];\n const terms = doc.terms().out('array') \|\| [];\n\n // Helper function to clean and split terms\n const cleanAndSplitTerm = (term: string): string[] => {\n // First normalize the string\n const normalized = normalizeString(term);\n\n // Split on word boundaries and filter out empty strings\n return normalized\n .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n .trim()\n .split(/\\s+/)\n .filter(word => word.length > 0);\n };\n\n // Process all elements\n const processedTerms = [\n ...nouns.flatMap(cleanAndSplitTerm),\n ...verbs.flatMap(cleanAndSplitTerm),\n ...topics.flatMap(cleanAndSplitTerm),\n ...terms.flatMap(cleanAndSplitTerm),\n ];\n\n // Remove duplicates\n return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n if (original.length === 0) {\n return simplified.length === 0 ? 1 : 0;\n }\n\n // Exact matching for short words (3 chars or less), substring matching for longer words\n const covered = original.filter(element =>\n simplified.some(s => {\n const elem = normalizeString(element);\n const simp = normalizeString(s);\n\n // For short words (3 chars or less), require exact match\n if (elem.length <= 3) {\n return elem === simp;\n }\n\n // For longer words, require substantial overlap (more than 60% of the longer word)\n const longer = elem.length > simp.length ? elem : simp;\n const shorter = elem.length > simp.length ? simp : elem;\n\n if (longer.includes(shorter)) {\n return shorter.length / longer.length > 0.6;\n }\n\n return false;\n }),\n );\n return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n const isInputInvalid =\n !run.input \|\|\n run.input.inputMessages.some((i: { content: string }) => i.content === null \|\| i.content === undefined);\n\n const isOutputInvalid =\n !run.output \|\| run.output.some((i: { content: string }) => i.content === null \|\| i.content === undefined);\n\n if (isInputInvalid \|\| isOutputInvalid) {\n throw new Error('Inputs cannot be null or undefined');\n }\n\n const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map(({ content }: { content: string }) => content).join(', ') \|\| '';\n\n const inputToProcess = input;\n const outputToProcess = output;\n\n const inputDoc = nlp(inputToProcess.trim());\n const outputDoc = nlp(outputToProcess.trim());\n\n // Extract and log elements\n const inputElements = extractElements(inputDoc);\n const outputElements = extractElements(outputDoc);\n\n return {\n inputElements,\n outputElements,\n missingElements: inputElements.filter(e => !outputElements.includes(e)),\n elementCounts: {\n input: inputElements.length,\n output: outputElements.length,\n },\n };\n })\n .generateScore(({ results }) => {\n const inputElements = results.preprocessStepResult?.inputElements;\n const outputElements = results.preprocessStepResult?.outputElements;\n\n return calculateCoverage({\n original: inputElements,\n simplified: outputElements,\n });\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const matcher = new SequenceMatcher(null, input, output);\n const ratio = matcher.ratio();\n\n // Get detailed operations\n const ops = matcher.getOpcodes();\n const changes = ops.filter(([op]) => op !== 'equal').length;\n\n // Calculate confidence based on text length difference\n const maxLength = Math.max(input.length, output.length);\n const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n const confidence = 1 - lengthDiff;\n\n return {\n ratio,\n confidence,\n changes,\n lengthDiff,\n };\n })\n .generateScore(({ results }) => {\n return results.preprocessStepResult?.ratio;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n\n if (!input && !output) {\n return {\n result: {\n referenceKeywords: new Set<string>(),\n responseKeywords: new Set<string>(),\n },\n };\n }\n\n const extractKeywords = (text: string) => {\n return keyword_extractor.extract(text, {\n language: 'english',\n remove_digits: true,\n return_changed_case: true,\n remove_duplicates: true,\n });\n };\n\n const referenceKeywords = new Set(extractKeywords(input));\n const responseKeywords = new Set(extractKeywords(output));\n return {\n referenceKeywords,\n responseKeywords,\n };\n })\n .analyze(async ({ results }) => {\n if (\n !results.preprocessStepResult?.referenceKeywords?.size &&\n !results.preprocessStepResult?.responseKeywords?.size\n ) {\n return {\n totalKeywordsLength: 0,\n matchedKeywordsLength: 0,\n };\n }\n\n const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n results.preprocessStepResult?.responseKeywords?.has(k),\n );\n\n return {\n totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n matchedKeywordsLength: matchedKeywords.length ?? 0,\n };\n })\n .generateScore(({ results }) => {\n if (!results.analyzeStepResult?.totalKeywordsLength) {\n return 1;\n }\n\n const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n ignoreCase?: boolean;\n ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') \|\| '';\n let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') \|\| '';\n\n if (ignoreCase) {\n processedInput = processedInput.toLowerCase();\n processedOutput = processedOutput.toLowerCase();\n }\n\n if (ignoreWhitespace) {\n processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n }\n\n return {\n processedInput,\n processedOutput,\n };\n })\n .generateScore(({ results }) => {\n const similarity = stringSimilarity.compareTwoStrings(\n results.preprocessStepResult?.processedInput,\n results.preprocessStepResult?.processedOutput,\n );\n\n return similarity;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n const { referenceTone } = config;\n\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Completeness',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n })\n .preprocess(async ({ run }) => {\n const sentiment = new Sentiment();\n const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const responseSentiment = sentiment.analyze(agentMessage);\n\n if (referenceTone) {\n // Compare sentiment with reference\n const referenceSentiment = sentiment.analyze(referenceTone);\n const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n return {\n score: normalizedScore,\n responseSentiment: responseSentiment.comparative,\n referenceSentiment: referenceSentiment.comparative,\n difference: sentimentDiff,\n };\n }\n\n // Evaluate sentiment stability across response\n const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) \|\| [agentMessage];\n const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n const stability = Math.max(0, 1 - variance);\n\n return {\n score: stability,\n avgSentiment,\n sentimentVariance: variance,\n };\n })\n .generateScore(({ results }) => {\n return results.preprocessStepResult?.score;\n });\n}\n","import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\n\ninterface ToolCallAccuracyOptions {\n expectedTool?: string;\n strictMode?: boolean;\n expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n if (strictMode) {\n return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n }\n\n const expectedIndices: number[] = [];\n for (const expectedTool of expectedOrder) {\n const index = actualTools.indexOf(expectedTool);\n if (index === -1) {\n return false;\n }\n expectedIndices.push(index);\n }\n\n for (let i = 1; i < expectedIndices.length; i++) {\n const currentIndex = expectedIndices[i];\n const prevIndex = expectedIndices[i - 1];\n if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n return false;\n }\n }\n\n return true;\n}\n\nfunction calculateAccuracy({\n expectedTool,\n actualTools,\n strictMode = false,\n expectedToolOrder,\n}: {\n expectedTool?: string;\n actualTools: string[];\n strictMode?: boolean;\n expectedToolOrder?: string[];\n}): number {\n if (actualTools.length === 0) {\n return 0;\n }\n\n if (expectedToolOrder && expectedToolOrder.length > 0) {\n return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n }\n\n if (!expectedTool) {\n return 0;\n }\n\n if (strictMode) {\n return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n }\n\n return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n if (!expectedTool && !expectedToolOrder) {\n throw new Error('Either expectedTool or expectedToolOrder must be provided');\n }\n\n const getDescription = () => {\n return expectedToolOrder\n ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n };\n\n return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n name: 'Tool Call Accuracy',\n description: getDescription(),\n })\n .preprocess(async ({ run }) => {\n const isInputInvalid = !run.input \|\| !run.input.inputMessages \|\| run.input.inputMessages.length === 0;\n const isOutputInvalid = !run.output \|\| run.output.length === 0;\n\n if (isInputInvalid \|\| isOutputInvalid) {\n throw new Error('Input and output messages cannot be null or empty');\n }\n\n const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n const correctToolCalled = expectedTool\n ? strictMode\n ? actualTools.length === 1 && actualTools[0] === expectedTool\n : actualTools.includes(expectedTool)\n : false;\n\n return {\n expectedTool,\n actualTools,\n strictMode,\n expectedToolOrder,\n hasToolCalls: actualTools.length > 0,\n correctToolCalled,\n toolCallInfos,\n correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n };\n })\n .generateScore(({ results }) => {\n const preprocessResult = results.preprocessStepResult;\n if (!preprocessResult) {\n return 0;\n }\n\n return calculateAccuracy({\n expectedTool: preprocessResult.expectedTool,\n actualTools: preprocessResult.actualTools,\n strictMode: preprocessResult.strictMode,\n expectedToolOrder: preprocessResult.expectedToolOrder,\n });\n });\n}\n"]}
1	+ {"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer"],"mappings":";;;;;;;;AAGA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAO,YAAA,CAAa;AAAA,IAClB,IAAA,EAAM,qBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAW,GAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAY,GAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;AC1HO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,2FAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAI,eAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,yBAAA;AAAA,IACN,WAAA,EACE,qHAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC5DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,WAAA,EAAa,qGAAA;AAAA,IACb,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAa,gBAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,aAAA;AAAA,IACN,WAAA,EACE,oJAAA;AAAA,IACF,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAI,SAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC3CA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOA,YAAAA,CAAa;AAAA,IAClB,IAAA,EAAM,2BAAA;AAAA,IACN,aAAa,cAAA,EAAe;AAAA,IAC5B,IAAA,EAAM;AAAA,GACP,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAI,gBAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.js","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n // Remove diacritics and convert to lowercase\n return str\n .normalize('NFD')\n .replace(/[\\u0300-\\u036f]/g, '')\n .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n // Get more specific elements and ensure they're arrays\n const nouns = doc.nouns().out('array') \|\| [];\n const verbs = doc.verbs().toInfinitive().out('array') \|\| [];\n const topics = doc.topics().out('array') \|\| [];\n const terms = doc.terms().out('array') \|\| [];\n\n // Helper function to clean and split terms\n const cleanAndSplitTerm = (term: string): string[] => {\n // First normalize the string\n const normalized = normalizeString(term);\n\n // Split on word boundaries and filter out empty strings\n return normalized\n .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n .trim()\n .split(/\\s+/)\n .filter(word => word.length > 0);\n };\n\n // Process all elements\n const processedTerms = [\n ...nouns.flatMap(cleanAndSplitTerm),\n ...verbs.flatMap(cleanAndSplitTerm),\n ...topics.flatMap(cleanAndSplitTerm),\n ...terms.flatMap(cleanAndSplitTerm),\n ];\n\n // Remove duplicates\n return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n if (original.length === 0) {\n return simplified.length === 0 ? 1 : 0;\n }\n\n // Exact matching for short words (3 chars or less), substring matching for longer words\n const covered = original.filter(element =>\n simplified.some(s => {\n const elem = normalizeString(element);\n const simp = normalizeString(s);\n\n // For short words (3 chars or less), require exact match\n if (elem.length <= 3) {\n return elem === simp;\n }\n\n // For longer words, require substantial overlap (more than 60% of the longer word)\n const longer = elem.length > simp.length ? elem : simp;\n const shorter = elem.length > simp.length ? simp : elem;\n\n if (longer.includes(shorter)) {\n return shorter.length / longer.length > 0.6;\n }\n\n return false;\n }),\n );\n return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n return createScorer({\n name: 'Completeness Scorer',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const isInputInvalid =\n !run.input \|\|\n run.input.inputMessages.some((i: { content: string }) => i.content === null \|\| i.content === undefined);\n\n const isOutputInvalid =\n !run.output \|\| run.output.some((i: { content: string }) => i.content === null \|\| i.content === undefined);\n\n if (isInputInvalid \|\| isOutputInvalid) {\n throw new Error('Inputs cannot be null or undefined');\n }\n\n const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map(({ content }: { content: string }) => content).join(', ') \|\| '';\n\n const inputToProcess = input;\n const outputToProcess = output;\n\n const inputDoc = nlp(inputToProcess.trim());\n const outputDoc = nlp(outputToProcess.trim());\n\n // Extract and log elements\n const inputElements = extractElements(inputDoc);\n const outputElements = extractElements(outputDoc);\n\n return {\n inputElements,\n outputElements,\n missingElements: inputElements.filter(e => !outputElements.includes(e)),\n elementCounts: {\n input: inputElements.length,\n output: outputElements.length,\n },\n };\n })\n .generateScore(({ results }) => {\n const inputElements = results.preprocessStepResult?.inputElements;\n const outputElements = results.preprocessStepResult?.outputElements;\n\n return calculateCoverage({\n original: inputElements,\n simplified: outputElements,\n });\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n return createScorer({\n name: 'Textual Difference Scorer',\n description: 'Calculate textual difference between input and output using sequence matching algorithms.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const matcher = new SequenceMatcher(null, input, output);\n const ratio = matcher.ratio();\n\n // Get detailed operations\n const ops = matcher.getOpcodes();\n const changes = ops.filter(([op]) => op !== 'equal').length;\n\n // Calculate confidence based on text length difference\n const maxLength = Math.max(input.length, output.length);\n const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n const confidence = 1 - lengthDiff;\n\n return {\n ratio,\n confidence,\n changes,\n lengthDiff,\n };\n })\n .generateScore(({ results }) => {\n return results.preprocessStepResult?.ratio;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n return createScorer({\n name: 'Keyword Coverage Scorer',\n description:\n 'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const output = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n\n if (!input && !output) {\n return {\n result: {\n referenceKeywords: new Set<string>(),\n responseKeywords: new Set<string>(),\n },\n };\n }\n\n const extractKeywords = (text: string) => {\n return keyword_extractor.extract(text, {\n language: 'english',\n remove_digits: true,\n return_changed_case: true,\n remove_duplicates: true,\n });\n };\n\n const referenceKeywords = new Set(extractKeywords(input));\n const responseKeywords = new Set(extractKeywords(output));\n return {\n referenceKeywords,\n responseKeywords,\n };\n })\n .analyze(async ({ results }) => {\n if (\n !results.preprocessStepResult?.referenceKeywords?.size &&\n !results.preprocessStepResult?.responseKeywords?.size\n ) {\n return {\n totalKeywordsLength: 0,\n matchedKeywordsLength: 0,\n };\n }\n\n const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n results.preprocessStepResult?.responseKeywords?.has(k),\n );\n\n return {\n totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n matchedKeywordsLength: matchedKeywords.length ?? 0,\n };\n })\n .generateScore(({ results }) => {\n if (!results.analyzeStepResult?.totalKeywordsLength) {\n return 1;\n }\n\n const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n ignoreCase?: boolean;\n ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n return createScorer({\n name: 'Content Similarity Scorer',\n description: 'Calculates content similarity between input and output messages using string comparison algorithms.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') \|\| '';\n let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') \|\| '';\n\n if (ignoreCase) {\n processedInput = processedInput.toLowerCase();\n processedOutput = processedOutput.toLowerCase();\n }\n\n if (ignoreWhitespace) {\n processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n }\n\n return {\n processedInput,\n processedOutput,\n };\n })\n .generateScore(({ results }) => {\n const similarity = stringSimilarity.compareTwoStrings(\n results.preprocessStepResult?.processedInput,\n results.preprocessStepResult?.processedOutput,\n );\n\n return similarity;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n const { referenceTone } = config;\n\n return createScorer({\n name: 'Tone Scorer',\n description:\n 'Analyzes the tone and sentiment of agent responses using sentiment analysis. Can compare against a reference tone or evaluate sentiment stability.',\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const sentiment = new Sentiment();\n const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') \|\| '';\n const responseSentiment = sentiment.analyze(agentMessage);\n\n if (referenceTone) {\n // Compare sentiment with reference\n const referenceSentiment = sentiment.analyze(referenceTone);\n const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n return {\n score: normalizedScore,\n responseSentiment: responseSentiment.comparative,\n referenceSentiment: referenceSentiment.comparative,\n difference: sentimentDiff,\n };\n }\n\n // Evaluate sentiment stability across response\n const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) \|\| [agentMessage];\n const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n const stability = Math.max(0, 1 - variance);\n\n return {\n score: stability,\n avgSentiment,\n sentimentVariance: variance,\n };\n })\n .generateScore(({ results }) => {\n return results.preprocessStepResult?.score;\n });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\ninterface ToolCallAccuracyOptions {\n expectedTool?: string;\n strictMode?: boolean;\n expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n if (strictMode) {\n return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n }\n\n const expectedIndices: number[] = [];\n for (const expectedTool of expectedOrder) {\n const index = actualTools.indexOf(expectedTool);\n if (index === -1) {\n return false;\n }\n expectedIndices.push(index);\n }\n\n for (let i = 1; i < expectedIndices.length; i++) {\n const currentIndex = expectedIndices[i];\n const prevIndex = expectedIndices[i - 1];\n if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n return false;\n }\n }\n\n return true;\n}\n\nfunction calculateAccuracy({\n expectedTool,\n actualTools,\n strictMode = false,\n expectedToolOrder,\n}: {\n expectedTool?: string;\n actualTools: string[];\n strictMode?: boolean;\n expectedToolOrder?: string[];\n}): number {\n if (actualTools.length === 0) {\n return 0;\n }\n\n if (expectedToolOrder && expectedToolOrder.length > 0) {\n return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n }\n\n if (!expectedTool) {\n return 0;\n }\n\n if (strictMode) {\n return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n }\n\n return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n if (!expectedTool && !expectedToolOrder) {\n throw new Error('Either expectedTool or expectedToolOrder must be provided');\n }\n\n const getDescription = () => {\n return expectedToolOrder\n ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n };\n\n return createScorer({\n name: 'Tool Call Accuracy Scorer',\n description: getDescription(),\n type: 'agent',\n })\n .preprocess(async ({ run }) => {\n const isInputInvalid = !run.input \|\| !run.input.inputMessages \|\| run.input.inputMessages.length === 0;\n const isOutputInvalid = !run.output \|\| run.output.length === 0;\n\n if (isInputInvalid \|\| isOutputInvalid) {\n throw new Error('Input and output messages cannot be null or empty');\n }\n\n const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n const correctToolCalled = expectedTool\n ? strictMode\n ? actualTools.length === 1 && actualTools[0] === expectedTool\n : actualTools.includes(expectedTool)\n : false;\n\n return {\n expectedTool,\n actualTools,\n strictMode,\n expectedToolOrder,\n hasToolCalls: actualTools.length > 0,\n correctToolCalled,\n toolCallInfos,\n correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n };\n })\n .generateScore(({ results }) => {\n const preprocessResult = results.preprocessStepResult;\n if (!preprocessResult) {\n return 0;\n }\n\n return calculateAccuracy({\n expectedTool: preprocessResult.expectedTool,\n actualTools: preprocessResult.actualTools,\n strictMode: preprocessResult.strictMode,\n expectedToolOrder: preprocessResult.expectedToolOrder,\n });\n });\n}\n"]}

package/dist/scorers/code/keyword-coverage/index.d.ts CHANGED Viewed

@@ -1,5 +1,4 @@
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
-export declare function createKeywordCoverageScorer(): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+export declare function createKeywordCoverageScorer(): import("@mastra/core/scores").MastraScorer<"Keyword Coverage Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
     result: {
         referenceKeywords: Set<string>;
         responseKeywords: Set<string>;

package/dist/scorers/code/keyword-coverage/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/keyword-coverage/index.ts"],"names":[],"mappings":"~~AACA~~,~~OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAG3F,~~wBAAgB,2BAA2B;;;;;;;;;;;;;;~~gDAgE1C~~"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/keyword-coverage/index.ts"],"names":[],"mappings":"AAGA,wBAAgB,2BAA2B;;;;;;;;;;;;;;gDAiE1C"}

package/dist/scorers/code/textual-difference/index.d.ts CHANGED Viewed

@@ -1,5 +1,4 @@
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
-export declare function createTextualDifferenceScorer(): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+export declare function createTextualDifferenceScorer(): import("@mastra/core/scores").MastraScorer<"Textual Difference Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
     ratio: number;
     confidence: number;
     changes: number;

package/dist/scorers/code/textual-difference/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/textual-difference/index.ts"],"names":[],"mappings":"~~AACA~~,~~OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAG3F,~~wBAAgB,6BAA6B;;;;;gDA+B5C"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/textual-difference/index.ts"],"names":[],"mappings":"AAGA,wBAAgB,6BAA6B;;;;;gDA+B5C"}

package/dist/scorers/code/tone/index.d.ts CHANGED Viewed

@@ -1,8 +1,7 @@
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
 interface ToneScorerConfig {
     referenceTone?: string;
 }
-export declare function createToneScorer(config?: ToneScorerConfig): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+export declare function createToneScorer(config?: ToneScorerConfig): import("@mastra/core/scores").MastraScorer<"Tone Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
     score: number;
     responseSentiment: number;
     referenceSentiment: number;

package/dist/scorers/code/tone/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/tone/index.ts"],"names":[],"mappings":"~~AACA~~,~~OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAG3F,~~UAAU,gBAAgB;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wBAAgB,gBAAgB,CAAC,MAAM,GAAE,gBAAqB;;;;;;;;;;;;;;~~gDA2C7D~~"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/tone/index.ts"],"names":[],"mappings":"AAGA,UAAU,gBAAgB;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,wBAAgB,gBAAgB,CAAC,MAAM,GAAE,gBAAqB;;;;;;;;;;;;;;gDA4C7D"}

package/dist/scorers/code/tool-call-accuracy/index.d.ts CHANGED Viewed

@@ -1,10 +1,9 @@
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
 interface ToolCallAccuracyOptions {
     expectedTool?: string;
     strictMode?: boolean;
     expectedToolOrder?: string[];
 }
-export declare function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+export declare function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions): import("@mastra/core/scores").MastraScorer<"Tool Call Accuracy Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
     expectedTool: string | undefined;
     actualTools: string[];
     strictMode: boolean;

package/dist/scorers/code/tool-call-accuracy/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":[],"mappings":"~~AAAA~~,~~OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAI3F,~~UAAU,uBAAuB;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAyDD,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,uBAAuB;;;;;;;;;~~gDAyDhF~~"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":[],"mappings":"AAEA,UAAU,uBAAuB;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAyDD,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,uBAAuB;;;;;;;;;gDA0DhF"}

package/dist/scorers/llm/answer-relevancy/index.d.ts CHANGED Viewed

@@ -1,11 +1,10 @@
 import type { MastraLanguageModel } from '@mastra/core/agent';
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
 export declare const DEFAULT_OPTIONS: Record<'uncertaintyWeight' | 'scale', number>;
 export declare const ANSWER_RELEVANCY_AGENT_INSTRUCTIONS = "\n    You are a balanced and nuanced answer relevancy evaluator. Your job is to determine if LLM outputs are relevant to the input, including handling partially relevant or uncertain cases.\n\n    Key Principles:\n    1. Evaluate whether the output addresses what the input is asking for\n    2. Consider both direct answers and related context\n    3. Prioritize relevance to the input over correctness\n    4. Recognize that responses can be partially relevant\n    5. Empty inputs or error messages should always be marked as \"no\"\n    6. Responses that discuss the type of information being asked show partial relevance\n";
 export declare function createAnswerRelevancyScorer({ model, options, }: {
     model: MastraLanguageModel;
     options?: Record<'uncertaintyWeight' | 'scale', number>;
-}): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+}): import("@mastra/core/scores").MastraScorer<"Answer Relevancy Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
     statements: string[];
 }> & Record<"analyzeStepResult", {
     results: {

package/dist/scorers/llm/answer-relevancy/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-relevancy/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;~~AAE9D~~,~~OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAM3F,~~eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,mBAAmB,GAAG,OAAO,EAAE,MAAM,CAGzE,CAAC;AAEF,eAAO,MAAM,mCAAmC,wnBAU/C,CAAC;AAMF,wBAAgB,2BAA2B,CAAC,EAC1C,KAAK,EACL,OAAyB,GAC1B,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC,mBAAmB,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC;CACzD;;;;;;;~~6FAyDA~~"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-relevancy/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAO9D,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,mBAAmB,GAAG,OAAO,EAAE,MAAM,CAGzE,CAAC;AAEF,eAAO,MAAM,mCAAmC,wnBAU/C,CAAC;AAMF,wBAAgB,2BAA2B,CAAC,EAC1C,KAAK,EACL,OAAyB,GAC1B,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC,mBAAmB,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC;CACzD;;;;;;;6FA0DA"}

package/dist/scorers/llm/answer-similarity/index.d.ts CHANGED Viewed

@@ -1,5 +1,4 @@
 import type { MastraLanguageModel } from '@mastra/core/agent';
-import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
 export interface AnswerSimilarityOptions {
     requireGroundTruth?: boolean;
     semanticThreshold?: number;
@@ -14,7 +13,7 @@ export declare const ANSWER_SIMILARITY_INSTRUCTIONS = "\nYou are a precise answe
 export declare function createAnswerSimilarityScorer({ model, options, }: {
     model: MastraLanguageModel;
     options?: AnswerSimilarityOptions;
-}): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+}): import("@mastra/core/scores").MastraScorer<"Answer Similarity Scorer", import("@mastra/core/scores").ScorerRunInputForAgent, import("@mastra/core/scores").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
     outputUnits: string[];
     groundTruthUnits: string[];
 }> & Record<"analyzeStepResult", {

package/dist/scorers/llm/answer-similarity/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-similarity/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;~~AAE9D~~,~~OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,~~MAAM,~~qBAAqB,CAAC;AAM3F,MAAM,~~WAAW,uBAAuB;IACtC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,eAAO,MAAM,iCAAiC,EAAE,QAAQ,CAAC,uBAAuB,CAQ/E,CAAC;AAEF,eAAO,MAAM,8BAA8B,+kBAU1C,CAAC;AA0BF,wBAAgB,4BAA4B,CAAC,EAC3C,KAAK,EACL,OAA2C,GAC5C,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC;;;;;;;;;;;;;;;;~~6FA4HA~~"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-similarity/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAO9D,MAAM,WAAW,uBAAuB;IACtC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,eAAO,MAAM,iCAAiC,EAAE,QAAQ,CAAC,uBAAuB,CAQ/E,CAAC;AAEF,eAAO,MAAM,8BAA8B,+kBAU1C,CAAC;AA0BF,wBAAgB,4BAA4B,CAAC,EAC3C,KAAK,EACL,OAA2C,GAC5C,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC;;;;;;;;;;;;;;;;6FA6HA"}

package/dist/scorers/llm/bias/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/bias/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAW3F,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,gBAAgB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAAE,KAAK,EAAE,aAAa,CAAC;IAAC,OAAO,CAAC,EAAE,iBAAiB,CAAA;CAAE;;;;;;;~~6FA+CzG~~"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/bias/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAW3F,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,gBAAgB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAAE,KAAK,EAAE,aAAa,CAAC;IAAC,OAAO,CAAC,EAAE,iBAAiB,CAAA;CAAE;;;;;;;6FAgDzG"}

package/dist/scorers/llm/context-precision/index.d.ts CHANGED Viewed

@@ -8,7 +8,7 @@ export interface ContextPrecisionMetricOptions {
 export declare function createContextPrecisionScorer({ model, options, }: {
     model: MastraLanguageModel;
     options: ContextPrecisionMetricOptions;
-}): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"analyzeStepResult", {
+}): import("@mastra/core/scores").MastraScorer<"Context Precision Scorer", ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"analyzeStepResult", {
     verdicts: {
         verdict: string;
         reason: string;