@mastra/evals 0.11.0 → 0.12.0-alpha.0

package/dist/metrics/llm/types.d.ts

@@ -0,0 +1,7 @@
+import type { MetricResult } from '@mastra/core/eval';
+export interface MetricResultWithReason extends MetricResult {
+    info: {
+        reason: string;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/metrics/llm/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/metrics/llm/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,WAAW,sBAAuB,SAAQ,YAAY;IAC1D,IAAI,EAAE;QACJ,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH"}

package/dist/metrics/llm/utils.d.ts

@@ -0,0 +1,14 @@
+export declare const roundToTwoDecimals: (num: number) => number;
+export declare function isCloserTo(value: number, target1: number, target2: number): boolean;
+export type TestCase = {
+    input: string;
+    output: string;
+    expectedResult: {
+        score: number;
+        reason?: string;
+    };
+};
+export type TestCaseWithContext = TestCase & {
+    context: string[];
+};
+//# sourceMappingURL=utils.d.ts.map

package/dist/metrics/llm/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/metrics/llm/utils.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,kBAAkB,GAAI,KAAK,MAAM,WAE7C,CAAC;AAEF,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAEnF;AAED,MAAM,MAAM,QAAQ,GAAG;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE;QACd,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,QAAQ,GAAG;IAC3C,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC"}

package/dist/metrics/nlp/completeness/index.d.ts

@@ -0,0 +1,21 @@
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
+interface CompletenessMetricResult extends MetricResult {
+    info: {
+        inputElements: string[];
+        outputElements: string[];
+        missingElements: string[];
+        elementCounts: {
+            input: number;
+            output: number;
+        };
+    };
+}
+export declare class CompletenessMetric extends Metric {
+    measure(input: string, output: string): Promise<CompletenessMetricResult>;
+    private extractElements;
+    private normalizeString;
+    private calculateCoverage;
+}
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/nlp/completeness/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/metrics/nlp/completeness/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGtD,UAAU,wBAAyB,SAAQ,YAAY;IACrD,IAAI,EAAE;QACJ,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,EAAE;YACb,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAChB,CAAC;KACH,CAAC;CACH;AAED,qBAAa,kBAAmB,SAAQ,MAAM;IACtC,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC;IAiC/E,OAAO,CAAC,eAAe;IAiCvB,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,iBAAiB;CA6B1B"}

package/dist/metrics/nlp/content-similarity/index.d.ts

@@ -0,0 +1,18 @@
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
+interface ContentSimilarityResult extends MetricResult {
+    info: {
+        similarity: number;
+    };
+}
+interface ContentSimilarityOptions {
+    ignoreCase?: boolean;
+    ignoreWhitespace?: boolean;
+}
+export declare class ContentSimilarityMetric extends Metric {
+    private options;
+    constructor(options?: ContentSimilarityOptions);
+    measure(input: string, output: string): Promise<ContentSimilarityResult>;
+}
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/nlp/content-similarity/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/metrics/nlp/content-similarity/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGtD,UAAU,uBAAwB,SAAQ,YAAY;IACpD,IAAI,EAAE;QACJ,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,UAAU,wBAAwB;IAChC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,qBAAa,uBAAwB,SAAQ,MAAM;IACjD,OAAO,CAAC,OAAO,CAA2B;gBAE9B,OAAO,GAAE,wBAA6B;IAS5C,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC;CAqB/E"}

package/dist/metrics/nlp/index.cjs

@@ -199,3 +199,5 @@ exports.ContentSimilarityMetric = ContentSimilarityMetric;
 exports.KeywordCoverageMetric = KeywordCoverageMetric;
 exports.TextualDifferenceMetric = TextualDifferenceMetric;
 exports.ToneConsistencyMetric = ToneConsistencyMetric;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/metrics/nlp/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/metrics/nlp/completeness/index.ts","../../../src/metrics/nlp/content-similarity/index.ts","../../../src/metrics/nlp/textual-difference/index.ts","../../../src/metrics/nlp/keyword-coverage/index.ts","../../../src/metrics/nlp/tone/index.ts"],"names":["Metric","nlp","stringSimilarity","SequenceMatcher","keyword_extractor","Sentiment"],"mappings":";;;;;;;;;;;;;;;;AAgBO,IAAM,kBAAA,GAAN,cAAiCA,YAAA,CAAO;AAAA,EAC7C,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAmD;AAE9E,IAAA,IAAI,UAAU,IAAA,IAAQ,KAAA,KAAU,UAAa,MAAA,KAAW,IAAA,IAAQ,WAAW,MAAA,EAAW;AACpF,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAGA,IAAA,KAAA,GAAQ,MAAM,IAAA,EAAK;AACnB,IAAA,MAAA,GAAS,OAAO,IAAA,EAAK;AAErB,IAAA,MAAM,QAAA,GAAWC,qBAAI,KAAK,CAAA;AAC1B,IAAA,MAAM,SAAA,GAAYA,qBAAI,MAAM,CAAA;AAG5B,IAAA,MAAM,aAAA,GAAgB,IAAA,CAAK,eAAA,CAAgB,QAAQ,CAAA;AACnD,IAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,eAAA,CAAgB,SAAS,CAAA;AAErD,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,iBAAA,CAAkB,aAAA,EAAe,cAAc,CAAA;AAErE,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,aAAA;AAAA,QACA,cAAA;AAAA,QACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,QACtE,aAAA,EAAe;AAAA,UACb,OAAO,aAAA,CAAc,MAAA;AAAA,UACrB,QAAQ,cAAA,CAAe;AAAA;AACzB;AACF,KACF;AAAA,EACF;AAAA,EAEQ,gBAAgB,GAAA,EAAoB;AAE1C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,IAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,IAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,MAAA,MAAM,UAAA,GAAa,IAAA,CAAK,eAAA,CAAgB,IAAI,CAAA;AAG5C,MAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,IACnC,CAAA;AAGA,IAAA,MAAM,cAAA,GAAiB;AAAA,MACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,KACpC;AAGA,IAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AAAA,EACpC;AAAA,EAEQ,gBAAgB,GAAA,EAAqB;AAE3C,IAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AAAA,EACjB;AAAA,EAEQ,iBAAA,CAAkB,UAAoB,UAAA,EAA8B;AAC1E,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,IACvC;AAGA,IAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,MAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,OAAO,CAAA;AACzC,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,CAAC,CAAA;AAGnC,QAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,UAAA,OAAO,IAAA,KAAS,IAAA;AAAA,QAClB;AAGA,QAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,QAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,UAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,QAC1C;AAEA,QAAA,OAAO,KAAA;AAAA,MACT,CAAC;AAAA,KACH;AACA,IAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AAAA,EACnC;AACF;ACzGO,IAAM,uBAAA,GAAN,cAAsCD,YAAAA,CAAO;AAAA,EAC1C,OAAA;AAAA,EAER,WAAA,CAAY,OAAA,GAAoC,EAAC,EAAG;AAClD,IAAA,KAAA,EAAM;AACN,IAAA,IAAA,CAAK,OAAA,GAAU;AAAA,MACb,UAAA,EAAY,IAAA;AAAA,MACZ,gBAAA,EAAkB,IAAA;AAAA,MAClB,GAAG;AAAA,KACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,IAAI,cAAA,GAAiB,KAAA;AACrB,IAAA,IAAI,eAAA,GAAkB,MAAA;AAEtB,IAAA,IAAI,IAAA,CAAK,QAAQ,UAAA,EAAY;AAC3B,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,IAAA,CAAK,QAAQ,gBAAA,EAAkB;AACjC,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,MAAM,UAAA,GAAaE,iCAAA,CAAiB,iBAAA,CAAkB,cAAA,EAAgB,eAAe,CAAA;AAErF,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,UAAA;AAAA,MACP,IAAA,EAAM,EAAE,UAAA;AAAW,KACrB;AAAA,EACF;AACF;ACnCO,IAAM,uBAAA,GAAN,cAAsCF,YAAAA,CAAO;AAAA,EAClD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,MAAM,OAAA,GAAU,IAAIG,uBAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,KAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,UAAA;AAAA,QACA,KAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA;AACF,KACF;AAAA,EACF;AACF;AC1BO,IAAM,qBAAA,GAAN,cAAoCH,YAAAA,CAAO;AAAA,EAChD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAgD;AAE3E,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,aAAA,EAAe,CAAA;AAAA,UACf,eAAA,EAAiB;AAAA;AACnB,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOI,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AAExD,IAAA,MAAM,eAAA,GAAkB,CAAC,GAAG,iBAAiB,CAAA,CAAE,OAAO,CAAA,CAAA,KAAK,gBAAA,CAAiB,GAAA,CAAI,CAAC,CAAC,CAAA;AAClF,IAAA,MAAM,gBAAgB,iBAAA,CAAkB,IAAA;AACxC,IAAA,MAAM,QAAA,GAAW,aAAA,GAAgB,CAAA,GAAI,eAAA,CAAgB,SAAS,aAAA,GAAgB,CAAA;AAE9E,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,eAAe,iBAAA,CAAkB,IAAA;AAAA,QACjC,iBAAiB,eAAA,CAAgB;AAAA;AACnC,KACF;AAAA,EACF;AACF;AC/BO,IAAM,qBAAA,GAAN,cAAoCJ,YAAAA,CAAO;AAAA,EACxC,SAAA,GAAY,IAAIK,0BAAA,EAAU;AAAA,EAElC,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAA+C;AAC1E,IAAA,MAAM,iBAAA,GAAoB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,KAAK,CAAA;AAEtD,IAAA,IAAI,MAAA,EAAQ;AAEV,MAAA,MAAM,kBAAA,GAAqB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,MAAM,CAAA;AACxD,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,UACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,UACvC,UAAA,EAAY;AAAA;AACd,OACF;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,KAAA,CAAM,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,KAAK,CAAA;AACzD,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,KAAK,SAAA,CAAU,OAAA,CAAQ,CAAC,CAAA,CAAE,WAAW,CAAA;AAC3E,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,YAAA;AAAA,QACA,iBAAA,EAAmB;AAAA;AACrB,KACF;AAAA,EACF;AACF","file":"index.cjs","sourcesContent":["import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport nlp from 'compromise';\n\ninterface CompletenessMetricResult extends MetricResult {\n  info: {\n    inputElements: string[];\n    outputElements: string[];\n    missingElements: string[];\n    elementCounts: {\n      input: number;\n      output: number;\n    };\n  };\n}\n\nexport class CompletenessMetric extends Metric {\n  async measure(input: string, output: string): Promise<CompletenessMetricResult> {\n    // Handle null/undefined inputs\n    if (input === null || input === undefined || output === null || output === undefined) {\n      throw new Error('Inputs cannot be null or undefined');\n    }\n\n    // Trim both inputs\n    input = input.trim();\n    output = output.trim();\n\n    const inputDoc = nlp(input);\n    const outputDoc = nlp(output);\n\n    // Extract and log elements\n    const inputElements = this.extractElements(inputDoc);\n    const outputElements = this.extractElements(outputDoc);\n    // Maybe we need a more sophisticated matching approach\n    const coverage = this.calculateCoverage(inputElements, outputElements);\n\n    return {\n      score: coverage,\n      info: {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      },\n    };\n  }\n\n  private extractElements(doc: any): string[] {\n    // Get more specific elements and ensure they're arrays\n    const nouns = doc.nouns().out('array') || [];\n    const verbs = doc.verbs().toInfinitive().out('array') || [];\n    const topics = doc.topics().out('array') || [];\n    const terms = doc.terms().out('array') || [];\n\n    // Helper function to clean and split terms\n    const cleanAndSplitTerm = (term: string): string[] => {\n      // First normalize the string\n      const normalized = this.normalizeString(term);\n\n      // Split on word boundaries and filter out empty strings\n      return normalized\n        .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n        .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n        .trim()\n        .split(/\\s+/)\n        .filter(word => word.length > 0);\n    };\n\n    // Process all elements\n    const processedTerms = [\n      ...nouns.flatMap(cleanAndSplitTerm),\n      ...verbs.flatMap(cleanAndSplitTerm),\n      ...topics.flatMap(cleanAndSplitTerm),\n      ...terms.flatMap(cleanAndSplitTerm),\n    ];\n\n    // Remove duplicates\n    return [...new Set(processedTerms)];\n  }\n\n  private normalizeString(str: string): string {\n    // Remove diacritics and convert to lowercase\n    return str\n      .normalize('NFD')\n      .replace(/[\\u0300-\\u036f]/g, '')\n      .toLowerCase();\n  }\n\n  private calculateCoverage(original: string[], simplified: string[]): number {\n    if (original.length === 0) {\n      return simplified.length === 0 ? 1 : 0;\n    }\n\n    // Exact matching for short words (3 chars or less), substring matching for longer words\n    const covered = original.filter(element =>\n      simplified.some(s => {\n        const elem = this.normalizeString(element);\n        const simp = this.normalizeString(s);\n\n        // For short words (3 chars or less), require exact match\n        if (elem.length <= 3) {\n          return elem === simp;\n        }\n\n        // For longer words, require substantial overlap (more than 60% of the longer word)\n        const longer = elem.length > simp.length ? elem : simp;\n        const shorter = elem.length > simp.length ? simp : elem;\n\n        if (longer.includes(shorter)) {\n          return shorter.length / longer.length > 0.6;\n        }\n\n        return false;\n      }),\n    );\n    return covered.length / original.length;\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityResult extends MetricResult {\n  info: {\n    similarity: number;\n  };\n}\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport class ContentSimilarityMetric extends Metric {\n  private options: ContentSimilarityOptions;\n\n  constructor(options: ContentSimilarityOptions = {}) {\n    super();\n    this.options = {\n      ignoreCase: true,\n      ignoreWhitespace: true,\n      ...options,\n    };\n  }\n\n  async measure(input: string, output: string): Promise<ContentSimilarityResult> {\n    let processedInput = input;\n    let processedOutput = output;\n\n    if (this.options.ignoreCase) {\n      processedInput = processedInput.toLowerCase();\n      processedOutput = processedOutput.toLowerCase();\n    }\n\n    if (this.options.ignoreWhitespace) {\n      processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n      processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n    }\n\n    const similarity = stringSimilarity.compareTwoStrings(processedInput, processedOutput);\n\n    return {\n      score: similarity,\n      info: { similarity },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport { SequenceMatcher } from 'difflib';\n\ninterface TextualDifferenceResult extends MetricResult {\n  info: {\n    ratio: number;\n    changes: number;\n    lengthDiff: number;\n    confidence: number;\n  };\n}\n\nexport class TextualDifferenceMetric extends Metric {\n  async measure(input: string, output: string): Promise<TextualDifferenceResult> {\n    const matcher = new SequenceMatcher(null, input, output);\n    const ratio = matcher.ratio();\n\n    // Get detailed operations\n    const ops = matcher.getOpcodes();\n    const changes = ops.filter(([op]) => op !== 'equal').length;\n\n    // Calculate confidence based on text length difference\n    const maxLength = Math.max(input.length, output.length);\n    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n    const confidence = 1 - lengthDiff;\n\n    return {\n      score: ratio,\n      info: {\n        confidence,\n        ratio,\n        changes,\n        lengthDiff,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport keyword_extractor from 'keyword-extractor';\n\ninterface KeywordCoverageResult extends MetricResult {\n  info: {\n    totalKeywords: number;\n    matchedKeywords: number;\n  };\n}\n\nexport class KeywordCoverageMetric extends Metric {\n  async measure(input: string, output: string): Promise<KeywordCoverageResult> {\n    // Handle empty strings case\n    if (!input && !output) {\n      return {\n        score: 1,\n        info: {\n          totalKeywords: 0,\n          matchedKeywords: 0,\n        },\n      };\n    }\n\n    const extractKeywords = (text: string) => {\n      return keyword_extractor.extract(text, {\n        language: 'english',\n        remove_digits: true,\n        return_changed_case: true,\n        remove_duplicates: true,\n      });\n    };\n\n    const referenceKeywords = new Set(extractKeywords(input));\n    const responseKeywords = new Set(extractKeywords(output));\n\n    const matchedKeywords = [...referenceKeywords].filter(k => responseKeywords.has(k));\n    const totalKeywords = referenceKeywords.size;\n    const coverage = totalKeywords > 0 ? matchedKeywords.length / totalKeywords : 0;\n\n    return {\n      score: coverage,\n      info: {\n        totalKeywords: referenceKeywords.size,\n        matchedKeywords: matchedKeywords.length,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport Sentiment from 'sentiment';\n\ninterface ToneConsitencyResult extends MetricResult {\n  info:\n    | {\n        responseSentiment: number;\n        referenceSentiment: number;\n        difference: number;\n      }\n    | {\n        avgSentiment: number;\n        sentimentVariance: number;\n      };\n}\n\nexport class ToneConsistencyMetric extends Metric {\n  private sentiment = new Sentiment();\n\n  async measure(input: string, output: string): Promise<ToneConsitencyResult> {\n    const responseSentiment = this.sentiment.analyze(input);\n\n    if (output) {\n      // Compare sentiment with reference\n      const referenceSentiment = this.sentiment.analyze(output);\n      const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n      const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n      return {\n        score: normalizedScore,\n        info: {\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        },\n      };\n    }\n\n    // Evaluate sentiment stability across response\n    const sentences = input.match(/[^.!?]+[.!?]+/g) || [input];\n    const sentiments = sentences.map(s => this.sentiment.analyze(s).comparative);\n    const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n    const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n    const stability = Math.max(0, 1 - variance);\n\n    return {\n      score: stability,\n      info: {\n        avgSentiment,\n        sentimentVariance: variance,\n      },\n    };\n  }\n}\n"]}

package/dist/metrics/nlp/index.d.ts

@@ -1,5 +1,6 @@
-export { CompletenessMetric_alias_1 as CompletenessMetric } from '../../_tsup-dts-rollup.js';
-export { ContentSimilarityMetric_alias_1 as ContentSimilarityMetric } from '../../_tsup-dts-rollup.js';
-export { TextualDifferenceMetric_alias_1 as TextualDifferenceMetric } from '../../_tsup-dts-rollup.js';
-export { KeywordCoverageMetric_alias_1 as KeywordCoverageMetric } from '../../_tsup-dts-rollup.js';
-export { ToneConsistencyMetric_alias_1 as ToneConsistencyMetric } from '../../_tsup-dts-rollup.js';
+export { CompletenessMetric } from './completeness';
+export { ContentSimilarityMetric } from './content-similarity';
+export { TextualDifferenceMetric } from './textual-difference';
+export { KeywordCoverageMetric } from './keyword-coverage';
+export { ToneConsistencyMetric } from './tone';
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/nlp/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/metrics/nlp/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,QAAQ,CAAC"}

package/dist/metrics/nlp/index.js

@@ -186,3 +186,5 @@ var ToneConsistencyMetric = class extends Metric {
 };
 
 export { CompletenessMetric, ContentSimilarityMetric, KeywordCoverageMetric, TextualDifferenceMetric, ToneConsistencyMetric };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/metrics/nlp/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/metrics/nlp/completeness/index.ts","../../../src/metrics/nlp/content-similarity/index.ts","../../../src/metrics/nlp/textual-difference/index.ts","../../../src/metrics/nlp/keyword-coverage/index.ts","../../../src/metrics/nlp/tone/index.ts"],"names":["Metric"],"mappings":";;;;;;;AAgBO,IAAM,kBAAA,GAAN,cAAiC,MAAA,CAAO;AAAA,EAC7C,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAmD;AAE9E,IAAA,IAAI,UAAU,IAAA,IAAQ,KAAA,KAAU,UAAa,MAAA,KAAW,IAAA,IAAQ,WAAW,MAAA,EAAW;AACpF,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAGA,IAAA,KAAA,GAAQ,MAAM,IAAA,EAAK;AACnB,IAAA,MAAA,GAAS,OAAO,IAAA,EAAK;AAErB,IAAA,MAAM,QAAA,GAAW,IAAI,KAAK,CAAA;AAC1B,IAAA,MAAM,SAAA,GAAY,IAAI,MAAM,CAAA;AAG5B,IAAA,MAAM,aAAA,GAAgB,IAAA,CAAK,eAAA,CAAgB,QAAQ,CAAA;AACnD,IAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,eAAA,CAAgB,SAAS,CAAA;AAErD,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,iBAAA,CAAkB,aAAA,EAAe,cAAc,CAAA;AAErE,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,aAAA;AAAA,QACA,cAAA;AAAA,QACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,QACtE,aAAA,EAAe;AAAA,UACb,OAAO,aAAA,CAAc,MAAA;AAAA,UACrB,QAAQ,cAAA,CAAe;AAAA;AACzB;AACF,KACF;AAAA,EACF;AAAA,EAEQ,gBAAgB,GAAA,EAAoB;AAE1C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,IAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,IAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,IAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,MAAA,MAAM,UAAA,GAAa,IAAA,CAAK,eAAA,CAAgB,IAAI,CAAA;AAG5C,MAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,IACnC,CAAA;AAGA,IAAA,MAAM,cAAA,GAAiB;AAAA,MACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,MACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,KACpC;AAGA,IAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AAAA,EACpC;AAAA,EAEQ,gBAAgB,GAAA,EAAqB;AAE3C,IAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AAAA,EACjB;AAAA,EAEQ,iBAAA,CAAkB,UAAoB,UAAA,EAA8B;AAC1E,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,IACvC;AAGA,IAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,MAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,OAAO,CAAA;AACzC,QAAA,MAAM,IAAA,GAAO,IAAA,CAAK,eAAA,CAAgB,CAAC,CAAA;AAGnC,QAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,UAAA,OAAO,IAAA,KAAS,IAAA;AAAA,QAClB;AAGA,QAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,QAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,UAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,QAC1C;AAEA,QAAA,OAAO,KAAA;AAAA,MACT,CAAC;AAAA,KACH;AACA,IAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AAAA,EACnC;AACF;ACzGO,IAAM,uBAAA,GAAN,cAAsCA,MAAAA,CAAO;AAAA,EAC1C,OAAA;AAAA,EAER,WAAA,CAAY,OAAA,GAAoC,EAAC,EAAG;AAClD,IAAA,KAAA,EAAM;AACN,IAAA,IAAA,CAAK,OAAA,GAAU;AAAA,MACb,UAAA,EAAY,IAAA;AAAA,MACZ,gBAAA,EAAkB,IAAA;AAAA,MAClB,GAAG;AAAA,KACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,IAAI,cAAA,GAAiB,KAAA;AACrB,IAAA,IAAI,eAAA,GAAkB,MAAA;AAEtB,IAAA,IAAI,IAAA,CAAK,QAAQ,UAAA,EAAY;AAC3B,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,IAAA,CAAK,QAAQ,gBAAA,EAAkB;AACjC,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,MAAM,UAAA,GAAa,gBAAA,CAAiB,iBAAA,CAAkB,cAAA,EAAgB,eAAe,CAAA;AAErF,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,UAAA;AAAA,MACP,IAAA,EAAM,EAAE,UAAA;AAAW,KACrB;AAAA,EACF;AACF;ACnCO,IAAM,uBAAA,GAAN,cAAsCA,MAAAA,CAAO;AAAA,EAClD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAkD;AAC7E,IAAA,MAAM,OAAA,GAAU,IAAI,eAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,KAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,UAAA;AAAA,QACA,KAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA;AACF,KACF;AAAA,EACF;AACF;AC1BO,IAAM,qBAAA,GAAN,cAAoCA,MAAAA,CAAO;AAAA,EAChD,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAAgD;AAE3E,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,aAAA,EAAe,CAAA;AAAA,UACf,eAAA,EAAiB;AAAA;AACnB,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AAExD,IAAA,MAAM,eAAA,GAAkB,CAAC,GAAG,iBAAiB,CAAA,CAAE,OAAO,CAAA,CAAA,KAAK,gBAAA,CAAiB,GAAA,CAAI,CAAC,CAAC,CAAA;AAClF,IAAA,MAAM,gBAAgB,iBAAA,CAAkB,IAAA;AACxC,IAAA,MAAM,QAAA,GAAW,aAAA,GAAgB,CAAA,GAAI,eAAA,CAAgB,SAAS,aAAA,GAAgB,CAAA;AAE9E,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,QAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,eAAe,iBAAA,CAAkB,IAAA;AAAA,QACjC,iBAAiB,eAAA,CAAgB;AAAA;AACnC,KACF;AAAA,EACF;AACF;AC/BO,IAAM,qBAAA,GAAN,cAAoCA,MAAAA,CAAO;AAAA,EACxC,SAAA,GAAY,IAAI,SAAA,EAAU;AAAA,EAElC,MAAM,OAAA,CAAQ,KAAA,EAAe,MAAA,EAA+C;AAC1E,IAAA,MAAM,iBAAA,GAAoB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,KAAK,CAAA;AAEtD,IAAA,IAAI,MAAA,EAAQ;AAEV,MAAA,MAAM,kBAAA,GAAqB,IAAA,CAAK,SAAA,CAAU,OAAA,CAAQ,MAAM,CAAA;AACxD,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,IAAA,EAAM;AAAA,UACJ,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,UACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,UACvC,UAAA,EAAY;AAAA;AACd,OACF;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,KAAA,CAAM,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,KAAK,CAAA;AACzD,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,KAAK,SAAA,CAAU,OAAA,CAAQ,CAAC,CAAA,CAAE,WAAW,CAAA;AAC3E,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,IAAA,EAAM;AAAA,QACJ,YAAA;AAAA,QACA,iBAAA,EAAmB;AAAA;AACrB,KACF;AAAA,EACF;AACF","file":"index.js","sourcesContent":["import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport nlp from 'compromise';\n\ninterface CompletenessMetricResult extends MetricResult {\n  info: {\n    inputElements: string[];\n    outputElements: string[];\n    missingElements: string[];\n    elementCounts: {\n      input: number;\n      output: number;\n    };\n  };\n}\n\nexport class CompletenessMetric extends Metric {\n  async measure(input: string, output: string): Promise<CompletenessMetricResult> {\n    // Handle null/undefined inputs\n    if (input === null || input === undefined || output === null || output === undefined) {\n      throw new Error('Inputs cannot be null or undefined');\n    }\n\n    // Trim both inputs\n    input = input.trim();\n    output = output.trim();\n\n    const inputDoc = nlp(input);\n    const outputDoc = nlp(output);\n\n    // Extract and log elements\n    const inputElements = this.extractElements(inputDoc);\n    const outputElements = this.extractElements(outputDoc);\n    // Maybe we need a more sophisticated matching approach\n    const coverage = this.calculateCoverage(inputElements, outputElements);\n\n    return {\n      score: coverage,\n      info: {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      },\n    };\n  }\n\n  private extractElements(doc: any): string[] {\n    // Get more specific elements and ensure they're arrays\n    const nouns = doc.nouns().out('array') || [];\n    const verbs = doc.verbs().toInfinitive().out('array') || [];\n    const topics = doc.topics().out('array') || [];\n    const terms = doc.terms().out('array') || [];\n\n    // Helper function to clean and split terms\n    const cleanAndSplitTerm = (term: string): string[] => {\n      // First normalize the string\n      const normalized = this.normalizeString(term);\n\n      // Split on word boundaries and filter out empty strings\n      return normalized\n        .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n        .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n        .trim()\n        .split(/\\s+/)\n        .filter(word => word.length > 0);\n    };\n\n    // Process all elements\n    const processedTerms = [\n      ...nouns.flatMap(cleanAndSplitTerm),\n      ...verbs.flatMap(cleanAndSplitTerm),\n      ...topics.flatMap(cleanAndSplitTerm),\n      ...terms.flatMap(cleanAndSplitTerm),\n    ];\n\n    // Remove duplicates\n    return [...new Set(processedTerms)];\n  }\n\n  private normalizeString(str: string): string {\n    // Remove diacritics and convert to lowercase\n    return str\n      .normalize('NFD')\n      .replace(/[\\u0300-\\u036f]/g, '')\n      .toLowerCase();\n  }\n\n  private calculateCoverage(original: string[], simplified: string[]): number {\n    if (original.length === 0) {\n      return simplified.length === 0 ? 1 : 0;\n    }\n\n    // Exact matching for short words (3 chars or less), substring matching for longer words\n    const covered = original.filter(element =>\n      simplified.some(s => {\n        const elem = this.normalizeString(element);\n        const simp = this.normalizeString(s);\n\n        // For short words (3 chars or less), require exact match\n        if (elem.length <= 3) {\n          return elem === simp;\n        }\n\n        // For longer words, require substantial overlap (more than 60% of the longer word)\n        const longer = elem.length > simp.length ? elem : simp;\n        const shorter = elem.length > simp.length ? simp : elem;\n\n        if (longer.includes(shorter)) {\n          return shorter.length / longer.length > 0.6;\n        }\n\n        return false;\n      }),\n    );\n    return covered.length / original.length;\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityResult extends MetricResult {\n  info: {\n    similarity: number;\n  };\n}\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport class ContentSimilarityMetric extends Metric {\n  private options: ContentSimilarityOptions;\n\n  constructor(options: ContentSimilarityOptions = {}) {\n    super();\n    this.options = {\n      ignoreCase: true,\n      ignoreWhitespace: true,\n      ...options,\n    };\n  }\n\n  async measure(input: string, output: string): Promise<ContentSimilarityResult> {\n    let processedInput = input;\n    let processedOutput = output;\n\n    if (this.options.ignoreCase) {\n      processedInput = processedInput.toLowerCase();\n      processedOutput = processedOutput.toLowerCase();\n    }\n\n    if (this.options.ignoreWhitespace) {\n      processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n      processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n    }\n\n    const similarity = stringSimilarity.compareTwoStrings(processedInput, processedOutput);\n\n    return {\n      score: similarity,\n      info: { similarity },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport { SequenceMatcher } from 'difflib';\n\ninterface TextualDifferenceResult extends MetricResult {\n  info: {\n    ratio: number;\n    changes: number;\n    lengthDiff: number;\n    confidence: number;\n  };\n}\n\nexport class TextualDifferenceMetric extends Metric {\n  async measure(input: string, output: string): Promise<TextualDifferenceResult> {\n    const matcher = new SequenceMatcher(null, input, output);\n    const ratio = matcher.ratio();\n\n    // Get detailed operations\n    const ops = matcher.getOpcodes();\n    const changes = ops.filter(([op]) => op !== 'equal').length;\n\n    // Calculate confidence based on text length difference\n    const maxLength = Math.max(input.length, output.length);\n    const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n    const confidence = 1 - lengthDiff;\n\n    return {\n      score: ratio,\n      info: {\n        confidence,\n        ratio,\n        changes,\n        lengthDiff,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport keyword_extractor from 'keyword-extractor';\n\ninterface KeywordCoverageResult extends MetricResult {\n  info: {\n    totalKeywords: number;\n    matchedKeywords: number;\n  };\n}\n\nexport class KeywordCoverageMetric extends Metric {\n  async measure(input: string, output: string): Promise<KeywordCoverageResult> {\n    // Handle empty strings case\n    if (!input && !output) {\n      return {\n        score: 1,\n        info: {\n          totalKeywords: 0,\n          matchedKeywords: 0,\n        },\n      };\n    }\n\n    const extractKeywords = (text: string) => {\n      return keyword_extractor.extract(text, {\n        language: 'english',\n        remove_digits: true,\n        return_changed_case: true,\n        remove_duplicates: true,\n      });\n    };\n\n    const referenceKeywords = new Set(extractKeywords(input));\n    const responseKeywords = new Set(extractKeywords(output));\n\n    const matchedKeywords = [...referenceKeywords].filter(k => responseKeywords.has(k));\n    const totalKeywords = referenceKeywords.size;\n    const coverage = totalKeywords > 0 ? matchedKeywords.length / totalKeywords : 0;\n\n    return {\n      score: coverage,\n      info: {\n        totalKeywords: referenceKeywords.size,\n        matchedKeywords: matchedKeywords.length,\n      },\n    };\n  }\n}\n","import { Metric } from '@mastra/core/eval';\nimport type { MetricResult } from '@mastra/core/eval';\nimport Sentiment from 'sentiment';\n\ninterface ToneConsitencyResult extends MetricResult {\n  info:\n    | {\n        responseSentiment: number;\n        referenceSentiment: number;\n        difference: number;\n      }\n    | {\n        avgSentiment: number;\n        sentimentVariance: number;\n      };\n}\n\nexport class ToneConsistencyMetric extends Metric {\n  private sentiment = new Sentiment();\n\n  async measure(input: string, output: string): Promise<ToneConsitencyResult> {\n    const responseSentiment = this.sentiment.analyze(input);\n\n    if (output) {\n      // Compare sentiment with reference\n      const referenceSentiment = this.sentiment.analyze(output);\n      const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n      const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n      return {\n        score: normalizedScore,\n        info: {\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        },\n      };\n    }\n\n    // Evaluate sentiment stability across response\n    const sentences = input.match(/[^.!?]+[.!?]+/g) || [input];\n    const sentiments = sentences.map(s => this.sentiment.analyze(s).comparative);\n    const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n    const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n    const stability = Math.max(0, 1 - variance);\n\n    return {\n      score: stability,\n      info: {\n        avgSentiment,\n        sentimentVariance: variance,\n      },\n    };\n  }\n}\n"]}

package/dist/metrics/nlp/keyword-coverage/index.d.ts

@@ -0,0 +1,13 @@
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
+interface KeywordCoverageResult extends MetricResult {
+    info: {
+        totalKeywords: number;
+        matchedKeywords: number;
+    };
+}
+export declare class KeywordCoverageMetric extends Metric {
+    measure(input: string, output: string): Promise<KeywordCoverageResult>;
+}
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/nlp/keyword-coverage/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/metrics/nlp/keyword-coverage/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGtD,UAAU,qBAAsB,SAAQ,YAAY;IAClD,IAAI,EAAE;QACJ,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAED,qBAAa,qBAAsB,SAAQ,MAAM;IACzC,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;CAoC7E"}

package/dist/metrics/nlp/textual-difference/index.d.ts

@@ -0,0 +1,15 @@
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
+interface TextualDifferenceResult extends MetricResult {
+    info: {
+        ratio: number;
+        changes: number;
+        lengthDiff: number;
+        confidence: number;
+    };
+}
+export declare class TextualDifferenceMetric extends Metric {
+    measure(input: string, output: string): Promise<TextualDifferenceResult>;
+}
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/nlp/textual-difference/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/metrics/nlp/textual-difference/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGtD,UAAU,uBAAwB,SAAQ,YAAY;IACpD,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,qBAAa,uBAAwB,SAAQ,MAAM;IAC3C,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC;CAuB/E"}

package/dist/metrics/nlp/tone/index.d.ts

@@ -0,0 +1,18 @@
+import { Metric } from '@mastra/core/eval';
+import type { MetricResult } from '@mastra/core/eval';
+interface ToneConsitencyResult extends MetricResult {
+    info: {
+        responseSentiment: number;
+        referenceSentiment: number;
+        difference: number;
+    } | {
+        avgSentiment: number;
+        sentimentVariance: number;
+    };
+}
+export declare class ToneConsistencyMetric extends Metric {
+    private sentiment;
+    measure(input: string, output: string): Promise<ToneConsitencyResult>;
+}
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/nlp/tone/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/metrics/nlp/tone/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGtD,UAAU,oBAAqB,SAAQ,YAAY;IACjD,IAAI,EACA;QACE,iBAAiB,EAAE,MAAM,CAAC;QAC1B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,GACD;QACE,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACP;AAED,qBAAa,qBAAsB,SAAQ,MAAM;IAC/C,OAAO,CAAC,SAAS,CAAmB;IAE9B,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;CAkC5E"}

package/dist/scorers/code/completeness/index.d.ts

@@ -0,0 +1,11 @@
+import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
+export declare function createCompletenessScorer(): import("@mastra/core/scores").MastraScorer<Record<"preprocessStepResult", {
+    inputElements: string[];
+    outputElements: string[];
+    missingElements: string[];
+    elementCounts: {
+        input: number;
+        output: number;
+    };
+}> & Record<"generateScoreStepResult", number>, ScorerRunInputForAgent, ScorerRunOutputForAgent>;
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/code/completeness/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/completeness/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AA0E3F,wBAAgB,wBAAwB;;;;;;;;iGAkDvC"}

package/dist/scorers/code/content-similarity/index.d.ts

@@ -0,0 +1,11 @@
+import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
+interface ContentSimilarityOptions {
+    ignoreCase?: boolean;
+    ignoreWhitespace?: boolean;
+}
+export declare function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace }?: ContentSimilarityOptions): import("@mastra/core/scores").MastraScorer<Record<"preprocessStepResult", {
+    processedInput: string;
+    processedOutput: string;
+}> & Record<"generateScoreStepResult", number>, ScorerRunInputForAgent, ScorerRunOutputForAgent>;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/code/content-similarity/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/content-similarity/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAG3F,UAAU,wBAAwB;IAChC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,wBAAgB,6BAA6B,CAC3C,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAE,wBAAuE;;;iGAkC1G"}