@mastra/evals 0.11.0 → 0.12.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachListeners.d.ts +4 -0
- package/dist/attachListeners.d.ts.map +1 -0
- package/dist/{chunk-2JVD5IX6.cjs → chunk-7QAUEU4L.cjs} +2 -0
- package/dist/chunk-7QAUEU4L.cjs.map +1 -0
- package/dist/{chunk-IS3BZTWE.cjs → chunk-EMMSS5I5.cjs} +2 -0
- package/dist/chunk-EMMSS5I5.cjs.map +1 -0
- package/dist/{chunk-U67V476Y.js → chunk-G3PMV62Z.js} +2 -0
- package/dist/chunk-G3PMV62Z.js.map +1 -0
- package/dist/{chunk-COBCYVZ7.cjs → chunk-IUSAD2BW.cjs} +2 -0
- package/dist/chunk-IUSAD2BW.cjs.map +1 -0
- package/dist/{chunk-UYXFD4VX.js → chunk-QTWX6TKR.js} +2 -0
- package/dist/chunk-QTWX6TKR.js.map +1 -0
- package/dist/{chunk-TXXJUIES.js → chunk-YGTIO3J5.js} +2 -0
- package/dist/chunk-YGTIO3J5.js.map +1 -0
- package/dist/constants.d.ts +2 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/{dist-ZXFGMR47.js → dist-66YSVXZH.js} +4 -2
- package/dist/dist-66YSVXZH.js.map +1 -0
- package/dist/{dist-JD6MNRVB.cjs → dist-6ZEQKKXY.cjs} +14 -12
- package/dist/dist-6ZEQKKXY.cjs.map +1 -0
- package/dist/evaluation.d.ts +8 -0
- package/dist/evaluation.d.ts.map +1 -0
- package/dist/index.cjs +3 -1
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -0
- package/dist/{magic-string.es-MNZ6ZGOL.js → magic-string.es-6JSI7KY4.js} +2 -0
- package/dist/magic-string.es-6JSI7KY4.js.map +1 -0
- package/dist/{magic-string.es-T2QO2IBJ.cjs → magic-string.es-NBXOXRCK.cjs} +2 -0
- package/dist/magic-string.es-NBXOXRCK.cjs.map +1 -0
- package/dist/metrics/index.d.ts +4 -0
- package/dist/metrics/index.d.ts.map +1 -0
- package/dist/metrics/judge/index.cjs +4 -2
- package/dist/metrics/judge/index.cjs.map +1 -0
- package/dist/metrics/judge/index.d.ts +7 -1
- package/dist/metrics/judge/index.d.ts.map +1 -0
- package/dist/metrics/judge/index.js +3 -1
- package/dist/metrics/judge/index.js.map +1 -0
- package/dist/metrics/llm/answer-relevancy/index.d.ts +16 -0
- package/dist/metrics/llm/answer-relevancy/index.d.ts.map +1 -0
- package/dist/metrics/llm/answer-relevancy/metricJudge.d.ts +20 -0
- package/dist/metrics/llm/answer-relevancy/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/answer-relevancy/prompts.d.ts +19 -0
- package/dist/metrics/llm/answer-relevancy/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/bias/index.d.ts +14 -0
- package/dist/metrics/llm/bias/index.d.ts.map +1 -0
- package/dist/metrics/llm/bias/metricJudge.d.ts +14 -0
- package/dist/metrics/llm/bias/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/bias/prompts.d.ts +14 -0
- package/dist/metrics/llm/bias/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/context-position/index.d.ts +16 -0
- package/dist/metrics/llm/context-position/index.d.ts.map +1 -0
- package/dist/metrics/llm/context-position/metricJudge.d.ts +20 -0
- package/dist/metrics/llm/context-position/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/context-position/prompts.d.ts +17 -0
- package/dist/metrics/llm/context-position/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/context-precision/index.d.ts +16 -0
- package/dist/metrics/llm/context-precision/index.d.ts.map +1 -0
- package/dist/metrics/llm/context-precision/metricJudge.d.ts +20 -0
- package/dist/metrics/llm/context-precision/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/context-precision/prompts.d.ts +17 -0
- package/dist/metrics/llm/context-precision/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/context-relevancy/index.d.ts +16 -0
- package/dist/metrics/llm/context-relevancy/index.d.ts.map +1 -0
- package/dist/metrics/llm/context-relevancy/metricJudge.d.ts +16 -0
- package/dist/metrics/llm/context-relevancy/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/context-relevancy/prompts.d.ts +13 -0
- package/dist/metrics/llm/context-relevancy/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/contextual-recall/index.d.ts +16 -0
- package/dist/metrics/llm/contextual-recall/index.d.ts.map +1 -0
- package/dist/metrics/llm/contextual-recall/metricJudge.d.ts +16 -0
- package/dist/metrics/llm/contextual-recall/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/contextual-recall/prompts.d.ts +13 -0
- package/dist/metrics/llm/contextual-recall/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/faithfulness/index.d.ts +16 -0
- package/dist/metrics/llm/faithfulness/index.d.ts.map +1 -0
- package/dist/metrics/llm/faithfulness/metricJudge.d.ts +22 -0
- package/dist/metrics/llm/faithfulness/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/faithfulness/prompts.d.ts +20 -0
- package/dist/metrics/llm/faithfulness/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/hallucination/index.d.ts +16 -0
- package/dist/metrics/llm/hallucination/index.d.ts.map +1 -0
- package/dist/metrics/llm/hallucination/metricJudge.d.ts +22 -0
- package/dist/metrics/llm/hallucination/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/hallucination/prompts.d.ts +17 -0
- package/dist/metrics/llm/hallucination/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/index.cjs +26 -24
- package/dist/metrics/llm/index.cjs.map +1 -0
- package/dist/metrics/llm/index.d.ts +12 -11
- package/dist/metrics/llm/index.d.ts.map +1 -0
- package/dist/metrics/llm/index.js +4 -2
- package/dist/metrics/llm/index.js.map +1 -0
- package/dist/metrics/llm/prompt-alignment/index.d.ts +33 -0
- package/dist/metrics/llm/prompt-alignment/index.d.ts.map +1 -0
- package/dist/metrics/llm/prompt-alignment/metricJudge.d.ts +20 -0
- package/dist/metrics/llm/prompt-alignment/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/prompt-alignment/prompts.d.ts +17 -0
- package/dist/metrics/llm/prompt-alignment/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/summarization/index.d.ts +19 -0
- package/dist/metrics/llm/summarization/index.d.ts.map +1 -0
- package/dist/metrics/llm/summarization/metricJudge.d.ts +34 -0
- package/dist/metrics/llm/summarization/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/summarization/prompts.d.ts +30 -0
- package/dist/metrics/llm/summarization/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/toxicity/index.d.ts +14 -0
- package/dist/metrics/llm/toxicity/index.d.ts.map +1 -0
- package/dist/metrics/llm/toxicity/metricJudge.d.ts +14 -0
- package/dist/metrics/llm/toxicity/metricJudge.d.ts.map +1 -0
- package/dist/metrics/llm/toxicity/prompts.d.ts +10 -0
- package/dist/metrics/llm/toxicity/prompts.d.ts.map +1 -0
- package/dist/metrics/llm/types.d.ts +7 -0
- package/dist/metrics/llm/types.d.ts.map +1 -0
- package/dist/metrics/llm/utils.d.ts +14 -0
- package/dist/metrics/llm/utils.d.ts.map +1 -0
- package/dist/metrics/nlp/completeness/index.d.ts +21 -0
- package/dist/metrics/nlp/completeness/index.d.ts.map +1 -0
- package/dist/metrics/nlp/content-similarity/index.d.ts +18 -0
- package/dist/metrics/nlp/content-similarity/index.d.ts.map +1 -0
- package/dist/metrics/nlp/index.cjs +2 -0
- package/dist/metrics/nlp/index.cjs.map +1 -0
- package/dist/metrics/nlp/index.d.ts +6 -5
- package/dist/metrics/nlp/index.d.ts.map +1 -0
- package/dist/metrics/nlp/index.js +2 -0
- package/dist/metrics/nlp/index.js.map +1 -0
- package/dist/metrics/nlp/keyword-coverage/index.d.ts +13 -0
- package/dist/metrics/nlp/keyword-coverage/index.d.ts.map +1 -0
- package/dist/metrics/nlp/textual-difference/index.d.ts +15 -0
- package/dist/metrics/nlp/textual-difference/index.d.ts.map +1 -0
- package/dist/metrics/nlp/tone/index.d.ts +18 -0
- package/dist/metrics/nlp/tone/index.d.ts.map +1 -0
- package/dist/scorers/code/completeness/index.d.ts +11 -0
- package/dist/scorers/code/completeness/index.d.ts.map +1 -0
- package/dist/scorers/code/content-similarity/index.d.ts +11 -0
- package/dist/scorers/code/content-similarity/index.d.ts.map +1 -0
- package/dist/scorers/code/index.cjs +139 -161
- package/dist/scorers/code/index.cjs.map +1 -0
- package/dist/scorers/code/index.d.ts +6 -5
- package/dist/scorers/code/index.d.ts.map +1 -0
- package/dist/scorers/code/index.js +139 -161
- package/dist/scorers/code/index.js.map +1 -0
- package/dist/scorers/code/keyword-coverage/index.d.ts +17 -0
- package/dist/scorers/code/keyword-coverage/index.d.ts.map +1 -0
- package/dist/scorers/code/textual-difference/index.d.ts +8 -0
- package/dist/scorers/code/textual-difference/index.d.ts.map +1 -0
- package/dist/scorers/code/tone/index.d.ts +21 -0
- package/dist/scorers/code/tone/index.d.ts.map +1 -0
- package/dist/scorers/index.d.ts +3 -0
- package/dist/scorers/index.d.ts.map +1 -0
- package/dist/scorers/llm/answer-relevancy/index.d.ts +16 -0
- package/dist/scorers/llm/answer-relevancy/index.d.ts.map +1 -0
- package/dist/scorers/llm/answer-relevancy/prompts.d.ts +13 -0
- package/dist/scorers/llm/answer-relevancy/prompts.d.ts.map +1 -0
- package/dist/scorers/llm/bias/index.d.ts +17 -0
- package/dist/scorers/llm/bias/index.d.ts.map +1 -0
- package/dist/scorers/llm/bias/prompts.d.ts +13 -0
- package/dist/scorers/llm/bias/prompts.d.ts.map +1 -0
- package/dist/scorers/llm/faithfulness/index.d.ts +16 -0
- package/dist/scorers/llm/faithfulness/index.d.ts.map +1 -0
- package/dist/scorers/llm/faithfulness/prompts.d.ts +20 -0
- package/dist/scorers/llm/faithfulness/prompts.d.ts.map +1 -0
- package/dist/scorers/llm/hallucination/index.d.ts +19 -0
- package/dist/scorers/llm/hallucination/index.d.ts.map +1 -0
- package/dist/scorers/llm/hallucination/prompts.d.ts +20 -0
- package/dist/scorers/llm/hallucination/prompts.d.ts.map +1 -0
- package/dist/scorers/llm/index.cjs +200 -207
- package/dist/scorers/llm/index.cjs.map +1 -0
- package/dist/scorers/llm/index.d.ts +6 -11
- package/dist/scorers/llm/index.d.ts.map +1 -0
- package/dist/scorers/llm/index.js +201 -208
- package/dist/scorers/llm/index.js.map +1 -0
- package/dist/scorers/llm/toxicity/index.d.ts +15 -0
- package/dist/scorers/llm/toxicity/index.d.ts.map +1 -0
- package/dist/scorers/llm/toxicity/prompts.d.ts +10 -0
- package/dist/scorers/llm/toxicity/prompts.d.ts.map +1 -0
- package/dist/scorers/utils.d.ts +59 -0
- package/dist/scorers/utils.d.ts.map +1 -0
- package/package.json +13 -12
- package/dist/_tsup-dts-rollup.d.cts +0 -984
- package/dist/_tsup-dts-rollup.d.ts +0 -984
- package/dist/index.d.cts +0 -3
- package/dist/metrics/judge/index.d.cts +0 -1
- package/dist/metrics/llm/index.d.cts +0 -11
- package/dist/metrics/nlp/index.d.cts +0 -5
- package/dist/scorers/code/index.d.cts +0 -5
- package/dist/scorers/llm/index.d.cts +0 -11
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"attachListeners.d.ts","sourceRoot":"","sources":["../src/attachListeners.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAO3C,wBAAsB,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,iBA0BpD;AAED,wBAAsB,WAAW,kBAOhC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/metrics/llm/utils.ts"],"names":[],"mappings":";;;AAAO,IAAM,kBAAA,GAAqB,CAAC,GAAA,KAAgB;AACjD,EAAA,OAAO,KAAK,KAAA,CAAA,CAAO,GAAA,GAAM,MAAA,CAAO,OAAA,IAAW,GAAG,CAAA,GAAI,GAAA;AACpD","file":"chunk-7QAUEU4L.cjs","sourcesContent":["export const roundToTwoDecimals = (num: number) => {\n return Math.round((num + Number.EPSILON) * 100) / 100;\n};\n\nexport function isCloserTo(value: number, target1: number, target2: number): boolean {\n return Math.abs(value - target1) < Math.abs(value - target2);\n}\n\nexport type TestCase = {\n input: string;\n output: string;\n expectedResult: {\n score: number;\n reason?: string;\n };\n};\n\nexport type TestCaseWithContext = TestCase & {\n context: string[];\n};\n"]}
|
|
@@ -33,3 +33,5 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
33
33
|
exports.__commonJS = __commonJS;
|
|
34
34
|
exports.__export = __export;
|
|
35
35
|
exports.__toESM = __toESM;
|
|
36
|
+
//# sourceMappingURL=chunk-EMMSS5I5.cjs.map
|
|
37
|
+
//# sourceMappingURL=chunk-EMMSS5I5.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"names":[],"mappings":"","file":"chunk-EMMSS5I5.cjs"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"names":[],"mappings":"","file":"chunk-G3PMV62Z.js"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/metrics/judge/index.ts"],"names":["Agent"],"mappings":";;;;;AAGO,IAAe,mBAAf,MAAgC;AAAA,EAClB,KAAA;AAAA,EAEnB,WAAA,CAAY,IAAA,EAAc,YAAA,EAAsB,KAAA,EAAsB;AACpE,IAAA,IAAA,CAAK,KAAA,GAAQ,IAAIA,WAAA,CAAM;AAAA,MACrB,IAAA,EAAM,qBAAqB,IAAI,CAAA,CAAA;AAAA,MAC/B,YAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AACF","file":"chunk-IUSAD2BW.cjs","sourcesContent":["import { Agent } from '@mastra/core/agent';\nimport type { LanguageModel } from '@mastra/core/llm';\n\nexport abstract class MastraAgentJudge {\n protected readonly agent: Agent;\n\n constructor(name: string, instructions: string, model: LanguageModel) {\n this.agent = new Agent({\n name: `Mastra Eval Judge ${name}`,\n instructions: instructions,\n model,\n });\n }\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/metrics/llm/utils.ts"],"names":[],"mappings":";AAAO,IAAM,kBAAA,GAAqB,CAAC,GAAA,KAAgB;AACjD,EAAA,OAAO,KAAK,KAAA,CAAA,CAAO,GAAA,GAAM,MAAA,CAAO,OAAA,IAAW,GAAG,CAAA,GAAI,GAAA;AACpD","file":"chunk-QTWX6TKR.js","sourcesContent":["export const roundToTwoDecimals = (num: number) => {\n return Math.round((num + Number.EPSILON) * 100) / 100;\n};\n\nexport function isCloserTo(value: number, target1: number, target2: number): boolean {\n return Math.abs(value - target1) < Math.abs(value - target2);\n}\n\nexport type TestCase = {\n input: string;\n output: string;\n expectedResult: {\n score: number;\n reason?: string;\n };\n};\n\nexport type TestCaseWithContext = TestCase & {\n context: string[];\n};\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/metrics/judge/index.ts"],"names":[],"mappings":";;;AAGO,IAAe,mBAAf,MAAgC;AAAA,EAClB,KAAA;AAAA,EAEnB,WAAA,CAAY,IAAA,EAAc,YAAA,EAAsB,KAAA,EAAsB;AACpE,IAAA,IAAA,CAAK,KAAA,GAAQ,IAAI,KAAA,CAAM;AAAA,MACrB,IAAA,EAAM,qBAAqB,IAAI,CAAA,CAAA;AAAA,MAC/B,YAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AACF","file":"chunk-YGTIO3J5.js","sourcesContent":["import { Agent } from '@mastra/core/agent';\nimport type { LanguageModel } from '@mastra/core/llm';\n\nexport abstract class MastraAgentJudge {\n protected readonly agent: Agent;\n\n constructor(name: string, instructions: string, model: LanguageModel) {\n this.agent = new Agent({\n name: `Mastra Eval Judge ${name}`,\n instructions: instructions,\n model,\n });\n }\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,qBAAqB,2BAA2B,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { __commonJS, __export, __toESM } from './chunk-
|
|
1
|
+
import { __commonJS, __export, __toESM } from './chunk-G3PMV62Z.js';
|
|
2
2
|
import { isatty } from 'tty';
|
|
3
3
|
|
|
4
4
|
// ../../node_modules/.pnpm/js-tokens@9.0.1/node_modules/js-tokens/index.js
|
|
@@ -12887,7 +12887,7 @@ function offsetToLineNumber(source, offset) {
|
|
|
12887
12887
|
return line + 1;
|
|
12888
12888
|
}
|
|
12889
12889
|
async function saveInlineSnapshots(environment, snapshots) {
|
|
12890
|
-
const MagicString = (await import('./magic-string.es-
|
|
12890
|
+
const MagicString = (await import('./magic-string.es-6JSI7KY4.js')).default;
|
|
12891
12891
|
const files = new Set(snapshots.map((i) => i.file));
|
|
12892
12892
|
await Promise.all(Array.from(files).map(async (file) => {
|
|
12893
12893
|
const snaps = snapshots.filter((i) => i.file === file);
|
|
@@ -16725,3 +16725,5 @@ chai/chai.js:
|
|
|
16725
16725
|
*/
|
|
16726
16726
|
|
|
16727
16727
|
export { afterAll, afterEach, assert2 as assert, assertType, beforeAll, beforeEach, bench, chai_exports as chai, createExpect, describe, globalExpect as expect, export_expectTypeOf as expectTypeOf, inject, it, onTestFailed, onTestFinished, should, suite, test3 as test, vi, vitest };
|
|
16728
|
+
//# sourceMappingURL=dist-66YSVXZH.js.map
|
|
16729
|
+
//# sourceMappingURL=dist-66YSVXZH.js.map
|