@mastra/evals 1.2.2 → 1.2.3-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/docs-evals-overview.md +2 -2
- package/dist/docs/references/reference-evals-noise-sensitivity.md +1 -1
- package/dist/scorers/llm/answer-relevancy/index.d.ts.map +1 -1
- package/dist/scorers/llm/answer-similarity/index.d.ts.map +1 -1
- package/dist/scorers/llm/bias/index.d.ts.map +1 -1
- package/dist/scorers/llm/context-precision/index.d.ts.map +1 -1
- package/dist/scorers/llm/context-relevance/index.d.ts.map +1 -1
- package/dist/scorers/llm/faithfulness/index.d.ts.map +1 -1
- package/dist/scorers/llm/hallucination/index.d.ts.map +1 -1
- package/dist/scorers/llm/noise-sensitivity/index.d.ts.map +1 -1
- package/dist/scorers/llm/prompt-alignment/index.d.ts.map +1 -1
- package/dist/scorers/llm/tool-call-accuracy/index.d.ts.map +1 -1
- package/dist/scorers/llm/toxicity/index.d.ts.map +1 -1
- package/dist/scorers/llm/trajectory/index.d.ts.map +1 -1
- package/dist/scorers/prebuilt/index.cjs +656 -132
- package/dist/scorers/prebuilt/index.cjs.map +1 -1
- package/dist/scorers/prebuilt/index.js +656 -132
- package/dist/scorers/prebuilt/index.js.map +1 -1
- package/package.json +9 -10
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# @mastra/evals
|
|
2
2
|
|
|
3
|
+
## 1.2.3-alpha.0
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Removed zod as a required peer dependency. Internal schemas now use plain JSON Schema objects instead of zod runtime. ([#16726](https://github.com/mastra-ai/mastra/pull/16726))
|
|
8
|
+
|
|
9
|
+
- Updated dependencies [[`c35b962`](https://github.com/mastra-ai/mastra/commit/c35b9625c7e854fcfdeee226a3338a750d0ff211), [`4084113`](https://github.com/mastra-ai/mastra/commit/408411370fc48a822e8b616b3b63f9409774e0e9)]:
|
|
10
|
+
- @mastra/core@1.37.0-alpha.8
|
|
11
|
+
|
|
3
12
|
## 1.2.2
|
|
4
13
|
|
|
5
14
|
### Patch Changes
|
package/dist/docs/SKILL.md
CHANGED
|
@@ -57,11 +57,11 @@ import { createAnswerRelevancyScorer, createToxicityScorer } from '@mastra/evals
|
|
|
57
57
|
export const evaluatedAgent = new Agent({
|
|
58
58
|
scorers: {
|
|
59
59
|
relevancy: {
|
|
60
|
-
scorer: createAnswerRelevancyScorer({ model: '
|
|
60
|
+
scorer: createAnswerRelevancyScorer({ model: '__OPENAI_MODEL_MINI__' }),
|
|
61
61
|
sampling: { type: 'ratio', rate: 0.5 },
|
|
62
62
|
},
|
|
63
63
|
safety: {
|
|
64
|
-
scorer: createToxicityScorer({ model: '
|
|
64
|
+
scorer: createToxicityScorer({ model: '__OPENAI_MODEL_MINI__' }),
|
|
65
65
|
sampling: { type: 'ratio', rate: 1 },
|
|
66
66
|
},
|
|
67
67
|
},
|
|
@@ -547,7 +547,7 @@ import { createNoiseSensitivityScorerLLM } from '@mastra/evals'
|
|
|
547
547
|
async function compareModelRobustness() {
|
|
548
548
|
const models = [
|
|
549
549
|
{ name: 'GPT-5.4', model: 'openai/gpt-5.4' },
|
|
550
|
-
{ name: 'GPT-5.4-mini', model: 'openai/gpt-5
|
|
550
|
+
{ name: 'GPT-5.4-mini', model: 'openai/gpt-5-mini' },
|
|
551
551
|
{ name: 'Claude', model: 'anthropic/claude-opus-4-6' },
|
|
552
552
|
]
|
|
553
553
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-relevancy/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-relevancy/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAGzF,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,mBAAmB,GAAG,OAAO,EAAE,MAAM,CAGzE,CAAC;AAEF,eAAO,MAAM,mCAAmC,wnBAU/C,CAAC;AAQF,wBAAgB,2BAA2B,CAAC,EAC1C,KAAK,EACL,OAAyB,GAC1B,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,mBAAmB,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC;CACzD;;;;;;;6FA2DA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-similarity/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-similarity/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAGzF,MAAM,WAAW,uBAAuB;IACtC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,eAAO,MAAM,iCAAiC,EAAE,QAAQ,CAAC,uBAAuB,CAQ/E,CAAC;AAEF,eAAO,MAAM,8BAA8B,+kBAU1C,CAAC;AA8BF,wBAAgB,4BAA4B,CAAC,EAC3C,KAAK,EACL,OAA2C,GAC5C,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC;;;;;;;;;;;;;;;;6FA8HA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/bias/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/bias/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAI1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAQzF,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,gBAAgB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAAE,KAAK,EAAE,iBAAiB,CAAC;IAAC,OAAO,CAAC,EAAE,iBAAiB,CAAA;CAAE;;;;;;;6FAmD7G"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/context-precision/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/context-precision/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC1F,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAS1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAOzF,MAAM,WAAW,6BAA6B;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,uBAAuB,KAAK,MAAM,EAAE,CAAC;CACjG;AA8BD,wBAAgB,4BAA4B,CAAC,EAC3C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,EAAE,6BAA6B,CAAC;CACxC;;;;;;6FAoGA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/context-relevance/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/context-relevance/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAE1F,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAS1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAGzF,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,uBAAuB,KAAK,MAAM,EAAE,CAAC;IAChG,SAAS,CAAC,EAAE;QACV,0BAA0B,CAAC,EAAE,MAAM,CAAC;QACpC,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,wBAAwB,CAAC,EAAE,MAAM,CAAC;KACnC,CAAC;CACH;AAyCD,wBAAgB,+BAA+B,CAAC,EAC9C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,EAAE,uBAAuB,CAAC;CAClC;;;;;;;;;;6FA8IA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/faithfulness/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/faithfulness/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAQzF,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAYD,wBAAgB,wBAAwB,CAAC,EACvC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,yBAAyB,CAAC;CACrC;;;;;;;6FAgEA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/hallucination/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/hallucination/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAIjE,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAQzF,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,0BAA0B,CAAC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACrC,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,aAAa,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,GAAG,gBAAgB,CAAC;CACpC;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,gBAAgB,KAAK,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AAEtF,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,YAAY,CAAC;CAC3B;AAED,wBAAgB,yBAAyB,CAAC,EACxC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,0BAA0B,CAAC;CACtC;;;;;;;;6FA8EA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/noise-sensitivity/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/noise-sensitivity/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAGzF,MAAM,WAAW,uBAAuB;IACtC,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE;QACR,aAAa,CAAC,EAAE;YACd,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,MAAM,CAAC,EAAE,MAAM,CAAC;SACjB,CAAC;QACF,SAAS,CAAC,EAAE;YACV,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;SAC/B,CAAC;QACF,oBAAoB,CAAC,EAAE,MAAM,CAAC;KAC/B,CAAC;CACH;AAkCD,wBAAgB,+BAA+B,CAAC,EAC9C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,EAAE,uBAAuB,CAAC;CAClC;;;;;;;;;;6FAsHA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/prompt-alignment/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/prompt-alignment/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAQ1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAGzF,MAAM,WAAW,sBAAsB;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC7C;AAyDD,wBAAgB,8BAA8B,CAAC,EAC7C,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAClC;;;;;;;;;;;;;;;;;;;;;;;;;;;6FAqHA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/tool-call-accuracy/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/tool-call-accuracy/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAU/C,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,iBAAiB,CAAC;IACzB,cAAc,EAAE,IAAI,EAAE,CAAC;CACxB;AAeD,wBAAgB,+BAA+B,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,EAAE,uBAAuB;;;;;;;;;;;6FA4EjG"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/toxicity/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/toxicity/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1D,OAAO,KAAK,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAGzF,MAAM,WAAW,qBAAqB;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,oBAAoB,CAAC,EACnC,KAAK,EACL,OAAO,GACR,EAAE;IACD,KAAK,EAAE,iBAAiB,CAAC;IACzB,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;;;;;6FAkDA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/trajectory/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/trajectory/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAyC,MAAM,oBAAoB,CAAC;AAE1G,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAK1D,MAAM,WAAW,4BAA4B;IAC3C,oCAAoC;IACpC,KAAK,EAAE,iBAAiB,CAAC;IACzB,sDAAsD;IACtD,kBAAkB,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,CAAC;CAClD;AAkED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,iCAAiC,CAAC,EAChD,KAAK,EACL,kBAAkB,EAAE,wBAAwB,GAC7C,EAAE,4BAA4B;;;;;;;;;;;;;;;6FAuG9B"}
|