@mastra/evals 1.10.2 → 1.10.4-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +2 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/docs-evals-overview.md +1 -1
- package/dist/docs/references/docs-evals-vitest-integration.md +71 -31
- package/dist/docs/references/docs-guides-build-an-eval-loop.md +395 -0
- package/dist/docs/references/reference-evals-trajectory-accuracy.md +20 -16
- package/dist/scorers/llm/faithfulness/index.d.ts.map +1 -1
- package/dist/scorers/llm/hallucination/index.d.ts.map +1 -1
- package/dist/scorers/prebuilt/index.cjs +9 -7
- package/dist/scorers/prebuilt/index.cjs.map +1 -1
- package/dist/scorers/prebuilt/index.js +9 -7
- package/dist/scorers/prebuilt/index.js.map +1 -1
- package/dist/scorers/utils.cjs +38 -22
- package/dist/scorers/utils.cjs.map +1 -1
- package/dist/scorers/utils.d.ts.map +1 -1
- package/dist/scorers/utils.js +38 -22
- package/dist/scorers/utils.js.map +1 -1
- package/package.json +11 -11
|
@@ -426,7 +426,7 @@ const analyzeOutputSchema$7 = {
|
|
|
426
426
|
"type": "object",
|
|
427
427
|
"properties": {
|
|
428
428
|
"groundTruthUnit": { "type": "string" },
|
|
429
|
-
"outputUnit": { "
|
|
429
|
+
"outputUnit": { "type": ["string", "null"] },
|
|
430
430
|
"matchType": {
|
|
431
431
|
"type": "string",
|
|
432
432
|
"enum": [
|
|
@@ -757,10 +757,11 @@ function createFaithfulnessScorer({ model, options }) {
|
|
|
757
757
|
});
|
|
758
758
|
}
|
|
759
759
|
}).generateScore(({ results }) => {
|
|
760
|
-
const
|
|
761
|
-
const
|
|
760
|
+
const verdicts = results.analyzeStepResult.verdicts;
|
|
761
|
+
const totalClaims = results.preprocessStepResult?.claims?.length || verdicts.length;
|
|
762
|
+
const supportedClaims = verdicts.filter((v) => v.verdict.toLowerCase().trim() === "yes").length;
|
|
762
763
|
if (totalClaims === 0) return 0;
|
|
763
|
-
return roundToTwoDecimals(supportedClaims / totalClaims * (options?.scale || 1));
|
|
764
|
+
return roundToTwoDecimals(Math.min(1, supportedClaims / totalClaims) * (options?.scale || 1));
|
|
764
765
|
}).generateReason({
|
|
765
766
|
description: "Reason about the results",
|
|
766
767
|
createPrompt: ({ run, results, score }) => {
|
|
@@ -1184,10 +1185,11 @@ function createHallucinationScorer({ model, options }) {
|
|
|
1184
1185
|
});
|
|
1185
1186
|
}
|
|
1186
1187
|
}).generateScore(({ results }) => {
|
|
1187
|
-
const
|
|
1188
|
-
const
|
|
1188
|
+
const verdicts = results.analyzeStepResult.verdicts;
|
|
1189
|
+
const totalStatements = results.preprocessStepResult?.claims?.length || verdicts.length;
|
|
1190
|
+
const contradictedStatements = verdicts.filter((v) => v.verdict.toLowerCase().trim() === "yes").length;
|
|
1189
1191
|
if (totalStatements === 0) return 0;
|
|
1190
|
-
return roundToTwoDecimals(contradictedStatements / totalStatements * (options?.scale || 1));
|
|
1192
|
+
return roundToTwoDecimals(Math.min(1, contradictedStatements / totalStatements) * (options?.scale || 1));
|
|
1191
1193
|
}).generateReason({
|
|
1192
1194
|
description: "Reason about the results",
|
|
1193
1195
|
createPrompt: async ({ run, results, score }) => {
|