@mastra/evals 1.4.0-alpha.0 → 1.5.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/dist/checks.cjs +56 -0
- package/dist/checks.cjs.map +1 -0
- package/dist/checks.d.ts +3 -0
- package/dist/checks.d.ts.map +1 -0
- package/dist/checks.js +3 -0
- package/dist/checks.js.map +1 -0
- package/dist/chunk-NXVFY4CK.cjs +233 -0
- package/dist/chunk-NXVFY4CK.cjs.map +1 -0
- package/dist/chunk-QRAONHNV.js +216 -0
- package/dist/chunk-QRAONHNV.js.map +1 -0
- package/dist/docs/SKILL.md +3 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/docs-evals-built-in-scorers.md +17 -1
- package/dist/docs/references/docs-evals-overview.md +1 -0
- package/dist/docs/references/docs-evals-quick-checks.md +136 -0
- package/dist/docs/references/reference-evals-checks.md +210 -0
- package/dist/scorers/code/checks/index.d.ts +214 -0
- package/dist/scorers/code/checks/index.d.ts.map +1 -0
- package/dist/scorers/code/index.d.ts +1 -0
- package/dist/scorers/code/index.d.ts.map +1 -1
- package/dist/scorers/prebuilt/index.cjs +56 -7
- package/dist/scorers/prebuilt/index.cjs.map +1 -1
- package/dist/scorers/prebuilt/index.js +8 -7
- package/dist/scorers/prebuilt/index.js.map +1 -1
- package/package.json +19 -7
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { calledTool, checks, didNotCall, equals, excludes, includes, matches, maxToolCalls, noToolErrors, similarity, toolOrder, usedNoTools } from '../../chunk-QRAONHNV.js';
|
|
1
2
|
import { getAssistantMessageFromRunOutput, getUserMessageFromRunInput, roundToTwoDecimals, extractToolCalls, getCombinedSystemPrompt, getTextContentFromMastraDBMessage, compareTrajectories, checkTrajectoryEfficiency, checkTrajectoryBlacklist, analyzeToolFailures, isScorerRunInputForAgent, isScorerRunOutputForAgent } from '../../chunk-BE5F2OUQ.js';
|
|
2
3
|
import { createScorer } from '@mastra/core/evals';
|
|
3
4
|
import nlp from 'compromise';
|
|
@@ -3666,9 +3667,9 @@ function calculateRatio(input, output) {
|
|
|
3666
3667
|
if (input.length === 0 || output.length === 0) {
|
|
3667
3668
|
return 0;
|
|
3668
3669
|
}
|
|
3669
|
-
const
|
|
3670
|
+
const matches2 = longestCommonSubsequence(input, output);
|
|
3670
3671
|
const total = input.length + output.length;
|
|
3671
|
-
return total > 0 ? 2 *
|
|
3672
|
+
return total > 0 ? 2 * matches2 / total : 0;
|
|
3672
3673
|
}
|
|
3673
3674
|
function longestCommonSubsequence(str1, str2) {
|
|
3674
3675
|
const m = str1.length;
|
|
@@ -3719,18 +3720,18 @@ function countChanges(input, output) {
|
|
|
3719
3720
|
return changes;
|
|
3720
3721
|
}
|
|
3721
3722
|
function findCommonWords(arr1, arr2) {
|
|
3722
|
-
let
|
|
3723
|
+
let matches2 = 0;
|
|
3723
3724
|
const used = /* @__PURE__ */ new Set();
|
|
3724
3725
|
for (let i = 0; i < arr1.length; i++) {
|
|
3725
3726
|
for (let j = 0; j < arr2.length; j++) {
|
|
3726
3727
|
if (!used.has(j) && arr1[i] === arr2[j]) {
|
|
3727
|
-
|
|
3728
|
+
matches2++;
|
|
3728
3729
|
used.add(j);
|
|
3729
3730
|
break;
|
|
3730
3731
|
}
|
|
3731
3732
|
}
|
|
3732
3733
|
}
|
|
3733
|
-
return
|
|
3734
|
+
return matches2;
|
|
3734
3735
|
}
|
|
3735
3736
|
function createTextualDifferenceScorer() {
|
|
3736
3737
|
return createScorer({
|
|
@@ -3832,11 +3833,11 @@ function createContentSimilarityScorer({ ignoreCase, ignoreWhitespace } = { igno
|
|
|
3832
3833
|
processedOutput
|
|
3833
3834
|
};
|
|
3834
3835
|
}).generateScore(({ results }) => {
|
|
3835
|
-
const
|
|
3836
|
+
const similarity2 = stringSimilarity.compareTwoStrings(
|
|
3836
3837
|
results.preprocessStepResult?.processedInput,
|
|
3837
3838
|
results.preprocessStepResult?.processedOutput
|
|
3838
3839
|
);
|
|
3839
|
-
return
|
|
3840
|
+
return similarity2;
|
|
3840
3841
|
});
|
|
3841
3842
|
}
|
|
3842
3843
|
function createToneScorer(config = {}) {
|