@mastra/evals 1.10.8-alpha.1 → 1.10.8-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/scorers/code/tool-call-accuracy/index.d.ts.map +1 -1
- package/dist/scorers/code/trajectory/index.d.ts.map +1 -1
- package/dist/scorers/llm/trajectory/index.d.ts.map +1 -1
- package/dist/scorers/prebuilt/index.cjs +11 -16
- package/dist/scorers/prebuilt/index.cjs.map +1 -1
- package/dist/scorers/prebuilt/index.js +12 -17
- package/dist/scorers/prebuilt/index.js.map +1 -1
- package/dist/scorers/utils.cjs +76 -42
- package/dist/scorers/utils.cjs.map +1 -1
- package/dist/scorers/utils.d.ts +14 -3
- package/dist/scorers/utils.d.ts.map +1 -1
- package/dist/scorers/utils.js +76 -43
- package/dist/scorers/utils.js.map +1 -1
- package/package.json +6 -7
package/dist/docs/SKILL.md
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":[],"mappings":"AAEA,UAAU,uBAAuB;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":[],"mappings":"AAEA,UAAU,uBAAuB;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAiDD,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,uBAAuB;;;;;;;;;gDA2DhF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/trajectory/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,2BAA2B,EAC3B,qBAAqB,EAEtB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/trajectory/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,2BAA2B,EAC3B,qBAAqB,EAEtB,MAAM,oBAAoB,CAAC;AAS5B,OAAO,KAAK,EACV,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EAC1B,MAAM,aAAa,CAAC;AAErB,UAAU,mCAAmC;IAC3C;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,CAAC;IACjD,kCAAkC;IAClC,iBAAiB,CAAC,EAAE,2BAA2B,CAAC;CACjD;AA2BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,kCAAkC,CAAC,OAAO,GAAE,mCAAwC;;;;;;;;;;;QA+DtE,KAAK;;;;;gDAclC;AAID;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,yCAAyC;IACzC,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,wCAAwC;IACxC,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,2CAA2C;IAC3C,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,gDAAgD;IAChD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AA2KF;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAClC,kGAAkG;IAClG,KAAK,EAAE,MAAM,CAAC;IACd,wFAAwF;IACxF,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,mDAAmD;IACnD,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,uDAAuD;IACvD,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,6BAA6B;IAC7B,YAAY,CAAC,EAAE,yBAAyB,CAAC;IACzC,wDAAwD;IACxD,MAAM,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACnC,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACrC,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,2BAA2B;IAC1C;;;OAGG;IACH,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC;;;;OAIG;IACH,OAAO,CAAC,EAAE,sBAAsB,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,GAAE,2BAAgC;;;;;;;6FA0NnF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/trajectory/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAyC,MAAM,oBAAoB,CAAC;AAE1G,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/trajectory/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAyC,MAAM,oBAAoB,CAAC;AAE1G,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAU1D,MAAM,WAAW,4BAA4B;IAC3C,oCAAoC;IACpC,KAAK,EAAE,iBAAiB,CAAC;IACzB,sDAAsD;IACtD,kBAAkB,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,CAAC;CAClD;AAkED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,iCAAiC,CAAC,EAChD,KAAK,EACL,kBAAkB,EAAE,wBAAwB,GAC7C,EAAE,4BAA4B;;;;;;;;;;;;;;;6FAuG9B"}
|
|
@@ -3814,14 +3814,13 @@ function formatStepDetails(step) {
|
|
|
3814
3814
|
if (step.toolResult !== void 0) parts.push(`result: ${JSON.stringify(step.toolResult)}`);
|
|
3815
3815
|
return parts.length > 0 ? ` (${parts.join(", ")})` : "";
|
|
3816
3816
|
}
|
|
3817
|
-
case "model_generation": return step.modelId ? ` (model: ${step.modelId})` : "";
|
|
3818
3817
|
case "workflow_step": return step.output !== void 0 ? ` (output: ${JSON.stringify(step.output)})` : "";
|
|
3819
3818
|
default: return "";
|
|
3820
3819
|
}
|
|
3821
3820
|
}
|
|
3822
3821
|
function formatTrajectory(trajectory, indent = 0) {
|
|
3823
3822
|
const prefix = " ".repeat(indent);
|
|
3824
|
-
return trajectory.steps.map((step, i) => {
|
|
3823
|
+
return require_scorers_utils.toToolLevelSteps(trajectory.steps).map((step, i) => {
|
|
3825
3824
|
let line = `${prefix}${i + 1}. [${step.stepType}] ${step.name}${formatStepDetails(step)}`;
|
|
3826
3825
|
if (step.children && step.children.length > 0) line += `\n${formatTrajectory({ steps: step.children }, indent + 1)}`;
|
|
3827
3826
|
return line;
|
|
@@ -4223,16 +4222,11 @@ function createToneScorer(config = {}) {
|
|
|
4223
4222
|
//#region src/scorers/code/tool-call-accuracy/index.ts
|
|
4224
4223
|
function checkToolOrder(actualTools, expectedOrder, strictMode = false) {
|
|
4225
4224
|
if (strictMode) return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);
|
|
4226
|
-
|
|
4225
|
+
let searchFrom = 0;
|
|
4227
4226
|
for (const expectedTool of expectedOrder) {
|
|
4228
|
-
const index = actualTools.indexOf(expectedTool);
|
|
4227
|
+
const index = actualTools.indexOf(expectedTool, searchFrom);
|
|
4229
4228
|
if (index === -1) return false;
|
|
4230
|
-
|
|
4231
|
-
}
|
|
4232
|
-
for (let i = 1; i < expectedIndices.length; i++) {
|
|
4233
|
-
const currentIndex = expectedIndices[i];
|
|
4234
|
-
const prevIndex = expectedIndices[i - 1];
|
|
4235
|
-
if (currentIndex !== void 0 && prevIndex !== void 0 && currentIndex <= prevIndex) return false;
|
|
4229
|
+
searchFrom = index + 1;
|
|
4236
4230
|
}
|
|
4237
4231
|
return true;
|
|
4238
4232
|
}
|
|
@@ -4356,7 +4350,7 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
|
|
|
4356
4350
|
actualTrajectory,
|
|
4357
4351
|
expectedTrajectory: void 0,
|
|
4358
4352
|
comparison: void 0,
|
|
4359
|
-
actualStepNames: actualTrajectory.steps.map((s) => s.name),
|
|
4353
|
+
actualStepNames: require_scorers_utils.toToolLevelSteps(actualTrajectory.steps).map((s) => s.name),
|
|
4360
4354
|
expectedStepNames: [],
|
|
4361
4355
|
error: "No expected trajectory provided (pass via options or dataset item expectedTrajectory)"
|
|
4362
4356
|
};
|
|
@@ -4371,7 +4365,7 @@ function createTrajectoryAccuracyScorerCode(options = {}) {
|
|
|
4371
4365
|
actualTrajectory,
|
|
4372
4366
|
expectedTrajectory: { steps: resolvedExpectedSteps },
|
|
4373
4367
|
comparison,
|
|
4374
|
-
actualStepNames: actualTrajectory.steps.map((s) => s.name),
|
|
4368
|
+
actualStepNames: require_scorers_utils.toToolLevelSteps(actualTrajectory.steps).map((s) => s.name),
|
|
4375
4369
|
expectedStepNames: resolvedExpectedSteps.map((s) => s.name)
|
|
4376
4370
|
};
|
|
4377
4371
|
}).generateScore(({ results }) => {
|
|
@@ -4394,12 +4388,13 @@ function evaluateNestedExpectations(expectedSteps, actualSteps, weights = {
|
|
|
4394
4388
|
blacklist: .1
|
|
4395
4389
|
}) {
|
|
4396
4390
|
const results = [];
|
|
4397
|
-
const
|
|
4391
|
+
const matchedSteps = /* @__PURE__ */ new Set();
|
|
4392
|
+
const toolLevelSteps = require_scorers_utils.toToolLevelSteps(actualSteps);
|
|
4393
|
+
const containerNames = new Set(actualSteps.filter((s) => s.stepType === "model_generation" || s.stepType === "processor_run").map((s) => s.name));
|
|
4398
4394
|
for (const expectedStep of expectedSteps) {
|
|
4399
4395
|
if (!expectedStep.children) continue;
|
|
4400
|
-
const
|
|
4401
|
-
|
|
4402
|
-
if (matchIndex >= 0) matchedIndices.add(matchIndex);
|
|
4396
|
+
const actualStep = (expectedStep.stepType === "model_generation" || expectedStep.stepType === "processor_run" || !expectedStep.stepType && containerNames.has(expectedStep.name) ? actualSteps : toolLevelSteps).find((s) => !matchedSteps.has(s) && s.name === expectedStep.name && (!expectedStep.stepType || s.stepType === expectedStep.stepType));
|
|
4397
|
+
if (actualStep) matchedSteps.add(actualStep);
|
|
4403
4398
|
if (!actualStep) {
|
|
4404
4399
|
const expectedStepCount = expectedStep.children.steps?.length ?? 0;
|
|
4405
4400
|
results.push({
|