@mastra/evals 1.10.8-alpha.0 → 1.10.8-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/scorers/code/tool-call-accuracy/index.d.ts.map +1 -1
- package/dist/scorers/code/trajectory/index.d.ts.map +1 -1
- package/dist/scorers/llm/trajectory/index.d.ts.map +1 -1
- package/dist/scorers/prebuilt/index.cjs +11 -16
- package/dist/scorers/prebuilt/index.cjs.map +1 -1
- package/dist/scorers/prebuilt/index.js +12 -17
- package/dist/scorers/prebuilt/index.js.map +1 -1
- package/dist/scorers/utils.cjs +80 -44
- package/dist/scorers/utils.cjs.map +1 -1
- package/dist/scorers/utils.d.ts +14 -3
- package/dist/scorers/utils.d.ts.map +1 -1
- package/dist/scorers/utils.js +80 -45
- package/dist/scorers/utils.js.map +1 -1
- package/package.json +6 -7
package/dist/scorers/utils.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { MastraDBMessage } from '@mastra/core/agent';
|
|
2
|
-
import type { ExpectedStep, ScorerRunInputForAgent, ScorerRunOutputForAgent, ScoringInput, TrajectoryExpectation, Trajectory } from '@mastra/core/evals';
|
|
2
|
+
import type { ExpectedStep, ScorerRunInputForAgent, ScorerRunOutputForAgent, ScoringInput, TrajectoryExpectation, TrajectoryStep, Trajectory } from '@mastra/core/evals';
|
|
3
3
|
import { RequestContext } from '@mastra/core/request-context';
|
|
4
4
|
export type ScorerRunInputForLLMJudge = ScorerRunInputForAgent | string | {
|
|
5
5
|
inputMessages?: unknown[];
|
|
@@ -571,13 +571,24 @@ export type ToolResultInfo = {
|
|
|
571
571
|
*/
|
|
572
572
|
export declare function extractToolResults(output: ScorerRunOutputForAgent): ToolResultInfo[];
|
|
573
573
|
export { extractTrajectory } from '@mastra/core/evals';
|
|
574
|
+
/**
|
|
575
|
+
* Returns the tool-level view of one sibling list of trajectory steps.
|
|
576
|
+
*
|
|
577
|
+
* Trace-derived trajectories (`extractTrajectoryFromTrace`) wrap tool calls in
|
|
578
|
+
* `model_generation` steps and interleave `processor_run` steps, while message-derived
|
|
579
|
+
* trajectories list tool calls flat. This drops `processor_run` steps and replaces each
|
|
580
|
+
* `model_generation` step with its (recursively flattened) children, so both shapes yield
|
|
581
|
+
* the same list. Other steps are returned as the same objects, with their children untouched.
|
|
582
|
+
*/
|
|
583
|
+
export declare function toToolLevelSteps(steps: TrajectoryStep[]): TrajectoryStep[];
|
|
574
584
|
/**
|
|
575
585
|
* Compares two trajectories and returns detailed comparison results.
|
|
576
586
|
*
|
|
577
587
|
* This is the core comparison logic used by trajectory scorers. It supports
|
|
578
588
|
* strict and non-strict ordering, optional step data comparison, and loop detection.
|
|
579
589
|
*
|
|
580
|
-
* @param
|
|
590
|
+
* @param rawActual - The trajectory the agent actually took. Trace-derived trajectories are
|
|
591
|
+
* compared at tool level (see {@link toToolLevelSteps}).
|
|
581
592
|
* @param expected - The expected trajectory to compare against
|
|
582
593
|
* @param options - Comparison configuration options
|
|
583
594
|
* @returns Detailed comparison results including match scores and diagnostics
|
|
@@ -592,7 +603,7 @@ export { extractTrajectory } from '@mastra/core/evals';
|
|
|
592
603
|
* // result.score = 1.0
|
|
593
604
|
* ```
|
|
594
605
|
*/
|
|
595
|
-
export declare function compareTrajectories(
|
|
606
|
+
export declare function compareTrajectories(rawActual: Trajectory, expected: Trajectory | {
|
|
596
607
|
steps: ExpectedStep[];
|
|
597
608
|
}, options?: {
|
|
598
609
|
ordering?: 'strict' | 'relaxed' | 'unordered';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/scorers/utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EACV,YAAY,EACZ,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,EACZ,qBAAqB,
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/scorers/utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EACV,YAAY,EACZ,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,EACZ,qBAAqB,EACrB,cAAc,EACd,UAAU,EACX,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,MAAM,MAAM,yBAAyB,GACjC,sBAAsB,GACtB,MAAM,GACN;IACE,aAAa,CAAC,EAAE,OAAO,EAAE,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AAEN,MAAM,MAAM,0BAA0B,GAClC,uBAAuB,GACvB,MAAM,GACN,OAAO,EAAE,GACT;IACE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AAEN;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAsBlF;AA0BD,eAAO,MAAM,wBAAwB,UAAW,OAAO,KAAG,KAAK,IAAI,sBAQlE,CAAC;AAYF,eAAO,MAAM,yBAAyB,WAAY,OAAO,KAAG,MAAM,IAAI,uBAErE,CAAC;AA8BF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,kBAAkB,QAAS,MAAM,WAE7C,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAEnF;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACrB,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,cAAc,EAAE;QACd,yBAAyB;QACzB,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,QAAQ,GAAG;IAC3C,oDAAoD;IACpD,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,aAAa,UACjB,MAAM,UACL,MAAM,sBACM,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,mBACtB,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KACnC,YAOF,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,0BAA0B,WAAY,OAAO,KAAG,MAAM,GAAG,SAarE,CAAC;AAIF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,kCAAkC,WACrC,OAAO,YACL;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,KACjC,MAAM,GAAG,SAoBX,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,6BAA6B,WAAY,OAAO,KAAG,MAAM,EAqDrE,CAAC;AAEF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,uBAAuB,WAAY,OAAO,KAAG,MAGzD,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,gCAAgC,YAAa,OAAO,uBA8BhE,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,eAAO,MAAM,yBAAyB,YAAa,uBAAuB,KAAG,MAAM,GAAG,SAgCrF,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,eAAO,MAAM,oBAAoB,uEAQ9B;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,GAAG,cAAc,GAAG,QAAQ,GAAG,cAAc,CAAC;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,KAAG;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC;IACxC,KAAK,EAAE,MAAM,GAAG,cAAc,GAAG,QAAQ,GAAG,cAAc,CAAC;IAC3D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;CAWnB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,wBAAgB,iBAAiB,CAAC,EAChC,OAAO,EACP,IAAI,EACJ,EAAmB,EACnB,eAAoB,EACpB,KAAK,GACN,EAAE;IACD,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;IACtC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,KAAK,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC1B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC7B,KAAK,EAAE,GAAG,CAAC;KACZ,CAAC,CAAC;IACH;;;;OAIG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC;CAC7C,GAAG,eAAe,CAoBlB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,eAAO,MAAM,kBAAkB,gHAQ5B;IACD,aAAa,CAAC,EAAE,sBAAsB,CAAC,eAAe,CAAC,CAAC;IACxD,MAAM,EAAE,uBAAuB,CAAC;IAChC,kBAAkB,CAAC,EAAE,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;IAClE,cAAc,CAAC,EAAE,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;IAC1D,oBAAoB,CAAC,EAAE,sBAAsB,CAAC,sBAAsB,CAAC,CAAC;IACtE,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,KAAG;IACF,KAAK,EAAE,sBAAsB,CAAC;IAC9B,MAAM,EAAE,uBAAuB,CAAC;IAChC,cAAc,EAAE,cAAc,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;CAaf,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,uBAAuB,wIASjC;IACD,aAAa,CAAC,EAAE,sBAAsB,CAAC,eAAe,CAAC,CAAC;IACxD,UAAU,EAAE,UAAU,CAAC;IACvB,kBAAkB,CAAC,EAAE,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;IAClE,cAAc,CAAC,EAAE,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;IAC1D,oBAAoB,CAAC,EAAE,sBAAsB,CAAC,sBAAsB,CAAC,CAAC;IACtE,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kBAAkB,CAAC,EAAE,qBAAqB,CAAC;CAC5C,KAAG;IACF,KAAK,EAAE,sBAAsB,CAAC;IAC9B,MAAM,EAAE,UAAU,CAAC;IACnB,cAAc,EAAE,cAAc,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,qBAAqB,CAAC;CAc5C,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,eAAe,GAAG,SAAS,oEA6BxE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,uBAAuB,GAAG;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,aAAa,EAAE,YAAY,EAAE,CAAA;CAAE,CA+BpH;AAED;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,oBAAoB,aAAc,sBAAsB,GAAG,SAAS,KAAG,MAAM,EAEzF,CAAC;AAEF;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,4BAA4B,cAAe,uBAAuB,KAAG,MAAM,EAEvF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,kCAAkC;IAClC,MAAM,EAAE,GAAG,CAAC;CACb,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,uBAAuB,GAAG,cAAc,EAAE,CAsBpF;AAID,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAEvD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CAW1E;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,UAAU,EACrB,QAAQ,EAAE,UAAU,GAAG;IAAE,KAAK,EAAE,YAAY,EAAE,CAAA;CAAE,EAChD,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,CAAC;IAC9C,kBAAkB,CAAC,EAAE,OAAO,CAAC;CACzB,GACL,0BAA0B,CA4D5B;AAED;;GAEG;AACH,MAAM,MAAM,0BAA0B,GAAG;IACvC,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,qCAAqC;IACrC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,yCAAyC;IACzC,gBAAgB,EAAE,MAAM,CAAC;IACzB,kEAAkE;IAClE,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,4DAA4D;IAC5D,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,yDAAyD;IACzD,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,yDAAyD;IACzD,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB,CAAC;AAqOF;;GAEG;AACH,MAAM,MAAM,0BAA0B,GAAG;IACvC,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,cAAc,EAAE,OAAO,CAAC;IACxB,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,eAAe,EAAE,OAAO,CAAC;IACzB,qCAAqC;IACrC,eAAe,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,kBAAkB,EAAE,OAAO,CAAC;IAC5B,qEAAqE;IACrE,cAAc,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD,CAAC;AAEF;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,UAAU,EAAE,UAAU,EACtB,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,OAAO,CAAC;CACvB,GACL,0BAA0B,CAwH5B;AAID;;GAEG;AACH,MAAM,MAAM,yBAAyB,GAAG;IACtC,sDAAsD;IACtD,KAAK,EAAE,MAAM,CAAC;IACd,mDAAmD;IACnD,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,4CAA4C;IAC5C,iBAAiB,EAAE,MAAM,EAAE,EAAE,CAAC;CAC/B,CAAC;AAEF;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,UAAU,EACtB,OAAO,GAAE;IACP,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,oBAAoB,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;CAC9B,GACL,yBAAyB,CA6D3B;AAID;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,qBAAqB,EAAE,OAAO,CAAC;IAC/B,wCAAwC;IACxC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,6CAA6C;IAC7C,mBAAmB,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,yBAAyB,GAAG;IACtC,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAC;IACd,qCAAqC;IACrC,QAAQ,EAAE,kBAAkB,EAAE,CAAC;IAC/B,+CAA+C;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,8CAA8C;IAC9C,mBAAmB,EAAE,MAAM,EAAE,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,UAAU,EAAE,UAAU,EACtB,OAAO,GAAE;IACP,iBAAiB,CAAC,EAAE,MAAM,CAAC;CACvB,GACL,yBAAyB,CA0E3B"}
|
package/dist/scorers/utils.js
CHANGED
|
@@ -703,12 +703,34 @@ function extractToolResults(output) {
|
|
|
703
703
|
return results;
|
|
704
704
|
}
|
|
705
705
|
/**
|
|
706
|
+
* Returns the tool-level view of one sibling list of trajectory steps.
|
|
707
|
+
*
|
|
708
|
+
* Trace-derived trajectories (`extractTrajectoryFromTrace`) wrap tool calls in
|
|
709
|
+
* `model_generation` steps and interleave `processor_run` steps, while message-derived
|
|
710
|
+
* trajectories list tool calls flat. This drops `processor_run` steps and replaces each
|
|
711
|
+
* `model_generation` step with its (recursively flattened) children, so both shapes yield
|
|
712
|
+
* the same list. Other steps are returned as the same objects, with their children untouched.
|
|
713
|
+
*/
|
|
714
|
+
function toToolLevelSteps(steps) {
|
|
715
|
+
const result = [];
|
|
716
|
+
for (const step of steps) {
|
|
717
|
+
if (step.stepType === "processor_run") continue;
|
|
718
|
+
if (step.stepType === "model_generation") {
|
|
719
|
+
if (step.children?.length) result.push(...toToolLevelSteps(step.children));
|
|
720
|
+
continue;
|
|
721
|
+
}
|
|
722
|
+
result.push(step);
|
|
723
|
+
}
|
|
724
|
+
return result;
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
706
727
|
* Compares two trajectories and returns detailed comparison results.
|
|
707
728
|
*
|
|
708
729
|
* This is the core comparison logic used by trajectory scorers. It supports
|
|
709
730
|
* strict and non-strict ordering, optional step data comparison, and loop detection.
|
|
710
731
|
*
|
|
711
|
-
* @param
|
|
732
|
+
* @param rawActual - The trajectory the agent actually took. Trace-derived trajectories are
|
|
733
|
+
* compared at tool level (see {@link toToolLevelSteps}).
|
|
712
734
|
* @param expected - The expected trajectory to compare against
|
|
713
735
|
* @param options - Comparison configuration options
|
|
714
736
|
* @returns Detailed comparison results including match scores and diagnostics
|
|
@@ -723,9 +745,14 @@ function extractToolResults(output) {
|
|
|
723
745
|
* // result.score = 1.0
|
|
724
746
|
* ```
|
|
725
747
|
*/
|
|
726
|
-
function compareTrajectories(
|
|
748
|
+
function compareTrajectories(rawActual, expected, options = {}) {
|
|
727
749
|
const { allowRepeatedSteps = true, ordering = "relaxed" } = options;
|
|
728
750
|
const normalizedExpected = { steps: expected.steps };
|
|
751
|
+
const containerNames = new Set(rawActual.steps.filter((s) => s.stepType === "model_generation" || s.stepType === "processor_run").map((s) => s.name));
|
|
752
|
+
const actual = normalizedExpected.steps.some((s) => s.stepType === "model_generation" || s.stepType === "processor_run" || !s.stepType && containerNames.has(s.name)) ? rawActual : {
|
|
753
|
+
...rawActual,
|
|
754
|
+
steps: toToolLevelSteps(rawActual.steps)
|
|
755
|
+
};
|
|
729
756
|
if (normalizedExpected.steps.length === 0) return {
|
|
730
757
|
score: actual.steps.length === 0 ? 1 : 0,
|
|
731
758
|
matchedSteps: 0,
|
|
@@ -945,7 +972,7 @@ function checkTrajectoryEfficiency(trajectory, options = {}) {
|
|
|
945
972
|
maxTotalTokens,
|
|
946
973
|
maxTotalDurationMs
|
|
947
974
|
})) if (limit !== void 0 && !isMeasurement(limit)) throw new Error(`Invalid ${name} budget: expected a finite nonnegative number`);
|
|
948
|
-
const totalSteps = trajectory.steps.length;
|
|
975
|
+
const totalSteps = toToolLevelSteps(trajectory.steps).length;
|
|
949
976
|
let totalTokens = 0;
|
|
950
977
|
let modelGenerations = 0;
|
|
951
978
|
const remaining = [...trajectory.steps];
|
|
@@ -966,12 +993,12 @@ function checkTrajectoryEfficiency(trajectory, options = {}) {
|
|
|
966
993
|
const redundantCalls = [];
|
|
967
994
|
const siblingLists = noRedundantCalls ? [trajectory.steps] : [];
|
|
968
995
|
while (siblingLists.length > 0) {
|
|
969
|
-
const steps = siblingLists.pop();
|
|
996
|
+
const steps = toToolLevelSteps(siblingLists.pop());
|
|
970
997
|
for (const step of steps) if (step.children?.length) siblingLists.push(step.children);
|
|
971
|
-
|
|
972
|
-
|
|
998
|
+
let prev;
|
|
999
|
+
for (let i = 0; i < steps.length; i++) {
|
|
973
1000
|
const curr = steps[i];
|
|
974
|
-
if (prev.name === curr.name && prev.stepType === curr.stepType && (prev.stepType === "tool_call" || prev.stepType === "mcp_tool_call")) {
|
|
1001
|
+
if (prev && prev.name === curr.name && prev.stepType === curr.stepType && (prev.stepType === "tool_call" || prev.stepType === "mcp_tool_call")) {
|
|
975
1002
|
const prevArgs = prev.toolArgs;
|
|
976
1003
|
const currArgs = curr.toolArgs;
|
|
977
1004
|
try {
|
|
@@ -981,6 +1008,7 @@ function checkTrajectoryEfficiency(trajectory, options = {}) {
|
|
|
981
1008
|
});
|
|
982
1009
|
} catch {}
|
|
983
1010
|
}
|
|
1011
|
+
prev = curr;
|
|
984
1012
|
}
|
|
985
1013
|
}
|
|
986
1014
|
const overStepBudget = maxSteps !== void 0 && totalSteps > maxSteps;
|
|
@@ -1011,16 +1039,19 @@ function checkTrajectoryBlacklist(trajectory, options = {}) {
|
|
|
1011
1039
|
const { blacklistedTools = [], blacklistedSequences = [] } = options;
|
|
1012
1040
|
const violatedTools = [];
|
|
1013
1041
|
const violatedSequences = [];
|
|
1014
|
-
const siblingLists = [];
|
|
1015
1042
|
const allNames = /* @__PURE__ */ new Set();
|
|
1016
|
-
const pending = [trajectory.steps];
|
|
1043
|
+
const pending = [...trajectory.steps];
|
|
1017
1044
|
while (pending.length > 0) {
|
|
1018
|
-
const
|
|
1045
|
+
const step = pending.pop();
|
|
1046
|
+
allNames.add(step.name);
|
|
1047
|
+
if (step.children?.length) pending.push(...step.children);
|
|
1048
|
+
}
|
|
1049
|
+
const siblingLists = [];
|
|
1050
|
+
const pendingLists = [trajectory.steps];
|
|
1051
|
+
while (pendingLists.length > 0) {
|
|
1052
|
+
const steps = toToolLevelSteps(pendingLists.pop());
|
|
1019
1053
|
siblingLists.push(steps.map((s) => s.name));
|
|
1020
|
-
for (const step of steps)
|
|
1021
|
-
allNames.add(step.name);
|
|
1022
|
-
if (step.children?.length) pending.push(step.children);
|
|
1023
|
-
}
|
|
1054
|
+
for (const step of steps) if (step.children?.length) pendingLists.push(step.children);
|
|
1024
1055
|
}
|
|
1025
1056
|
for (const forbidden of blacklistedTools) if (allNames.has(forbidden)) violatedTools.push(forbidden);
|
|
1026
1057
|
const containsSequence = (names, sequence) => {
|
|
@@ -1051,42 +1082,46 @@ function analyzeToolFailures(trajectory, options = {}) {
|
|
|
1051
1082
|
const { maxRetriesPerTool = 2 } = options;
|
|
1052
1083
|
const patterns = [];
|
|
1053
1084
|
let totalRetries = 0;
|
|
1054
|
-
|
|
1055
|
-
|
|
1085
|
+
let totalToolCalls = 0;
|
|
1086
|
+
const pending = [trajectory.steps];
|
|
1087
|
+
while (pending.length > 0) {
|
|
1088
|
+
const steps = toToolLevelSteps(pending.shift());
|
|
1089
|
+
for (const step of steps) if (step.children?.length) pending.push(step.children);
|
|
1090
|
+
const toolCallSteps = steps.filter((s) => s.stepType === "tool_call" || s.stepType === "mcp_tool_call");
|
|
1091
|
+
totalToolCalls += toolCallSteps.length;
|
|
1092
|
+
let i = 0;
|
|
1093
|
+
while (i < toolCallSteps.length) {
|
|
1094
|
+
const currentTool = toolCallSteps[i];
|
|
1095
|
+
let retryCount = 0;
|
|
1096
|
+
let j = i + 1;
|
|
1097
|
+
while (j < toolCallSteps.length && toolCallSteps[j].name === currentTool.name) {
|
|
1098
|
+
if (toolCallSteps[j - 1].success === false) retryCount++;
|
|
1099
|
+
j++;
|
|
1100
|
+
}
|
|
1101
|
+
if (retryCount > 0) {
|
|
1102
|
+
const nextDifferentTool = j < toolCallSteps.length ? toolCallSteps[j] : void 0;
|
|
1103
|
+
const lastSuccess = toolCallSteps[j - 1].success !== false;
|
|
1104
|
+
patterns.push({
|
|
1105
|
+
toolName: currentTool.name,
|
|
1106
|
+
retryCount,
|
|
1107
|
+
fellBackToAlternative: nextDifferentTool !== void 0 && !lastSuccess,
|
|
1108
|
+
alternativeTool: nextDifferentTool !== void 0 && !lastSuccess ? nextDifferentTool.name : void 0,
|
|
1109
|
+
eventuallySucceeded: lastSuccess
|
|
1110
|
+
});
|
|
1111
|
+
totalRetries += retryCount;
|
|
1112
|
+
}
|
|
1113
|
+
i = j;
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
if (totalToolCalls === 0) return {
|
|
1056
1117
|
score: 1,
|
|
1057
1118
|
patterns: [],
|
|
1058
1119
|
totalRetries: 0,
|
|
1059
1120
|
excessiveRetryTools: []
|
|
1060
1121
|
};
|
|
1061
|
-
let i = 0;
|
|
1062
|
-
while (i < toolCallSteps.length) {
|
|
1063
|
-
const currentTool = toolCallSteps[i];
|
|
1064
|
-
let retryCount = 0;
|
|
1065
|
-
let j = i + 1;
|
|
1066
|
-
while (j < toolCallSteps.length && toolCallSteps[j].name === currentTool.name) {
|
|
1067
|
-
if (toolCallSteps[j - 1].success === false) retryCount++;
|
|
1068
|
-
j++;
|
|
1069
|
-
}
|
|
1070
|
-
if (retryCount > 0) {
|
|
1071
|
-
const nextDifferentTool = j < toolCallSteps.length ? toolCallSteps[j] : void 0;
|
|
1072
|
-
const lastSuccess = toolCallSteps[j - 1].success !== false;
|
|
1073
|
-
patterns.push({
|
|
1074
|
-
toolName: currentTool.name,
|
|
1075
|
-
retryCount,
|
|
1076
|
-
fellBackToAlternative: nextDifferentTool !== void 0 && !lastSuccess,
|
|
1077
|
-
alternativeTool: nextDifferentTool !== void 0 && !lastSuccess ? nextDifferentTool.name : void 0,
|
|
1078
|
-
eventuallySucceeded: lastSuccess
|
|
1079
|
-
});
|
|
1080
|
-
totalRetries += retryCount;
|
|
1081
|
-
}
|
|
1082
|
-
i = j;
|
|
1083
|
-
}
|
|
1084
1122
|
const excessiveRetryTools = patterns.filter((p) => p.retryCount > maxRetriesPerTool).map((p) => p.toolName);
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
const excessRetries = patterns.reduce((sum, p) => sum + Math.max(0, p.retryCount - maxRetriesPerTool), 0);
|
|
1088
|
-
score = Math.max(0, 1 - excessRetries * .2);
|
|
1089
|
-
}
|
|
1123
|
+
const excessRetries = patterns.reduce((sum, p) => sum + Math.max(0, p.retryCount - maxRetriesPerTool), 0);
|
|
1124
|
+
const score = Math.max(0, 1 - excessRetries * .2);
|
|
1090
1125
|
return {
|
|
1091
1126
|
score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
|
|
1092
1127
|
patterns,
|
|
@@ -1095,6 +1130,6 @@ function analyzeToolFailures(trajectory, options = {}) {
|
|
|
1095
1130
|
};
|
|
1096
1131
|
}
|
|
1097
1132
|
//#endregion
|
|
1098
|
-
export { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, createAgentTestRun, createTestMessage, createTestRun, createToolInvocation, createTrajectoryTestRun, extractAgentResponseMessages, extractInputMessages, extractToolCalls, extractToolResults, extractTrajectory, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getConversationHistoryFromRunInput, getReasoningFromRunOutput, getSystemMessagesFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isCloserTo, isScorerRunInputForAgent, isScorerRunOutputForAgent, mergeToolInvocations, roundToTwoDecimals };
|
|
1133
|
+
export { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, createAgentTestRun, createTestMessage, createTestRun, createToolInvocation, createTrajectoryTestRun, extractAgentResponseMessages, extractInputMessages, extractToolCalls, extractToolResults, extractTrajectory, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getConversationHistoryFromRunInput, getReasoningFromRunOutput, getSystemMessagesFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isCloserTo, isScorerRunInputForAgent, isScorerRunOutputForAgent, mergeToolInvocations, roundToTwoDecimals, toToolLevelSteps };
|
|
1099
1134
|
|
|
1100
1135
|
//# sourceMappingURL=utils.js.map
|