@memberjunction/testing-engine 6.2.0-edge.0 → 6.2.0-edge.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/drivers/PromptEvalDriver.js +3 -3
- package/dist/engine/TestEngine.js +7 -7
- package/dist/eval/corpus.d.ts +8 -0
- package/dist/eval/corpus.d.ts.map +1 -1
- package/dist/eval/corpus.js +26 -10
- package/dist/eval/corpus.js.map +1 -1
- package/dist/eval/decision.d.ts +8 -0
- package/dist/eval/decision.d.ts.map +1 -1
- package/dist/eval/decision.js +23 -7
- package/dist/eval/decision.js.map +1 -1
- package/dist/eval/expectation.d.ts +2 -0
- package/dist/eval/expectation.d.ts.map +1 -1
- package/dist/eval/expectation.js +9 -5
- package/dist/eval/expectation.js.map +1 -1
- package/dist/eval/history.d.ts +2 -0
- package/dist/eval/history.d.ts.map +1 -1
- package/dist/eval/history.js +5 -1
- package/dist/eval/history.js.map +1 -1
- package/dist/eval/matchers.d.ts +7 -0
- package/dist/eval/matchers.d.ts.map +1 -1
- package/dist/eval/matchers.js +11 -3
- package/dist/eval/matchers.js.map +1 -1
- package/dist/eval/subAgentTrace.d.ts +8 -6
- package/dist/eval/subAgentTrace.d.ts.map +1 -1
- package/dist/eval/subAgentTrace.js +15 -11
- package/dist/eval/subAgentTrace.js.map +1 -1
- package/dist/eval/wellFormed.d.ts +2 -0
- package/dist/eval/wellFormed.d.ts.map +1 -1
- package/dist/eval/wellFormed.js +5 -1
- package/dist/eval/wellFormed.js.map +1 -1
- package/dist/oracles/AgentDecisionOracle.js +7 -7
- package/dist/oracles/TraceSubAgentValidatorOracle.js +3 -3
- package/dist/utils/cost-calculator.d.ts +34 -0
- package/dist/utils/cost-calculator.d.ts.map +1 -1
- package/dist/utils/cost-calculator.js +40 -8
- package/dist/utils/cost-calculator.js.map +1 -1
- package/dist/utils/execution-context.d.ts +6 -0
- package/dist/utils/execution-context.d.ts.map +1 -1
- package/dist/utils/execution-context.js +15 -3
- package/dist/utils/execution-context.js.map +1 -1
- package/dist/utils/result-formatter.d.ts +33 -0
- package/dist/utils/result-formatter.d.ts.map +1 -1
- package/dist/utils/result-formatter.js +66 -22
- package/dist/utils/result-formatter.js.map +1 -1
- package/dist/utils/scoring.d.ts +20 -0
- package/dist/utils/scoring.d.ts.map +1 -1
- package/dist/utils/scoring.js +35 -7
- package/dist/utils/scoring.js.map +1 -1
- package/dist/utils/variable-resolver.d.ts +20 -1
- package/dist/utils/variable-resolver.d.ts.map +1 -1
- package/dist/utils/variable-resolver.js +58 -20
- package/dist/utils/variable-resolver.js.map +1 -1
- package/package.json +10 -10
package/dist/eval/matchers.js
CHANGED
|
@@ -40,7 +40,7 @@ function deepEqual(a, b) {
|
|
|
40
40
|
return aKeys.every((k, i) => k === bKeys[i] && deepEqual(a[k], b[k]));
|
|
41
41
|
}
|
|
42
42
|
/** Applies one matcher to one value, returning why it failed rather than just that it did. */
|
|
43
|
-
export function
|
|
43
|
+
export function ApplyMatcher(matcher, value) {
|
|
44
44
|
switch (matcher.kind) {
|
|
45
45
|
case 'absent':
|
|
46
46
|
return value === undefined || value === null
|
|
@@ -105,15 +105,23 @@ export function applyMatcher(matcher, value) {
|
|
|
105
105
|
return { passed: false, detail: 'unknown matcher kind' };
|
|
106
106
|
}
|
|
107
107
|
}
|
|
108
|
+
/** @deprecated Use {@link ApplyMatcher}. */
|
|
109
|
+
export function applyMatcher(matcher, value) {
|
|
110
|
+
return ApplyMatcher(matcher, value);
|
|
111
|
+
}
|
|
108
112
|
/** Runs every expectation against one params object. */
|
|
109
|
-
export function
|
|
113
|
+
export function MatchParams(expectations, params) {
|
|
110
114
|
return expectations.map((expectation) => {
|
|
111
115
|
const value = params[expectation.param];
|
|
112
116
|
if (expectation.optional && (value === undefined || value === null)) {
|
|
113
117
|
return { param: expectation.param, matcherKind: expectation.matcher.kind, passed: true };
|
|
114
118
|
}
|
|
115
|
-
const { passed, detail } =
|
|
119
|
+
const { passed, detail } = ApplyMatcher(expectation.matcher, value);
|
|
116
120
|
return { param: expectation.param, matcherKind: expectation.matcher.kind, passed, detail };
|
|
117
121
|
});
|
|
118
122
|
}
|
|
123
|
+
/** @deprecated Use {@link MatchParams}. */
|
|
124
|
+
export function matchParams(expectations, params) {
|
|
125
|
+
return MatchParams(expectations, params);
|
|
126
|
+
}
|
|
119
127
|
//# sourceMappingURL=matchers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/eval/matchers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA6CH,SAAS,QAAQ,CAAC,KAAc;IAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3E,OAAO,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,SAAS,CAAC,CAAU,EAAE,CAAU;IACrC,4FAA4F;IAC5F,yFAAyF;IACzF,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACV,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACzE,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,SAAS,CAAE,CAA6B,CAAC,CAAC,CAAC,EAAG,CAA6B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpI,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,YAAY,CAAC,OAAqB,EAAE,KAAc;IAC9D,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,QAAQ;YACT,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBACxC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,wBAAwB,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC/E,KAAK,OAAO;YACR,OAAO,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;gBAClC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACnG,KAAK,UAAU,CAAC,CAAC,CAAC;YACd,mFAAmF;YACnF,MAAM,EAAE,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBAC1D,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;oBACzC,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC;wBAC7E,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC;YAC5C,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,mCAAmC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACnH,CAAC;QACD,KAAK,gBAAgB;YACjB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBACvD,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,oCAAoC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC3F,KAAK,oBAAoB;YACrB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;gBACzF,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC3G,KAAK,OAAO;YACR,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAClE,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC3G,KAAK,OAAO,CAAC,CAAC,CAAC;YACX,kFAAkF;YAClF,8DAA8D;YAC9D,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACD,UAAU,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iCAAkC,KAAe,CAAC,OAAO,EAAE,EAAE,CAAC;YAClG,CAAC;YACD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBACtD,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,SAAS,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACnH,CAAC;QACD,KAAK,aAAa,CAAC,CAAC,CAAC;YACjB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;YAClF,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC;YAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC;YAClE,OAAO,QAAQ,IAAI,QAAQ;gBACvB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,OAAO,CAAC,GAAG,IAAI,GAAG,SAAS,KAAK,EAAE,EAAE;gBACnG,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACZ,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC;YAC7D,OAAO,MAAM,KAAK,OAAO,CAAC,IAAI;gBAC1B,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,OAAO,CAAC,IAAI,SAAS,MAAM,EAAE,EAAE,CAAC;QACpF,CAAC;QACD;YACI,gFAAgF;YAChF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;IACjE,CAAC;AACL,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,WAAW,CAAC,YAAgC,EAAE,MAA+B;IACzF,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;QACpC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,WAAW,CAAC,QAAQ,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC7F,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACpE,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC/F,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
1
|
+
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/eval/matchers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA6CH,SAAS,QAAQ,CAAC,KAAc;IAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3E,OAAO,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,SAAS,CAAC,CAAU,EAAE,CAAU;IACrC,4FAA4F;IAC5F,yFAAyF;IACzF,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACV,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACzE,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,SAAS,CAAE,CAA6B,CAAC,CAAC,CAAC,EAAG,CAA6B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpI,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,YAAY,CAAC,OAAqB,EAAE,KAAc;IAC9D,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,QAAQ;YACT,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBACxC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,wBAAwB,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC/E,KAAK,OAAO;YACR,OAAO,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;gBAClC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACnG,KAAK,UAAU,CAAC,CAAC,CAAC;YACd,mFAAmF;YACnF,MAAM,EAAE,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBAC1D,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;oBACzC,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC;wBAC7E,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC;YAC5C,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,mCAAmC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACnH,CAAC;QACD,KAAK,gBAAgB;YACjB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBACvD,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,oCAAoC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC3F,KAAK,oBAAoB;YACrB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;gBACzF,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC3G,KAAK,OAAO;YACR,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAClE,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC3G,KAAK,OAAO,CAAC,CAAC,CAAC;YACX,kFAAkF;YAClF,8DAA8D;YAC9D,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACD,UAAU,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iCAAkC,KAAe,CAAC,OAAO,EAAE,EAAE,CAAC;YAClG,CAAC;YACD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBACtD,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,SAAS,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACnH,CAAC;QACD,KAAK,aAAa,CAAC,CAAC,CAAC;YACjB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;YAClF,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC;YAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC;YAClE,OAAO,QAAQ,IAAI,QAAQ;gBACvB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,OAAO,CAAC,GAAG,IAAI,GAAG,SAAS,KAAK,EAAE,EAAE;gBACnG,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACZ,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC;YAC7D,OAAO,MAAM,KAAK,OAAO,CAAC,IAAI;gBAC1B,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;gBAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,OAAO,CAAC,IAAI,SAAS,MAAM,EAAE,EAAE,CAAC;QACpF,CAAC;QACD;YACI,gFAAgF;YAChF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;IACjE,CAAC;AACL,CAAC;AAED,4CAA4C;AAC5C,MAAM,UAAU,YAAY,CAAC,OAAqB,EAAE,KAAc;IAC9D,OAAO,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;AACxC,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,WAAW,CAAC,YAAgC,EAAE,MAA+B;IACzF,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;QACpC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,WAAW,CAAC,QAAQ,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC7F,CAAC;QACD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACpE,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC/F,CAAC,CAAC,CAAC;AACP,CAAC;AAED,2CAA2C;AAC3C,MAAM,UAAU,WAAW,CAAC,YAAgC,EAAE,MAA+B;IACzF,OAAO,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AAC7C,CAAC"}
|
|
@@ -11,15 +11,15 @@
|
|
|
11
11
|
export interface SubAgentTraceFacts {
|
|
12
12
|
/** Every sub-agent dispatched anywhere in the run tree, in dispatch order, duplicates kept. */
|
|
13
13
|
dispatchedAgents: string[];
|
|
14
|
-
/** Loop iterations on the top-level run. See {@link SubAgentTraceConfig.
|
|
14
|
+
/** Loop iterations on the top-level run. See {@link SubAgentTraceConfig.MinIterations}. */
|
|
15
15
|
iterations: number;
|
|
16
16
|
}
|
|
17
17
|
/** The oracle's configuration, exactly as the shipped research-agent tests already supply it. */
|
|
18
18
|
export interface SubAgentTraceConfig {
|
|
19
19
|
/** Every one of these must have been dispatched. Compared case-insensitively. */
|
|
20
|
-
|
|
20
|
+
RequiredAgents?: string[];
|
|
21
21
|
/** None of these may have been dispatched — the "didn't over-delegate" check. */
|
|
22
|
-
|
|
22
|
+
ForbiddenAgents?: string[];
|
|
23
23
|
/**
|
|
24
24
|
* Minimum loop iterations.
|
|
25
25
|
*
|
|
@@ -27,13 +27,13 @@ export interface SubAgentTraceConfig {
|
|
|
27
27
|
* prompt per pass, so prompt steps are the only honest proxy the run tree offers. Sub-agent
|
|
28
28
|
* prompt steps are excluded — they are the *child's* iterations, not the parent's.
|
|
29
29
|
*/
|
|
30
|
-
|
|
30
|
+
MinIterations?: number;
|
|
31
31
|
}
|
|
32
32
|
export interface SubAgentTraceResult {
|
|
33
33
|
passed: boolean;
|
|
34
|
-
|
|
34
|
+
Score: number;
|
|
35
35
|
message: string;
|
|
36
|
-
|
|
36
|
+
Details: {
|
|
37
37
|
dispatchedAgents: string[];
|
|
38
38
|
missingAgents: string[];
|
|
39
39
|
forbiddenDispatched: string[];
|
|
@@ -48,5 +48,7 @@ export interface SubAgentTraceResult {
|
|
|
48
48
|
* is typed by a human while the one in the trace comes from the catalog — a casing difference is
|
|
49
49
|
* never the failure anyone means to report.
|
|
50
50
|
*/
|
|
51
|
+
export declare function EvaluateSubAgentTrace(facts: SubAgentTraceFacts, config: SubAgentTraceConfig): SubAgentTraceResult;
|
|
52
|
+
/** @deprecated Use {@link EvaluateSubAgentTrace}. */
|
|
51
53
|
export declare function evaluateSubAgentTrace(facts: SubAgentTraceFacts, config: SubAgentTraceConfig): SubAgentTraceResult;
|
|
52
54
|
//# sourceMappingURL=subAgentTrace.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"subAgentTrace.d.ts","sourceRoot":"","sources":["../../src/eval/subAgentTrace.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,gCAAgC;AAChC,MAAM,WAAW,kBAAkB;IAC/B,+FAA+F;IAC/F,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,iGAAiG;AACjG,MAAM,WAAW,mBAAmB;IAChC,iFAAiF;IACjF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,iFAAiF;IACjF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,mBAAmB;IAChC,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACL,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,mBAAmB,EAAE,MAAM,EAAE,CAAC;QAC9B,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACL;AAID;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,mBAAmB,GAAG,mBAAmB,CA+CjH"}
|
|
1
|
+
{"version":3,"file":"subAgentTrace.d.ts","sourceRoot":"","sources":["../../src/eval/subAgentTrace.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,gCAAgC;AAChC,MAAM,WAAW,kBAAkB;IAC/B,+FAA+F;IAC/F,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,iGAAiG;AACjG,MAAM,WAAW,mBAAmB;IAChC,iFAAiF;IACjF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,iFAAiF;IACjF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,mBAAmB;IAChC,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACL,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,mBAAmB,EAAE,MAAM,EAAE,CAAC;QAC9B,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACL;AAID;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,mBAAmB,GAAG,mBAAmB,CA+CjH;AAED,qDAAqD;AACrD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,mBAAmB,GAAG,mBAAmB,CAEjH"}
|
|
@@ -15,17 +15,17 @@ const normalize = (name) => name.trim().toLowerCase();
|
|
|
15
15
|
* is typed by a human while the one in the trace comes from the catalog — a casing difference is
|
|
16
16
|
* never the failure anyone means to report.
|
|
17
17
|
*/
|
|
18
|
-
export function
|
|
18
|
+
export function EvaluateSubAgentTrace(facts, config) {
|
|
19
19
|
const dispatched = new Set(facts.dispatchedAgents.map(normalize));
|
|
20
|
-
const missingAgents = (config.
|
|
21
|
-
const forbiddenDispatched = (config.
|
|
22
|
-
const iterationsShort = config.
|
|
20
|
+
const missingAgents = (config.RequiredAgents ?? []).filter((name) => !dispatched.has(normalize(name)));
|
|
21
|
+
const forbiddenDispatched = (config.ForbiddenAgents ?? []).filter((name) => dispatched.has(normalize(name)));
|
|
22
|
+
const iterationsShort = config.MinIterations !== undefined && facts.iterations < config.MinIterations;
|
|
23
23
|
const details = {
|
|
24
24
|
dispatchedAgents: facts.dispatchedAgents,
|
|
25
25
|
missingAgents,
|
|
26
26
|
forbiddenDispatched,
|
|
27
27
|
iterations: facts.iterations,
|
|
28
|
-
minIterations: config.
|
|
28
|
+
minIterations: config.MinIterations
|
|
29
29
|
};
|
|
30
30
|
const problems = [];
|
|
31
31
|
if (missingAgents.length > 0) {
|
|
@@ -35,27 +35,31 @@ export function evaluateSubAgentTrace(facts, config) {
|
|
|
35
35
|
problems.push(`dispatched forbidden sub-agent(s): ${forbiddenDispatched.join(', ')}`);
|
|
36
36
|
}
|
|
37
37
|
if (iterationsShort) {
|
|
38
|
-
problems.push(`expected at least ${config.
|
|
38
|
+
problems.push(`expected at least ${config.MinIterations} iteration(s), saw ${facts.iterations}`);
|
|
39
39
|
}
|
|
40
40
|
if (problems.length > 0) {
|
|
41
41
|
// Partial credit, so a run that got two of three required sub-agents scores above one that
|
|
42
42
|
// got none. The three checks are weighted equally and only the ones configured count.
|
|
43
43
|
const checks = [];
|
|
44
|
-
if (config.
|
|
44
|
+
if (config.RequiredAgents?.length) {
|
|
45
45
|
checks.push(missingAgents.length === 0);
|
|
46
46
|
}
|
|
47
|
-
if (config.
|
|
47
|
+
if (config.ForbiddenAgents?.length) {
|
|
48
48
|
checks.push(forbiddenDispatched.length === 0);
|
|
49
49
|
}
|
|
50
|
-
if (config.
|
|
50
|
+
if (config.MinIterations !== undefined) {
|
|
51
51
|
checks.push(!iterationsShort);
|
|
52
52
|
}
|
|
53
53
|
const score = checks.length === 0 ? 0 : checks.filter(Boolean).length / checks.length;
|
|
54
|
-
return { passed: false, score, message: problems.join('; '), details };
|
|
54
|
+
return { passed: false, Score: score, message: problems.join('; '), Details: details };
|
|
55
55
|
}
|
|
56
56
|
const summary = facts.dispatchedAgents.length > 0
|
|
57
57
|
? `dispatched ${facts.dispatchedAgents.length} sub-agent step(s): ${[...new Set(facts.dispatchedAgents)].join(', ')}`
|
|
58
58
|
: 'no sub-agents dispatched (none required)';
|
|
59
|
-
return { passed: true,
|
|
59
|
+
return { passed: true, Score: 1, message: `${summary}; ${facts.iterations} iteration(s)`, Details: details };
|
|
60
|
+
}
|
|
61
|
+
/** @deprecated Use {@link EvaluateSubAgentTrace}. */
|
|
62
|
+
export function evaluateSubAgentTrace(facts, config) {
|
|
63
|
+
return EvaluateSubAgentTrace(facts, config);
|
|
60
64
|
}
|
|
61
65
|
//# sourceMappingURL=subAgentTrace.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"subAgentTrace.js","sourceRoot":"","sources":["../../src/eval/subAgentTrace.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAuCH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAEtE;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAyB,EAAE,MAA2B;IACxF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;IAElE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvG,MAAM,mBAAmB,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,eAAe,GAAG,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC;IAEtG,MAAM,OAAO,GAAG;QACZ,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,aAAa;QACb,mBAAmB;QACnB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,aAAa,EAAE,MAAM,CAAC,aAAa;KACtC,CAAC;IAEF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC,2CAA2C,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC,sCAAsC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,IAAI,eAAe,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,aAAa,sBAAsB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IACrG,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,2FAA2F;QAC3F,sFAAsF;QACtF,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC;QAClC,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACtF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"subAgentTrace.js","sourceRoot":"","sources":["../../src/eval/subAgentTrace.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAuCH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAEtE;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAyB,EAAE,MAA2B;IACxF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;IAElE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvG,MAAM,mBAAmB,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,eAAe,GAAG,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC;IAEtG,MAAM,OAAO,GAAG;QACZ,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,aAAa;QACb,mBAAmB;QACnB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,aAAa,EAAE,MAAM,CAAC,aAAa;KACtC,CAAC;IAEF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC,2CAA2C,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC,sCAAsC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,IAAI,eAAe,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,aAAa,sBAAsB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IACrG,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,2FAA2F;QAC3F,sFAAsF;QACtF,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC;QAClC,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACtF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC3F,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC7C,CAAC,CAAC,cAAc,KAAK,CAAC,gBAAgB,CAAC,MAAM,uBAAuB,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACrH,CAAC,CAAC,0CAA0C,CAAC;IACjD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,KAAK,KAAK,CAAC,UAAU,eAAe,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AACjH,CAAC;AAED,qDAAqD;AACrD,MAAM,UAAU,qBAAqB,CAAC,KAAyB,EAAE,MAA2B;IACxF,OAAO,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AAChD,CAAC"}
|
|
@@ -37,5 +37,7 @@ export interface WellFormedResult {
|
|
|
37
37
|
};
|
|
38
38
|
}
|
|
39
39
|
/** Judges whether a turn produced usable output. */
|
|
40
|
+
export declare function EvaluateWellFormed(input: WellFormedInput, config?: WellFormedConfig): WellFormedResult;
|
|
41
|
+
/** @deprecated Use {@link EvaluateWellFormed}. */
|
|
40
42
|
export declare function evaluateWellFormed(input: WellFormedInput, config?: WellFormedConfig): WellFormedResult;
|
|
41
43
|
//# sourceMappingURL=wellFormed.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wellFormed.d.ts","sourceRoot":"","sources":["../../src/eval/wellFormed.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAanD,wEAAwE;AACxE,MAAM,WAAW,eAAe;IAC5B,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,qFAAqF;IACrF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,gBAAgB;IAC7B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;;OAKG;IACH,yBAAyB,CAAC,EAAE,MAAM,EAAE,CAAC;CACxC;AAWD,MAAM,WAAW,gBAAgB;IAC7B,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACL,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC/B,QAAQ,EAAE,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACvC,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;QAC/B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,CAAC;CACL;AAED,oDAAoD;AACpD,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,GAAE,gBAAqB,GAAG,gBAAgB,CAqD1G"}
|
|
1
|
+
{"version":3,"file":"wellFormed.d.ts","sourceRoot":"","sources":["../../src/eval/wellFormed.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAanD,wEAAwE;AACxE,MAAM,WAAW,eAAe;IAC5B,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,qFAAqF;IACrF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,gBAAgB;IAC7B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;;OAKG;IACH,yBAAyB,CAAC,EAAE,MAAM,EAAE,CAAC;CACxC;AAWD,MAAM,WAAW,gBAAgB;IAC7B,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACL,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC/B,QAAQ,EAAE,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACvC,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;QAC/B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,CAAC;CACL;AAED,oDAAoD;AACpD,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,GAAE,gBAAqB,GAAG,gBAAgB,CAqD1G;AAED,kDAAkD;AAClD,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,GAAE,gBAAqB,GAAG,gBAAgB,CAE1G"}
|
package/dist/eval/wellFormed.js
CHANGED
|
@@ -32,7 +32,7 @@ const DEFAULT_UNACCEPTABLE_FINISH_REASONS = [
|
|
|
32
32
|
'max_tokens'
|
|
33
33
|
];
|
|
34
34
|
/** Judges whether a turn produced usable output. */
|
|
35
|
-
export function
|
|
35
|
+
export function EvaluateWellFormed(input, config = {}) {
|
|
36
36
|
const { decision } = input;
|
|
37
37
|
const details = {
|
|
38
38
|
kind: decision.kind,
|
|
@@ -80,4 +80,8 @@ export function evaluateWellFormed(input, config = {}) {
|
|
|
80
80
|
}
|
|
81
81
|
return { passed: true, score: 1, message: `usable ${decision.encoding} response (${decision.kind})`, details };
|
|
82
82
|
}
|
|
83
|
+
/** @deprecated Use {@link EvaluateWellFormed}. */
|
|
84
|
+
export function evaluateWellFormed(input, config = {}) {
|
|
85
|
+
return EvaluateWellFormed(input, config);
|
|
86
|
+
}
|
|
83
87
|
//# sourceMappingURL=wellFormed.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wellFormed.js","sourceRoot":"","sources":["../../src/eval/wellFormed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAGjE;;;;;;;;GAQG;AACH,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAS,oBAAoB,CAAC,CAAC;AA6BtE,MAAM,mCAAmC,GAAG;IACxC,yBAAyB;IACzB,gBAAgB;IAChB,QAAQ;IACR,YAAY;IACZ,QAAQ;IACR,YAAY;CACf,CAAC;AAeF,oDAAoD;AACpD,MAAM,UAAU,kBAAkB,CAAC,KAAsB,EAAE,SAA2B,EAAE;IACpF,MAAM,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IAC3B,MAAM,OAAO,GAAG;QACZ,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,IAAI;QACnC,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;KAC3C,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,OAAe,EAAoB,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAEpG,uEAAuE;IACvE,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,qBAAqB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,+BAA+B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,wCAAwC,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,yBAAyB,IAAI,mCAAmC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3H,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACxD,IAAI,MAAM,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,+BAA+B,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvE,MAAM,SAAS,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QAChH,OAAO,IAAI,CAAC,SAAS;YACjB,CAAC,CAAC,kBAAkB,QAAQ,CAAC,QAAQ,qDAAqD,SAAS,2BAA2B;YAC9H,CAAC,CAAC,kBAAkB,QAAQ,CAAC,QAAQ,mCAAmC,CAAC,CAAC;IAClF,CAAC;IAED,8FAA8F;IAC9F,+FAA+F;IAC/F,8FAA8F;IAC9F,+FAA+F;IAC/F,4FAA4F;IAC5F,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,2EAA2E,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,MAAM,CAAC,eAAe,KAAK,IAAI,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ;YACtC,8FAA8F;YAC9F,yEAAyE;YACzE,CAAC,CAAC,iEAAiE;YACnE,CAAC,CAAC,kCAAkC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,UAAU,QAAQ,CAAC,QAAQ,cAAc,QAAQ,CAAC,IAAI,GAAG,EAAE,OAAO,EAAE,CAAC;AACnH,CAAC"}
|
|
1
|
+
{"version":3,"file":"wellFormed.js","sourceRoot":"","sources":["../../src/eval/wellFormed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAGjE;;;;;;;;GAQG;AACH,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAS,oBAAoB,CAAC,CAAC;AA6BtE,MAAM,mCAAmC,GAAG;IACxC,yBAAyB;IACzB,gBAAgB;IAChB,QAAQ;IACR,YAAY;IACZ,QAAQ;IACR,YAAY;CACf,CAAC;AAeF,oDAAoD;AACpD,MAAM,UAAU,kBAAkB,CAAC,KAAsB,EAAE,SAA2B,EAAE;IACpF,MAAM,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IAC3B,MAAM,OAAO,GAAG;QACZ,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,IAAI;QACnC,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;KAC3C,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,OAAe,EAAoB,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAEpG,uEAAuE;IACvE,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,qBAAqB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,+BAA+B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,wCAAwC,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,yBAAyB,IAAI,mCAAmC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3H,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACxD,IAAI,MAAM,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,+BAA+B,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvE,MAAM,SAAS,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QAChH,OAAO,IAAI,CAAC,SAAS;YACjB,CAAC,CAAC,kBAAkB,QAAQ,CAAC,QAAQ,qDAAqD,SAAS,2BAA2B;YAC9H,CAAC,CAAC,kBAAkB,QAAQ,CAAC,QAAQ,mCAAmC,CAAC,CAAC;IAClF,CAAC;IAED,8FAA8F;IAC9F,+FAA+F;IAC/F,8FAA8F;IAC9F,+FAA+F;IAC/F,4FAA4F;IAC5F,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,2EAA2E,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,MAAM,CAAC,eAAe,KAAK,IAAI,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ;YACtC,8FAA8F;YAC9F,yEAAyE;YACzE,CAAC,CAAC,iEAAiE;YACnE,CAAC,CAAC,kCAAkC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,UAAU,QAAQ,CAAC,QAAQ,cAAc,QAAQ,CAAC,IAAI,GAAG,EAAE,OAAO,EAAE,CAAC;AACnH,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,kBAAkB,CAAC,KAAsB,EAAE,SAA2B,EAAE;IACpF,OAAO,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AAC7C,CAAC"}
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
* @fileoverview Agent-decision oracles — the centrepiece of the prompt-eval corpus (test plan §3.2).
|
|
3
3
|
* @module @memberjunction/testing-engine
|
|
4
4
|
*/
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
5
|
+
import { NormalizeDecision } from '../eval/decision.js';
|
|
6
|
+
import { EvaluateCorpusExpectation } from '../eval/corpus.js';
|
|
7
|
+
import { EvaluateWellFormed } from '../eval/wellFormed.js';
|
|
8
8
|
function readActual(input) {
|
|
9
9
|
const actual = input.actualOutput;
|
|
10
10
|
// A driver that produced nothing at all still has to be judged rather than crash the run —
|
|
@@ -36,12 +36,12 @@ export class AgentDecisionOracle {
|
|
|
36
36
|
return { oracleType: this.type, passed: false, score: 0, message: "Configuration requires a 'kind' (the expected decision)" };
|
|
37
37
|
}
|
|
38
38
|
const actual = readActual(input);
|
|
39
|
-
const observed =
|
|
39
|
+
const observed = NormalizeDecision(actual.turn);
|
|
40
40
|
// evaluateCorpusExpectation, NOT evaluateDecision: only the former unwraps `anyOf`.
|
|
41
41
|
// `anyOf` is a corpus-level kind — a case that legitimately admits several decisions —
|
|
42
42
|
// and evaluateDecision would compare the literal string 'anyOf' against the observed
|
|
43
43
|
// kind and fail every time, which is a scored failure the model can never avoid.
|
|
44
|
-
const evaluation =
|
|
44
|
+
const evaluation = EvaluateCorpusExpectation((input.test?.Name ?? 'case'), expectation, observed);
|
|
45
45
|
return {
|
|
46
46
|
oracleType: this.type,
|
|
47
47
|
passed: evaluation.passed,
|
|
@@ -95,8 +95,8 @@ export class ResponseWellFormedOracle {
|
|
|
95
95
|
async evaluate(input, config) {
|
|
96
96
|
try {
|
|
97
97
|
const actual = readActual(input);
|
|
98
|
-
const observed =
|
|
99
|
-
const result =
|
|
98
|
+
const observed = NormalizeDecision(actual.turn);
|
|
99
|
+
const result = EvaluateWellFormed({ decision: observed, finishReason: actual.finishReason, executionError: actual.executionError }, config);
|
|
100
100
|
return { oracleType: this.type, passed: result.passed, score: result.score, message: result.message, details: result.details };
|
|
101
101
|
}
|
|
102
102
|
catch (error) {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* @module @memberjunction/testing-engine
|
|
4
4
|
*/
|
|
5
5
|
import { RunView } from '@memberjunction/core';
|
|
6
|
-
import {
|
|
6
|
+
import { EvaluateSubAgentTrace } from '../eval/subAgentTrace.js';
|
|
7
7
|
/**
|
|
8
8
|
* Validates which sub-agents an agent run dispatched, and how many iterations it took.
|
|
9
9
|
*
|
|
@@ -44,8 +44,8 @@ export class TraceSubAgentValidatorOracle {
|
|
|
44
44
|
return { oracleType: this.type, passed: false, score: 0, message: 'No agent run entity provided' };
|
|
45
45
|
}
|
|
46
46
|
const facts = await this.collectFacts(agentRun.ID, input.contextUser);
|
|
47
|
-
const result =
|
|
48
|
-
return { oracleType: this.type, passed: result.passed, score: result.
|
|
47
|
+
const result = EvaluateSubAgentTrace(facts, config);
|
|
48
|
+
return { oracleType: this.type, passed: result.passed, score: result.Score, message: result.message, details: result.Details };
|
|
49
49
|
}
|
|
50
50
|
catch (error) {
|
|
51
51
|
return {
|
|
@@ -13,6 +13,8 @@ import { MJAIAgentRunEntity } from '@memberjunction/core-entities';
|
|
|
13
13
|
* @param contextUser - User context
|
|
14
14
|
* @returns Total cost in dollars
|
|
15
15
|
*/
|
|
16
|
+
export declare function CalculateAgentRunCost(agentRun: MJAIAgentRunEntity, contextUser: UserInfo): Promise<number>;
|
|
17
|
+
/** @deprecated Use {@link CalculateAgentRunCost}. */
|
|
16
18
|
export declare function calculateAgentRunCost(agentRun: MJAIAgentRunEntity, contextUser: UserInfo): Promise<number>;
|
|
17
19
|
/**
|
|
18
20
|
* Calculate cost from multiple test runs.
|
|
@@ -20,6 +22,8 @@ export declare function calculateAgentRunCost(agentRun: MJAIAgentRunEntity, cont
|
|
|
20
22
|
* @param testRunCosts - Array of individual test run costs
|
|
21
23
|
* @returns Total cost
|
|
22
24
|
*/
|
|
25
|
+
export declare function AggregateTestRunCosts(testRunCosts: number[]): number;
|
|
26
|
+
/** @deprecated Use {@link AggregateTestRunCosts}. */
|
|
23
27
|
export declare function aggregateTestRunCosts(testRunCosts: number[]): number;
|
|
24
28
|
/**
|
|
25
29
|
* Calculate average cost per test.
|
|
@@ -28,6 +32,8 @@ export declare function aggregateTestRunCosts(testRunCosts: number[]): number;
|
|
|
28
32
|
* @param testCount - Number of tests
|
|
29
33
|
* @returns Average cost per test
|
|
30
34
|
*/
|
|
35
|
+
export declare function CalculateAverageCost(totalCost: number, testCount: number): number;
|
|
36
|
+
/** @deprecated Use {@link CalculateAverageCost}. */
|
|
31
37
|
export declare function calculateAverageCost(totalCost: number, testCount: number): number;
|
|
32
38
|
/**
|
|
33
39
|
* Format cost for display.
|
|
@@ -36,6 +42,8 @@ export declare function calculateAverageCost(totalCost: number, testCount: numbe
|
|
|
36
42
|
* @param precision - Decimal places (default: 4)
|
|
37
43
|
* @returns Formatted cost string
|
|
38
44
|
*/
|
|
45
|
+
export declare function FormatCost(cost: number, precision?: number): string;
|
|
46
|
+
/** @deprecated Use {@link FormatCost}. */
|
|
39
47
|
export declare function formatCost(cost: number, precision?: number): string;
|
|
40
48
|
/**
|
|
41
49
|
* Calculate cost per oracle evaluation.
|
|
@@ -45,6 +53,13 @@ export declare function formatCost(cost: number, precision?: number): string;
|
|
|
45
53
|
* @param oracleResults - Oracle results with cost details
|
|
46
54
|
* @returns Map of oracle type to cost
|
|
47
55
|
*/
|
|
56
|
+
export declare function CalculateOracleCosts(oracleResults: Array<{
|
|
57
|
+
oracleType: string;
|
|
58
|
+
details?: {
|
|
59
|
+
llmCost?: number;
|
|
60
|
+
};
|
|
61
|
+
}>): Map<string, number>;
|
|
62
|
+
/** @deprecated Use {@link CalculateOracleCosts}. */
|
|
48
63
|
export declare function calculateOracleCosts(oracleResults: Array<{
|
|
49
64
|
oracleType: string;
|
|
50
65
|
details?: {
|
|
@@ -58,6 +73,17 @@ export declare function calculateOracleCosts(oracleResults: Array<{
|
|
|
58
73
|
* @param oracleCosts - Costs by oracle type
|
|
59
74
|
* @returns Cost breakdown
|
|
60
75
|
*/
|
|
76
|
+
export declare function CalculateCostBreakdown(testRunCost: number, oracleCosts: Map<string, number>): {
|
|
77
|
+
agentExecution: number;
|
|
78
|
+
oracleEvaluation: number;
|
|
79
|
+
total: number;
|
|
80
|
+
breakdown: Array<{
|
|
81
|
+
component: string;
|
|
82
|
+
cost: number;
|
|
83
|
+
percentage: number;
|
|
84
|
+
}>;
|
|
85
|
+
};
|
|
86
|
+
/** @deprecated Use {@link CalculateCostBreakdown}. */
|
|
61
87
|
export declare function calculateCostBreakdown(testRunCost: number, oracleCosts: Map<string, number>): {
|
|
62
88
|
agentExecution: number;
|
|
63
89
|
oracleEvaluation: number;
|
|
@@ -75,6 +101,8 @@ export declare function calculateCostBreakdown(testRunCost: number, oracleCosts:
|
|
|
75
101
|
* @param avgCostPerTest - Average cost per test
|
|
76
102
|
* @returns Estimated total cost
|
|
77
103
|
*/
|
|
104
|
+
export declare function EstimateSuiteCost(testCount: number, avgCostPerTest: number): number;
|
|
105
|
+
/** @deprecated Use {@link EstimateSuiteCost}. */
|
|
78
106
|
export declare function estimateSuiteCost(testCount: number, avgCostPerTest: number): number;
|
|
79
107
|
/**
|
|
80
108
|
* Calculate cost efficiency metrics.
|
|
@@ -84,6 +112,12 @@ export declare function estimateSuiteCost(testCount: number, avgCostPerTest: num
|
|
|
84
112
|
* @param totalTests - Total number of tests
|
|
85
113
|
* @returns Cost efficiency metrics
|
|
86
114
|
*/
|
|
115
|
+
export declare function CalculateCostEfficiency(totalCost: number, passedTests: number, totalTests: number): {
|
|
116
|
+
costPerTest: number;
|
|
117
|
+
costPerPassedTest: number;
|
|
118
|
+
costEfficiencyRatio: number;
|
|
119
|
+
};
|
|
120
|
+
/** @deprecated Use {@link CalculateCostEfficiency}. */
|
|
87
121
|
export declare function calculateCostEfficiency(totalCost: number, passedTests: number, totalTests: number): {
|
|
88
122
|
costPerTest: number;
|
|
89
123
|
costPerPassedTest: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cost-calculator.d.ts","sourceRoot":"","sources":["../../src/utils/cost-calculator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAW,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAuB,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAExF;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACvC,QAAQ,EAAE,kBAAkB,EAC5B,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,CAEpE;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjF;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAU,GAAG,MAAM,CAUtE;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAChC,aAAa,EAAE,KAAK,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAClC,CAAC,GACH,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CASrB;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAClC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACjC;IACC,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7E,CAuBA;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,MAAM,CAEnF;AAED;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACnC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,GACnB;IACC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;CAC/B,CAUA"}
|
|
1
|
+
{"version":3,"file":"cost-calculator.d.ts","sourceRoot":"","sources":["../../src/utils/cost-calculator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAW,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAuB,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAExF;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACvC,QAAQ,EAAE,kBAAkB,EAC5B,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED,qDAAqD;AACrD,wBAAsB,qBAAqB,CACvC,QAAQ,EAAE,kBAAkB,EAC5B,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,MAAM,CAAC,CAEjB;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,CAEpE;AAED,qDAAqD;AACrD,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,CAEpE;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjF;AAED,oDAAoD;AACpD,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjF;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAU,GAAG,MAAM,CAUtE;AAED,0CAA0C;AAC1C,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAU,GAAG,MAAM,CAEtE;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAChC,aAAa,EAAE,KAAK,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAClC,CAAC,GACH,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CASrB;AAED,oDAAoD;AACpD,wBAAgB,oBAAoB,CAChC,aAAa,EAAE,KAAK,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAClC,CAAC,GACH,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAErB;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAClC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACjC;IACC,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7E,CAuBA;AAED,sDAAsD;AACtD,wBAAgB,sBAAsB,CAClC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACjC;IACC,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7E,CAEA;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,MAAM,CAEnF;AAED,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,MAAM,CAEnF;AAED;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACnC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,GACnB;IACC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;CAC/B,CAUA;AAED,uDAAuD;AACvD,wBAAgB,uBAAuB,CACnC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,GACnB;IACC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;CAC/B,CAEA"}
|
|
@@ -11,19 +11,27 @@
|
|
|
11
11
|
* @param contextUser - User context
|
|
12
12
|
* @returns Total cost in dollars
|
|
13
13
|
*/
|
|
14
|
-
export async function
|
|
14
|
+
export async function CalculateAgentRunCost(agentRun, contextUser) {
|
|
15
15
|
// Agent run already has TotalCost calculated
|
|
16
16
|
return agentRun.TotalCost || 0;
|
|
17
17
|
}
|
|
18
|
+
/** @deprecated Use {@link CalculateAgentRunCost}. */
|
|
19
|
+
export async function calculateAgentRunCost(agentRun, contextUser) {
|
|
20
|
+
return CalculateAgentRunCost(agentRun, contextUser);
|
|
21
|
+
}
|
|
18
22
|
/**
|
|
19
23
|
* Calculate cost from multiple test runs.
|
|
20
24
|
*
|
|
21
25
|
* @param testRunCosts - Array of individual test run costs
|
|
22
26
|
* @returns Total cost
|
|
23
27
|
*/
|
|
24
|
-
export function
|
|
28
|
+
export function AggregateTestRunCosts(testRunCosts) {
|
|
25
29
|
return testRunCosts.reduce((sum, cost) => sum + cost, 0);
|
|
26
30
|
}
|
|
31
|
+
/** @deprecated Use {@link AggregateTestRunCosts}. */
|
|
32
|
+
export function aggregateTestRunCosts(testRunCosts) {
|
|
33
|
+
return AggregateTestRunCosts(testRunCosts);
|
|
34
|
+
}
|
|
27
35
|
/**
|
|
28
36
|
* Calculate average cost per test.
|
|
29
37
|
*
|
|
@@ -31,9 +39,13 @@ export function aggregateTestRunCosts(testRunCosts) {
|
|
|
31
39
|
* @param testCount - Number of tests
|
|
32
40
|
* @returns Average cost per test
|
|
33
41
|
*/
|
|
34
|
-
export function
|
|
42
|
+
export function CalculateAverageCost(totalCost, testCount) {
|
|
35
43
|
return testCount > 0 ? totalCost / testCount : 0;
|
|
36
44
|
}
|
|
45
|
+
/** @deprecated Use {@link CalculateAverageCost}. */
|
|
46
|
+
export function calculateAverageCost(totalCost, testCount) {
|
|
47
|
+
return CalculateAverageCost(totalCost, testCount);
|
|
48
|
+
}
|
|
37
49
|
/**
|
|
38
50
|
* Format cost for display.
|
|
39
51
|
*
|
|
@@ -41,7 +53,7 @@ export function calculateAverageCost(totalCost, testCount) {
|
|
|
41
53
|
* @param precision - Decimal places (default: 4)
|
|
42
54
|
* @returns Formatted cost string
|
|
43
55
|
*/
|
|
44
|
-
export function
|
|
56
|
+
export function FormatCost(cost, precision = 4) {
|
|
45
57
|
if (cost === 0) {
|
|
46
58
|
return '$0.00';
|
|
47
59
|
}
|
|
@@ -50,6 +62,10 @@ export function formatCost(cost, precision = 4) {
|
|
|
50
62
|
}
|
|
51
63
|
return `$${cost.toFixed(precision)}`;
|
|
52
64
|
}
|
|
65
|
+
/** @deprecated Use {@link FormatCost}. */
|
|
66
|
+
export function formatCost(cost, precision = 4) {
|
|
67
|
+
return FormatCost(cost, precision);
|
|
68
|
+
}
|
|
53
69
|
/**
|
|
54
70
|
* Calculate cost per oracle evaluation.
|
|
55
71
|
*
|
|
@@ -58,7 +74,7 @@ export function formatCost(cost, precision = 4) {
|
|
|
58
74
|
* @param oracleResults - Oracle results with cost details
|
|
59
75
|
* @returns Map of oracle type to cost
|
|
60
76
|
*/
|
|
61
|
-
export function
|
|
77
|
+
export function CalculateOracleCosts(oracleResults) {
|
|
62
78
|
const costs = new Map();
|
|
63
79
|
for (const result of oracleResults) {
|
|
64
80
|
const cost = result.details?.llmCost || 0;
|
|
@@ -66,6 +82,10 @@ export function calculateOracleCosts(oracleResults) {
|
|
|
66
82
|
}
|
|
67
83
|
return costs;
|
|
68
84
|
}
|
|
85
|
+
/** @deprecated Use {@link CalculateOracleCosts}. */
|
|
86
|
+
export function calculateOracleCosts(oracleResults) {
|
|
87
|
+
return CalculateOracleCosts(oracleResults);
|
|
88
|
+
}
|
|
69
89
|
/**
|
|
70
90
|
* Calculate cost breakdown by component.
|
|
71
91
|
*
|
|
@@ -73,7 +93,7 @@ export function calculateOracleCosts(oracleResults) {
|
|
|
73
93
|
* @param oracleCosts - Costs by oracle type
|
|
74
94
|
* @returns Cost breakdown
|
|
75
95
|
*/
|
|
76
|
-
export function
|
|
96
|
+
export function CalculateCostBreakdown(testRunCost, oracleCosts) {
|
|
77
97
|
const oracleTotal = Array.from(oracleCosts.values()).reduce((sum, c) => sum + c, 0);
|
|
78
98
|
const agentExecution = Math.max(0, testRunCost - oracleTotal);
|
|
79
99
|
const breakdown = [
|
|
@@ -95,6 +115,10 @@ export function calculateCostBreakdown(testRunCost, oracleCosts) {
|
|
|
95
115
|
breakdown
|
|
96
116
|
};
|
|
97
117
|
}
|
|
118
|
+
/** @deprecated Use {@link CalculateCostBreakdown}. */
|
|
119
|
+
export function calculateCostBreakdown(testRunCost, oracleCosts) {
|
|
120
|
+
return CalculateCostBreakdown(testRunCost, oracleCosts);
|
|
121
|
+
}
|
|
98
122
|
/**
|
|
99
123
|
* Estimate cost for test suite.
|
|
100
124
|
*
|
|
@@ -102,9 +126,13 @@ export function calculateCostBreakdown(testRunCost, oracleCosts) {
|
|
|
102
126
|
* @param avgCostPerTest - Average cost per test
|
|
103
127
|
* @returns Estimated total cost
|
|
104
128
|
*/
|
|
105
|
-
export function
|
|
129
|
+
export function EstimateSuiteCost(testCount, avgCostPerTest) {
|
|
106
130
|
return testCount * avgCostPerTest;
|
|
107
131
|
}
|
|
132
|
+
/** @deprecated Use {@link EstimateSuiteCost}. */
|
|
133
|
+
export function estimateSuiteCost(testCount, avgCostPerTest) {
|
|
134
|
+
return EstimateSuiteCost(testCount, avgCostPerTest);
|
|
135
|
+
}
|
|
108
136
|
/**
|
|
109
137
|
* Calculate cost efficiency metrics.
|
|
110
138
|
*
|
|
@@ -113,7 +141,7 @@ export function estimateSuiteCost(testCount, avgCostPerTest) {
|
|
|
113
141
|
* @param totalTests - Total number of tests
|
|
114
142
|
* @returns Cost efficiency metrics
|
|
115
143
|
*/
|
|
116
|
-
export function
|
|
144
|
+
export function CalculateCostEfficiency(totalCost, passedTests, totalTests) {
|
|
117
145
|
const costPerTest = totalTests > 0 ? totalCost / totalTests : 0;
|
|
118
146
|
const costPerPassedTest = passedTests > 0 ? totalCost / passedTests : 0;
|
|
119
147
|
const costEfficiencyRatio = totalCost > 0 ? passedTests / totalCost : 0;
|
|
@@ -123,4 +151,8 @@ export function calculateCostEfficiency(totalCost, passedTests, totalTests) {
|
|
|
123
151
|
costEfficiencyRatio
|
|
124
152
|
};
|
|
125
153
|
}
|
|
154
|
+
/** @deprecated Use {@link CalculateCostEfficiency}. */
|
|
155
|
+
export function calculateCostEfficiency(totalCost, passedTests, totalTests) {
|
|
156
|
+
return CalculateCostEfficiency(totalCost, passedTests, totalTests);
|
|
157
|
+
}
|
|
126
158
|
//# sourceMappingURL=cost-calculator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cost-calculator.js","sourceRoot":"","sources":["../../src/utils/cost-calculator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACvC,QAA4B,EAC5B,WAAqB;IAErB,6CAA6C;IAC7C,OAAO,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;AACnC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,YAAsB;IACxD,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB,EAAE,SAAiB;IACrE,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,YAAoB,CAAC;IAC1D,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;QACb,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;QAChB,OAAO,IAAI,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IACvC,CAAC;IAED,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;AACzC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAChC,aAGE;IAEF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAExC,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;QAC1C,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CAClC,WAAmB,EACnB,WAAgC;IAOhC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,WAAW,CAAC,CAAC;IAE9D,MAAM,SAAS,GAAG;QACd;YACI,SAAS,EAAE,iBAAiB;YAC5B,IAAI,EAAE,cAAc;YACpB,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SACzE;QACD,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACxD,SAAS,EAAE,WAAW,IAAI,EAAE;YAC5B,IAAI;YACJ,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SAC/D,CAAC,CAAC;KACN,CAAC;IAEF,OAAO;QACH,cAAc;QACd,gBAAgB,EAAE,WAAW;QAC7B,KAAK,EAAE,WAAW;QAClB,SAAS;KACZ,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAiB,EAAE,cAAsB;IACvE,OAAO,SAAS,GAAG,cAAc,CAAC;AACtC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CACnC,SAAiB,EACjB,WAAmB,EACnB,UAAkB;IAMlB,MAAM,WAAW,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,MAAM,iBAAiB,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,mBAAmB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAExE,OAAO;QACH,WAAW;QACX,iBAAiB;QACjB,mBAAmB;KACtB,CAAC;AACN,CAAC"}
|
|
1
|
+
{"version":3,"file":"cost-calculator.js","sourceRoot":"","sources":["../../src/utils/cost-calculator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACvC,QAA4B,EAC5B,WAAqB;IAErB,6CAA6C;IAC7C,OAAO,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,qDAAqD;AACrD,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACvC,QAA4B,EAC5B,WAAqB;IAErB,OAAO,qBAAqB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AACxD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,YAAsB;IACxD,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED,qDAAqD;AACrD,MAAM,UAAU,qBAAqB,CAAC,YAAsB;IACxD,OAAO,qBAAqB,CAAC,YAAY,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB,EAAE,SAAiB;IACrE,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,oBAAoB,CAAC,SAAiB,EAAE,SAAiB;IACrE,OAAO,oBAAoB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;AACtD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,YAAoB,CAAC;IAC1D,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;QACb,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;QAChB,OAAO,IAAI,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IACvC,CAAC;IAED,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;AACzC,CAAC;AAED,0CAA0C;AAC1C,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,YAAoB,CAAC;IAC1D,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAChC,aAGE;IAEF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAExC,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;QAC1C,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,oBAAoB,CAChC,aAGE;IAEF,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CAClC,WAAmB,EACnB,WAAgC;IAOhC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,WAAW,CAAC,CAAC;IAE9D,MAAM,SAAS,GAAG;QACd;YACI,SAAS,EAAE,iBAAiB;YAC5B,IAAI,EAAE,cAAc;YACpB,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SACzE;QACD,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACxD,SAAS,EAAE,WAAW,IAAI,EAAE;YAC5B,IAAI;YACJ,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SAC/D,CAAC,CAAC;KACN,CAAC;IAEF,OAAO;QACH,cAAc;QACd,gBAAgB,EAAE,WAAW;QAC7B,KAAK,EAAE,WAAW;QAClB,SAAS;KACZ,CAAC;AACN,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,sBAAsB,CAClC,WAAmB,EACnB,WAAgC;IAOhC,OAAO,sBAAsB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;AAC5D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAiB,EAAE,cAAsB;IACvE,OAAO,SAAS,GAAG,cAAc,CAAC;AACtC,CAAC;AAED,iDAAiD;AACjD,MAAM,UAAU,iBAAiB,CAAC,SAAiB,EAAE,cAAsB;IACvE,OAAO,iBAAiB,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AACxD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CACnC,SAAiB,EACjB,WAAmB,EACnB,UAAkB;IAMlB,MAAM,WAAW,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,MAAM,iBAAiB,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,mBAAmB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAExE,OAAO;QACH,WAAW;QACX,iBAAiB;QACjB,mBAAmB;KACtB,CAAC;AACN,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,uBAAuB,CACnC,SAAiB,EACjB,WAAmB,EACnB,UAAkB;IAMlB,OAAO,uBAAuB,CAAC,SAAS,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AACvE,CAAC"}
|
|
@@ -9,15 +9,21 @@ import { RunContextDetails } from '@memberjunction/testing-engine-base';
|
|
|
9
9
|
*
|
|
10
10
|
* @returns RunContextDetails object with all available context
|
|
11
11
|
*/
|
|
12
|
+
export declare function GatherExecutionContext(): RunContextDetails;
|
|
13
|
+
/** @deprecated Use {@link GatherExecutionContext}. */
|
|
12
14
|
export declare function gatherExecutionContext(): RunContextDetails;
|
|
13
15
|
/**
|
|
14
16
|
* Get the hostname of the current machine.
|
|
15
17
|
* @returns Machine hostname
|
|
16
18
|
*/
|
|
19
|
+
export declare function GetMachineName(): string;
|
|
20
|
+
/** @deprecated Use {@link GetMachineName}. */
|
|
17
21
|
export declare function getMachineName(): string;
|
|
18
22
|
/**
|
|
19
23
|
* Get the machine identifier (MAC address).
|
|
20
24
|
* @returns MAC address or undefined if not available
|
|
21
25
|
*/
|
|
26
|
+
export declare function GetMachineIdentifier(): string | undefined;
|
|
27
|
+
/** @deprecated Use {@link GetMachineIdentifier}. */
|
|
22
28
|
export declare function getMachineIdentifier(): string | undefined;
|
|
23
29
|
//# sourceMappingURL=execution-context.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execution-context.d.ts","sourceRoot":"","sources":["../../src/utils/execution-context.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AA+HxE;;;;;GAKG;AACH,wBAAgB,sBAAsB,IAAI,iBAAiB,CAY1D;AAED;;;GAGG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,GAAG,SAAS,CAEzD"}
|
|
1
|
+
{"version":3,"file":"execution-context.d.ts","sourceRoot":"","sources":["../../src/utils/execution-context.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AA+HxE;;;;;GAKG;AACH,wBAAgB,sBAAsB,IAAI,iBAAiB,CAY1D;AAED,sDAAsD;AACtD,wBAAgB,sBAAsB,IAAI,iBAAiB,CAE1D;AAED;;;GAGG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED,8CAA8C;AAC9C,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,GAAG,SAAS,CAEzD;AAED,oDAAoD;AACpD,wBAAgB,oBAAoB,IAAI,MAAM,GAAG,SAAS,CAEzD"}
|
|
@@ -119,7 +119,7 @@ function detectCIProvider() {
|
|
|
119
119
|
*
|
|
120
120
|
* @returns RunContextDetails object with all available context
|
|
121
121
|
*/
|
|
122
|
-
export function
|
|
122
|
+
export function GatherExecutionContext() {
|
|
123
123
|
const ciInfo = detectCIProvider();
|
|
124
124
|
return {
|
|
125
125
|
osType: os.platform(),
|
|
@@ -131,18 +131,30 @@ export function gatherExecutionContext() {
|
|
|
131
131
|
...ciInfo
|
|
132
132
|
};
|
|
133
133
|
}
|
|
134
|
+
/** @deprecated Use {@link GatherExecutionContext}. */
|
|
135
|
+
export function gatherExecutionContext() {
|
|
136
|
+
return GatherExecutionContext();
|
|
137
|
+
}
|
|
134
138
|
/**
|
|
135
139
|
* Get the hostname of the current machine.
|
|
136
140
|
* @returns Machine hostname
|
|
137
141
|
*/
|
|
138
|
-
export function
|
|
142
|
+
export function GetMachineName() {
|
|
139
143
|
return os.hostname();
|
|
140
144
|
}
|
|
145
|
+
/** @deprecated Use {@link GetMachineName}. */
|
|
146
|
+
export function getMachineName() {
|
|
147
|
+
return GetMachineName();
|
|
148
|
+
}
|
|
141
149
|
/**
|
|
142
150
|
* Get the machine identifier (MAC address).
|
|
143
151
|
* @returns MAC address or undefined if not available
|
|
144
152
|
*/
|
|
145
|
-
export function
|
|
153
|
+
export function GetMachineIdentifier() {
|
|
146
154
|
return getMachineId();
|
|
147
155
|
}
|
|
156
|
+
/** @deprecated Use {@link GetMachineIdentifier}. */
|
|
157
|
+
export function getMachineIdentifier() {
|
|
158
|
+
return GetMachineIdentifier();
|
|
159
|
+
}
|
|
148
160
|
//# sourceMappingURL=execution-context.js.map
|