@memberjunction/testing-engine 3.4.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/variable-resolver.test.js +14 -16
- package/dist/__tests__/variable-resolver.test.js.map +1 -1
- package/dist/drivers/AgentEvalDriver.d.ts +2 -2
- package/dist/drivers/AgentEvalDriver.js +17 -19
- package/dist/drivers/AgentEvalDriver.js.map +1 -1
- package/dist/drivers/BaseTestDriver.d.ts +1 -1
- package/dist/drivers/BaseTestDriver.js +8 -12
- package/dist/drivers/BaseTestDriver.js.map +1 -1
- package/dist/drivers/index.d.ts +2 -2
- package/dist/drivers/index.js +2 -18
- package/dist/drivers/index.js.map +1 -1
- package/dist/engine/TestEngine.d.ts +3 -3
- package/dist/engine/TestEngine.js +32 -36
- package/dist/engine/TestEngine.js.map +1 -1
- package/dist/index.d.ts +15 -15
- package/dist/index.js +15 -31
- package/dist/index.js.map +1 -1
- package/dist/oracles/ExactMatchOracle.d.ts +2 -2
- package/dist/oracles/ExactMatchOracle.js +1 -5
- package/dist/oracles/ExactMatchOracle.js.map +1 -1
- package/dist/oracles/IOracle.d.ts +1 -1
- package/dist/oracles/IOracle.js +1 -2
- package/dist/oracles/IOracle.js.map +1 -1
- package/dist/oracles/LLMJudgeOracle.d.ts +2 -2
- package/dist/oracles/LLMJudgeOracle.js +8 -12
- package/dist/oracles/LLMJudgeOracle.js.map +1 -1
- package/dist/oracles/SQLValidatorOracle.d.ts +2 -2
- package/dist/oracles/SQLValidatorOracle.js +3 -7
- package/dist/oracles/SQLValidatorOracle.js.map +1 -1
- package/dist/oracles/SchemaValidatorOracle.d.ts +2 -2
- package/dist/oracles/SchemaValidatorOracle.js +1 -5
- package/dist/oracles/SchemaValidatorOracle.js.map +1 -1
- package/dist/oracles/TraceValidatorOracle.d.ts +2 -2
- package/dist/oracles/TraceValidatorOracle.js +3 -7
- package/dist/oracles/TraceValidatorOracle.js.map +1 -1
- package/dist/oracles/index.d.ts +6 -6
- package/dist/oracles/index.js +6 -22
- package/dist/oracles/index.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -2
- package/dist/types.js.map +1 -1
- package/dist/utils/cost-calculator.js +8 -19
- package/dist/utils/cost-calculator.js.map +1 -1
- package/dist/utils/execution-context.js +4 -33
- package/dist/utils/execution-context.js.map +1 -1
- package/dist/utils/result-formatter.d.ts +1 -1
- package/dist/utils/result-formatter.js +16 -30
- package/dist/utils/result-formatter.js.map +1 -1
- package/dist/utils/scoring.d.ts +1 -1
- package/dist/utils/scoring.js +7 -17
- package/dist/utils/scoring.js.map +1 -1
- package/dist/utils/variable-resolver.d.ts +3 -3
- package/dist/utils/variable-resolver.d.ts.map +1 -1
- package/dist/utils/variable-resolver.js +6 -11
- package/dist/utils/variable-resolver.js.map +1 -1
- package/package.json +19 -18
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SQLValidatorOracle.js","sourceRoot":"","sources":["../../src/oracles/SQLValidatorOracle.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"SQLValidatorOracle.js","sourceRoot":"","sources":["../../src/oracles/SQLValidatorOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAkC,MAAM,sBAAsB,CAAC;AAIhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAM,OAAO,kBAAkB;IAA/B;QACa,SAAI,GAAG,cAAc,CAAC;IA6NnC,CAAC;IA3NG;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,MAAoB;QACnD,IAAI,CAAC;YACD,6CAA6C;YAC7C,MAAM,WAAW,GAAK,KAAK,CAAC,cAAsB,EAAE,cAIjD,IAAI,EAAE,CAAC;YAEV,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,gEAAgE;iBAC5E,CAAC;YACN,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,eAAe;YAC/D,MAAM,OAAO,GAOR,EAAE,CAAC;YAER,gCAAgC;YAChC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACnC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAC5C,UAAU,EACV,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,WAAW,CACpB,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC9B,CAAC;YAED,0CAA0C;YAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;YACzD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;YAClC,MAAM,KAAK,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;YAEzE,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM;gBACN,KAAK;gBACL,OAAO,EAAE,UAAU;oBACf,CAAC,CAAC,GAAG,WAAW,IAAI,UAAU,uBAAuB;oBACrD,CAAC,CAAC,mCAAmC,WAAW,IAAI,UAAU,GAAG;gBACrE,OAAO,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;aACpC,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,yBAA0B,KAAe,CAAC,OAAO,EAAE;aAC/D,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAC3B,UAA0E,EAC1E,YAAqB,EACrB,WAAqB;QASrB,IAAI,CAAC;YACD,0DAA0D;YAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YAEjE,+CAA+C;YAC/C,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAgC,CAAC;YAE7D,wBAAwB;YACxB,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,UAAU,CAC5C,GAAG,EACH,SAAS,EAAG,oDAAoD;YAChE,EAAE,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,gBAAgB,EAAE,EAC3D,WAAW,CACd,CAAC;YAEF,qEAAqE;YACrE,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAEtD,+BAA+B;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;YAE5E,OAAO;gBACH,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,gBAAgB;gBACvD,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,MAAM;gBACN,QAAQ,EAAE,UAAU,CAAC,cAAc;gBACnC,MAAM,EAAE,YAAY;aACvB,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,gBAAgB;gBACvD,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,UAAU,CAAC,cAAc;gBACnC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAG,KAAe,CAAC,OAAO;aAClC,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,iBAAiB,CAAC,GAAW,EAAE,YAAqB;QACxD,IAAI,CAAC,YAAY,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;YACpD,OAAO,GAAG,CAAC;QACf,CAAC;QAED,IAAI,MAAM,GAAG,GAAG,CAAC;QACjB,MAAM,SAAS,GAAG,YAAuC,CAAC;QAE1D,uDAAuD;QACvD,MAAM,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAE9E,2CAA2C;YAC3C,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,cAAc,CAAC,CAAC;YAEhE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,sCAAsC;gBACtC,MAAM,YAAY,GAAG,OAAO,KAAK,KAAK,QAAQ;oBAC1C,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG;oBAClC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAEpB,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,SAAS,EAAE,EAAE,GAAG,CAAC,EAAE,YAAY,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,aAAa,CAAC,OAAkB;QACpC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,iDAAiD;QACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAA4B,CAAC;YAClD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAE9B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC;YAED,OAAO,GAAG,CAAC;QACf,CAAC;QAED,gBAAgB;QAChB,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACK,cAAc,CAAC,QAAiB,EAAE,MAAe;QACrD,wBAAwB;QACxB,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS,CAAC;QACnD,CAAC;QAED,iBAAiB;QACjB,IAAI,OAAO,QAAQ,KAAK,SAAS,EAAE,CAAC;YAChC,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;QACxC,CAAC;QAED,gBAAgB;QAChB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;QACvC,CAAC;QAED,gBAAgB;QAChB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;QACvC,CAAC;QAED,sBAAsB;QACtB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ"}
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* @fileoverview Schema validation oracle implementation
|
|
3
3
|
* @module @memberjunction/testing-engine
|
|
4
4
|
*/
|
|
5
|
-
import { IOracle } from './IOracle';
|
|
6
|
-
import { OracleInput, OracleConfig, OracleResult } from '../types';
|
|
5
|
+
import { IOracle } from './IOracle.js';
|
|
6
|
+
import { OracleInput, OracleConfig, OracleResult } from '../types.js';
|
|
7
7
|
/**
|
|
8
8
|
* Schema Validator Oracle.
|
|
9
9
|
*
|
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/**
|
|
3
2
|
* @fileoverview Schema validation oracle implementation
|
|
4
3
|
* @module @memberjunction/testing-engine
|
|
5
4
|
*/
|
|
6
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.SchemaValidatorOracle = void 0;
|
|
8
5
|
/**
|
|
9
6
|
* Schema Validator Oracle.
|
|
10
7
|
*
|
|
@@ -33,7 +30,7 @@ exports.SchemaValidatorOracle = void 0;
|
|
|
33
30
|
* }, {});
|
|
34
31
|
* ```
|
|
35
32
|
*/
|
|
36
|
-
class SchemaValidatorOracle {
|
|
33
|
+
export class SchemaValidatorOracle {
|
|
37
34
|
constructor() {
|
|
38
35
|
this.type = 'schema-validate';
|
|
39
36
|
}
|
|
@@ -189,5 +186,4 @@ class SchemaValidatorOracle {
|
|
|
189
186
|
return 'unknown';
|
|
190
187
|
}
|
|
191
188
|
}
|
|
192
|
-
exports.SchemaValidatorOracle = SchemaValidatorOracle;
|
|
193
189
|
//# sourceMappingURL=SchemaValidatorOracle.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SchemaValidatorOracle.js","sourceRoot":"","sources":["../../src/oracles/SchemaValidatorOracle.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"SchemaValidatorOracle.js","sourceRoot":"","sources":["../../src/oracles/SchemaValidatorOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,OAAO,qBAAqB;IAAlC;QACa,SAAI,GAAG,iBAAiB,CAAC;IAgMtC,CAAC;IA9LG;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,MAAoB;QACnD,IAAI,CAAC;YACD,wCAAwC;YACxC,MAAM,MAAM,GAAI,KAAK,CAAC,cAAsB,EAAE,cAAc,CAAC;YAC7D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,gDAAgD;iBAC5D,CAAC;YACN,CAAC;YAED,wCAAwC;YACxC,MAAM,gBAAgB,GAAG,IAAI,CAAC,qBAAqB,CAC/C,KAAK,CAAC,YAAY,EAClB,MAAM,EACN,MAAM,CAAC,MAAiB,CAC3B,CAAC;YAEF,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChC,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,GAAG;oBACV,OAAO,EAAE,gCAAgC;iBAC5C,CAAC;YACN,CAAC;iBAAM,CAAC;gBACJ,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,6BAA6B,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;oBACnE,OAAO,EAAE,EAAE,gBAAgB,EAAE;iBAChC,CAAC;YACN,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,4BAA6B,KAAe,CAAC,OAAO,EAAE;aAClE,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,qBAAqB,CACzB,IAAa,EACb,MAA+B,EAC/B,SAAkB,KAAK;QAEvB,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,+CAA+C;QAC/C,oDAAoD;QACpD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAEzD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,aAAa,CACjB,KAAc,EACd,MAA+B,EAC/B,IAAY,EACZ,MAAgB,EAChB,MAAe;QAEf,aAAa;QACb,MAAM,YAAY,GAAG,MAAM,CAAC,IAAc,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAEvC,IAAI,YAAY,IAAI,UAAU,KAAK,YAAY,EAAE,CAAC;YAC9C,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,mBAAmB,YAAY,SAAS,UAAU,EAAE,CAAC,CAAC;YACzE,OAAO;QACX,CAAC;QAED,wCAAwC;QACxC,IAAI,UAAU,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,KAAgC,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAoB,IAAI,EAAE,CAAC;YACnD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAqD,IAAI,EAAE,CAAC;YAEtF,wBAAwB;YACxB,KAAK,MAAM,YAAY,IAAI,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,CAAC,YAAY,IAAI,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,gCAAgC,YAAY,GAAG,CAAC,CAAC;gBACxE,CAAC;YACL,CAAC;YAED,sBAAsB;YACtB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3C,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC/E,CAAC;qBAAM,IAAI,MAAM,EAAE,CAAC;oBAChB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,0BAA0B,GAAG,GAAG,CAAC,CAAC;gBACzD,CAAC;YACL,CAAC;QACL,CAAC;QAED,oBAAoB;QACpB,IAAI,UAAU,KAAK,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACzC,MAAM,GAAG,GAAG,KAAkB,CAAC;YAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,KAAgC,CAAC;YAE3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;QAED,oBAAoB;QACpB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,UAAU,GAAG,MAAM,CAAC,IAAiB,CAAC;YAC5C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CACP,GAAG,IAAI,2BAA2B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,GAAG,CAC7E,CAAC;YACN,CAAC;QACL,CAAC;QAED,wBAAwB;QACxB,IAAI,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,OAAiB,CAAC,CAAC;YACrD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAe,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,mCAAmC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;QAED,4BAA4B;QAC5B,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,KAAe,CAAC;YAE5B,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,GAAI,MAAM,CAAC,OAAkB,EAAE,CAAC;gBAC7D,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,WAAW,GAAG,yBAAyB,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YAChF,CAAC;YAED,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,GAAI,MAAM,CAAC,OAAkB,EAAE,CAAC;gBAC7D,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,WAAW,GAAG,4BAA4B,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACnF,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,OAAO,CAAC,KAAc;QAC1B,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,KAAK,CAAC;QAE5B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC1D,CAAC;QAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtB,OAAO,QAAQ,CAAC;QACpB,CAAC;QAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtB,OAAO,QAAQ,CAAC;QACpB,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ"}
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* @fileoverview Trace validation oracle implementation
|
|
3
3
|
* @module @memberjunction/testing-engine
|
|
4
4
|
*/
|
|
5
|
-
import { IOracle } from './IOracle';
|
|
6
|
-
import { OracleInput, OracleConfig, OracleResult } from '../types';
|
|
5
|
+
import { IOracle } from './IOracle.js';
|
|
6
|
+
import { OracleInput, OracleConfig, OracleResult } from '../types.js';
|
|
7
7
|
/**
|
|
8
8
|
* Trace Validator Oracle.
|
|
9
9
|
*
|
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/**
|
|
3
2
|
* @fileoverview Trace validation oracle implementation
|
|
4
3
|
* @module @memberjunction/testing-engine
|
|
5
4
|
*/
|
|
6
|
-
|
|
7
|
-
exports.TraceValidatorOracle = void 0;
|
|
8
|
-
const core_1 = require("@memberjunction/core");
|
|
5
|
+
import { RunView } from '@memberjunction/core';
|
|
9
6
|
/**
|
|
10
7
|
* Trace Validator Oracle.
|
|
11
8
|
*
|
|
@@ -29,7 +26,7 @@ const core_1 = require("@memberjunction/core");
|
|
|
29
26
|
* });
|
|
30
27
|
* ```
|
|
31
28
|
*/
|
|
32
|
-
class TraceValidatorOracle {
|
|
29
|
+
export class TraceValidatorOracle {
|
|
33
30
|
constructor() {
|
|
34
31
|
this.type = 'trace-no-errors';
|
|
35
32
|
}
|
|
@@ -66,7 +63,7 @@ class TraceValidatorOracle {
|
|
|
66
63
|
};
|
|
67
64
|
}
|
|
68
65
|
// Load agent run steps
|
|
69
|
-
const rv = new
|
|
66
|
+
const rv = new RunView();
|
|
70
67
|
const stepsResult = await rv.RunView({
|
|
71
68
|
EntityName: 'MJ: AI Agent Run Steps',
|
|
72
69
|
ExtraFilter: `AgentRunID='${agentRun.ID}'`,
|
|
@@ -155,5 +152,4 @@ class TraceValidatorOracle {
|
|
|
155
152
|
}
|
|
156
153
|
}
|
|
157
154
|
}
|
|
158
|
-
exports.TraceValidatorOracle = TraceValidatorOracle;
|
|
159
155
|
//# sourceMappingURL=TraceValidatorOracle.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TraceValidatorOracle.js","sourceRoot":"","sources":["../../src/oracles/TraceValidatorOracle.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"TraceValidatorOracle.js","sourceRoot":"","sources":["../../src/oracles/TraceValidatorOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAK/C;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,OAAO,oBAAoB;IAAjC;QACa,SAAI,GAAG,iBAAiB,CAAC;IAwItC,CAAC;IAtIG;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,MAAoB;QACnD,IAAI,CAAC;YACD,uBAAuB;YACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,YAAgC,CAAC;YACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,8BAA8B;iBAC1C,CAAC;YACN,CAAC;YAED,yBAAyB;YACzB,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClE,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,iCAAiC,QAAQ,CAAC,MAAM,EAAE;oBAC3D,OAAO,EAAE;wBACL,UAAU,EAAE,QAAQ,CAAC,EAAE;wBACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;qBAC1B;iBACJ,CAAC;YACN,CAAC;YAED,uBAAuB;YACvB,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,OAAO,CAAuB;gBACvD,UAAU,EAAE,wBAAwB;gBACpC,WAAW,EAAE,eAAe,QAAQ,CAAC,EAAE,GAAG;gBAC1C,OAAO,EAAE,cAAc;gBACvB,UAAU,EAAE,eAAe;aAC9B,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;YAEtB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACvB,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,mCAAmC,WAAW,CAAC,YAAY,EAAE;iBACzE,CAAC;YACN,CAAC;YAED,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,IAAI,EAAE,CAAC;YAExC,+BAA+B;YAC/B,MAAM,aAAa,GAAG,MAAM,CAAC,aAAuB,CAAC;YACrD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAkB,CAAC;YAE3C,IAAI,aAAa,IAAI,KAAK,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;gBAChD,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,qBAAqB,aAAa,eAAe,KAAK,CAAC,MAAM,EAAE;oBACxE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,aAAa,EAAE;iBACtD,CAAC;YACN,CAAC;YAED,IAAI,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBACtC,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,oBAAoB,QAAQ,eAAe,KAAK,CAAC,MAAM,EAAE;oBAClE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE;iBACjD,CAAC;YACN,CAAC;YAED,6BAA6B;YAC7B,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,kBAAkB;YACxE,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;YAE9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;oBAC3B,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,QAAQ,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC9E,CAAC;gBAED,2DAA2D;gBAC3D,uEAAuE;YAC3E,CAAC;YAED,mBAAmB;YACnB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,kBAAkB,MAAM,CAAC,MAAM,WAAW;oBACnD,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE;iBACzD,CAAC;YACN,CAAC;YAED,IAAI,CAAC,aAAa,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxC,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,GAAG;oBACV,OAAO,EAAE,kBAAkB,QAAQ,CAAC,MAAM,aAAa;oBACvD,OAAO,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE;iBACjD,CAAC;YACN,CAAC;YAED,UAAU;YACV,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,OAAO,EAAE,oBAAoB,KAAK,CAAC,MAAM,UAAU;gBACnD,OAAO,EAAE;oBACL,SAAS,EAAE,KAAK,CAAC,MAAM;oBACvB,QAAQ,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;iBACvD;aACJ,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,2BAA4B,KAAe,CAAC,OAAO,EAAE;aACjE,CAAC;QACN,CAAC;IACL,CAAC;CACJ"}
|
package/dist/oracles/index.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Oracle implementations for test evaluation
|
|
3
3
|
*/
|
|
4
|
-
export * from './IOracle';
|
|
5
|
-
export * from './SchemaValidatorOracle';
|
|
6
|
-
export * from './TraceValidatorOracle';
|
|
7
|
-
export * from './LLMJudgeOracle';
|
|
8
|
-
export * from './ExactMatchOracle';
|
|
9
|
-
export * from './SQLValidatorOracle';
|
|
4
|
+
export * from './IOracle.js';
|
|
5
|
+
export * from './SchemaValidatorOracle.js';
|
|
6
|
+
export * from './TraceValidatorOracle.js';
|
|
7
|
+
export * from './LLMJudgeOracle.js';
|
|
8
|
+
export * from './ExactMatchOracle.js';
|
|
9
|
+
export * from './SQLValidatorOracle.js';
|
|
10
10
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/oracles/index.js
CHANGED
|
@@ -1,26 +1,10 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/**
|
|
3
2
|
* Oracle implementations for test evaluation
|
|
4
3
|
*/
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
Object.defineProperty(o, k2, desc);
|
|
12
|
-
}) : (function(o, m, k, k2) {
|
|
13
|
-
if (k2 === undefined) k2 = k;
|
|
14
|
-
o[k2] = m[k];
|
|
15
|
-
}));
|
|
16
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
17
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
18
|
-
};
|
|
19
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
|
-
__exportStar(require("./IOracle"), exports);
|
|
21
|
-
__exportStar(require("./SchemaValidatorOracle"), exports);
|
|
22
|
-
__exportStar(require("./TraceValidatorOracle"), exports);
|
|
23
|
-
__exportStar(require("./LLMJudgeOracle"), exports);
|
|
24
|
-
__exportStar(require("./ExactMatchOracle"), exports);
|
|
25
|
-
__exportStar(require("./SQLValidatorOracle"), exports);
|
|
4
|
+
export * from './IOracle.js';
|
|
5
|
+
export * from './SchemaValidatorOracle.js';
|
|
6
|
+
export * from './TraceValidatorOracle.js';
|
|
7
|
+
export * from './LLMJudgeOracle.js';
|
|
8
|
+
export * from './ExactMatchOracle.js';
|
|
9
|
+
export * from './SQLValidatorOracle.js';
|
|
26
10
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/oracles/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/oracles/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,WAAW,CAAC;AAC1B,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { UserInfo } from '@memberjunction/core';
|
|
8
8
|
import { TestEntity, TestRunEntity, AIAgentRunEntity } from '@memberjunction/core-entities';
|
|
9
|
-
import { IOracle } from './oracles/IOracle';
|
|
9
|
+
import { IOracle } from './oracles/IOracle.js';
|
|
10
10
|
export { TestLogMessage, TestProgress, TestRunOptions, SuiteRunOptions, OracleResult, TestRunResult, TestSuiteRunResult, ScoringWeights, ValidationResult, ValidationError, ValidationWarning, RunContextDetails, OracleConfig, TestVariableDataType, TestVariableValueSource, TestVariablePossibleValue, TestVariableDefinition, TestTypeVariablesSchema, TestVariableOverride, TestVariablesConfig, TestSuiteVariablesConfig, ResolvedTestVariables, TestVariableValue } from '@memberjunction/testing-engine-base';
|
|
11
11
|
import { TestRunOptions, OracleResult, ResolvedTestVariables } from '@memberjunction/testing-engine-base';
|
|
12
12
|
/**
|
package/dist/types.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/**
|
|
3
2
|
* Core type definitions for the Testing Engine
|
|
4
3
|
*
|
|
5
4
|
* Note: UI-safe types are defined in @memberjunction/testing-engine-base
|
|
6
5
|
* This file contains execution-specific types that depend on engine internals
|
|
7
6
|
*/
|
|
8
|
-
|
|
7
|
+
export {};
|
|
9
8
|
//# sourceMappingURL=types.js.map
|
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/**
|
|
3
2
|
* @fileoverview Cost calculation utilities for test execution
|
|
4
3
|
* @module @memberjunction/testing-engine
|
|
5
4
|
*/
|
|
6
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.calculateCostEfficiency = exports.estimateSuiteCost = exports.calculateCostBreakdown = exports.calculateOracleCosts = exports.formatCost = exports.calculateAverageCost = exports.aggregateTestRunCosts = exports.calculateAgentRunCost = void 0;
|
|
8
5
|
/**
|
|
9
6
|
* Calculate total cost from AI Agent Run.
|
|
10
7
|
*
|
|
@@ -14,21 +11,19 @@ exports.calculateCostEfficiency = exports.estimateSuiteCost = exports.calculateC
|
|
|
14
11
|
* @param contextUser - User context
|
|
15
12
|
* @returns Total cost in dollars
|
|
16
13
|
*/
|
|
17
|
-
async function calculateAgentRunCost(agentRun, contextUser) {
|
|
14
|
+
export async function calculateAgentRunCost(agentRun, contextUser) {
|
|
18
15
|
// Agent run already has TotalCost calculated
|
|
19
16
|
return agentRun.TotalCost || 0;
|
|
20
17
|
}
|
|
21
|
-
exports.calculateAgentRunCost = calculateAgentRunCost;
|
|
22
18
|
/**
|
|
23
19
|
* Calculate cost from multiple test runs.
|
|
24
20
|
*
|
|
25
21
|
* @param testRunCosts - Array of individual test run costs
|
|
26
22
|
* @returns Total cost
|
|
27
23
|
*/
|
|
28
|
-
function aggregateTestRunCosts(testRunCosts) {
|
|
24
|
+
export function aggregateTestRunCosts(testRunCosts) {
|
|
29
25
|
return testRunCosts.reduce((sum, cost) => sum + cost, 0);
|
|
30
26
|
}
|
|
31
|
-
exports.aggregateTestRunCosts = aggregateTestRunCosts;
|
|
32
27
|
/**
|
|
33
28
|
* Calculate average cost per test.
|
|
34
29
|
*
|
|
@@ -36,10 +31,9 @@ exports.aggregateTestRunCosts = aggregateTestRunCosts;
|
|
|
36
31
|
* @param testCount - Number of tests
|
|
37
32
|
* @returns Average cost per test
|
|
38
33
|
*/
|
|
39
|
-
function calculateAverageCost(totalCost, testCount) {
|
|
34
|
+
export function calculateAverageCost(totalCost, testCount) {
|
|
40
35
|
return testCount > 0 ? totalCost / testCount : 0;
|
|
41
36
|
}
|
|
42
|
-
exports.calculateAverageCost = calculateAverageCost;
|
|
43
37
|
/**
|
|
44
38
|
* Format cost for display.
|
|
45
39
|
*
|
|
@@ -47,7 +41,7 @@ exports.calculateAverageCost = calculateAverageCost;
|
|
|
47
41
|
* @param precision - Decimal places (default: 4)
|
|
48
42
|
* @returns Formatted cost string
|
|
49
43
|
*/
|
|
50
|
-
function formatCost(cost, precision = 4) {
|
|
44
|
+
export function formatCost(cost, precision = 4) {
|
|
51
45
|
if (cost === 0) {
|
|
52
46
|
return '$0.00';
|
|
53
47
|
}
|
|
@@ -56,7 +50,6 @@ function formatCost(cost, precision = 4) {
|
|
|
56
50
|
}
|
|
57
51
|
return `$${cost.toFixed(precision)}`;
|
|
58
52
|
}
|
|
59
|
-
exports.formatCost = formatCost;
|
|
60
53
|
/**
|
|
61
54
|
* Calculate cost per oracle evaluation.
|
|
62
55
|
*
|
|
@@ -65,7 +58,7 @@ exports.formatCost = formatCost;
|
|
|
65
58
|
* @param oracleResults - Oracle results with cost details
|
|
66
59
|
* @returns Map of oracle type to cost
|
|
67
60
|
*/
|
|
68
|
-
function calculateOracleCosts(oracleResults) {
|
|
61
|
+
export function calculateOracleCosts(oracleResults) {
|
|
69
62
|
const costs = new Map();
|
|
70
63
|
for (const result of oracleResults) {
|
|
71
64
|
const cost = result.details?.llmCost || 0;
|
|
@@ -73,7 +66,6 @@ function calculateOracleCosts(oracleResults) {
|
|
|
73
66
|
}
|
|
74
67
|
return costs;
|
|
75
68
|
}
|
|
76
|
-
exports.calculateOracleCosts = calculateOracleCosts;
|
|
77
69
|
/**
|
|
78
70
|
* Calculate cost breakdown by component.
|
|
79
71
|
*
|
|
@@ -81,7 +73,7 @@ exports.calculateOracleCosts = calculateOracleCosts;
|
|
|
81
73
|
* @param oracleCosts - Costs by oracle type
|
|
82
74
|
* @returns Cost breakdown
|
|
83
75
|
*/
|
|
84
|
-
function calculateCostBreakdown(testRunCost, oracleCosts) {
|
|
76
|
+
export function calculateCostBreakdown(testRunCost, oracleCosts) {
|
|
85
77
|
const oracleTotal = Array.from(oracleCosts.values()).reduce((sum, c) => sum + c, 0);
|
|
86
78
|
const agentExecution = Math.max(0, testRunCost - oracleTotal);
|
|
87
79
|
const breakdown = [
|
|
@@ -103,7 +95,6 @@ function calculateCostBreakdown(testRunCost, oracleCosts) {
|
|
|
103
95
|
breakdown
|
|
104
96
|
};
|
|
105
97
|
}
|
|
106
|
-
exports.calculateCostBreakdown = calculateCostBreakdown;
|
|
107
98
|
/**
|
|
108
99
|
* Estimate cost for test suite.
|
|
109
100
|
*
|
|
@@ -111,10 +102,9 @@ exports.calculateCostBreakdown = calculateCostBreakdown;
|
|
|
111
102
|
* @param avgCostPerTest - Average cost per test
|
|
112
103
|
* @returns Estimated total cost
|
|
113
104
|
*/
|
|
114
|
-
function estimateSuiteCost(testCount, avgCostPerTest) {
|
|
105
|
+
export function estimateSuiteCost(testCount, avgCostPerTest) {
|
|
115
106
|
return testCount * avgCostPerTest;
|
|
116
107
|
}
|
|
117
|
-
exports.estimateSuiteCost = estimateSuiteCost;
|
|
118
108
|
/**
|
|
119
109
|
* Calculate cost efficiency metrics.
|
|
120
110
|
*
|
|
@@ -123,7 +113,7 @@ exports.estimateSuiteCost = estimateSuiteCost;
|
|
|
123
113
|
* @param totalTests - Total number of tests
|
|
124
114
|
* @returns Cost efficiency metrics
|
|
125
115
|
*/
|
|
126
|
-
function calculateCostEfficiency(totalCost, passedTests, totalTests) {
|
|
116
|
+
export function calculateCostEfficiency(totalCost, passedTests, totalTests) {
|
|
127
117
|
const costPerTest = totalTests > 0 ? totalCost / totalTests : 0;
|
|
128
118
|
const costPerPassedTest = passedTests > 0 ? totalCost / passedTests : 0;
|
|
129
119
|
const costEfficiencyRatio = totalCost > 0 ? passedTests / totalCost : 0;
|
|
@@ -133,5 +123,4 @@ function calculateCostEfficiency(totalCost, passedTests, totalTests) {
|
|
|
133
123
|
costEfficiencyRatio
|
|
134
124
|
};
|
|
135
125
|
}
|
|
136
|
-
exports.calculateCostEfficiency = calculateCostEfficiency;
|
|
137
126
|
//# sourceMappingURL=cost-calculator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cost-calculator.js","sourceRoot":"","sources":["../../src/utils/cost-calculator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cost-calculator.js","sourceRoot":"","sources":["../../src/utils/cost-calculator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACvC,QAA0B,EAC1B,WAAqB;IAErB,6CAA6C;IAC7C,OAAO,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;AACnC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,YAAsB;IACxD,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB,EAAE,SAAiB;IACrE,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,YAAoB,CAAC;IAC1D,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;QACb,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;QAChB,OAAO,IAAI,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IACvC,CAAC;IAED,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;AACzC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAChC,aAGE;IAEF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAExC,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;QAC1C,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CAClC,WAAmB,EACnB,WAAgC;IAOhC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,WAAW,CAAC,CAAC;IAE9D,MAAM,SAAS,GAAG;QACd;YACI,SAAS,EAAE,iBAAiB;YAC5B,IAAI,EAAE,cAAc;YACpB,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SACzE;QACD,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACxD,SAAS,EAAE,WAAW,IAAI,EAAE;YAC5B,IAAI;YACJ,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SAC/D,CAAC,CAAC;KACN,CAAC;IAEF,OAAO;QACH,cAAc;QACd,gBAAgB,EAAE,WAAW;QAC7B,KAAK,EAAE,WAAW;QAClB,SAAS;KACZ,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAiB,EAAE,cAAsB;IACvE,OAAO,SAAS,GAAG,cAAc,CAAC;AACtC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CACnC,SAAiB,EACjB,WAAmB,EACnB,UAAkB;IAMlB,MAAM,WAAW,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,MAAM,iBAAiB,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,mBAAmB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAExE,OAAO;QACH,WAAW;QACX,iBAAiB;QACjB,mBAAmB;KACtB,CAAC;AACN,CAAC"}
|
|
@@ -1,34 +1,8 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/**
|
|
3
2
|
* @fileoverview Utility functions for gathering execution context
|
|
4
3
|
* @module @memberjunction/testing-engine
|
|
5
4
|
*/
|
|
6
|
-
|
|
7
|
-
if (k2 === undefined) k2 = k;
|
|
8
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
9
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
10
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
11
|
-
}
|
|
12
|
-
Object.defineProperty(o, k2, desc);
|
|
13
|
-
}) : (function(o, m, k, k2) {
|
|
14
|
-
if (k2 === undefined) k2 = k;
|
|
15
|
-
o[k2] = m[k];
|
|
16
|
-
}));
|
|
17
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
18
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
19
|
-
}) : function(o, v) {
|
|
20
|
-
o["default"] = v;
|
|
21
|
-
});
|
|
22
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
23
|
-
if (mod && mod.__esModule) return mod;
|
|
24
|
-
var result = {};
|
|
25
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
26
|
-
__setModuleDefault(result, mod);
|
|
27
|
-
return result;
|
|
28
|
-
};
|
|
29
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
30
|
-
exports.getMachineIdentifier = exports.getMachineName = exports.gatherExecutionContext = void 0;
|
|
31
|
-
const os = __importStar(require("os"));
|
|
5
|
+
import * as os from 'os';
|
|
32
6
|
/**
|
|
33
7
|
* Get the primary MAC address for use as a machine identifier.
|
|
34
8
|
* Returns the first non-internal, non-loopback MAC address found.
|
|
@@ -145,7 +119,7 @@ function detectCIProvider() {
|
|
|
145
119
|
*
|
|
146
120
|
* @returns RunContextDetails object with all available context
|
|
147
121
|
*/
|
|
148
|
-
function gatherExecutionContext() {
|
|
122
|
+
export function gatherExecutionContext() {
|
|
149
123
|
const ciInfo = detectCIProvider();
|
|
150
124
|
return {
|
|
151
125
|
osType: os.platform(),
|
|
@@ -157,21 +131,18 @@ function gatherExecutionContext() {
|
|
|
157
131
|
...ciInfo
|
|
158
132
|
};
|
|
159
133
|
}
|
|
160
|
-
exports.gatherExecutionContext = gatherExecutionContext;
|
|
161
134
|
/**
|
|
162
135
|
* Get the hostname of the current machine.
|
|
163
136
|
* @returns Machine hostname
|
|
164
137
|
*/
|
|
165
|
-
function getMachineName() {
|
|
138
|
+
export function getMachineName() {
|
|
166
139
|
return os.hostname();
|
|
167
140
|
}
|
|
168
|
-
exports.getMachineName = getMachineName;
|
|
169
141
|
/**
|
|
170
142
|
* Get the machine identifier (MAC address).
|
|
171
143
|
* @returns MAC address or undefined if not available
|
|
172
144
|
*/
|
|
173
|
-
function getMachineIdentifier() {
|
|
145
|
+
export function getMachineIdentifier() {
|
|
174
146
|
return getMachineId();
|
|
175
147
|
}
|
|
176
|
-
exports.getMachineIdentifier = getMachineIdentifier;
|
|
177
148
|
//# sourceMappingURL=execution-context.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execution-context.js","sourceRoot":"","sources":["../../src/utils/execution-context.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"execution-context.js","sourceRoot":"","sources":["../../src/utils/execution-context.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAGzB;;;;GAIG;AACH,SAAS,YAAY;IACjB,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,SAAS;YAAE,SAAS;QAEzB,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;YAC5B,sDAAsD;YACtD,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,GAAG,KAAK,mBAAmB,EAAE,CAAC;gBACpE,SAAS;YACb,CAAC;YACD,OAAO,KAAK,CAAC,GAAG,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY;IACjB,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,SAAS;YAAE,SAAS;QAEzB,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;YAC5B,wCAAwC;YACxC,IAAI,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC5C,SAAS;YACb,CAAC;YACD,OAAO,KAAK,CAAC,OAAO,CAAC;QACzB,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB;IAOrB,iBAAiB;IACjB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,MAAM,EAAE,CAAC;QACxC,OAAO;YACH,UAAU,EAAE,gBAAgB;YAC5B,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;YACvC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;YAC1C,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;YAClE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB;SACzC,CAAC;IACN,CAAC;IAED,eAAe;IACf,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QAClC,OAAO;YACH,UAAU,EAAE,cAAc;YAC1B,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB;YAC5C,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;YAC1C,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;YACtC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,oCAAoC;SAC7D,CAAC;IACN,CAAC;IAED,UAAU;IACV,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;QAC1B,OAAO;YACH,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;YAChC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;YACrC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW;YACzD,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS;SAClC,CAAC;IACN,CAAC;IAED,WAAW;IACX,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QAClC,OAAO;YACH,UAAU,EAAE,UAAU;YACtB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;YAC1C,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB;YACzC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;YACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB;SACzC,CAAC;IACN,CAAC;IAED,YAAY;IACZ,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,MAAM,EAAE,CAAC;QACnC,OAAO;YACH,UAAU,EAAE,WAAW;YACvB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;YACtC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS;YAClC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;YACtC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB;SAC7C,CAAC;IACN,CAAC;IAED,YAAY;IACZ,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO;YACH,UAAU,EAAE,WAAW;YACvB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;YACrC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB;YAC5C,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;YACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;SACtG,CAAC;IACN,CAAC;IAED,YAAY;IACZ,OAAO,EAAE,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB;IAClC,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;IAElC,OAAO;QACH,MAAM,EAAE,EAAE,CAAC,QAAQ,EAAE;QACrB,SAAS,EAAE,EAAE,CAAC,OAAO,EAAE;QACvB,WAAW,EAAE,OAAO,CAAC,OAAO;QAC5B,QAAQ,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,eAAe,EAAE,CAAC,QAAQ;QAC1D,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,eAAe,EAAE,CAAC,MAAM;QACtD,SAAS,EAAE,YAAY,EAAE;QACzB,GAAG,MAAM;KACZ,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc;IAC1B,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB;IAChC,OAAO,YAAY,EAAE,CAAC;AAC1B,CAAC"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @fileoverview Result formatting utilities for test output
|
|
3
3
|
* @module @memberjunction/testing-engine
|
|
4
4
|
*/
|
|
5
|
-
import { TestRunResult, TestSuiteRunResult, OracleResult } from '../types';
|
|
5
|
+
import { TestRunResult, TestSuiteRunResult, OracleResult } from '../types.js';
|
|
6
6
|
/**
|
|
7
7
|
* Format test run result as human-readable text.
|
|
8
8
|
*
|