@memberjunction/testing-cli 5.37.0 → 5.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/compare.d.ts +36 -11
- package/dist/commands/compare.d.ts.map +1 -1
- package/dist/commands/compare.js +464 -29
- package/dist/commands/compare.js.map +1 -1
- package/dist/commands/run.d.ts +5 -0
- package/dist/commands/run.d.ts.map +1 -1
- package/dist/commands/run.js +137 -7
- package/dist/commands/run.js.map +1 -1
- package/dist/commands/suite.d.ts +10 -0
- package/dist/commands/suite.d.ts.map +1 -1
- package/dist/commands/suite.js +94 -6
- package/dist/commands/suite.js.map +1 -1
- package/dist/types.d.ts +34 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/config-loader.d.ts.map +1 -1
- package/dist/utils/config-loader.js +5 -3
- package/dist/utils/config-loader.js.map +1 -1
- package/dist/utils/oracle-module-loader.d.ts +25 -0
- package/dist/utils/oracle-module-loader.d.ts.map +1 -0
- package/dist/utils/oracle-module-loader.js +135 -0
- package/dist/utils/oracle-module-loader.js.map +1 -0
- package/package.json +7 -7
package/dist/commands/run.js
CHANGED
|
@@ -8,6 +8,7 @@ import { SpinnerManager } from '../utils/spinner-manager.js';
|
|
|
8
8
|
import { loadCLIConfig } from '../utils/config-loader.js';
|
|
9
9
|
import { initializeMJProvider, closeMJProvider, getContextUser } from '../lib/mj-provider.js';
|
|
10
10
|
import { parseVariableFlags, getTestVariablesSchema } from '../utils/variable-parser.js';
|
|
11
|
+
import { loadOraclesModule } from '../utils/oracle-module-loader.js';
|
|
11
12
|
/**
|
|
12
13
|
* Run command - Execute a single test or filtered set of tests
|
|
13
14
|
*/
|
|
@@ -36,6 +37,14 @@ export class RunCommand {
|
|
|
36
37
|
// Get engine instance
|
|
37
38
|
const engine = TestEngine.Instance;
|
|
38
39
|
await engine.Config(false, contextUser);
|
|
40
|
+
// Plug in user-supplied oracles before resolving the test.
|
|
41
|
+
if (flags.oraclesModule) {
|
|
42
|
+
const summary = await loadOraclesModule(flags.oraclesModule, engine);
|
|
43
|
+
console.log(`Loaded oracle module ${summary.modulePath} ` +
|
|
44
|
+
`(registered: ${summary.registered.join(', ') || 'none'}` +
|
|
45
|
+
(summary.skipped.length ? `; skipped: ${summary.skipped.length}` : '') +
|
|
46
|
+
')');
|
|
47
|
+
}
|
|
39
48
|
let test;
|
|
40
49
|
if (testId) {
|
|
41
50
|
// Run specific test by ID
|
|
@@ -75,15 +84,47 @@ export class RunCommand {
|
|
|
75
84
|
// Parse variables from --var flags
|
|
76
85
|
const variablesSchema = getTestVariablesSchema(engine, test.ID);
|
|
77
86
|
const variables = parseVariableFlags(flags.var, variablesSchema);
|
|
78
|
-
// Dry run mode
|
|
87
|
+
// Dry run mode — validate everything that would execute, but don't execute.
|
|
88
|
+
// Catches misconfigured oracles, missing variables, and unregistered drivers
|
|
89
|
+
// before burning LLM credits on a doomed run.
|
|
79
90
|
if (flags.dryRun) {
|
|
80
|
-
|
|
81
|
-
console.log(
|
|
82
|
-
console.log(
|
|
83
|
-
|
|
84
|
-
|
|
91
|
+
const issues = this.dryRunValidation(test, engine);
|
|
92
|
+
console.log('');
|
|
93
|
+
console.log(` Test: ${test.Name}`);
|
|
94
|
+
console.log(` ID: ${test.ID}`);
|
|
95
|
+
console.log(` Type: ${test.Type}`);
|
|
96
|
+
console.log(` Environment: ${environment}`);
|
|
97
|
+
console.log(` Status: ${test.Status}`);
|
|
98
|
+
if (variables && Object.keys(variables).length > 0) {
|
|
99
|
+
console.log(` Variables: ${JSON.stringify(variables)}`);
|
|
100
|
+
}
|
|
101
|
+
console.log('');
|
|
102
|
+
if (issues.errors.length > 0) {
|
|
103
|
+
console.log(` Errors (${issues.errors.length}):`);
|
|
104
|
+
for (const e of issues.errors)
|
|
105
|
+
console.log(` ✗ ${e}`);
|
|
106
|
+
console.log('');
|
|
107
|
+
}
|
|
108
|
+
if (issues.warnings.length > 0) {
|
|
109
|
+
console.log(` Warnings (${issues.warnings.length}):`);
|
|
110
|
+
for (const w of issues.warnings)
|
|
111
|
+
console.log(` ⚠ ${w}`);
|
|
112
|
+
console.log('');
|
|
113
|
+
}
|
|
114
|
+
if (issues.info.length > 0) {
|
|
115
|
+
console.log(` Configuration:`);
|
|
116
|
+
for (const i of issues.info)
|
|
117
|
+
console.log(` • ${i}`);
|
|
118
|
+
console.log('');
|
|
119
|
+
}
|
|
120
|
+
if (issues.errors.length === 0) {
|
|
121
|
+
console.log(OutputFormatter.formatSuccess('Dry run passed — test is ready to execute.'));
|
|
85
122
|
}
|
|
86
|
-
|
|
123
|
+
else {
|
|
124
|
+
console.error(OutputFormatter.formatError(`Dry run found ${issues.errors.length} error(s) — fix before executing.`));
|
|
125
|
+
}
|
|
126
|
+
await closeMJProvider();
|
|
127
|
+
process.exit(issues.errors.length === 0 ? 0 : 1);
|
|
87
128
|
}
|
|
88
129
|
// Execute test
|
|
89
130
|
this.spinner.start(`Running test: ${test.Name}...`);
|
|
@@ -139,5 +180,94 @@ export class RunCommand {
|
|
|
139
180
|
process.exit(1);
|
|
140
181
|
}
|
|
141
182
|
}
|
|
183
|
+
/**
|
|
184
|
+
* Validate that a test is ready to execute without actually running it.
|
|
185
|
+
* Catches misconfigured oracles, malformed JSON, and missing drivers.
|
|
186
|
+
*/
|
|
187
|
+
dryRunValidation(test, engine) {
|
|
188
|
+
const errors = [];
|
|
189
|
+
const warnings = [];
|
|
190
|
+
const info = [];
|
|
191
|
+
if (test.Status !== 'Active') {
|
|
192
|
+
warnings.push(`Test status is '${test.Status}' — only 'Active' tests run by default`);
|
|
193
|
+
}
|
|
194
|
+
// Verify the test type and driver exist
|
|
195
|
+
const testType = engine.GetTestTypeByID(test.TypeID);
|
|
196
|
+
if (!testType) {
|
|
197
|
+
errors.push(`TestType not found for TypeID ${test.TypeID}`);
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
info.push(`Driver: ${testType.DriverClass}`);
|
|
201
|
+
}
|
|
202
|
+
// Validate Configuration JSON parses
|
|
203
|
+
let config = null;
|
|
204
|
+
if (test.Configuration) {
|
|
205
|
+
try {
|
|
206
|
+
config = JSON.parse(test.Configuration);
|
|
207
|
+
}
|
|
208
|
+
catch (e) {
|
|
209
|
+
errors.push(`Configuration is not valid JSON: ${e.message}`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
warnings.push('No Configuration defined — test will use defaults');
|
|
214
|
+
}
|
|
215
|
+
// Validate InputDefinition JSON parses
|
|
216
|
+
if (test.InputDefinition) {
|
|
217
|
+
try {
|
|
218
|
+
JSON.parse(test.InputDefinition);
|
|
219
|
+
}
|
|
220
|
+
catch (e) {
|
|
221
|
+
errors.push(`InputDefinition is not valid JSON: ${e.message}`);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
// Validate ExpectedOutcomes JSON parses
|
|
225
|
+
if (test.ExpectedOutcomes) {
|
|
226
|
+
try {
|
|
227
|
+
JSON.parse(test.ExpectedOutcomes);
|
|
228
|
+
}
|
|
229
|
+
catch (e) {
|
|
230
|
+
errors.push(`ExpectedOutcomes is not valid JSON: ${e.message}`);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
// Verify each declared oracle exists in the registry
|
|
234
|
+
if (config?.oracles && Array.isArray(config.oracles)) {
|
|
235
|
+
const registeredOracles = engine.GetOracleTypes();
|
|
236
|
+
const oracleSummary = [];
|
|
237
|
+
for (const oracle of config.oracles) {
|
|
238
|
+
if (!oracle.type) {
|
|
239
|
+
errors.push('Oracle entry missing required "type" field');
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
if (!registeredOracles.includes(oracle.type)) {
|
|
243
|
+
errors.push(`Oracle type '${oracle.type}' is not registered. ` +
|
|
244
|
+
`Available: ${registeredOracles.join(', ')}`);
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
const w = typeof oracle.weight === 'number' ? oracle.weight : 1;
|
|
248
|
+
oracleSummary.push(`${oracle.type} (weight ${w})`);
|
|
249
|
+
}
|
|
250
|
+
if (oracleSummary.length > 0) {
|
|
251
|
+
info.push(`Oracles: ${oracleSummary.join(', ')}`);
|
|
252
|
+
}
|
|
253
|
+
// Sanity-check scoringWeights matches oracles (sum should be ~1.0)
|
|
254
|
+
if (config.scoringWeights) {
|
|
255
|
+
const declaredOracleTypes = new Set(config.oracles.map(o => o.type));
|
|
256
|
+
for (const weightedType of Object.keys(config.scoringWeights)) {
|
|
257
|
+
if (!declaredOracleTypes.has(weightedType)) {
|
|
258
|
+
warnings.push(`scoringWeights references oracle '${weightedType}' that isn't in the oracles array`);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
const sum = Object.values(config.scoringWeights).reduce((a, b) => a + b, 0);
|
|
262
|
+
if (Math.abs(sum - 1.0) > 0.001) {
|
|
263
|
+
warnings.push(`scoringWeights sum to ${sum.toFixed(3)}, expected ~1.0`);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
else if (config) {
|
|
268
|
+
warnings.push('No oracles defined in Configuration — test scoring will use driver defaults');
|
|
269
|
+
}
|
|
270
|
+
return { errors, warnings, info };
|
|
271
|
+
}
|
|
142
272
|
}
|
|
143
273
|
//# sourceMappingURL=run.js.map
|
package/dist/commands/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/commands/run.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAC;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/commands/run.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAC;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AACtF,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAElE;;GAEG;AACH,MAAM,OAAO,UAAU;IAAvB;QACY,YAAO,GAAG,IAAI,cAAc,EAAE,CAAC;IAsR3C,CAAC;IApRG;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,MAA0B,EAAE,KAAe,EAAE,WAAsB;QAC7E,IAAI,CAAC;YACD,4DAA4D;YAC5D,MAAM,oBAAoB,EAAE,CAAC;YAE7B,wDAAwD;YACxD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACf,WAAW,GAAG,MAAM,cAAc,EAAE,CAAC;YACzC,CAAC;YAED,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,aAAa,IAAI,SAAS,CAAC;YACjE,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,kBAAkB,CAAC;YAEnE,sBAAsB;YACtB,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;YACnC,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAExC,2DAA2D;YAC3D,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;gBACrE,OAAO,CAAC,GAAG,CACP,wBAAwB,OAAO,CAAC,UAAU,GAAG;oBACzC,gBAAgB,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE;oBACzD,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtE,GAAG,CACV,CAAC;YACN,CAAC;YAED,IAAI,IAAI,CAAC;YAET,IAAI,MAAM,EAAE,CAAC;gBACT,0BAA0B;gBAC1B,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAClC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACR,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,mBAAmB,MAAM,EAAE,CAAC,CAAC,CAAC;oBACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACpB,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACpB,mBAAmB;gBACnB,IAAI,GAAG,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACxC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACR,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC5E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACpB,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrB,6CAA6C;gBAC7C,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,gDAAgD,CAAC,CAAC,CAAC;gBAC7F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;iBAAM,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACzD,sBAAsB;gBACtB,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,6CAA6C,CAAC,CAAC,CAAC;gBAC1F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;iBAAM,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;gBACnB,gBAAgB;gBAChB,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,8CAA8C,CAAC,CAAC,CAAC;gBAC3F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,+CAA+C,CAAC,CAAC,CAAC;gBAC5F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;YAED,mCAAmC;YACnC,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YAChE,MAAM,SAAS,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAEjE,4EAA4E;YAC5E,6EAA6E;YAC7E,8CAA8C;YAC9C,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3C,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3C,OAAO,CAAC,GAAG,CAAC,kBAAkB,WAAW,EAAE,CAAC,CAAC;gBAC7C,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC7C,IAAI,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACjD,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAC/D,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAEhB,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3B,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;oBACnD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM;wBAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;oBACzD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACpB,CAAC;gBACD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;oBACvD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ;wBAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;oBAC3D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACpB,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;oBAChC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI;wBAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;oBACvD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACpB,CAAC;gBAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,aAAa,CAAC,4CAA4C,CAAC,CAAC,CAAC;gBAC7F,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,iBAAiB,MAAM,CAAC,MAAM,CAAC,MAAM,mCAAmC,CAAC,CAAC,CAAC;gBACzH,CAAC;gBAED,MAAM,eAAe,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,CAAC;YAED,eAAe;YACf,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC;YAEpD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE;gBACzC,WAAW;gBACX,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,SAAS;aACZ,EAAE,WAAW,CAAC,CAAC;YAEhB,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAEpB,mEAAmE;YACnE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,wDAAwD;gBACxD,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,CAAC,OAAO,MAAM,CAAC,MAAM,aAAa,CAAC,CAAC,CAAC;gBAE3E,IAAI,SAAS,GAAG,IAAI,CAAC;gBACrB,KAAK,MAAM,eAAe,IAAI,MAAM,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAG,eAAe,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;oBACzE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAEpB,IAAI,eAAe,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;wBACtC,SAAS,GAAG,KAAK,CAAC;oBACtB,CAAC;gBACL,CAAC;gBAED,+DAA+D;gBAC/D,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;oBACf,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC/E,CAAC;gBAED,qBAAqB;gBACrB,MAAM,eAAe,EAAE,CAAC;gBAExB,kEAAkE;gBAClE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACJ,gBAAgB;gBAChB,MAAM,MAAM,GAAG,eAAe,CAAC,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAChE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAEpB,6BAA6B;gBAC7B,eAAe,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBAElD,qBAAqB;gBACrB,MAAM,eAAe,EAAE,CAAC;gBAExB,6BAA6B;gBAC7B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,oBAAoB,EAAE,KAAc,CAAC,CAAC,CAAC;YAEjF,iCAAiC;YACjC,IAAI,CAAC;gBACD,MAAM,eAAe,EAAE,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACL,wBAAwB;YAC5B,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,gBAAgB,CAAC,IAAkL,EAAE,MAAkB;QAC3N,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC,mBAAmB,IAAI,CAAC,MAAM,wCAAwC,CAAC,CAAC;QAC1F,CAAC;QAED,wCAAwC;QACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,iCAAiC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAChE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACjD,CAAC;QAED,qCAAqC;QACrC,IAAI,MAAM,GAA6I,IAAI,CAAC;QAC5J,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC;gBACD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC5C,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,IAAI,CAAC,oCAAqC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,QAAQ,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;QAED,uCAAuC;QACvC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,IAAI,CAAC;gBACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACrC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,IAAI,CAAC,sCAAuC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9E,CAAC;QACL,CAAC;QAED,wCAAwC;QACxC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,IAAI,CAAC;gBACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACtC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,CAAC,IAAI,CAAC,uCAAwC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/E,CAAC;QACL,CAAC;QAED,qDAAqD;QACrD,IAAI,MAAM,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;YAClD,MAAM,aAAa,GAAa,EAAE,CAAC;YACnC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAClC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;oBAC1D,SAAS;gBACb,CAAC;gBACD,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3C,MAAM,CAAC,IAAI,CACP,gBAAgB,MAAM,CAAC,IAAI,uBAAuB;wBAClD,cAAc,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC/C,CAAC;oBACF,SAAS;gBACb,CAAC;gBACD,MAAM,CAAC,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChE,aAAa,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;YACvD,CAAC;YACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,YAAY,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACtD,CAAC;YAED,mEAAmE;YACnE,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;gBACxB,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACrE,KAAK,MAAM,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;oBAC5D,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;wBACzC,QAAQ,CAAC,IAAI,CACT,qCAAqC,YAAY,mCAAmC,CACvF,CAAC;oBACN,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC5E,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,KAAK,EAAE,CAAC;oBAC9B,QAAQ,CAAC,IAAI,CAAC,yBAAyB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAChB,QAAQ,CAAC,IAAI,CAAC,6EAA6E,CAAC,CAAC;QACjG,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACtC,CAAC;CACJ"}
|
package/dist/commands/suite.d.ts
CHANGED
|
@@ -17,5 +17,15 @@ export declare class SuiteCommand {
|
|
|
17
17
|
* @param contextUser - Optional user context (will be fetched if not provided)
|
|
18
18
|
*/
|
|
19
19
|
execute(suiteId: string | undefined, flags: SuiteFlags, contextUser?: UserInfo): Promise<void>;
|
|
20
|
+
/**
|
|
21
|
+
* Build a flaky-test summary by grouping iteration results by testId and
|
|
22
|
+
* computing score variance. A test is "flaky" if its scores vary by more
|
|
23
|
+
* than VARIANCE_THRESHOLD across iterations OR if its statuses are mixed
|
|
24
|
+
* (some Passed, some Failed).
|
|
25
|
+
*
|
|
26
|
+
* Variance threshold of 0.3 is the plan-recommended cutoff — small enough
|
|
27
|
+
* to catch real instability, large enough to ignore minor LLM judge noise.
|
|
28
|
+
*/
|
|
29
|
+
private buildFlakyReport;
|
|
20
30
|
}
|
|
21
31
|
//# sourceMappingURL=suite.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"suite.d.ts","sourceRoot":"","sources":["../../src/commands/suite.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"suite.d.ts","sourceRoot":"","sources":["../../src/commands/suite.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAQtC;;GAEG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;;;OAMG;IACG,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA0HpG;;;;;;;;OAQG;IACH,OAAO,CAAC,gBAAgB;CA6D3B"}
|
package/dist/commands/suite.js
CHANGED
|
@@ -8,6 +8,7 @@ import { SpinnerManager } from '../utils/spinner-manager.js';
|
|
|
8
8
|
import { loadCLIConfig } from '../utils/config-loader.js';
|
|
9
9
|
import { initializeMJProvider, closeMJProvider, getContextUser } from '../lib/mj-provider.js';
|
|
10
10
|
import { parseVariableFlags } from '../utils/variable-parser.js';
|
|
11
|
+
import { loadOraclesModule } from '../utils/oracle-module-loader.js';
|
|
11
12
|
/**
|
|
12
13
|
* Suite command - Execute a test suite
|
|
13
14
|
*/
|
|
@@ -42,6 +43,17 @@ export class SuiteCommand {
|
|
|
42
43
|
console.log(`TestEngine configured. Test Types loaded: ${engine.TestTypes?.length || 0}`);
|
|
43
44
|
console.log(`Test Suites loaded: ${engine.TestSuites?.length || 0}`);
|
|
44
45
|
console.log(`Tests loaded: ${engine.Tests?.length || 0}`);
|
|
46
|
+
// Plug in user-supplied oracles before resolving the suite. They
|
|
47
|
+
// register at the engine level via `engine.RegisterOracle()` and
|
|
48
|
+
// become available to every test in the suite that references the
|
|
49
|
+
// matching oracle `type`.
|
|
50
|
+
if (flags.oraclesModule) {
|
|
51
|
+
const summary = await loadOraclesModule(flags.oraclesModule, engine);
|
|
52
|
+
console.log(`Loaded oracle module ${summary.modulePath} ` +
|
|
53
|
+
`(registered: ${summary.registered.join(', ') || 'none'}` +
|
|
54
|
+
(summary.skipped.length ? `; skipped: ${summary.skipped.length}` : '') +
|
|
55
|
+
')');
|
|
56
|
+
}
|
|
45
57
|
let suite;
|
|
46
58
|
if (suiteId) {
|
|
47
59
|
// Run specific suite by ID
|
|
@@ -71,19 +83,34 @@ export class SuiteCommand {
|
|
|
71
83
|
// Note: Suite variables apply to all tests - type conversion happens per-test
|
|
72
84
|
const variables = parseVariableFlags(flags.var);
|
|
73
85
|
// Execute suite
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
86
|
+
const flakyMsg = flags.flakyCheck && flags.flakyCheck > 1
|
|
87
|
+
? ` (flaky-check: each test ×${flags.flakyCheck})`
|
|
88
|
+
: '';
|
|
89
|
+
this.spinner.start(`Running test suite: ${suite.Name}${flakyMsg}...`);
|
|
77
90
|
const result = await engine.RunSuite(suite.ID, {
|
|
78
91
|
verbose: flags.verbose,
|
|
79
|
-
variables
|
|
92
|
+
variables,
|
|
93
|
+
delayBetweenTests: flags.delay,
|
|
94
|
+
parallel: flags.parallel,
|
|
95
|
+
maxParallel: flags.maxParallel,
|
|
96
|
+
repeatCountOverride: flags.flakyCheck && flags.flakyCheck > 1 ? flags.flakyCheck : undefined,
|
|
80
97
|
}, contextUser);
|
|
81
98
|
this.spinner.stop();
|
|
99
|
+
// If --flaky-check was used, compute per-test variance and report flaky tests.
|
|
100
|
+
// The engine returns multiple results per test (one per iteration); we group by
|
|
101
|
+
// testId and compute score variance to identify inconsistent tests.
|
|
102
|
+
let flakyReport = '';
|
|
103
|
+
if (flags.flakyCheck && flags.flakyCheck > 1) {
|
|
104
|
+
flakyReport = this.buildFlakyReport(result.testResults, flags.flakyCheck);
|
|
105
|
+
}
|
|
82
106
|
// Format and display result
|
|
83
107
|
const output = OutputFormatter.formatSuiteResult(result, format);
|
|
84
108
|
console.log(output);
|
|
85
|
-
|
|
86
|
-
|
|
109
|
+
if (flakyReport)
|
|
110
|
+
console.log(flakyReport);
|
|
111
|
+
// Write to file if requested (include flaky report in markdown/console output)
|
|
112
|
+
const fileOutput = flakyReport && format !== 'json' ? output + '\n' + flakyReport : output;
|
|
113
|
+
OutputFormatter.writeToFile(fileOutput, flags.output);
|
|
87
114
|
// Clean up resources
|
|
88
115
|
await closeMJProvider();
|
|
89
116
|
// Exit with appropriate code (non-zero if any test failed)
|
|
@@ -102,5 +129,66 @@ export class SuiteCommand {
|
|
|
102
129
|
process.exit(1);
|
|
103
130
|
}
|
|
104
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* Build a flaky-test summary by grouping iteration results by testId and
|
|
134
|
+
* computing score variance. A test is "flaky" if its scores vary by more
|
|
135
|
+
* than VARIANCE_THRESHOLD across iterations OR if its statuses are mixed
|
|
136
|
+
* (some Passed, some Failed).
|
|
137
|
+
*
|
|
138
|
+
* Variance threshold of 0.3 is the plan-recommended cutoff — small enough
|
|
139
|
+
* to catch real instability, large enough to ignore minor LLM judge noise.
|
|
140
|
+
*/
|
|
141
|
+
buildFlakyReport(testResults, iterations) {
|
|
142
|
+
const VARIANCE_THRESHOLD = 0.3;
|
|
143
|
+
// Group by testId — when --flaky-check N is used, each test produces N entries
|
|
144
|
+
const byTest = new Map();
|
|
145
|
+
for (const r of testResults) {
|
|
146
|
+
const entry = byTest.get(r.testId) ?? { name: r.testName, scores: [], statuses: [] };
|
|
147
|
+
entry.scores.push(r.score);
|
|
148
|
+
entry.statuses.push(r.status);
|
|
149
|
+
byTest.set(r.testId, entry);
|
|
150
|
+
}
|
|
151
|
+
const rows = [];
|
|
152
|
+
for (const [, entry] of byTest) {
|
|
153
|
+
// Skip tests that didn't actually run multiple times (e.g. if an iteration errored)
|
|
154
|
+
if (entry.scores.length < 2)
|
|
155
|
+
continue;
|
|
156
|
+
const max = Math.max(...entry.scores);
|
|
157
|
+
const min = Math.min(...entry.scores);
|
|
158
|
+
const variance = max - min;
|
|
159
|
+
const uniqueStatuses = new Set(entry.statuses);
|
|
160
|
+
const mixedStatus = uniqueStatuses.size > 1;
|
|
161
|
+
const flaky = variance > VARIANCE_THRESHOLD || mixedStatus;
|
|
162
|
+
rows.push({ ...entry, variance, mixedStatus, flaky });
|
|
163
|
+
}
|
|
164
|
+
const flakyRows = rows.filter(r => r.flaky).sort((a, b) => b.variance - a.variance);
|
|
165
|
+
const lines = [];
|
|
166
|
+
lines.push('');
|
|
167
|
+
lines.push(' Flaky Test Detection');
|
|
168
|
+
lines.push(' ─────────────────────────────────────────');
|
|
169
|
+
lines.push(` Each test ran ${iterations}× (variance threshold: ${VARIANCE_THRESHOLD})`);
|
|
170
|
+
lines.push('');
|
|
171
|
+
if (flakyRows.length === 0) {
|
|
172
|
+
lines.push(' ✓ No flaky tests detected — all tests produced consistent results.');
|
|
173
|
+
lines.push('');
|
|
174
|
+
return lines.join('\n');
|
|
175
|
+
}
|
|
176
|
+
lines.push(` ⚠ ${flakyRows.length} flaky test(s) detected:`);
|
|
177
|
+
lines.push('');
|
|
178
|
+
for (const r of flakyRows) {
|
|
179
|
+
const reasons = [];
|
|
180
|
+
if (r.variance > VARIANCE_THRESHOLD) {
|
|
181
|
+
reasons.push(`variance ${(r.variance * 100).toFixed(0)}%`);
|
|
182
|
+
}
|
|
183
|
+
if (r.mixedStatus) {
|
|
184
|
+
reasons.push(`mixed: ${r.statuses.join('/')}`);
|
|
185
|
+
}
|
|
186
|
+
const scoresStr = r.scores.map(s => (s * 100).toFixed(0) + '%').join(', ');
|
|
187
|
+
lines.push(` [FLAKY] ${r.name}`);
|
|
188
|
+
lines.push(` scores: ${scoresStr} (${reasons.join(', ')})`);
|
|
189
|
+
}
|
|
190
|
+
lines.push('');
|
|
191
|
+
return lines.join('\n');
|
|
192
|
+
}
|
|
105
193
|
}
|
|
106
194
|
//# sourceMappingURL=suite.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"suite.js","sourceRoot":"","sources":["../../src/commands/suite.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAC;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"suite.js","sourceRoot":"","sources":["../../src/commands/suite.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAC;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAElE;;GAEG;AACH,MAAM,OAAO,YAAY;IAAzB;QACY,YAAO,GAAG,IAAI,cAAc,EAAE,CAAC;IAyM3C,CAAC;IAvMG;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,OAA2B,EAAE,KAAiB,EAAE,WAAsB;QAChF,IAAI,CAAC;YACD,4DAA4D;YAC5D,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC3C,MAAM,oBAAoB,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;YAEpD,wDAAwD;YACxD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACf,WAAW,GAAG,MAAM,cAAc,EAAE,CAAC;YACzC,CAAC;YAED,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,aAAa,IAAI,SAAS,CAAC;YAEjE,sBAAsB;YACtB,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;YACzC,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,6CAA6C,MAAM,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1F,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC;YACrE,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC;YAE1D,iEAAiE;YACjE,iEAAiE;YACjE,kEAAkE;YAClE,0BAA0B;YAC1B,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;gBACrE,OAAO,CAAC,GAAG,CACP,wBAAwB,OAAO,CAAC,UAAU,GAAG;oBACzC,gBAAgB,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE;oBACzD,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtE,GAAG,CACV,CAAC;YACN,CAAC;YAED,IAAI,KAAK,CAAC;YAEV,IAAI,OAAO,EAAE,CAAC;gBACV,2BAA2B;gBAC3B,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAC;gBACnD,KAAK,GAAG,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBACzC,IAAI,CAAC,KAAK,EAAE,CAAC;oBACT,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,yBAAyB,OAAO,EAAE,CAAC,CAAC,CAAC;oBAC/E,OAAO,CAAC,KAAK,CAAC,qBAAqB,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;oBAC/F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACpB,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACpB,oBAAoB;gBACpB,OAAO,CAAC,GAAG,CAAC,8BAA8B,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;gBACxD,KAAK,GAAG,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC9C,IAAI,CAAC,KAAK,EAAE,CAAC;oBACT,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,yBAAyB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;oBAClF,OAAO,CAAC,KAAK,CAAC,qBAAqB,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;oBAC/F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACpB,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,iCAAiC,CAAC,CAAC,CAAC;gBAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;YAED,mCAAmC;YACnC,8EAA8E;YAC9E,MAAM,SAAS,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAEhD,gBAAgB;YAChB,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC;gBACrD,CAAC,CAAC,6BAA6B,KAAK,CAAC,UAAU,GAAG;gBAClD,CAAC,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,uBAAuB,KAAK,CAAC,IAAI,GAAG,QAAQ,KAAK,CAAC,CAAC;YAEtE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE;gBAC3C,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,SAAS;gBACT,iBAAiB,EAAE,KAAK,CAAC,KAAK;gBAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,mBAAmB,EAAE,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;aAC/F,EAAE,WAAW,CAAC,CAAC;YAEhB,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAEpB,+EAA+E;YAC/E,gFAAgF;YAChF,oEAAoE;YACpE,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;YAC9E,CAAC;YAED,4BAA4B;YAC5B,MAAM,MAAM,GAAG,eAAe,CAAC,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,IAAI,WAAW;gBAAE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAE1C,+EAA+E;YAC/E,MAAM,UAAU,GAAG,WAAW,IAAI,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;YAC3F,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAEtD,qBAAqB;YACrB,MAAM,eAAe,EAAE,CAAC;YAExB,2DAA2D;YAC3D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,0BAA0B,EAAE,KAAc,CAAC,CAAC,CAAC;YAEvF,iCAAiC;YACjC,IAAI,CAAC;gBACD,MAAM,eAAe,EAAE,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACL,wBAAwB;YAC5B,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACK,gBAAgB,CAAC,WAAuF,EAAE,UAAkB;QAChI,MAAM,kBAAkB,GAAG,GAAG,CAAC;QAE/B,+EAA+E;QAC/E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkE,CAAC;QACzF,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;YACrF,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC3B,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC9B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;QAID,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YAC7B,oFAAoF;YACpF,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,GAAG,GAAG,GAAG,CAAC;YAC3B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,GAAG,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,QAAQ,GAAG,kBAAkB,IAAI,WAAW,CAAC;YAC3D,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEpF,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,mBAAmB,UAAU,0BAA0B,kBAAkB,GAAG,CAAC,CAAC;QACzF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;YACnF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,SAAS,CAAC,MAAM,0BAA0B,CAAC,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YACxB,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,IAAI,CAAC,CAAC,QAAQ,GAAG,kBAAkB,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC/D,CAAC;YACD,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnD,CAAC;YACD,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,qBAAqB,SAAS,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1E,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;CACJ"}
|
package/dist/types.d.ts
CHANGED
|
@@ -32,6 +32,13 @@ export interface CommonFlags {
|
|
|
32
32
|
format?: OutputFormat;
|
|
33
33
|
output?: string;
|
|
34
34
|
verbose?: boolean;
|
|
35
|
+
/**
|
|
36
|
+
* Path to a CommonJS or ESM module that exports custom `IOracle` classes
|
|
37
|
+
* or instances. Each exported oracle is registered on the engine before
|
|
38
|
+
* the test/suite runs — used by non-MJ adopters (Mode C) to plug their
|
|
39
|
+
* own oracle types without modifying TestingFramework.
|
|
40
|
+
*/
|
|
41
|
+
oraclesModule?: string;
|
|
35
42
|
}
|
|
36
43
|
/**
|
|
37
44
|
* Flags for run command
|
|
@@ -64,6 +71,21 @@ export interface SuiteFlags extends CommonFlags {
|
|
|
64
71
|
* Can be specified multiple times for multiple variables
|
|
65
72
|
*/
|
|
66
73
|
var?: string[];
|
|
74
|
+
/**
|
|
75
|
+
* Maximum number of parallel workers (default 4).
|
|
76
|
+
*/
|
|
77
|
+
maxParallel?: number;
|
|
78
|
+
/**
|
|
79
|
+
* Delay in milliseconds between test executions.
|
|
80
|
+
* Useful for avoiding rate limits (e.g., Auth0 brute-force protection).
|
|
81
|
+
*/
|
|
82
|
+
delay?: number;
|
|
83
|
+
/**
|
|
84
|
+
* Run each test N times to detect flakiness via score variance.
|
|
85
|
+
* Tests with score variance > 0.3 across iterations are flagged [FLAKY].
|
|
86
|
+
* Recommended: 3 (statistical minimum), 5 (more reliable detection).
|
|
87
|
+
*/
|
|
88
|
+
flakyCheck?: number;
|
|
67
89
|
}
|
|
68
90
|
/**
|
|
69
91
|
* Flags for list command
|
|
@@ -113,5 +135,17 @@ export interface CompareFlags extends CommonFlags {
|
|
|
113
135
|
version?: string[];
|
|
114
136
|
commit?: string[];
|
|
115
137
|
diffOnly?: boolean;
|
|
138
|
+
/** Compare the two most recent completed suite runs */
|
|
139
|
+
latest?: boolean;
|
|
140
|
+
/** Compare two results.json files directly (no DB needed). Takes two file paths. */
|
|
141
|
+
fromJson?: string[];
|
|
142
|
+
/**
|
|
143
|
+
* Filter suite runs by a single tag. Matches against `MJTestSuiteRunEntity.Tags`,
|
|
144
|
+
* which is a JSON array string (e.g., `["staging-nightly","sonnet-4.6"]`).
|
|
145
|
+
* Used to isolate runs from a specific source environment when an archive MJ
|
|
146
|
+
* holds multiple sources side-by-side. DB mode only — `--from-json` ignores it
|
|
147
|
+
* because `results.json` does not currently emit the Tags field.
|
|
148
|
+
*/
|
|
149
|
+
tag?: string;
|
|
116
150
|
}
|
|
117
151
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,QAAS,SAAQ,WAAW;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,UAAW,SAAQ,WAAW;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,SAAU,SAAQ,WAAW;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,aAAc,SAAQ,WAAW;IAC9C,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC7C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC7C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,uDAAuD;IACvD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,oFAAoF;IACpF,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;;;;;OAMG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;CAChB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-loader.d.ts","sourceRoot":"","sources":["../../src/utils/config-loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAMrC,MAAM,WAAW,QAAQ;IAErB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE;QACN,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,aAAa,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;QAChD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IAGF,QAAQ,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACL;AAID;;;;GAIG;AACH,wBAAsB,YAAY,IAAI,OAAO,CAAC,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"config-loader.d.ts","sourceRoot":"","sources":["../../src/utils/config-loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAMrC,MAAM,WAAW,QAAQ;IAErB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE;QACN,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,aAAa,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;QAChD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IAGF,QAAQ,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACL;AAID;;;;GAIG;AACH,wBAAsB,YAAY,IAAI,OAAO,CAAC,QAAQ,CAAC,CAwBtD;AAED;;;;GAIG;AACH,wBAAgB,aAAa,IAAI,SAAS,CAqBzC"}
|
|
@@ -18,10 +18,12 @@ export async function loadMJConfig() {
|
|
|
18
18
|
if (cachedConfig) {
|
|
19
19
|
return cachedConfig;
|
|
20
20
|
}
|
|
21
|
-
// Clear any existing require cache for mj.config.cjs to ensure env vars are re-evaluated
|
|
22
|
-
//
|
|
21
|
+
// Clear any existing require cache for mj.config.cjs to ensure env vars are re-evaluated.
|
|
22
|
+
// require.cache is only available in CommonJS; in ESM, cosmiconfig's cache:false handles this.
|
|
23
23
|
const configPath = path.resolve(process.cwd(), 'mj.config.cjs');
|
|
24
|
-
|
|
24
|
+
if (typeof require !== 'undefined') {
|
|
25
|
+
delete require.cache[configPath];
|
|
26
|
+
}
|
|
25
27
|
// Create a new explorer instance to ensure fresh config load with current env vars
|
|
26
28
|
const explorer = cosmiconfig('mj', {
|
|
27
29
|
cache: false // Disable caching to ensure fresh load
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../../src/utils/config-loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AAGxB,mDAAmD;AACnD,wEAAwE;AACxE,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAgC1F,IAAI,YAAY,GAAoB,IAAI,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY;IAC9B,IAAI,YAAY,EAAE,CAAC;QACf,OAAO,YAAY,CAAC;IACxB,CAAC;IAED,
|
|
1
|
+
{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../../src/utils/config-loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AAGxB,mDAAmD;AACnD,wEAAwE;AACxE,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAgC1F,IAAI,YAAY,GAAoB,IAAI,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY;IAC9B,IAAI,YAAY,EAAE,CAAC;QACf,OAAO,YAAY,CAAC;IACxB,CAAC;IAED,0FAA0F;IAC1F,+FAA+F;IAC/F,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC,CAAC;IAChE,IAAI,OAAO,OAAO,KAAK,WAAW,EAAE,CAAC;QACjC,OAAO,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IAED,mFAAmF;IACnF,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE;QAC/B,KAAK,EAAE,KAAK,CAAE,uCAAuC;KACxD,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,CAAC;IAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,0FAA0F,CAAC,CAAC;IAChH,CAAC;IAED,YAAY,GAAG,MAAM,CAAC,MAAkB,CAAC;IACzC,OAAO,YAAY,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa;IACzB,iDAAiD;IACjD,8DAA8D;IAC9D,MAAM,aAAa,GAAG,YAAY,EAAE,OAAO,IAAI,EAAE,CAAC;IAElD,OAAO;QACH,kBAAkB,EAAE,aAAa,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,KAAK;QACxF,aAAa,EAAE,aAAa,CAAC,aAAa,IAAI,SAAS;QACvD,QAAQ,EAAE,aAAa,CAAC,QAAQ,IAAI,KAAK;QACzC,QAAQ,EAAE,aAAa,CAAC,QAAQ,IAAI,KAAK;QACzC,gBAAgB,EAAE,aAAa,CAAC,gBAAgB,IAAI,CAAC;QACrD,OAAO,EAAE,aAAa,CAAC,OAAO,IAAI,MAAM,EAAG,YAAY;QACvD,QAAQ,EAAE,YAAY,EAAE,QAAQ,IAAI;YAChC,IAAI,EAAE,YAAY,EAAE,MAAM,IAAI,WAAW;YACzC,IAAI,EAAE,YAAY,EAAE,UAAU;YAC9B,IAAI,EAAE,OAAO,YAAY,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,MAAM;YACrG,QAAQ,EAAE,YAAY,EAAE,UAAU;YAClC,QAAQ,EAAE,YAAY,EAAE,UAAU;YAClC,MAAM,EAAE,YAAY,EAAE,UAAU,IAAI,MAAM;SAC7C;KACJ,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { TestEngine } from '@memberjunction/testing-engine';
|
|
2
|
+
export interface LoadedOracleSummary {
|
|
3
|
+
/** Module path that was loaded. */
|
|
4
|
+
modulePath: string;
|
|
5
|
+
/** Names of oracles that were successfully registered. */
|
|
6
|
+
registered: string[];
|
|
7
|
+
/** Export keys we walked past because they didn't look like oracles. */
|
|
8
|
+
skipped: string[];
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Load a user-supplied oracle module and register every `IOracle` export
|
|
12
|
+
* onto the provided TestEngine instance. Returns a summary so the caller
|
|
13
|
+
* can log what happened.
|
|
14
|
+
*
|
|
15
|
+
* Exports recognized:
|
|
16
|
+
* - A class implementing `IOracle` — instantiated with no args.
|
|
17
|
+
* - An object instance with `{ type: string, evaluate: fn }` — used as-is.
|
|
18
|
+
*
|
|
19
|
+
* Anything else (constants, helper functions, unrelated exports) is silently
|
|
20
|
+
* skipped — callers like the CLI surface this as a warning, not an error.
|
|
21
|
+
*
|
|
22
|
+
* @throws if the module file doesn't exist or the import itself fails.
|
|
23
|
+
*/
|
|
24
|
+
export declare function loadOraclesModule(modulePath: string, engine: TestEngine): Promise<LoadedOracleSummary>;
|
|
25
|
+
//# sourceMappingURL=oracle-module-loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oracle-module-loader.d.ts","sourceRoot":"","sources":["../../src/utils/oracle-module-loader.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAC;AA8BjE,MAAM,WAAW,mBAAmB;IAChC,mCAAmC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,wEAAwE;IACxE,OAAO,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACnC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,UAAU,GACnB,OAAO,CAAC,mBAAmB,CAAC,CAsF9B"}
|