@memberjunction/testing-engine 0.0.1 → 2.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +403 -29
- package/dist/drivers/AgentEvalDriver.d.ts +197 -0
- package/dist/drivers/AgentEvalDriver.d.ts.map +1 -0
- package/dist/drivers/AgentEvalDriver.js +370 -0
- package/dist/drivers/AgentEvalDriver.js.map +1 -0
- package/dist/drivers/BaseTestDriver.d.ts +145 -0
- package/dist/drivers/BaseTestDriver.d.ts.map +1 -0
- package/dist/drivers/BaseTestDriver.js +266 -0
- package/dist/drivers/BaseTestDriver.js.map +1 -0
- package/dist/drivers/index.d.ts +6 -0
- package/dist/drivers/index.d.ts.map +1 -0
- package/dist/drivers/index.js +22 -0
- package/dist/drivers/index.js.map +1 -0
- package/dist/engine/TestEngine.d.ts +148 -0
- package/dist/engine/TestEngine.d.ts.map +1 -0
- package/dist/engine/TestEngine.js +490 -0
- package/dist/engine/TestEngine.js.map +1 -0
- package/dist/index.d.ts +20 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +42 -0
- package/dist/index.js.map +1 -0
- package/dist/oracles/ExactMatchOracle.d.ts +98 -0
- package/dist/oracles/ExactMatchOracle.d.ts.map +1 -0
- package/dist/oracles/ExactMatchOracle.js +355 -0
- package/dist/oracles/ExactMatchOracle.js.map +1 -0
- package/dist/oracles/IOracle.d.ts +47 -0
- package/dist/oracles/IOracle.d.ts.map +1 -0
- package/dist/oracles/IOracle.js +7 -0
- package/dist/oracles/IOracle.js.map +1 -0
- package/dist/oracles/LLMJudgeOracle.d.ts +65 -0
- package/dist/oracles/LLMJudgeOracle.d.ts.map +1 -0
- package/dist/oracles/LLMJudgeOracle.js +214 -0
- package/dist/oracles/LLMJudgeOracle.js.map +1 -0
- package/dist/oracles/SQLValidatorOracle.d.ts +78 -0
- package/dist/oracles/SQLValidatorOracle.d.ts.map +1 -0
- package/dist/oracles/SQLValidatorOracle.js +215 -0
- package/dist/oracles/SQLValidatorOracle.js.map +1 -0
- package/dist/oracles/SchemaValidatorOracle.d.ts +61 -0
- package/dist/oracles/SchemaValidatorOracle.d.ts.map +1 -0
- package/dist/oracles/SchemaValidatorOracle.js +193 -0
- package/dist/oracles/SchemaValidatorOracle.js.map +1 -0
- package/dist/oracles/TraceValidatorOracle.d.ts +41 -0
- package/dist/oracles/TraceValidatorOracle.d.ts.map +1 -0
- package/dist/oracles/TraceValidatorOracle.js +159 -0
- package/dist/oracles/TraceValidatorOracle.js.map +1 -0
- package/dist/oracles/index.d.ts +10 -0
- package/dist/oracles/index.d.ts.map +1 -0
- package/dist/oracles/index.js +26 -0
- package/dist/oracles/index.js.map +1 -0
- package/dist/types.d.ts +428 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +6 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/cost-calculator.d.ts +92 -0
- package/dist/utils/cost-calculator.d.ts.map +1 -0
- package/dist/utils/cost-calculator.js +137 -0
- package/dist/utils/cost-calculator.js.map +1 -0
- package/dist/utils/result-formatter.d.ts +98 -0
- package/dist/utils/result-formatter.d.ts.map +1 -0
- package/dist/utils/result-formatter.js +252 -0
- package/dist/utils/result-formatter.js.map +1 -0
- package/dist/utils/scoring.d.ts +64 -0
- package/dist/utils/scoring.d.ts.map +1 -0
- package/dist/utils/scoring.js +140 -0
- package/dist/utils/scoring.js.map +1 -0
- package/package.json +36 -7
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* @fileoverview Main testing engine orchestrator
|
|
4
|
+
* @module @memberjunction/testing-engine
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.TestEngine = void 0;
|
|
8
|
+
const core_1 = require("@memberjunction/core");
|
|
9
|
+
const global_1 = require("@memberjunction/global");
|
|
10
|
+
const testing_engine_base_1 = require("@memberjunction/testing-engine-base");
|
|
11
|
+
const BaseTestDriver_1 = require("../drivers/BaseTestDriver");
|
|
12
|
+
const SchemaValidatorOracle_1 = require("../oracles/SchemaValidatorOracle");
|
|
13
|
+
const TraceValidatorOracle_1 = require("../oracles/TraceValidatorOracle");
|
|
14
|
+
const LLMJudgeOracle_1 = require("../oracles/LLMJudgeOracle");
|
|
15
|
+
const ExactMatchOracle_1 = require("../oracles/ExactMatchOracle");
|
|
16
|
+
const SQLValidatorOracle_1 = require("../oracles/SQLValidatorOracle");
|
|
17
|
+
/**
|
|
18
|
+
* Main testing engine that orchestrates test execution.
|
|
19
|
+
*
|
|
20
|
+
* Extends TestEngineBase (UI-safe metadata cache) with execution capabilities.
|
|
21
|
+
* Follows singleton pattern like SchedulingEngine.
|
|
22
|
+
*
|
|
23
|
+
* Responsibilities:
|
|
24
|
+
* - Load and instantiate test drivers via ClassFactory
|
|
25
|
+
* - Manage oracle registry
|
|
26
|
+
* - Execute individual tests and test suites
|
|
27
|
+
* - Create and update TestRun entities
|
|
28
|
+
* - Track execution timing and costs
|
|
29
|
+
* - Handle errors and logging
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* ```typescript
|
|
33
|
+
* const engine = TestEngine.Instance;
|
|
34
|
+
* await engine.Config(false, contextUser);
|
|
35
|
+
*
|
|
36
|
+
* const result = await engine.RunTest('test-id', { verbose: true }, contextUser);
|
|
37
|
+
* console.log(`Test ${result.status}: Score ${result.score}`);
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
class TestEngine extends testing_engine_base_1.TestEngineBase {
|
|
41
|
+
/**
|
|
42
|
+
* Get singleton instance
|
|
43
|
+
*/
|
|
44
|
+
static get Instance() {
|
|
45
|
+
return super.getInstance();
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Private constructor for singleton
|
|
49
|
+
*/
|
|
50
|
+
constructor() {
|
|
51
|
+
super();
|
|
52
|
+
this._driverCache = new Map();
|
|
53
|
+
this._oracleRegistry = new Map();
|
|
54
|
+
}
|
|
55
|
+
async AdditionalLoading(contextUser) {
|
|
56
|
+
try {
|
|
57
|
+
await super.AdditionalLoading(contextUser);
|
|
58
|
+
// Register built-in oracles
|
|
59
|
+
await this.registerBuiltInOracles();
|
|
60
|
+
this.log('TestEngine configured successfully');
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
this.logError('Failed to configure TestEngine', error);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Run a single test.
|
|
68
|
+
*
|
|
69
|
+
* @param testId - ID of the test to run
|
|
70
|
+
* @param options - Test execution options
|
|
71
|
+
* @param contextUser - User context
|
|
72
|
+
* @param suiteRunId - Optional suite run ID if part of a suite
|
|
73
|
+
* @returns Test run result (or array of results if RepeatCount > 1)
|
|
74
|
+
*/
|
|
75
|
+
async RunTest(testId, options, contextUser, suiteRunId, suiteTestSequence) {
|
|
76
|
+
const startTime = Date.now();
|
|
77
|
+
this.log(`Starting test execution: ${testId}`, options.verbose);
|
|
78
|
+
try {
|
|
79
|
+
// Progress: Loading test
|
|
80
|
+
options.progressCallback?.({
|
|
81
|
+
step: 'loading_test',
|
|
82
|
+
percentage: 10,
|
|
83
|
+
message: 'Loading test configuration',
|
|
84
|
+
metadata: { testId }
|
|
85
|
+
});
|
|
86
|
+
// Load test entity from cache
|
|
87
|
+
const test = await this.loadTest(testId);
|
|
88
|
+
if (!test) {
|
|
89
|
+
throw new Error(`Test not found: ${testId}`);
|
|
90
|
+
}
|
|
91
|
+
// Check RepeatCount and branch to repeated execution if needed
|
|
92
|
+
if (test.RepeatCount && test.RepeatCount > 1) {
|
|
93
|
+
return await this.runRepeatedTest(test, test.RepeatCount, options, contextUser, suiteRunId, suiteTestSequence, startTime);
|
|
94
|
+
}
|
|
95
|
+
// Single execution - delegate to helper method
|
|
96
|
+
return await this.runSingleTestIteration(test, suiteRunId, suiteTestSequence, options, contextUser, startTime);
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
this.logError(`Test execution failed: ${testId}`, error);
|
|
100
|
+
throw error;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Run a test suite.
|
|
105
|
+
*
|
|
106
|
+
* @param suiteId - ID of the test suite to run
|
|
107
|
+
* @param options - Test execution options
|
|
108
|
+
* @param contextUser - User context
|
|
109
|
+
* @returns Test suite run result
|
|
110
|
+
*/
|
|
111
|
+
async RunSuite(suiteId, options, contextUser) {
|
|
112
|
+
const startTime = Date.now();
|
|
113
|
+
this.log(`Starting test suite execution: ${suiteId}`, options.verbose);
|
|
114
|
+
try {
|
|
115
|
+
// Load suite entity from cache
|
|
116
|
+
const suite = await this.loadSuite(suiteId);
|
|
117
|
+
if (!suite) {
|
|
118
|
+
throw new Error(`Test suite not found: ${suiteId}`);
|
|
119
|
+
}
|
|
120
|
+
// Load suite tests from cache
|
|
121
|
+
const tests = await this.loadSuiteTests(suiteId);
|
|
122
|
+
if (tests.length === 0) {
|
|
123
|
+
throw new Error(`No tests found in suite: ${suiteId}`);
|
|
124
|
+
}
|
|
125
|
+
// Create TestSuiteRun entity
|
|
126
|
+
const suiteRun = await this.createSuiteRun(suite, contextUser);
|
|
127
|
+
// Execute tests
|
|
128
|
+
const testResults = [];
|
|
129
|
+
let testSequence = 1; // Track suite execution order (1-based)
|
|
130
|
+
for (const test of tests) {
|
|
131
|
+
try {
|
|
132
|
+
const result = await this.RunTest(test.ID, options, contextUser, suiteRun.ID, testSequence);
|
|
133
|
+
testSequence++; // Increment for next test in suite
|
|
134
|
+
// Handle both single result and array of results (if RepeatCount > 1)
|
|
135
|
+
if (Array.isArray(result)) {
|
|
136
|
+
testResults.push(...result);
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
testResults.push(result);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
this.logError(`Test failed in suite: ${test.Name}`, error);
|
|
144
|
+
// Continue with remaining tests
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
// Update TestSuiteRun entity with results
|
|
148
|
+
await this.updateSuiteRun(suiteRun, testResults, startTime);
|
|
149
|
+
// Calculate suite-level metrics
|
|
150
|
+
const passedTests = testResults.filter(r => r.status === 'Passed').length;
|
|
151
|
+
const failedTests = testResults.filter(r => r.status === 'Failed').length;
|
|
152
|
+
const totalScore = testResults.reduce((sum, r) => sum + r.score, 0);
|
|
153
|
+
const avgScore = testResults.length > 0 ? totalScore / testResults.length : 0;
|
|
154
|
+
const result = {
|
|
155
|
+
suiteRunId: suiteRun.ID,
|
|
156
|
+
suiteId: suite.ID,
|
|
157
|
+
suiteName: suite.Name,
|
|
158
|
+
status: suiteRun.Status,
|
|
159
|
+
passedTests,
|
|
160
|
+
failedTests,
|
|
161
|
+
totalTests: testResults.length,
|
|
162
|
+
averageScore: avgScore,
|
|
163
|
+
testResults,
|
|
164
|
+
durationMs: Date.now() - startTime,
|
|
165
|
+
totalCost: testResults.reduce((sum, r) => sum + r.totalCost, 0),
|
|
166
|
+
startedAt: suiteRun.StartedAt,
|
|
167
|
+
completedAt: suiteRun.CompletedAt
|
|
168
|
+
};
|
|
169
|
+
this.log(`Suite completed: ${result.status} (${passedTests}/${testResults.length} passed)`, options.verbose);
|
|
170
|
+
return result;
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
this.logError(`Suite execution failed: ${suiteId}`, error);
|
|
174
|
+
throw error;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Register a custom oracle.
|
|
179
|
+
*
|
|
180
|
+
* @param oracle - Oracle implementation
|
|
181
|
+
*/
|
|
182
|
+
RegisterOracle(oracle) {
|
|
183
|
+
this._oracleRegistry.set(oracle.type, oracle);
|
|
184
|
+
this.log(`Registered oracle: ${oracle.type}`);
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Get registered oracle by type.
|
|
188
|
+
*
|
|
189
|
+
* @param type - Oracle type
|
|
190
|
+
* @returns Oracle instance or undefined
|
|
191
|
+
*/
|
|
192
|
+
GetOracle(type) {
|
|
193
|
+
return this._oracleRegistry.get(type);
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Get all registered oracle types.
|
|
197
|
+
*
|
|
198
|
+
* @returns Array of oracle type names
|
|
199
|
+
*/
|
|
200
|
+
GetOracleTypes() {
|
|
201
|
+
return Array.from(this._oracleRegistry.keys());
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Register built-in oracles.
|
|
205
|
+
* @private
|
|
206
|
+
*/
|
|
207
|
+
async registerBuiltInOracles() {
|
|
208
|
+
this.RegisterOracle(new SchemaValidatorOracle_1.SchemaValidatorOracle());
|
|
209
|
+
this.RegisterOracle(new TraceValidatorOracle_1.TraceValidatorOracle());
|
|
210
|
+
this.RegisterOracle(new LLMJudgeOracle_1.LLMJudgeOracle());
|
|
211
|
+
this.RegisterOracle(new ExactMatchOracle_1.ExactMatchOracle());
|
|
212
|
+
this.RegisterOracle(new SQLValidatorOracle_1.SQLValidatorOracle());
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Get or create test driver instance.
|
|
216
|
+
* @private
|
|
217
|
+
*/
|
|
218
|
+
async getDriver(testType, contextUser) {
|
|
219
|
+
// Check cache
|
|
220
|
+
const cached = this._driverCache.get(testType.ID);
|
|
221
|
+
if (cached) {
|
|
222
|
+
return cached;
|
|
223
|
+
}
|
|
224
|
+
// Create new instance via ClassFactory
|
|
225
|
+
const driver = global_1.MJGlobal.Instance.ClassFactory.CreateInstance(BaseTestDriver_1.BaseTestDriver, testType.DriverClass);
|
|
226
|
+
if (!driver) {
|
|
227
|
+
throw new Error(`Failed to create driver: ${testType.DriverClass}`);
|
|
228
|
+
}
|
|
229
|
+
// Cache and return
|
|
230
|
+
this._driverCache.set(testType.ID, driver);
|
|
231
|
+
return driver;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Get test entity from cache.
|
|
235
|
+
* @private
|
|
236
|
+
*/
|
|
237
|
+
async loadTest(testId) {
|
|
238
|
+
// Use cached test instead of loading from DB
|
|
239
|
+
const test = this.GetTestByID(testId);
|
|
240
|
+
if (!test) {
|
|
241
|
+
throw new Error(`Test not found in cache: ${testId}`);
|
|
242
|
+
}
|
|
243
|
+
return test;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Get suite entity from cache.
|
|
247
|
+
* @private
|
|
248
|
+
*/
|
|
249
|
+
async loadSuite(suiteId) {
|
|
250
|
+
// Use cached suite instead of loading from DB
|
|
251
|
+
const suite = this.GetTestSuiteByID(suiteId);
|
|
252
|
+
if (!suite) {
|
|
253
|
+
throw new Error(`Test suite not found in cache: ${suiteId}`);
|
|
254
|
+
}
|
|
255
|
+
return suite;
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Get tests for a suite from cache (sorted by sequence).
|
|
259
|
+
* @private
|
|
260
|
+
*/
|
|
261
|
+
async loadSuiteTests(suiteId) {
|
|
262
|
+
// Use cached test suite tests and tests instead of querying DB
|
|
263
|
+
return this.GetTestsForSuite(suiteId);
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Create TestRun entity.
|
|
267
|
+
* @private
|
|
268
|
+
*/
|
|
269
|
+
async createTestRun(test, contextUser, suiteRunId, sequence) {
|
|
270
|
+
const md = new core_1.Metadata();
|
|
271
|
+
const testRun = await md.GetEntityObject('MJ: Test Runs', contextUser);
|
|
272
|
+
testRun.NewRecord();
|
|
273
|
+
testRun.TestID = test.ID;
|
|
274
|
+
testRun.RunByUserID = contextUser.ID;
|
|
275
|
+
testRun.Status = 'Running';
|
|
276
|
+
testRun.StartedAt = new Date();
|
|
277
|
+
// Set suite run ID if part of a suite
|
|
278
|
+
if (suiteRunId) {
|
|
279
|
+
testRun.TestSuiteRunID = suiteRunId;
|
|
280
|
+
}
|
|
281
|
+
// Set sequence if provided (for suite test order or repeat iterations)
|
|
282
|
+
// sequence will be:
|
|
283
|
+
// - Suite test position (1, 2, 3...) for tests in a suite
|
|
284
|
+
// - Iteration number (1, 2, 3...) for repeated tests
|
|
285
|
+
// - null for standalone, non-repeated tests
|
|
286
|
+
if (sequence != null) {
|
|
287
|
+
testRun.Sequence = sequence;
|
|
288
|
+
}
|
|
289
|
+
const saved = await testRun.Save();
|
|
290
|
+
if (!saved) {
|
|
291
|
+
const errorMsg = testRun.LatestResult?.Message || 'Unknown error';
|
|
292
|
+
throw new Error(`Failed to create TestRun entity: ${errorMsg}`);
|
|
293
|
+
}
|
|
294
|
+
return testRun;
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Create TestSuiteRun entity.
|
|
298
|
+
* @private
|
|
299
|
+
*/
|
|
300
|
+
async createSuiteRun(suite, contextUser) {
|
|
301
|
+
const md = new core_1.Metadata();
|
|
302
|
+
const suiteRun = await md.GetEntityObject('MJ: Test Suite Runs', contextUser);
|
|
303
|
+
suiteRun.NewRecord();
|
|
304
|
+
suiteRun.SuiteID = suite.ID;
|
|
305
|
+
suiteRun.RunByUserID = contextUser.ID;
|
|
306
|
+
suiteRun.Status = 'Running';
|
|
307
|
+
suiteRun.StartedAt = new Date();
|
|
308
|
+
const saved = await suiteRun.Save();
|
|
309
|
+
if (!saved) {
|
|
310
|
+
const errorMsg = suiteRun.LatestResult?.Message || 'Unknown error';
|
|
311
|
+
throw new Error(`Failed to create TestSuiteRun entity: ${errorMsg}`);
|
|
312
|
+
}
|
|
313
|
+
return suiteRun;
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Update TestRun entity with results.
|
|
317
|
+
* @private
|
|
318
|
+
*/
|
|
319
|
+
async updateTestRun(testRun, result, startTime) {
|
|
320
|
+
testRun.Status = result.status;
|
|
321
|
+
testRun.Score = result.score;
|
|
322
|
+
testRun.PassedChecks = result.passedChecks;
|
|
323
|
+
testRun.FailedChecks = result.failedChecks;
|
|
324
|
+
testRun.TotalChecks = result.totalChecks;
|
|
325
|
+
testRun.TargetType = result.targetType;
|
|
326
|
+
testRun.TargetLogID = result.targetLogId;
|
|
327
|
+
testRun.InputData = result.inputData ? JSON.stringify(result.inputData) : null;
|
|
328
|
+
testRun.ExpectedOutputData = result.expectedOutput ? JSON.stringify(result.expectedOutput) : null;
|
|
329
|
+
testRun.ActualOutputData = result.actualOutput ? JSON.stringify(result.actualOutput) : null;
|
|
330
|
+
testRun.ResultDetails = result.oracleResults ? JSON.stringify(result.oracleResults) : null;
|
|
331
|
+
testRun.CostUSD = result.totalCost || 0;
|
|
332
|
+
testRun.DurationSeconds = (Date.now() - startTime) / 1000;
|
|
333
|
+
testRun.CompletedAt = new Date();
|
|
334
|
+
const saved = await testRun.Save();
|
|
335
|
+
if (!saved) {
|
|
336
|
+
this.logError('Failed to update TestRun entity', new Error(testRun.LatestResult?.Message));
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Update TestSuiteRun entity with results.
|
|
341
|
+
* @private
|
|
342
|
+
*/
|
|
343
|
+
async updateSuiteRun(suiteRun, testResults, startTime) {
|
|
344
|
+
const passedTests = testResults.filter(r => r.status === 'Passed').length;
|
|
345
|
+
const totalTests = testResults.length;
|
|
346
|
+
suiteRun.Status = passedTests === totalTests ? 'Completed' : 'Failed';
|
|
347
|
+
suiteRun.PassedTests = passedTests;
|
|
348
|
+
suiteRun.FailedTests = totalTests - passedTests;
|
|
349
|
+
suiteRun.TotalTests = totalTests;
|
|
350
|
+
suiteRun.TotalCostUSD = testResults.reduce((sum, r) => sum + r.totalCost, 0);
|
|
351
|
+
suiteRun.TotalDurationSeconds = (Date.now() - startTime) / 1000;
|
|
352
|
+
suiteRun.CompletedAt = new Date();
|
|
353
|
+
const saved = await suiteRun.Save();
|
|
354
|
+
if (!saved) {
|
|
355
|
+
this.logError('Failed to update TestSuiteRun entity', new Error(suiteRun.LatestResult?.Message));
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Run a test multiple times for statistical analysis.
|
|
360
|
+
* @private
|
|
361
|
+
*/
|
|
362
|
+
async runRepeatedTest(test, repeatCount, options, contextUser, suiteRunId, suiteTestSequence, startTime) {
|
|
363
|
+
const results = [];
|
|
364
|
+
this.log(`Running test ${repeatCount} times for statistical analysis`, options.verbose);
|
|
365
|
+
for (let iteration = 1; iteration <= repeatCount; iteration++) {
|
|
366
|
+
this.log(`Running iteration ${iteration} of ${repeatCount}`, options.verbose);
|
|
367
|
+
const result = await this.runSingleTestIteration(test, suiteRunId, iteration, options, contextUser, Date.now() // Each iteration gets its own start time
|
|
368
|
+
);
|
|
369
|
+
results.push(result);
|
|
370
|
+
// Small delay between iterations to avoid rate limiting
|
|
371
|
+
if (iteration < repeatCount) {
|
|
372
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
this.log(`Completed ${repeatCount} iterations`, options.verbose);
|
|
376
|
+
return results;
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Run a single test iteration (extracted from RunTest for reuse).
|
|
380
|
+
* @private
|
|
381
|
+
*/
|
|
382
|
+
async runSingleTestIteration(test, suiteRunId, sequence, options, contextUser, startTime) {
|
|
383
|
+
// Get test type
|
|
384
|
+
const testType = this.GetTestTypeByID(test.TypeID);
|
|
385
|
+
if (!testType) {
|
|
386
|
+
throw new Error(`Test type not found: ${test.TypeID}`);
|
|
387
|
+
}
|
|
388
|
+
// Progress: Initializing driver
|
|
389
|
+
options.progressCallback?.({
|
|
390
|
+
step: 'initializing_driver',
|
|
391
|
+
percentage: 20,
|
|
392
|
+
message: `Initializing ${testType.DriverClass} driver`,
|
|
393
|
+
metadata: {
|
|
394
|
+
testName: test.Name,
|
|
395
|
+
driverType: testType.DriverClass
|
|
396
|
+
}
|
|
397
|
+
});
|
|
398
|
+
// Get or create driver
|
|
399
|
+
const driver = await this.getDriver(testType, contextUser);
|
|
400
|
+
// Create TestRun entity
|
|
401
|
+
const testRun = await this.createTestRun(test, contextUser, suiteRunId, sequence);
|
|
402
|
+
// Progress: Executing test
|
|
403
|
+
options.progressCallback?.({
|
|
404
|
+
step: 'executing_test',
|
|
405
|
+
percentage: 40,
|
|
406
|
+
message: 'Running test driver',
|
|
407
|
+
metadata: {
|
|
408
|
+
testName: test.Name,
|
|
409
|
+
testRun,
|
|
410
|
+
driverType: testType.DriverClass
|
|
411
|
+
}
|
|
412
|
+
});
|
|
413
|
+
// Execute test via driver
|
|
414
|
+
this.log(`Executing test via ${testType.DriverClass}`, options.verbose);
|
|
415
|
+
const driverResult = await driver.Execute({
|
|
416
|
+
test,
|
|
417
|
+
testRun,
|
|
418
|
+
contextUser,
|
|
419
|
+
options,
|
|
420
|
+
oracleRegistry: this._oracleRegistry
|
|
421
|
+
});
|
|
422
|
+
// Progress: Evaluating oracles
|
|
423
|
+
const oracleCount = driverResult.oracleResults?.length || 0;
|
|
424
|
+
options.progressCallback?.({
|
|
425
|
+
step: 'evaluating_oracles',
|
|
426
|
+
percentage: 70,
|
|
427
|
+
message: `Evaluated ${oracleCount} oracle${oracleCount !== 1 ? 's' : ''}`,
|
|
428
|
+
metadata: {
|
|
429
|
+
testName: test.Name,
|
|
430
|
+
testRun
|
|
431
|
+
}
|
|
432
|
+
});
|
|
433
|
+
// Update TestRun entity with results
|
|
434
|
+
await this.updateTestRun(testRun, driverResult, startTime);
|
|
435
|
+
// Convert to TestRunResult
|
|
436
|
+
const result = {
|
|
437
|
+
testRunId: testRun.ID,
|
|
438
|
+
testId: test.ID,
|
|
439
|
+
testName: test.Name,
|
|
440
|
+
status: driverResult.status,
|
|
441
|
+
score: driverResult.score,
|
|
442
|
+
passedChecks: driverResult.passedChecks,
|
|
443
|
+
failedChecks: driverResult.failedChecks,
|
|
444
|
+
totalChecks: driverResult.totalChecks,
|
|
445
|
+
oracleResults: driverResult.oracleResults,
|
|
446
|
+
targetType: driverResult.targetType,
|
|
447
|
+
targetLogId: driverResult.targetLogId,
|
|
448
|
+
durationMs: Date.now() - startTime,
|
|
449
|
+
totalCost: driverResult.totalCost || 0,
|
|
450
|
+
startedAt: testRun.StartedAt,
|
|
451
|
+
completedAt: testRun.CompletedAt
|
|
452
|
+
};
|
|
453
|
+
// Add sequence if this is a repeated test iteration
|
|
454
|
+
if (sequence && sequence > 1) {
|
|
455
|
+
result.sequence = sequence;
|
|
456
|
+
}
|
|
457
|
+
// Progress: Complete
|
|
458
|
+
options.progressCallback?.({
|
|
459
|
+
step: 'complete',
|
|
460
|
+
percentage: 100,
|
|
461
|
+
message: `Test ${result.status}`,
|
|
462
|
+
metadata: {
|
|
463
|
+
testName: test.Name,
|
|
464
|
+
testRun
|
|
465
|
+
}
|
|
466
|
+
});
|
|
467
|
+
this.log(`Test completed: ${result.status} (Score: ${result.score})`, options.verbose);
|
|
468
|
+
return result;
|
|
469
|
+
}
|
|
470
|
+
/**
|
|
471
|
+
* Log execution progress.
|
|
472
|
+
* @private
|
|
473
|
+
*/
|
|
474
|
+
log(message, verboseOnly = false) {
|
|
475
|
+
(0, core_1.LogStatusEx)({
|
|
476
|
+
message: `[TestEngine] ${message}`,
|
|
477
|
+
verboseOnly,
|
|
478
|
+
isVerboseEnabled: () => (0, core_1.IsVerboseLoggingEnabled)()
|
|
479
|
+
});
|
|
480
|
+
}
|
|
481
|
+
/**
|
|
482
|
+
* Log errors.
|
|
483
|
+
* @private
|
|
484
|
+
*/
|
|
485
|
+
logError(message, error) {
|
|
486
|
+
(0, core_1.LogError)(`[TestEngine] ${message}`, undefined, error);
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
exports.TestEngine = TestEngine;
|
|
490
|
+
//# sourceMappingURL=TestEngine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TestEngine.js","sourceRoot":"","sources":["../../src/engine/TestEngine.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,+CAM8B;AAQ9B,mDAAkD;AAClD,6EAAqE;AACrE,8DAA2D;AAE3D,4EAAyE;AACzE,0EAAuE;AACvE,8DAA2D;AAC3D,kEAA+D;AAC/D,sEAAmE;AAQnE;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAa,UAAW,SAAQ,oCAAc;IAI1C;;OAEG;IACI,MAAM,KAAK,QAAQ;QACtB,OAAO,KAAK,CAAC,WAAW,EAAc,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH;QACI,KAAK,EAAE,CAAC;QAdJ,iBAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QACjD,oBAAe,GAAG,IAAI,GAAG,EAAmB,CAAC;IAcrD,CAAC;IAEkB,KAAK,CAAC,iBAAiB,CAAC,WAAsB;QAC7D,IAAI,CAAC;YACD,MAAM,KAAK,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAE3C,4BAA4B;YAC5B,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAEpC,IAAI,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,CAAC,gCAAgC,EAAE,KAAc,CAAC,CAAC;QACpE,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACI,KAAK,CAAC,OAAO,CAChB,MAAc,EACd,OAAuB,EACvB,WAAqB,EACrB,UAA0B,EAC1B,iBAAiC;QAEjC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,4BAA4B,MAAM,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAEhE,IAAI,CAAC;YACD,yBAAyB;YACzB,OAAO,CAAC,gBAAgB,EAAE,CAAC;gBACvB,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,EAAE;gBACd,OAAO,EAAE,4BAA4B;gBACrC,QAAQ,EAAE,EAAE,MAAM,EAAE;aACvB,CAAC,CAAC;YAEH,8BAA8B;YAC9B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,EAAE,CAAC,CAAC;YACjD,CAAC;YAED,+DAA+D;YAC/D,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBAC3C,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,CAAC,CAAC;YAC9H,CAAC;YAED,+CAA+C;YAC/C,OAAO,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,UAAU,EAAE,iBAAiB,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QAEnH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,CAAC,0BAA0B,MAAM,EAAE,EAAE,KAAc,CAAC,CAAC;YAClE,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,QAAQ,CACjB,OAAe,EACf,OAAuB,EACvB,WAAqB;QAErB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,kCAAkC,OAAO,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAEvE,IAAI,CAAC;YACD,+BAA+B;YAC/B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,yBAAyB,OAAO,EAAE,CAAC,CAAC;YACxD,CAAC;YAED,8BAA8B;YAC9B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YACjD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAC;YAC3D,CAAC;YAED,6BAA6B;YAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAE/D,gBAAgB;YAChB,MAAM,WAAW,GAAoB,EAAE,CAAC;YACxC,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,wCAAwC;YAC9D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;oBAC5F,YAAY,EAAE,CAAC,CAAC,mCAAmC;oBAEnD,sEAAsE;oBACtE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;wBACxB,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;oBAChC,CAAC;yBAAM,CAAC;wBACJ,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC7B,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,IAAI,CAAC,QAAQ,CAAC,yBAAyB,IAAI,CAAC,IAAI,EAAE,EAAE,KAAc,CAAC,CAAC;oBACpE,gCAAgC;gBACpC,CAAC;YACL,CAAC;YAED,0CAA0C;YAC1C,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;YAE5D,gCAAgC;YAChC,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;YAC1E,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;YAC1E,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACpE,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAE9E,MAAM,MAAM,GAAuB;gBAC/B,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,OAAO,EAAE,KAAK,CAAC,EAAE;gBACjB,SAAS,EAAE,KAAK,CAAC,IAAI;gBACrB,MAAM,EAAE,QAAQ,CAAC,MAAsE;gBACvF,WAAW;gBACX,WAAW;gBACX,UAAU,EAAE,WAAW,CAAC,MAAM;gBAC9B,YAAY,EAAE,QAAQ;gBACtB,WAAW;gBACX,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;gBAC/D,SAAS,EAAE,QAAQ,CAAC,SAAU;gBAC9B,WAAW,EAAE,QAAQ,CAAC,WAAY;aACrC,CAAC;YAEF,IAAI,CAAC,GAAG,CACJ,oBAAoB,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,WAAW,CAAC,MAAM,UAAU,EACjF,OAAO,CAAC,OAAO,CAClB,CAAC;YACF,OAAO,MAAM,CAAC;QAElB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,CAAC,2BAA2B,OAAO,EAAE,EAAE,KAAc,CAAC,CAAC;YACpE,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,MAAe;QACjC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC9C,IAAI,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACI,SAAS,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACI,cAAc;QACjB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,sBAAsB;QAChC,IAAI,CAAC,cAAc,CAAC,IAAI,6CAAqB,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,cAAc,CAAC,IAAI,2CAAoB,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,cAAc,CAAC,IAAI,+BAAc,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,cAAc,CAAC,IAAI,mCAAgB,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,cAAc,CAAC,IAAI,uCAAkB,EAAE,CAAC,CAAC;IAClD,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,SAAS,CAAC,QAAwB,EAAE,WAAqB;QACnE,cAAc;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAClD,IAAI,MAAM,EAAE,CAAC;YACT,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,uCAAuC;QACvC,MAAM,MAAM,GAAG,iBAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CACxD,+BAAc,EACd,QAAQ,CAAC,WAAW,CACvB,CAAC;QAEF,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,mBAAmB;QACnB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC3C,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,QAAQ,CAAC,MAAc;QACjC,6CAA6C;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,SAAS,CAAC,OAAe;QACnC,8CAA8C;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,EAAE,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,cAAc,CAAC,OAAe;QACxC,+DAA+D;QAC/D,OAAO,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CACvB,IAAgB,EAChB,WAAqB,EACrB,UAA0B,EAC1B,QAAwB;QAExB,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,eAAe,CAAgB,eAAe,EAAE,WAAW,CAAC,CAAC;QACtF,OAAO,CAAC,SAAS,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;QAC3B,OAAO,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,sCAAsC;QACtC,IAAI,UAAU,EAAE,CAAC;YACb,OAAO,CAAC,cAAc,GAAG,UAAU,CAAC;QACxC,CAAC;QAED,uEAAuE;QACvE,oBAAoB;QACpB,0DAA0D;QAC1D,qDAAqD;QACrD,4CAA4C;QAC5C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAChC,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,EAAE,OAAO,IAAI,eAAe,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,cAAc,CACxB,KAAsB,EACtB,WAAqB;QAErB,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,eAAe,CACrC,qBAAqB,EACrB,WAAW,CACd,CAAC;QACF,QAAQ,CAAC,SAAS,EAAE,CAAC;QACrB,QAAQ,CAAC,OAAO,GAAG,KAAK,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,WAAW,GAAG,WAAW,CAAC,EAAE,CAAC;QACtC,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC;QAC5B,QAAQ,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAEhC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,EAAE,OAAO,IAAI,eAAe,CAAC;YACnE,MAAM,IAAI,KAAK,CAAC,yCAAyC,QAAQ,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CACvB,OAAsB,EACtB,MAA6B,EAC7B,SAAiB;QAEjB,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC/B,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC7B,OAAO,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QAC3C,OAAO,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QAC3C,OAAO,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACzC,OAAO,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACvC,OAAO,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACzC,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/E,OAAO,CAAC,kBAAkB,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClG,OAAO,CAAC,gBAAgB,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5F,OAAO,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3F,OAAO,CAAC,OAAO,GAAG,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;QACxC,OAAO,CAAC,eAAe,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;QAC1D,OAAO,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;QAEjC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,iCAAiC,EAAE,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/F,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,cAAc,CACxB,QAA4B,EAC5B,WAA4B,EAC5B,SAAiB;QAEjB,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC1E,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;QAEtC,QAAQ,CAAC,MAAM,GAAG,WAAW,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;QACtE,QAAQ,CAAC,WAAW,GAAG,WAAW,CAAC;QACnC,QAAQ,CAAC,WAAW,GAAG,UAAU,GAAG,WAAW,CAAC;QAChD,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;QACjC,QAAQ,CAAC,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAC7E,QAAQ,CAAC,oBAAoB,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;QAChE,QAAQ,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;QAElC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,sCAAsC,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;QACrG,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,eAAe,CACzB,IAAgB,EAChB,WAAmB,EACnB,OAAuB,EACvB,WAAqB,EACrB,UAAqC,EACrC,iBAA4C,EAC5C,SAAiB;QAEjB,MAAM,OAAO,GAAoB,EAAE,CAAC;QAEpC,IAAI,CAAC,GAAG,CAAC,gBAAgB,WAAW,iCAAiC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAExF,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,WAAW,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC,GAAG,CAAC,qBAAqB,SAAS,OAAO,WAAW,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;YAE9E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAC5C,IAAI,EACJ,UAAU,EACV,SAAS,EACT,OAAO,EACP,WAAW,EACX,IAAI,CAAC,GAAG,EAAE,CAAC,yCAAyC;aACvD,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAErB,wDAAwD;YACxD,IAAI,SAAS,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,aAAa,WAAW,aAAa,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QACjE,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,sBAAsB,CAChC,IAAgB,EAChB,UAAqC,EACrC,QAAuB,EACvB,OAAuB,EACvB,WAAqB,EACrB,SAAiB;QAEjB,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,gCAAgC;QAChC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACvB,IAAI,EAAE,qBAAqB;YAC3B,UAAU,EAAE,EAAE;YACd,OAAO,EAAE,gBAAgB,QAAQ,CAAC,WAAW,SAAS;YACtD,QAAQ,EAAE;gBACN,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,UAAU,EAAE,QAAQ,CAAC,WAAW;aACnC;SACJ,CAAC,CAAC;QAEH,uBAAuB;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAE3D,wBAAwB;QACxB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QAElF,2BAA2B;QAC3B,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACvB,IAAI,EAAE,gBAAgB;YACtB,UAAU,EAAE,EAAE;YACd,OAAO,EAAE,qBAAqB;YAC9B,QAAQ,EAAE;gBACN,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,OAAO;gBACP,UAAU,EAAE,QAAQ,CAAC,WAAW;aACnC;SACJ,CAAC,CAAC;QAEH,0BAA0B;QAC1B,IAAI,CAAC,GAAG,CAAC,sBAAsB,QAAQ,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QACxE,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;YACtC,IAAI;YACJ,OAAO;YACP,WAAW;YACX,OAAO;YACP,cAAc,EAAE,IAAI,CAAC,eAAe;SACvC,CAAC,CAAC;QAEH,+BAA+B;QAC/B,MAAM,WAAW,GAAG,YAAY,CAAC,aAAa,EAAE,MAAM,IAAI,CAAC,CAAC;QAC5D,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACvB,IAAI,EAAE,oBAAoB;YAC1B,UAAU,EAAE,EAAE;YACd,OAAO,EAAE,aAAa,WAAW,UAAU,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACzE,QAAQ,EAAE;gBACN,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,OAAO;aACV;SACJ,CAAC,CAAC;QAEH,qCAAqC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;QAE3D,2BAA2B;QAC3B,MAAM,MAAM,GAAkB;YAC1B,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,YAAY,CAAC,MAAM;YAC3B,KAAK,EAAE,YAAY,CAAC,KAAK;YACzB,YAAY,EAAE,YAAY,CAAC,YAAY;YACvC,YAAY,EAAE,YAAY,CAAC,YAAY;YACvC,WAAW,EAAE,YAAY,CAAC,WAAW;YACrC,aAAa,EAAE,YAAY,CAAC,aAAa;YACzC,UAAU,EAAE,YAAY,CAAC,UAAU;YACnC,WAAW,EAAE,YAAY,CAAC,WAAW;YACrC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,SAAS,EAAE,YAAY,CAAC,SAAS,IAAI,CAAC;YACtC,SAAS,EAAE,OAAO,CAAC,SAAU;YAC7B,WAAW,EAAE,OAAO,CAAC,WAAY;SACpC,CAAC;QAEF,oDAAoD;QACpD,IAAI,QAAQ,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC/B,CAAC;QAED,qBAAqB;QACrB,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACvB,IAAI,EAAE,UAAU;YAChB,UAAU,EAAE,GAAG;YACf,OAAO,EAAE,QAAQ,MAAM,CAAC,MAAM,EAAE;YAChC,QAAQ,EAAE;gBACN,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,OAAO;aACV;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,KAAK,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QACvF,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,GAAG,CAAC,OAAe,EAAE,cAAuB,KAAK;QACrD,IAAA,kBAAW,EAAC;YACR,OAAO,EAAE,gBAAgB,OAAO,EAAE;YAClC,WAAW;YACX,gBAAgB,EAAE,GAAG,EAAE,CAAC,IAAA,8BAAuB,GAAE;SACpD,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACK,QAAQ,CAAC,OAAe,EAAE,KAAa;QAC3C,IAAA,eAAQ,EAAC,gBAAgB,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC1D,CAAC;CACJ;AAtjBD,gCAsjBC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MemberJunction Testing Engine
|
|
3
|
+
*
|
|
4
|
+
* Core execution engine for the MemberJunction Testing Framework.
|
|
5
|
+
* Provides test drivers, oracles, and execution orchestration.
|
|
6
|
+
*/
|
|
7
|
+
export * from './engine/TestEngine';
|
|
8
|
+
export * from './drivers/BaseTestDriver';
|
|
9
|
+
export * from './drivers/AgentEvalDriver';
|
|
10
|
+
export * from './oracles/IOracle';
|
|
11
|
+
export * from './oracles/SchemaValidatorOracle';
|
|
12
|
+
export * from './oracles/TraceValidatorOracle';
|
|
13
|
+
export * from './oracles/LLMJudgeOracle';
|
|
14
|
+
export * from './oracles/ExactMatchOracle';
|
|
15
|
+
export * from './oracles/SQLValidatorOracle';
|
|
16
|
+
export * from './types';
|
|
17
|
+
export * from './utils/scoring';
|
|
18
|
+
export * from './utils/cost-calculator';
|
|
19
|
+
export * from './utils/result-formatter';
|
|
20
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,cAAc,qBAAqB,CAAC;AAGpC,cAAc,0BAA0B,CAAC;AAGzC,cAAc,2BAA2B,CAAC;AAG1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,iCAAiC,CAAC;AAChD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,8BAA8B,CAAC;AAG7C,cAAc,SAAS,CAAC;AAGxB,cAAc,iBAAiB,CAAC;AAChC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* MemberJunction Testing Engine
|
|
4
|
+
*
|
|
5
|
+
* Core execution engine for the MemberJunction Testing Framework.
|
|
6
|
+
* Provides test drivers, oracles, and execution orchestration.
|
|
7
|
+
*/
|
|
8
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
|
+
if (k2 === undefined) k2 = k;
|
|
10
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
11
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
12
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
13
|
+
}
|
|
14
|
+
Object.defineProperty(o, k2, desc);
|
|
15
|
+
}) : (function(o, m, k, k2) {
|
|
16
|
+
if (k2 === undefined) k2 = k;
|
|
17
|
+
o[k2] = m[k];
|
|
18
|
+
}));
|
|
19
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
20
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
21
|
+
};
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
// Main engine
|
|
24
|
+
__exportStar(require("./engine/TestEngine"), exports);
|
|
25
|
+
// Base classes
|
|
26
|
+
__exportStar(require("./drivers/BaseTestDriver"), exports);
|
|
27
|
+
// Concrete drivers
|
|
28
|
+
__exportStar(require("./drivers/AgentEvalDriver"), exports);
|
|
29
|
+
// Oracle interface and implementations
|
|
30
|
+
__exportStar(require("./oracles/IOracle"), exports);
|
|
31
|
+
__exportStar(require("./oracles/SchemaValidatorOracle"), exports);
|
|
32
|
+
__exportStar(require("./oracles/TraceValidatorOracle"), exports);
|
|
33
|
+
__exportStar(require("./oracles/LLMJudgeOracle"), exports);
|
|
34
|
+
__exportStar(require("./oracles/ExactMatchOracle"), exports);
|
|
35
|
+
__exportStar(require("./oracles/SQLValidatorOracle"), exports);
|
|
36
|
+
// Types and interfaces
|
|
37
|
+
__exportStar(require("./types"), exports);
|
|
38
|
+
// Utilities
|
|
39
|
+
__exportStar(require("./utils/scoring"), exports);
|
|
40
|
+
__exportStar(require("./utils/cost-calculator"), exports);
|
|
41
|
+
__exportStar(require("./utils/result-formatter"), exports);
|
|
42
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;;AAEH,cAAc;AACd,sDAAoC;AAEpC,eAAe;AACf,2DAAyC;AAEzC,mBAAmB;AACnB,4DAA0C;AAE1C,uCAAuC;AACvC,oDAAkC;AAClC,kEAAgD;AAChD,iEAA+C;AAC/C,2DAAyC;AACzC,6DAA2C;AAC3C,+DAA6C;AAE7C,uBAAuB;AACvB,0CAAwB;AAExB,YAAY;AACZ,kDAAgC;AAChC,0DAAwC;AACxC,2DAAyC"}
|