@matware/e2e-runner 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +37 -6
- package/.claude-plugin/plugin.json +17 -3
- package/LICENSE +190 -0
- package/README.md +61 -526
- package/bin/cli.js +5 -4
- package/commands/capture.md +45 -0
- package/package.json +1 -1
- package/src/actions.js +151 -0
- package/src/ai-generate.js +81 -0
- package/src/app-pool.js +339 -0
- package/src/config.js +125 -7
- package/src/dashboard.js +75 -8
- package/src/db.js +63 -7
- package/src/index.js +6 -4
- package/src/learner-sqlite.js +154 -0
- package/src/learner.js +70 -3
- package/src/mcp-tools.js +251 -32
- package/src/narrate.js +28 -0
- package/src/pool-manager.js +22 -16
- package/src/pool.js +301 -31
- package/src/reporter.js +4 -1
- package/src/runner.js +335 -55
- package/src/visual-diff.js +446 -0
- package/templates/dashboard/js/api.js +2 -0
- package/templates/dashboard/js/utils.js +20 -0
- package/templates/dashboard/js/view-live.js +40 -2
- package/templates/dashboard/js/view-runs.js +161 -57
- package/templates/dashboard/js/websocket.js +6 -0
- package/templates/dashboard/styles/components.css +7 -0
- package/templates/dashboard/styles/view-live.css +24 -1
- package/templates/dashboard/styles/view-runs.css +36 -0
- package/templates/dashboard/template.html +24 -9
- package/templates/dashboard.html +322 -310
package/src/runner.js
CHANGED
|
@@ -9,12 +9,14 @@ import fs from 'fs';
|
|
|
9
9
|
import path from 'path';
|
|
10
10
|
import http from 'http';
|
|
11
11
|
import https from 'https';
|
|
12
|
-
import { connectToPool } from './pool.js';
|
|
12
|
+
import { connectToPool, getCachedDriver, disconnectFromPool } from './pool.js';
|
|
13
13
|
import { getPoolUrls, selectPool, releasePending } from './pool-manager.js';
|
|
14
|
+
import { forkAppInstance, destroyFork, isAppPoolEnabled } from './app-pool.js';
|
|
14
15
|
import { executeAction } from './actions.js';
|
|
15
16
|
import { narrateAction } from './narrate.js';
|
|
16
17
|
import { log, colors as C } from './logger.js';
|
|
17
18
|
import { resolveTestData, validateActionTypes } from './module-resolver.js';
|
|
19
|
+
import { compareImages } from './visual-diff.js';
|
|
18
20
|
import { ensureProject, getVariables } from './db.js';
|
|
19
21
|
|
|
20
22
|
function sleep(ms) {
|
|
@@ -115,7 +117,7 @@ function getByPath(obj, dotPath) {
|
|
|
115
117
|
}
|
|
116
118
|
|
|
117
119
|
/** Fetches an auth token by POSTing credentials to a login endpoint. */
|
|
118
|
-
function fetchAuthToken(endpoint, credentials, tokenPath) {
|
|
120
|
+
export function fetchAuthToken(endpoint, credentials, tokenPath) {
|
|
119
121
|
return new Promise((resolve, reject) => {
|
|
120
122
|
const url = new URL(endpoint);
|
|
121
123
|
const transport = url.protocol === 'https:' ? https : http;
|
|
@@ -123,7 +125,7 @@ function fetchAuthToken(endpoint, credentials, tokenPath) {
|
|
|
123
125
|
|
|
124
126
|
const req = transport.request(url, {
|
|
125
127
|
method: 'POST',
|
|
126
|
-
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
|
|
128
|
+
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body), 'Accept': '*/*', 'User-Agent': '@matware/e2e-runner' },
|
|
127
129
|
timeout: 15000,
|
|
128
130
|
}, (res) => {
|
|
129
131
|
let data = '';
|
|
@@ -155,6 +157,14 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
155
157
|
let browser = null;
|
|
156
158
|
let context = null;
|
|
157
159
|
let page = null;
|
|
160
|
+
let cdpSession = null;
|
|
161
|
+
let appFork = null;
|
|
162
|
+
|
|
163
|
+
// ── Multi-tab registry ────────────────────────────────────────────────────
|
|
164
|
+
// Maps label → page. The "default" label is the initial page.
|
|
165
|
+
// activePage tracks the current tab; page always points to it.
|
|
166
|
+
const tabRegistry = new Map();
|
|
167
|
+
let activeTabLabel = 'default';
|
|
158
168
|
|
|
159
169
|
const result = {
|
|
160
170
|
name: test.name,
|
|
@@ -169,8 +179,20 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
169
179
|
const pendingBodies = [];
|
|
170
180
|
|
|
171
181
|
try {
|
|
172
|
-
|
|
182
|
+
// Fork an isolated app instance if app pool is enabled
|
|
183
|
+
let effectiveConfig = config;
|
|
184
|
+
if (isAppPoolEnabled(config)) {
|
|
185
|
+
appFork = await forkAppInstance(config, test.name);
|
|
186
|
+
// Override baseUrl to point to this test's isolated app instance
|
|
187
|
+
// Use dockerBaseUrl when Chrome runs inside Docker (default setup)
|
|
188
|
+
effectiveConfig = { ...config, baseUrl: appFork.dockerBaseUrl };
|
|
189
|
+
result.appFork = { forkId: appFork.forkId, baseUrl: appFork.baseUrl, port: appFork.port, forkTimeMs: appFork.forkTimeMs };
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const driverOpts = { poolDriver: config.poolDriver || 'auto', maxSessions: config.maxSessions || 10 };
|
|
193
|
+
const chosenPool = await selectPool(getPoolUrls(config), 2000, 60000, driverOpts);
|
|
173
194
|
result.poolUrl = chosenPool;
|
|
195
|
+
result.poolDriver = getCachedDriver(chosenPool);
|
|
174
196
|
const poolLabel = chosenPool.replace('ws://', '').replace('wss://', '');
|
|
175
197
|
const isMultiPool = getPoolUrls(config).length > 1;
|
|
176
198
|
if (isMultiPool) {
|
|
@@ -182,6 +204,42 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
182
204
|
context = await browser.createBrowserContext();
|
|
183
205
|
page = await context.newPage();
|
|
184
206
|
await page.setViewport(config.viewport);
|
|
207
|
+
tabRegistry.set('default', page);
|
|
208
|
+
|
|
209
|
+
// CDP screencast — streams browser frames as JPEG to the dashboard
|
|
210
|
+
// Only attempt on browserless pools; generic CDP pools (Lightpanda) break on createCDPSession
|
|
211
|
+
const poolDriver = getCachedDriver(chosenPool);
|
|
212
|
+
if (config.screencast && poolDriver !== 'cdp') {
|
|
213
|
+
try {
|
|
214
|
+
const raceTimeout = (promise, ms) => Promise.race([
|
|
215
|
+
promise,
|
|
216
|
+
new Promise((_, reject) => { const t = setTimeout(() => reject(new Error('CDP timeout')), ms); t.unref(); }),
|
|
217
|
+
]);
|
|
218
|
+
cdpSession = await raceTimeout(page.createCDPSession(), 5000);
|
|
219
|
+
let frameCount = 0;
|
|
220
|
+
const everyNth = config.screencastEveryNthFrame || 1;
|
|
221
|
+
cdpSession.on('Page.screencastFrame', (frame) => {
|
|
222
|
+
frameCount++;
|
|
223
|
+
cdpSession.send('Page.screencastFrameAck', { sessionId: frame.sessionId }).catch(() => {});
|
|
224
|
+
if (everyNth > 1 && frameCount % everyNth !== 0) return;
|
|
225
|
+
progressFn({
|
|
226
|
+
event: 'test:frame',
|
|
227
|
+
name: test.name,
|
|
228
|
+
data: frame.data,
|
|
229
|
+
metadata: frame.metadata,
|
|
230
|
+
});
|
|
231
|
+
});
|
|
232
|
+
await raceTimeout(cdpSession.send('Page.startScreencast', {
|
|
233
|
+
format: 'jpeg',
|
|
234
|
+
quality: config.screencastQuality || 60,
|
|
235
|
+
maxWidth: config.screencastMaxWidth || 800,
|
|
236
|
+
maxHeight: config.screencastMaxHeight || 600,
|
|
237
|
+
everyNthFrame: 1,
|
|
238
|
+
}), 5000);
|
|
239
|
+
} catch {
|
|
240
|
+
cdpSession = null;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
185
243
|
|
|
186
244
|
page.on('console', (msg) => {
|
|
187
245
|
result.consoleLogs.push({ type: msg.type(), text: msg.text() });
|
|
@@ -225,9 +283,9 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
225
283
|
});
|
|
226
284
|
|
|
227
285
|
// Auto-inject auth token into localStorage (runs BEFORE beforeEach hooks)
|
|
228
|
-
if (
|
|
229
|
-
const storageKey =
|
|
230
|
-
await page.goto(
|
|
286
|
+
if (effectiveConfig.authToken) {
|
|
287
|
+
const storageKey = effectiveConfig.authStorageKey || 'accessToken';
|
|
288
|
+
await page.goto(effectiveConfig.baseUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
|
|
231
289
|
await page.evaluate((key, token) => {
|
|
232
290
|
localStorage.setItem(key, token);
|
|
233
291
|
}, storageKey, config.authToken);
|
|
@@ -243,14 +301,14 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
243
301
|
|
|
244
302
|
// Run beforeEach hook
|
|
245
303
|
if (hooks.beforeEach?.length) {
|
|
246
|
-
await executeHookActions(page, hooks.beforeEach,
|
|
304
|
+
await executeHookActions(page, hooks.beforeEach, effectiveConfig);
|
|
247
305
|
}
|
|
248
306
|
|
|
249
307
|
// Auto-capture baseline screenshot if test has "expect" (BEFORE actions)
|
|
250
308
|
if (test.expect && page) {
|
|
251
309
|
try {
|
|
252
310
|
const safeName = test.name.replace(/[^a-zA-Z0-9_\-. ]/g, '_');
|
|
253
|
-
const baselinePath = path.join(
|
|
311
|
+
const baselinePath = path.join(effectiveConfig.screenshotsDir, `baseline-${safeName}-${Date.now()}.png`);
|
|
254
312
|
await page.screenshot({ path: baselinePath, fullPage: true });
|
|
255
313
|
result.baselineScreenshot = baselinePath;
|
|
256
314
|
} catch { /* page may not be ready */ }
|
|
@@ -258,8 +316,8 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
258
316
|
|
|
259
317
|
for (let i = 0; i < test.actions.length; i++) {
|
|
260
318
|
const action = test.actions[i];
|
|
261
|
-
const maxActionRetries = action.retries ??
|
|
262
|
-
const actionRetryDelay =
|
|
319
|
+
const maxActionRetries = action.retries ?? effectiveConfig.actionRetries ?? 0;
|
|
320
|
+
const actionRetryDelay = effectiveConfig.actionRetryDelay ?? 500;
|
|
263
321
|
let lastError = null;
|
|
264
322
|
|
|
265
323
|
for (let attempt = 0; attempt <= maxActionRetries; attempt++) {
|
|
@@ -273,8 +331,113 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
273
331
|
throw new Error(`assert_no_network_errors failed: ${result.networkErrors.length} error(s): ${summary}`);
|
|
274
332
|
}
|
|
275
333
|
actionResult = null;
|
|
334
|
+
|
|
335
|
+
// ── Multi-tab actions (intercepted here, not in actions.js) ──────
|
|
336
|
+
} else if (action.type === 'open_tab') {
|
|
337
|
+
const label = action.text || `tab-${tabRegistry.size}`;
|
|
338
|
+
const newPage = await context.newPage();
|
|
339
|
+
await newPage.setViewport(config.viewport);
|
|
340
|
+
tabRegistry.set(label, newPage);
|
|
341
|
+
activeTabLabel = label;
|
|
342
|
+
page = newPage;
|
|
343
|
+
// Navigate inside the new tab
|
|
344
|
+
actionResult = await executeAction(page, action, effectiveConfig);
|
|
345
|
+
|
|
346
|
+
} else if (action.type === 'switch_tab') {
|
|
347
|
+
// value: label, title regex, URL substring, or numeric index
|
|
348
|
+
const target = action.value;
|
|
349
|
+
let found = false;
|
|
350
|
+
|
|
351
|
+
// 1. By label (exact match)
|
|
352
|
+
if (tabRegistry.has(target)) {
|
|
353
|
+
page = tabRegistry.get(target);
|
|
354
|
+
activeTabLabel = target;
|
|
355
|
+
found = true;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// 2. By numeric index
|
|
359
|
+
if (!found && /^\d+$/.test(target)) {
|
|
360
|
+
const idx = parseInt(target);
|
|
361
|
+
const labels = [...tabRegistry.keys()];
|
|
362
|
+
if (idx >= 0 && idx < labels.length) {
|
|
363
|
+
activeTabLabel = labels[idx];
|
|
364
|
+
page = tabRegistry.get(activeTabLabel);
|
|
365
|
+
found = true;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// 3. By title or URL match (substring or regex)
|
|
370
|
+
if (!found) {
|
|
371
|
+
for (const [label, p] of tabRegistry) {
|
|
372
|
+
try {
|
|
373
|
+
const title = await p.title();
|
|
374
|
+
const url = p.url();
|
|
375
|
+
const regex = new RegExp(target, 'i');
|
|
376
|
+
if (regex.test(title) || regex.test(url) || url.includes(target)) {
|
|
377
|
+
page = p;
|
|
378
|
+
activeTabLabel = label;
|
|
379
|
+
found = true;
|
|
380
|
+
break;
|
|
381
|
+
}
|
|
382
|
+
} catch { /* page may be closed */ }
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
if (!found) {
|
|
387
|
+
throw new Error(`switch_tab failed: no tab matching "${target}" (labels: ${[...tabRegistry.keys()].join(', ')})`);
|
|
388
|
+
}
|
|
389
|
+
// Bring tab to front
|
|
390
|
+
await page.bringToFront();
|
|
391
|
+
actionResult = null;
|
|
392
|
+
|
|
393
|
+
} else if (action.type === 'close_tab') {
|
|
394
|
+
const targetLabel = action.value || activeTabLabel;
|
|
395
|
+
if (targetLabel === 'default' && tabRegistry.size > 1) {
|
|
396
|
+
throw new Error('close_tab: cannot close the default tab while other tabs are open');
|
|
397
|
+
}
|
|
398
|
+
const targetPage = tabRegistry.get(targetLabel);
|
|
399
|
+
if (!targetPage) {
|
|
400
|
+
throw new Error(`close_tab failed: no tab with label "${targetLabel}"`);
|
|
401
|
+
}
|
|
402
|
+
tabRegistry.delete(targetLabel);
|
|
403
|
+
if (!targetPage.isClosed()) {
|
|
404
|
+
await targetPage.close();
|
|
405
|
+
}
|
|
406
|
+
// Switch to the last remaining tab
|
|
407
|
+
if (activeTabLabel === targetLabel) {
|
|
408
|
+
const remaining = [...tabRegistry.keys()];
|
|
409
|
+
activeTabLabel = remaining[remaining.length - 1] || 'default';
|
|
410
|
+
page = tabRegistry.get(activeTabLabel);
|
|
411
|
+
if (page) await page.bringToFront();
|
|
412
|
+
}
|
|
413
|
+
actionResult = null;
|
|
414
|
+
|
|
415
|
+
} else if (action.type === 'assert_tab_count') {
|
|
416
|
+
action.__tabCount = tabRegistry.size;
|
|
417
|
+
actionResult = await executeAction(page, action, effectiveConfig);
|
|
418
|
+
|
|
419
|
+
} else if (action.type === 'wait_for_tab') {
|
|
420
|
+
// Wait for a new tab/popup to be opened (e.g. by window.open, target=_blank)
|
|
421
|
+
const label = action.text || `tab-${tabRegistry.size}`;
|
|
422
|
+
const waitTimeout = action.timeout || config.defaultTimeout || 10000;
|
|
423
|
+
const newTarget = await new Promise((resolve, reject) => {
|
|
424
|
+
const timer = setTimeout(() => reject(new Error(`wait_for_tab: no new tab appeared after ${waitTimeout}ms`)), waitTimeout);
|
|
425
|
+
context.once('targetcreated', (target) => {
|
|
426
|
+
clearTimeout(timer);
|
|
427
|
+
resolve(target);
|
|
428
|
+
});
|
|
429
|
+
});
|
|
430
|
+
const newPage = await newTarget.page();
|
|
431
|
+
if (newPage) {
|
|
432
|
+
await newPage.setViewport(config.viewport);
|
|
433
|
+
tabRegistry.set(label, newPage);
|
|
434
|
+
activeTabLabel = label;
|
|
435
|
+
page = newPage;
|
|
436
|
+
}
|
|
437
|
+
actionResult = null;
|
|
438
|
+
|
|
276
439
|
} else {
|
|
277
|
-
actionResult = await executeAction(page, action,
|
|
440
|
+
actionResult = await executeAction(page, action, effectiveConfig);
|
|
278
441
|
}
|
|
279
442
|
const actionDuration = Date.now() - actionStart;
|
|
280
443
|
const actionEntry = {
|
|
@@ -323,15 +486,40 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
323
486
|
result.expect = test.expect;
|
|
324
487
|
try {
|
|
325
488
|
const safeName = test.name.replace(/[^a-zA-Z0-9_\-. ]/g, '_');
|
|
326
|
-
const verifyPath = path.join(
|
|
489
|
+
const verifyPath = path.join(effectiveConfig.screenshotsDir, `verify-${safeName}-${Date.now()}.png`);
|
|
327
490
|
await page.screenshot({ path: verifyPath, fullPage: true });
|
|
328
491
|
result.verificationScreenshot = verifyPath;
|
|
492
|
+
|
|
493
|
+
// Auto visual comparison: compare baseline vs verification screenshot
|
|
494
|
+
if (result.baselineScreenshot && result.verificationScreenshot) {
|
|
495
|
+
try {
|
|
496
|
+
const diffPath = path.join(effectiveConfig.screenshotsDir, `diff-${safeName}-${Date.now()}.png`);
|
|
497
|
+
const threshold = effectiveConfig.verificationThreshold ?? 0.02;
|
|
498
|
+
const visualResult = compareImages(result.baselineScreenshot, result.verificationScreenshot, {
|
|
499
|
+
threshold: 0.1,
|
|
500
|
+
diffOutputPath: diffPath,
|
|
501
|
+
maskRegions: test.expect?.maskRegions || [],
|
|
502
|
+
});
|
|
503
|
+
result.visualDiff = {
|
|
504
|
+
diffPercentage: visualResult.diffPercentage,
|
|
505
|
+
differentPixels: visualResult.differentPixels,
|
|
506
|
+
totalPixels: visualResult.totalPixels,
|
|
507
|
+
matchPercentage: visualResult.matchPercentage,
|
|
508
|
+
diffImagePath: visualResult.diffImagePath,
|
|
509
|
+
threshold,
|
|
510
|
+
passed: visualResult.diffPercentage <= threshold,
|
|
511
|
+
};
|
|
512
|
+
if (result.visualDiff.diffImagePath) {
|
|
513
|
+
result.diffScreenshot = result.visualDiff.diffImagePath;
|
|
514
|
+
}
|
|
515
|
+
} catch { /* visual diff is best-effort, never blocks the test */ }
|
|
516
|
+
}
|
|
329
517
|
} catch { /* page may be dead */ }
|
|
330
518
|
}
|
|
331
519
|
|
|
332
520
|
// Run afterEach hook (success path)
|
|
333
521
|
if (hooks.afterEach?.length) {
|
|
334
|
-
await executeHookActions(page, hooks.afterEach,
|
|
522
|
+
await executeHookActions(page, hooks.afterEach, effectiveConfig);
|
|
335
523
|
}
|
|
336
524
|
} catch (error) {
|
|
337
525
|
result.success = false;
|
|
@@ -339,7 +527,7 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
339
527
|
|
|
340
528
|
// Run afterEach hook (failure path)
|
|
341
529
|
if (page && hooks.afterEach?.length) {
|
|
342
|
-
try { await executeHookActions(page, hooks.afterEach,
|
|
530
|
+
try { await executeHookActions(page, hooks.afterEach, effectiveConfig); } catch { /* */ }
|
|
343
531
|
}
|
|
344
532
|
|
|
345
533
|
if (page) {
|
|
@@ -351,6 +539,11 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
351
539
|
} catch { /* page may be dead */ }
|
|
352
540
|
}
|
|
353
541
|
} finally {
|
|
542
|
+
// Stop screencast before disconnecting
|
|
543
|
+
if (cdpSession) {
|
|
544
|
+
try { await cdpSession.send('Page.stopScreencast'); } catch { /* */ }
|
|
545
|
+
try { await cdpSession.detach(); } catch { /* */ }
|
|
546
|
+
}
|
|
354
547
|
// Flush pending response body reads before disconnecting
|
|
355
548
|
if (pendingBodies.length > 0) {
|
|
356
549
|
try { await Promise.allSettled(pendingBodies); } catch { /* */ }
|
|
@@ -363,12 +556,68 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
363
556
|
try { await context.close(); } catch { /* */ }
|
|
364
557
|
}
|
|
365
558
|
if (browser) {
|
|
366
|
-
try { browser.
|
|
559
|
+
try { await disconnectFromPool(browser, result.poolUrl); } catch { /* */ }
|
|
367
560
|
}
|
|
368
561
|
// Release local pending counter so selectPool() knows this slot is free
|
|
369
562
|
if (result.poolUrl) {
|
|
370
563
|
releasePending(result.poolUrl);
|
|
371
564
|
}
|
|
565
|
+
// Destroy the app fork after the test completes
|
|
566
|
+
if (appFork) {
|
|
567
|
+
try { await destroyFork(appFork.forkId); } catch { /* best effort */ }
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
return result;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Majority voting — runs a test N times in parallel and uses majority vote for pass/fail.
|
|
576
|
+
* If majority passes but not unanimously, marks as flaky.
|
|
577
|
+
*/
|
|
578
|
+
async function runTestWithVoting(test, config, hooks, votingCount, testTimeout, progressFn) {
|
|
579
|
+
const votes = [];
|
|
580
|
+
const promises = [];
|
|
581
|
+
|
|
582
|
+
for (let v = 0; v < votingCount; v++) {
|
|
583
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
584
|
+
const timer = setTimeout(() => reject(new Error(`Test timed out after ${testTimeout}ms`)), testTimeout);
|
|
585
|
+
timer.unref();
|
|
586
|
+
});
|
|
587
|
+
promises.push(
|
|
588
|
+
Promise.race([runTest(test, config, hooks, progressFn), timeoutPromise])
|
|
589
|
+
.catch(error => ({
|
|
590
|
+
name: test.name,
|
|
591
|
+
startTime: new Date().toISOString(),
|
|
592
|
+
endTime: new Date().toISOString(),
|
|
593
|
+
actions: [],
|
|
594
|
+
success: false,
|
|
595
|
+
error: error.message,
|
|
596
|
+
consoleLogs: [],
|
|
597
|
+
networkErrors: [],
|
|
598
|
+
networkLogs: [],
|
|
599
|
+
}))
|
|
600
|
+
);
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
const results = await Promise.all(promises);
|
|
604
|
+
const passCount = results.filter(r => r.success).length;
|
|
605
|
+
const majorityPassed = passCount > votingCount / 2;
|
|
606
|
+
|
|
607
|
+
// Pick the representative result: a passing one if majority passed, failing one otherwise
|
|
608
|
+
const representative = majorityPassed
|
|
609
|
+
? results.find(r => r.success) || results[0]
|
|
610
|
+
: results.find(r => !r.success) || results[0];
|
|
611
|
+
|
|
612
|
+
const result = { ...representative };
|
|
613
|
+
result.success = majorityPassed;
|
|
614
|
+
result.voting = { total: votingCount, passed: passCount, failed: votingCount - passCount };
|
|
615
|
+
result.attempt = 1;
|
|
616
|
+
result.maxAttempts = 1;
|
|
617
|
+
|
|
618
|
+
// Non-unanimous pass = flaky
|
|
619
|
+
if (majorityPassed && passCount < votingCount) {
|
|
620
|
+
result.flaky = true;
|
|
372
621
|
}
|
|
373
622
|
|
|
374
623
|
return result;
|
|
@@ -377,6 +626,7 @@ export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
|
|
|
377
626
|
/** Runs tests in parallel with limited concurrency, retries, timeouts, and hooks */
|
|
378
627
|
export async function runTestsParallel(tests, config, suiteHooks = {}) {
|
|
379
628
|
const hooks = mergeHooks(config.hooks, suiteHooks);
|
|
629
|
+
const driverOpts = { poolDriver: config.poolDriver || 'auto', maxSessions: config.maxSessions || 10 };
|
|
380
630
|
|
|
381
631
|
// Run beforeAll hook
|
|
382
632
|
if (hooks.beforeAll?.length) {
|
|
@@ -389,7 +639,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
|
|
|
389
639
|
log('🪝', `${C.dim}Running beforeAll hook...${C.reset}`);
|
|
390
640
|
let browser = null;
|
|
391
641
|
try {
|
|
392
|
-
const hookPool = await selectPool(getPoolUrls(config));
|
|
642
|
+
const hookPool = await selectPool(getPoolUrls(config), 2000, 60000, driverOpts);
|
|
393
643
|
browser = await connectToPool(hookPool, config.connectRetries, config.connectRetryDelay);
|
|
394
644
|
const page = await browser.newPage();
|
|
395
645
|
await page.setViewport(config.viewport);
|
|
@@ -399,7 +649,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
|
|
|
399
649
|
log('❌', `${C.red}beforeAll hook failed: ${error.message}${C.reset}`);
|
|
400
650
|
throw error;
|
|
401
651
|
} finally {
|
|
402
|
-
if (browser) try { browser
|
|
652
|
+
if (browser) try { await disconnectFromPool(browser, hookPool); } catch { /* */ }
|
|
403
653
|
}
|
|
404
654
|
}
|
|
405
655
|
|
|
@@ -414,8 +664,27 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
|
|
|
414
664
|
);
|
|
415
665
|
log('✅', `${C.dim}Auth token acquired (${config.authToken.length} chars)${C.reset}`);
|
|
416
666
|
} catch (error) {
|
|
417
|
-
|
|
418
|
-
|
|
667
|
+
// Docker-internal hostname (nginx, api, etc.) → retry with localhost from host machine
|
|
668
|
+
if (error.message && error.message.includes('ENOTFOUND')) {
|
|
669
|
+
const url = new URL(config.authLoginEndpoint);
|
|
670
|
+
if (!url.hostname.includes('.')) {
|
|
671
|
+
const localhostUrl = `http://localhost${url.port && url.port !== '80' ? ':' + url.port : ''}${url.pathname}${url.search}`;
|
|
672
|
+
log('🔄', `${C.dim}Docker hostname "${url.hostname}" not reachable from host, retrying with ${localhostUrl}...${C.reset}`);
|
|
673
|
+
try {
|
|
674
|
+
config.authToken = await fetchAuthToken(localhostUrl, config.authCredentials, config.authTokenPath || 'token');
|
|
675
|
+
log('✅', `${C.dim}Auth token acquired via localhost fallback (${config.authToken.length} chars)${C.reset}`);
|
|
676
|
+
} catch (retryErr) {
|
|
677
|
+
log('❌', `${C.red}Auth auto-login failed (localhost fallback): ${retryErr.message}${C.reset}`);
|
|
678
|
+
throw retryErr;
|
|
679
|
+
}
|
|
680
|
+
} else {
|
|
681
|
+
log('❌', `${C.red}Auth auto-login failed: ${error.message}${C.reset}`);
|
|
682
|
+
throw error;
|
|
683
|
+
}
|
|
684
|
+
} else {
|
|
685
|
+
log('❌', `${C.red}Auth auto-login failed: ${error.message}${C.reset}`);
|
|
686
|
+
throw error;
|
|
687
|
+
}
|
|
419
688
|
}
|
|
420
689
|
}
|
|
421
690
|
|
|
@@ -446,40 +715,49 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
|
|
|
446
715
|
log('▶▶▶', `${C.cyan}${test.name}${C.reset} ${C.dim}(${activeCount} active)${C.reset}`);
|
|
447
716
|
_progress({ event: 'test:start', name: test.name, serial: test.serial || false, activeCount, queueRemaining: queue.length });
|
|
448
717
|
|
|
449
|
-
const maxAttempts = (test.retries ?? config.retries ?? 0) + 1;
|
|
450
718
|
const testTimeout = test.timeout ?? config.testTimeout ?? 60000;
|
|
719
|
+
const votingCount = test.voting ?? config.voting ?? 0;
|
|
720
|
+
const testHooks = test._suiteHooks ? mergeHooks(config.hooks, test._suiteHooks) : hooks;
|
|
451
721
|
let result;
|
|
452
722
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
723
|
+
if (votingCount > 1) {
|
|
724
|
+
// Majority voting: run N times in parallel, majority wins
|
|
725
|
+
log('🗳️', `${C.dim}${test.name}: voting ${votingCount}x in parallel${C.reset}`);
|
|
726
|
+
result = await runTestWithVoting(test, config, testHooks, votingCount, testTimeout, _progress);
|
|
727
|
+
} else {
|
|
728
|
+
// Standard sequential retry
|
|
729
|
+
const maxAttempts = (test.retries ?? config.retries ?? 0) + 1;
|
|
730
|
+
|
|
731
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
732
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
733
|
+
const timer = setTimeout(() => reject(new Error(`Test timed out after ${testTimeout}ms`)), testTimeout);
|
|
734
|
+
timer.unref();
|
|
735
|
+
});
|
|
736
|
+
|
|
737
|
+
try {
|
|
738
|
+
result = await Promise.race([runTest(test, config, testHooks, _progress), timeoutPromise]);
|
|
739
|
+
} catch (error) {
|
|
740
|
+
result = {
|
|
741
|
+
name: test.name,
|
|
742
|
+
startTime: new Date().toISOString(),
|
|
743
|
+
endTime: new Date().toISOString(),
|
|
744
|
+
actions: [],
|
|
745
|
+
success: false,
|
|
746
|
+
error: error.message,
|
|
747
|
+
consoleLogs: [],
|
|
748
|
+
networkErrors: [],
|
|
749
|
+
networkLogs: [],
|
|
750
|
+
};
|
|
751
|
+
}
|
|
475
752
|
|
|
476
|
-
|
|
477
|
-
|
|
753
|
+
result.attempt = attempt;
|
|
754
|
+
result.maxAttempts = maxAttempts;
|
|
478
755
|
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
756
|
+
if (result.success || attempt === maxAttempts) break;
|
|
757
|
+
log('🔄', `${C.yellow}${test.name}${C.reset} failed, retrying (${attempt}/${maxAttempts})...`);
|
|
758
|
+
_progress({ event: 'test:retry', name: test.name, attempt, maxAttempts });
|
|
759
|
+
await sleep(config.retryDelay || 1000);
|
|
760
|
+
}
|
|
483
761
|
}
|
|
484
762
|
|
|
485
763
|
results.push(result);
|
|
@@ -489,11 +767,13 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
|
|
|
489
767
|
_progress({ event: 'test:complete', name: test.name, success: result.success, duration: timeDiff(result.startTime, result.endTime), error: result.error, consoleLogs: result.consoleLogs, networkErrors: result.networkErrors, networkLogs: result.networkLogs, errorScreenshot: result.errorScreenshot, screenshots, poolUrl: result.poolUrl || null });
|
|
490
768
|
|
|
491
769
|
if (result.success) {
|
|
492
|
-
const
|
|
493
|
-
|
|
770
|
+
const votingInfo = result.voting ? ` ${C.yellow}(voting: ${result.voting.passed}/${result.voting.total} passed${result.flaky ? ', flaky' : ''})${C.reset}` : '';
|
|
771
|
+
const retryInfo = !result.voting && result.attempt > 1 ? ` ${C.yellow}(flaky, passed on attempt ${result.attempt}/${result.maxAttempts})${C.reset}` : '';
|
|
772
|
+
log('✅', `${C.green}${test.name}${C.reset} ${C.dim}(${timeDiff(result.startTime, result.endTime)})${C.reset}${votingInfo}${retryInfo}`);
|
|
494
773
|
} else {
|
|
495
|
-
const
|
|
496
|
-
|
|
774
|
+
const votingInfo = result.voting ? ` (voting: ${result.voting.passed}/${result.voting.total} passed)` : '';
|
|
775
|
+
const attempts = !result.voting && result.maxAttempts > 1 ? ` (${result.maxAttempts} attempts)` : '';
|
|
776
|
+
log('❌', `${C.red}${test.name}${C.reset}: ${result.error}${votingInfo}${attempts}`);
|
|
497
777
|
}
|
|
498
778
|
|
|
499
779
|
const consoleIssues = result.consoleLogs?.filter(l => l.type === 'error' || l.type === 'warning').length || 0;
|
|
@@ -524,7 +804,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
|
|
|
524
804
|
log('🪝', `${C.dim}Running afterAll hook...${C.reset}`);
|
|
525
805
|
let browser = null;
|
|
526
806
|
try {
|
|
527
|
-
const hookPool = await selectPool(getPoolUrls(config));
|
|
807
|
+
const hookPool = await selectPool(getPoolUrls(config), 2000, 60000, driverOpts);
|
|
528
808
|
browser = await connectToPool(hookPool, config.connectRetries, config.connectRetryDelay);
|
|
529
809
|
const page = await browser.newPage();
|
|
530
810
|
await page.setViewport(config.viewport);
|
|
@@ -533,7 +813,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
|
|
|
533
813
|
} catch (error) {
|
|
534
814
|
log('⚠️', `${C.yellow}afterAll hook failed: ${error.message}${C.reset}`);
|
|
535
815
|
} finally {
|
|
536
|
-
if (browser) try { browser
|
|
816
|
+
if (browser) try { await disconnectFromPool(browser, hookPool); } catch { /* */ }
|
|
537
817
|
}
|
|
538
818
|
}
|
|
539
819
|
|