@matware/e2e-runner 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +37 -6
- package/.claude-plugin/plugin.json +17 -3
- package/LICENSE +190 -0
- package/README.md +151 -527
- package/agents/test-creator.md +4 -2
- package/agents/test-improver.md +5 -3
- package/bin/cli.js +84 -20
- package/commands/capture.md +45 -0
- package/package.json +3 -2
- package/skills/e2e-testing/SKILL.md +3 -2
- package/skills/e2e-testing/references/action-types.md +22 -4
- package/skills/e2e-testing/references/test-json-format.md +23 -0
- package/src/actions.js +321 -14
- package/src/ai-generate.js +81 -0
- package/src/app-pool.js +339 -0
- package/src/config.js +131 -7
- package/src/dashboard.js +209 -11
- package/src/db.js +74 -7
- package/src/index.js +6 -4
- package/src/learner-sqlite.js +154 -0
- package/src/learner.js +70 -3
- package/src/mcp-tools.js +259 -34
- package/src/module-analysis.js +247 -0
- package/src/module-resolver.js +35 -2
- package/src/narrate.js +42 -1
- package/src/pool-manager.js +68 -17
- package/src/pool.js +464 -37
- package/src/reporter.js +4 -1
- package/src/runner.js +410 -63
- package/src/visual-diff.js +515 -0
- package/src/websocket.js +14 -3
- package/src/wizard.js +184 -0
- package/templates/build-dashboard.js +3 -0
- package/templates/dashboard/js/api.js +62 -3
- package/templates/dashboard/js/init.js +46 -0
- package/templates/dashboard/js/keyboard.js +8 -7
- package/templates/dashboard/js/quicksearch.js +277 -0
- package/templates/dashboard/js/state.js +61 -7
- package/templates/dashboard/js/toast.js +1 -1
- package/templates/dashboard/js/utils.js +20 -0
- package/templates/dashboard/js/view-live.js +240 -9
- package/templates/dashboard/js/view-runs.js +540 -94
- package/templates/dashboard/js/view-tests.js +157 -16
- package/templates/dashboard/js/view-tools.js +234 -0
- package/templates/dashboard/js/view-watch.js +2 -2
- package/templates/dashboard/js/websocket.js +36 -0
- package/templates/dashboard/styles/base.css +489 -53
- package/templates/dashboard/styles/components.css +719 -77
- package/templates/dashboard/styles/view-live.css +463 -59
- package/templates/dashboard/styles/view-runs.css +793 -155
- package/templates/dashboard/styles/view-tests.css +440 -77
- package/templates/dashboard/styles/view-tools.css +206 -0
- package/templates/dashboard/styles/view-watch.css +198 -41
- package/templates/dashboard/template.html +369 -56
- package/templates/dashboard.html +5375 -901
- package/templates/docker-compose-lightpanda.yml +7 -0
package/src/dashboard.js
CHANGED
|
@@ -16,11 +16,13 @@ import { createRequire } from 'module';
|
|
|
16
16
|
import { createWebSocketServer } from './websocket.js';
|
|
17
17
|
import { getPoolUrls, getAggregatedPoolStatus, waitForAnyPool } from './pool-manager.js';
|
|
18
18
|
import { runTestsParallel, loadAllSuites, loadTestSuite, listSuites } from './runner.js';
|
|
19
|
+
import { runModuleAnalysis } from './module-analysis.js';
|
|
19
20
|
import { generateReport, generateJUnitXML, saveReport, persistRun, loadHistory, loadHistoryRun } from './reporter.js';
|
|
20
21
|
import { listProjects as dbListProjects, listProjectsWithSparklines as dbListProjectsWithSparklines, getProjectRuns as dbGetProjectRuns, getRunDetail as dbGetRunDetail, getAllRuns as dbGetAllRuns, getRunCount as dbGetRunCount, getProjectScreenshotsDir as dbGetProjectScreenshotsDir, getProjectTestsDir as dbGetProjectTestsDir, getProjectCwd as dbGetProjectCwd, lookupScreenshotHash as dbLookupScreenshotHash, ensureProject as dbEnsureProject, getNetworkLogs as dbGetNetworkLogs, listVariables as dbListVariables, setVariable as dbSetVariable, deleteVariable as dbDeleteVariable, closeDb } from './db.js';
|
|
21
22
|
import { loadConfig } from './config.js';
|
|
22
23
|
import { log, colors as C } from './logger.js';
|
|
23
|
-
import { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights, getHealthSnapshot } from './learner-sqlite.js';
|
|
24
|
+
import { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights, getHealthSnapshot, getActionHealthScores } from './learner-sqlite.js';
|
|
25
|
+
import { compareImages, isBlankImage } from './visual-diff.js';
|
|
24
26
|
import { handleSyncRoutes } from './sync/hub-routes.js';
|
|
25
27
|
import { migrateSyncSchema } from './sync/schema.js';
|
|
26
28
|
|
|
@@ -88,11 +90,16 @@ export async function startDashboard(config) {
|
|
|
88
90
|
const url = new URL(req.url, `http://localhost:${port}`);
|
|
89
91
|
const pathname = url.pathname;
|
|
90
92
|
|
|
91
|
-
// CORS —
|
|
93
|
+
// CORS — allow same-origin (Origin's host matches the Host header)
|
|
94
|
+
// and the explicit whitelist (localhost/127.0.0.1 on dashboard port).
|
|
92
95
|
const allowedOrigins = [`http://localhost:${port}`, `http://127.0.0.1:${port}`];
|
|
93
96
|
const origin = req.headers.origin;
|
|
94
|
-
if (origin
|
|
95
|
-
|
|
97
|
+
if (origin) {
|
|
98
|
+
let allowOrigin = allowedOrigins.includes(origin);
|
|
99
|
+
if (!allowOrigin && req.headers.host) {
|
|
100
|
+
try { allowOrigin = new URL(origin).host === req.headers.host; } catch { /* */ }
|
|
101
|
+
}
|
|
102
|
+
if (allowOrigin) res.setHeader('Access-Control-Allow-Origin', origin);
|
|
96
103
|
}
|
|
97
104
|
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
|
|
98
105
|
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Accept, Mcp-Session-Id');
|
|
@@ -120,7 +127,7 @@ export async function startDashboard(config) {
|
|
|
120
127
|
// API: pool status + dashboard state
|
|
121
128
|
if (pathname === '/api/status') {
|
|
122
129
|
const poolUrls = getPoolUrls(config);
|
|
123
|
-
const aggregated = await getAggregatedPoolStatus(poolUrls);
|
|
130
|
+
const aggregated = await getAggregatedPoolStatus(poolUrls, { poolDriver: config.poolDriver, maxSessions: config.maxSessions });
|
|
124
131
|
jsonResponse(res, {
|
|
125
132
|
pool: aggregated,
|
|
126
133
|
poolUrls,
|
|
@@ -305,6 +312,38 @@ export async function startDashboard(config) {
|
|
|
305
312
|
return;
|
|
306
313
|
}
|
|
307
314
|
|
|
315
|
+
// API: DB — project config warnings (Docker hostname detection)
|
|
316
|
+
const configWarningsMatch = pathname.match(/^\/api\/db\/projects\/(\d+)\/config-warnings$/);
|
|
317
|
+
if (configWarningsMatch) {
|
|
318
|
+
try {
|
|
319
|
+
const projectId = parseInt(configWarningsMatch[1], 10);
|
|
320
|
+
const projectCwd = dbGetProjectCwd(projectId);
|
|
321
|
+
if (!projectCwd) { jsonResponse(res, { warnings: [] }); return; }
|
|
322
|
+
const cfg = await loadConfig({}, projectCwd);
|
|
323
|
+
const warnings = [];
|
|
324
|
+
const checkDockerHostname = (url, label) => {
|
|
325
|
+
try {
|
|
326
|
+
const parsed = new URL(url);
|
|
327
|
+
if (!parsed.hostname.includes('.') && parsed.hostname !== 'localhost' && parsed.hostname !== '127') {
|
|
328
|
+
warnings.push({
|
|
329
|
+
type: 'docker-hostname',
|
|
330
|
+
field: label,
|
|
331
|
+
hostname: parsed.hostname,
|
|
332
|
+
url,
|
|
333
|
+
message: `"${parsed.hostname}" looks like a Docker-internal hostname. The runner will auto-fallback to localhost for auth, but baseUrl requests go through Chrome in Docker (which can resolve it). If tests fail with ENOTFOUND, ensure Chrome pool is on the same Docker network.`,
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
} catch {}
|
|
337
|
+
};
|
|
338
|
+
if (cfg.baseUrl) checkDockerHostname(cfg.baseUrl, 'baseUrl');
|
|
339
|
+
if (cfg.authLoginEndpoint) checkDockerHostname(cfg.authLoginEndpoint, 'authLoginEndpoint');
|
|
340
|
+
jsonResponse(res, { warnings });
|
|
341
|
+
} catch (error) {
|
|
342
|
+
jsonResponse(res, { warnings: [], error: error.message });
|
|
343
|
+
}
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
|
|
308
347
|
// API: DB — project screenshots list
|
|
309
348
|
const projectScreenshotsMatch = pathname.match(/^\/api\/db\/projects\/(\d+)\/screenshots$/);
|
|
310
349
|
if (projectScreenshotsMatch) {
|
|
@@ -323,6 +362,28 @@ export async function startDashboard(config) {
|
|
|
323
362
|
return;
|
|
324
363
|
}
|
|
325
364
|
|
|
365
|
+
// API: DB — scan a project's screenshots for blank (uniform-color) images
|
|
366
|
+
const blankScanMatch = pathname.match(/^\/api\/db\/projects\/(\d+)\/screenshots\/blank-scan$/);
|
|
367
|
+
if (blankScanMatch) {
|
|
368
|
+
try {
|
|
369
|
+
const projectId = parseInt(blankScanMatch[1], 10);
|
|
370
|
+
const dir = dbGetProjectScreenshotsDir(projectId);
|
|
371
|
+
if (!dir || !fs.existsSync(dir)) { jsonResponse(res, { blanks: [], scanned: 0 }); return; }
|
|
372
|
+
// Only PNGs are decodable; other formats are skipped (never flagged).
|
|
373
|
+
const files = fs.readdirSync(dir).filter(f => /\.png$/i.test(f)).sort();
|
|
374
|
+
const blanks = [];
|
|
375
|
+
for (const f of files) {
|
|
376
|
+
const fp = path.join(dir, f);
|
|
377
|
+
const r = isBlankImage(fp);
|
|
378
|
+
if (r.blank) blanks.push({ name: f, path: fp, color: r.color, brightness: r.brightness });
|
|
379
|
+
}
|
|
380
|
+
jsonResponse(res, { blanks, scanned: files.length });
|
|
381
|
+
} catch (error) {
|
|
382
|
+
jsonResponse(res, { error: error.message }, 500);
|
|
383
|
+
}
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
386
|
+
|
|
326
387
|
// API: DB — project suites list
|
|
327
388
|
const projectSuitesMatch = pathname.match(/^\/api\/db\/projects\/(\d+)\/suites$/);
|
|
328
389
|
if (projectSuitesMatch) {
|
|
@@ -425,6 +486,68 @@ export async function startDashboard(config) {
|
|
|
425
486
|
return;
|
|
426
487
|
}
|
|
427
488
|
|
|
489
|
+
// API: Tools — proxy to MCP tool handlers
|
|
490
|
+
// Generic helper: resolve projectId from POST body → cwd, then call dispatchTool.
|
|
491
|
+
if (pathname.startsWith('/api/tool/') && req.method === 'POST') {
|
|
492
|
+
const tool = pathname.replace('/api/tool/', '');
|
|
493
|
+
const map = { capture: 'e2e_capture', analyze: 'e2e_analyze', 'issue-verify': 'e2e_issue' };
|
|
494
|
+
const mcpName = map[tool];
|
|
495
|
+
if (!mcpName) { jsonResponse(res, { error: 'Unknown tool: ' + tool }, 400); return; }
|
|
496
|
+
let body = '';
|
|
497
|
+
let oversize = false;
|
|
498
|
+
req.on('data', chunk => { body += chunk; if (body.length > MAX_BODY) { oversize = true; req.destroy(); } });
|
|
499
|
+
req.on('end', async () => {
|
|
500
|
+
if (oversize) { jsonResponse(res, { error: 'Payload too large' }, 413); return; }
|
|
501
|
+
try {
|
|
502
|
+
const args = body ? JSON.parse(body) : {};
|
|
503
|
+
if (args.projectId) {
|
|
504
|
+
const pcwd = dbGetProjectCwd(parseInt(args.projectId, 10));
|
|
505
|
+
if (pcwd) args.cwd = pcwd;
|
|
506
|
+
delete args.projectId;
|
|
507
|
+
}
|
|
508
|
+
if (tool === 'issue-verify') args.mode = 'verify';
|
|
509
|
+
const result = await dispatchTool(mcpName, args);
|
|
510
|
+
// dispatchTool returns MCP-style { content:[{type,text}], isError? }.
|
|
511
|
+
// Convert to a friendlier shape for the dashboard.
|
|
512
|
+
let payload = result;
|
|
513
|
+
if (result && Array.isArray(result.content)) {
|
|
514
|
+
const text = result.content.map(c => c.text || '').join('\n');
|
|
515
|
+
let parsed = null; try { parsed = JSON.parse(text); } catch { /* */ }
|
|
516
|
+
payload = parsed || { text };
|
|
517
|
+
if (result.isError) payload.error = payload.error || payload.text || 'Tool returned error';
|
|
518
|
+
}
|
|
519
|
+
jsonResponse(res, payload);
|
|
520
|
+
} catch (error) {
|
|
521
|
+
jsonResponse(res, { error: error.message }, 500);
|
|
522
|
+
}
|
|
523
|
+
});
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
// API: Tools — module analysis
|
|
528
|
+
// Reads all tests + modules in a project, finds repeated 3-8-action
|
|
529
|
+
// subsequences that appear in 2+ tests (extraction candidates), and
|
|
530
|
+
// counts current module usage. Returns a report ready for the
|
|
531
|
+
// dashboard to display + a prompt the user can paste into Claude Code
|
|
532
|
+
// to ask the test-improver agent for deeper analysis.
|
|
533
|
+
const modAnalysisMatch = pathname.match(/^\/api\/tools\/module-analysis\/(\d+)$/);
|
|
534
|
+
if (modAnalysisMatch) {
|
|
535
|
+
try {
|
|
536
|
+
const projectId = parseInt(modAnalysisMatch[1], 10);
|
|
537
|
+
const cwd = dbGetProjectCwd(projectId);
|
|
538
|
+
const testsDir = dbGetProjectTestsDir(projectId);
|
|
539
|
+
if (!cwd || !testsDir || !fs.existsSync(testsDir)) {
|
|
540
|
+
jsonResponse(res, { error: 'Project tests directory not found' }, 404);
|
|
541
|
+
return;
|
|
542
|
+
}
|
|
543
|
+
const modulesDir = path.join(cwd, 'e2e', 'modules');
|
|
544
|
+
jsonResponse(res, runModuleAnalysis(testsDir, modulesDir, projectId));
|
|
545
|
+
} catch (error) {
|
|
546
|
+
jsonResponse(res, { error: error.message }, 500);
|
|
547
|
+
}
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
|
|
428
551
|
// API: DB — project variables (set/upsert)
|
|
429
552
|
if (projectVarsMatch && req.method === 'PUT') {
|
|
430
553
|
let body = '';
|
|
@@ -498,6 +621,9 @@ export async function startDashboard(config) {
|
|
|
498
621
|
case 'trends':
|
|
499
622
|
data = getTestTrends(projectId, days);
|
|
500
623
|
break;
|
|
624
|
+
case 'actions':
|
|
625
|
+
data = getActionHealthScores(projectId, days);
|
|
626
|
+
break;
|
|
501
627
|
default:
|
|
502
628
|
jsonResponse(res, { error: `Unknown learnings category: ${category}` }, 400);
|
|
503
629
|
return;
|
|
@@ -509,6 +635,71 @@ export async function startDashboard(config) {
|
|
|
509
635
|
return;
|
|
510
636
|
}
|
|
511
637
|
|
|
638
|
+
// API: delete screenshots — { paths: [...] }, each validated against known dirs
|
|
639
|
+
if (pathname === '/api/screenshots/delete' && req.method === 'POST') {
|
|
640
|
+
let body = '';
|
|
641
|
+
let oversize = false;
|
|
642
|
+
req.on('data', chunk => { body += chunk; if (body.length > MAX_BODY) { oversize = true; req.destroy(); } });
|
|
643
|
+
req.on('end', () => {
|
|
644
|
+
if (oversize) { jsonResponse(res, { error: 'Payload too large' }, 413); return; }
|
|
645
|
+
try {
|
|
646
|
+
const { paths } = body ? JSON.parse(body) : {};
|
|
647
|
+
if (!Array.isArray(paths) || !paths.length) { jsonResponse(res, { error: 'Missing paths array' }, 400); return; }
|
|
648
|
+
// Build the allow-list of directories deletions may touch.
|
|
649
|
+
const allowedDirs = [path.resolve(config.screenshotsDir)];
|
|
650
|
+
try {
|
|
651
|
+
for (const p of dbListProjects()) {
|
|
652
|
+
const dir = p.screenshots_dir || path.join(p.cwd, 'e2e', 'screenshots');
|
|
653
|
+
allowedDirs.push(path.resolve(dir));
|
|
654
|
+
}
|
|
655
|
+
} catch { /* db may be unavailable */ }
|
|
656
|
+
let deleted = 0;
|
|
657
|
+
const failed = [];
|
|
658
|
+
for (const raw of paths) {
|
|
659
|
+
try {
|
|
660
|
+
if (typeof raw !== 'string' || !path.isAbsolute(raw)) { failed.push({ path: raw, error: 'Invalid path' }); continue; }
|
|
661
|
+
const real = fs.realpathSync(raw);
|
|
662
|
+
const inAllowed = allowedDirs.some(dir => real.startsWith(dir + path.sep) || real === dir);
|
|
663
|
+
if (!inAllowed) { failed.push({ path: raw, error: 'Access denied' }); continue; }
|
|
664
|
+
if (!/\.(png|jpg|jpeg|gif|webp)$/i.test(real)) { failed.push({ path: raw, error: 'Not an image' }); continue; }
|
|
665
|
+
fs.unlinkSync(real);
|
|
666
|
+
deleted++;
|
|
667
|
+
} catch (e) {
|
|
668
|
+
failed.push({ path: raw, error: e.message });
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
jsonResponse(res, { deleted, failed });
|
|
672
|
+
} catch (error) {
|
|
673
|
+
jsonResponse(res, { error: error.message }, 500);
|
|
674
|
+
}
|
|
675
|
+
});
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// API: visual diff — compare two screenshots on demand
|
|
680
|
+
if (pathname === '/api/visual-diff') {
|
|
681
|
+
try {
|
|
682
|
+
const baseline = url.searchParams.get('baseline');
|
|
683
|
+
const current = url.searchParams.get('current');
|
|
684
|
+
const thresholdParam = url.searchParams.get('threshold');
|
|
685
|
+
if (!baseline || !current) {
|
|
686
|
+
jsonResponse(res, { error: 'Missing baseline or current parameter' }, 400); return;
|
|
687
|
+
}
|
|
688
|
+
if (!fs.existsSync(baseline)) { jsonResponse(res, { error: `Baseline not found: ${baseline}` }, 404); return; }
|
|
689
|
+
if (!fs.existsSync(current)) { jsonResponse(res, { error: `Current not found: ${current}` }, 404); return; }
|
|
690
|
+
|
|
691
|
+
const diffPath = path.join(config.screenshotsDir, `api-diff-${Date.now()}.png`);
|
|
692
|
+
const result = compareImages(baseline, current, {
|
|
693
|
+
threshold: thresholdParam ? parseFloat(thresholdParam) : 0.1,
|
|
694
|
+
diffOutputPath: diffPath,
|
|
695
|
+
});
|
|
696
|
+
jsonResponse(res, { ...result, diffImagePath: diffPath });
|
|
697
|
+
} catch (error) {
|
|
698
|
+
jsonResponse(res, { error: error.message }, 500);
|
|
699
|
+
}
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
|
|
512
703
|
// API: serve screenshot by hash (e.g. /api/screenshot-hash/a3f2b1c9)
|
|
513
704
|
const ssHashMatch = pathname.match(/^\/api\/screenshot-hash\/([a-f0-9]{8})$/);
|
|
514
705
|
if (ssHashMatch) {
|
|
@@ -663,7 +854,9 @@ export async function startDashboard(config) {
|
|
|
663
854
|
if (oversize) { jsonResponse(res, { error: 'Payload too large' }, 413); return; }
|
|
664
855
|
try {
|
|
665
856
|
const data = JSON.parse(body);
|
|
666
|
-
|
|
857
|
+
if (data.event !== 'test:frame') {
|
|
858
|
+
bufferLiveEvent(data);
|
|
859
|
+
}
|
|
667
860
|
wss.broadcast(JSON.stringify(data));
|
|
668
861
|
} catch { /* */ }
|
|
669
862
|
jsonResponse(res, { ok: true });
|
|
@@ -711,8 +904,9 @@ export async function startDashboard(config) {
|
|
|
711
904
|
}
|
|
712
905
|
}, 30000);
|
|
713
906
|
|
|
907
|
+
const devOrigins = process.env.NODE_ENV === 'production' ? [] : ['http://localhost:5173', 'http://127.0.0.1:5173'];
|
|
714
908
|
const wss = createWebSocketServer(server, {
|
|
715
|
-
allowedOrigins: [`http://localhost:${port}`, `http://127.0.0.1:${port}
|
|
909
|
+
allowedOrigins: [`http://localhost:${port}`, `http://127.0.0.1:${port}`, ...devOrigins],
|
|
716
910
|
onConnect(socket) {
|
|
717
911
|
// Replay live state for new/reconnected clients
|
|
718
912
|
for (const rid of Object.keys(liveEventBuffers)) {
|
|
@@ -727,7 +921,7 @@ export async function startDashboard(config) {
|
|
|
727
921
|
const poolUrls = getPoolUrls(config);
|
|
728
922
|
const pollInterval = setInterval(async () => {
|
|
729
923
|
try {
|
|
730
|
-
const aggregated = await getAggregatedPoolStatus(poolUrls);
|
|
924
|
+
const aggregated = await getAggregatedPoolStatus(poolUrls, { poolDriver: config.poolDriver, maxSessions: config.maxSessions });
|
|
731
925
|
wss.broadcast(JSON.stringify({ event: 'pool:status', data: aggregated }));
|
|
732
926
|
} catch { /* */ }
|
|
733
927
|
}, 5000);
|
|
@@ -765,21 +959,25 @@ export async function startDashboard(config) {
|
|
|
765
959
|
runConfig.triggeredBy = 'dashboard';
|
|
766
960
|
if (params.concurrency) runConfig.concurrency = params.concurrency;
|
|
767
961
|
if (params.baseUrl) runConfig.baseUrl = params.baseUrl;
|
|
962
|
+
if (params.screencast !== undefined) runConfig.screencast = params.screencast;
|
|
768
963
|
|
|
769
964
|
// Wire up onProgress to broadcast WS events
|
|
770
965
|
runConfig.onProgress = (data) => {
|
|
771
|
-
|
|
966
|
+
// Don't buffer screencast frames — they're ephemeral and high volume
|
|
967
|
+
if (data.event !== 'test:frame') {
|
|
968
|
+
bufferLiveEvent(data);
|
|
969
|
+
}
|
|
772
970
|
wss.broadcast(JSON.stringify(data));
|
|
773
971
|
};
|
|
774
972
|
|
|
775
973
|
let tests, hooks;
|
|
776
974
|
if (params.suite) {
|
|
777
|
-
({ tests, hooks } = loadTestSuite(params.suite, runConfig.testsDir));
|
|
975
|
+
({ tests, hooks } = loadTestSuite(params.suite, runConfig.testsDir, runConfig.modulesDir));
|
|
778
976
|
} else {
|
|
779
977
|
({ tests, hooks } = loadAllSuites(runConfig.testsDir, runConfig.modulesDir, runConfig.exclude));
|
|
780
978
|
}
|
|
781
979
|
|
|
782
|
-
await waitForAnyPool(getPoolUrls(runConfig));
|
|
980
|
+
await waitForAnyPool(getPoolUrls(runConfig), 30000, { poolDriver: runConfig.poolDriver, maxSessions: runConfig.maxSessions });
|
|
783
981
|
const results = await runTestsParallel(tests, runConfig, hooks || {});
|
|
784
982
|
const report = generateReport(results);
|
|
785
983
|
const suiteName = params.suite || null;
|
package/src/db.js
CHANGED
|
@@ -199,6 +199,24 @@ function migrate(db) {
|
|
|
199
199
|
CREATE INDEX IF NOT EXISTS idx_al_project ON api_learnings(project_id);
|
|
200
200
|
CREATE INDEX IF NOT EXISTS idx_al_endpoint ON api_learnings(endpoint);
|
|
201
201
|
|
|
202
|
+
CREATE TABLE IF NOT EXISTS action_health (
|
|
203
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
204
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
205
|
+
run_id INTEGER REFERENCES runs(id) ON DELETE CASCADE,
|
|
206
|
+
test_name TEXT NOT NULL,
|
|
207
|
+
action_index INTEGER NOT NULL,
|
|
208
|
+
action_type TEXT NOT NULL,
|
|
209
|
+
selector TEXT,
|
|
210
|
+
success INTEGER NOT NULL,
|
|
211
|
+
duration_ms INTEGER,
|
|
212
|
+
console_errors_after INTEGER DEFAULT 0,
|
|
213
|
+
network_errors_after INTEGER DEFAULT 0,
|
|
214
|
+
page_url TEXT,
|
|
215
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
216
|
+
);
|
|
217
|
+
CREATE INDEX IF NOT EXISTS idx_ah_project ON action_health(project_id);
|
|
218
|
+
CREATE INDEX IF NOT EXISTS idx_ah_selector ON action_health(selector);
|
|
219
|
+
|
|
202
220
|
CREATE TABLE IF NOT EXISTS error_patterns (
|
|
203
221
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
204
222
|
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
@@ -255,6 +273,28 @@ function migrate(db) {
|
|
|
255
273
|
db.exec('ALTER TABLE test_results ADD COLUMN pool_url TEXT');
|
|
256
274
|
}
|
|
257
275
|
|
|
276
|
+
// Add pool_driver column to runs for driver visibility
|
|
277
|
+
try {
|
|
278
|
+
db.prepare('SELECT pool_driver FROM runs LIMIT 0').run();
|
|
279
|
+
} catch {
|
|
280
|
+
db.exec('ALTER TABLE runs ADD COLUMN pool_driver TEXT');
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// Add visual diff columns to test_results
|
|
284
|
+
const trCols = db.pragma('table_info(test_results)').map(c => c.name);
|
|
285
|
+
if (!trCols.includes('baseline_screenshot')) {
|
|
286
|
+
db.exec('ALTER TABLE test_results ADD COLUMN baseline_screenshot TEXT');
|
|
287
|
+
}
|
|
288
|
+
if (!trCols.includes('verification_screenshot')) {
|
|
289
|
+
db.exec('ALTER TABLE test_results ADD COLUMN verification_screenshot TEXT');
|
|
290
|
+
}
|
|
291
|
+
if (!trCols.includes('diff_screenshot')) {
|
|
292
|
+
db.exec('ALTER TABLE test_results ADD COLUMN diff_screenshot TEXT');
|
|
293
|
+
}
|
|
294
|
+
if (!trCols.includes('visual_diff_json')) {
|
|
295
|
+
db.exec('ALTER TABLE test_results ADD COLUMN visual_diff_json TEXT');
|
|
296
|
+
}
|
|
297
|
+
|
|
258
298
|
// Migrations: add metadata columns to screenshot_hashes
|
|
259
299
|
const ssColumns = db.pragma('table_info(screenshot_hashes)').map(c => c.name);
|
|
260
300
|
if (!ssColumns.includes('test_name')) {
|
|
@@ -336,18 +376,18 @@ export function getScreenshotHashes(filePaths) {
|
|
|
336
376
|
}
|
|
337
377
|
|
|
338
378
|
/** Save a run + its test results in a single transaction. Returns the run's DB id. */
|
|
339
|
-
export function saveRun(projectId, report, runId, suiteName, triggeredBy) {
|
|
379
|
+
export function saveRun(projectId, report, runId, suiteName, triggeredBy, poolDriver) {
|
|
340
380
|
const d = getDb();
|
|
341
381
|
const { summary, results, generatedAt } = report;
|
|
342
382
|
|
|
343
383
|
const insertRun = d.prepare(`
|
|
344
|
-
INSERT INTO runs (project_id, run_id, total, passed, failed, pass_rate, duration, generated_at, suite_name, triggered_by)
|
|
345
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
384
|
+
INSERT INTO runs (project_id, run_id, total, passed, failed, pass_rate, duration, generated_at, suite_name, triggered_by, pool_driver)
|
|
385
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
346
386
|
`);
|
|
347
387
|
|
|
348
388
|
const insertTest = d.prepare(`
|
|
349
|
-
INSERT INTO test_results (run_id, name, success, error, start_time, end_time, duration_ms, attempt, max_attempts, error_screenshot, console_logs, network_errors, screenshots, network_logs, actions_json, pool_url)
|
|
350
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
389
|
+
INSERT INTO test_results (run_id, name, success, error, start_time, end_time, duration_ms, attempt, max_attempts, error_screenshot, console_logs, network_errors, screenshots, network_logs, actions_json, pool_url, baseline_screenshot, verification_screenshot, diff_screenshot, visual_diff_json)
|
|
390
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
351
391
|
`);
|
|
352
392
|
|
|
353
393
|
const insertHash = d.prepare('INSERT OR IGNORE INTO screenshot_hashes (hash, file_path, project_id, run_id, test_name, step_index, page_url, screenshot_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)');
|
|
@@ -364,6 +404,7 @@ export function saveRun(projectId, report, runId, suiteName, triggeredBy) {
|
|
|
364
404
|
generatedAt,
|
|
365
405
|
suiteName || null,
|
|
366
406
|
triggeredBy || null,
|
|
407
|
+
poolDriver || null,
|
|
367
408
|
);
|
|
368
409
|
const runDbId = runInfo.lastInsertRowid;
|
|
369
410
|
|
|
@@ -388,6 +429,8 @@ export function saveRun(projectId, report, runId, suiteName, triggeredBy) {
|
|
|
388
429
|
narrative: a.narrative || undefined,
|
|
389
430
|
error: a.error || undefined,
|
|
390
431
|
actionRetries: a.actionRetries || undefined,
|
|
432
|
+
autoScreenshot: a.autoScreenshot || undefined,
|
|
433
|
+
screenshot: a.result?.screenshot || undefined,
|
|
391
434
|
}));
|
|
392
435
|
|
|
393
436
|
insertTest.run(
|
|
@@ -407,6 +450,10 @@ export function saveRun(projectId, report, runId, suiteName, triggeredBy) {
|
|
|
407
450
|
r.networkLogs?.length ? JSON.stringify(r.networkLogs) : null,
|
|
408
451
|
actionsCondensed.length ? JSON.stringify(actionsCondensed) : null,
|
|
409
452
|
r.poolUrl || null,
|
|
453
|
+
r.baselineScreenshot || null,
|
|
454
|
+
r.verificationScreenshot || null,
|
|
455
|
+
r.diffScreenshot || null,
|
|
456
|
+
r.visualDiff ? JSON.stringify(r.visualDiff) : null,
|
|
410
457
|
);
|
|
411
458
|
|
|
412
459
|
// Register screenshot hashes with metadata
|
|
@@ -416,6 +463,15 @@ export function saveRun(projectId, report, runId, suiteName, triggeredBy) {
|
|
|
416
463
|
const actionIdx = r.actions.indexOf(a);
|
|
417
464
|
insertHash.run(computeScreenshotHash(a.result.screenshot), a.result.screenshot, projectId, runDbId, r.name, actionIdx, null, 'action');
|
|
418
465
|
}
|
|
466
|
+
|
|
467
|
+
// Auto-captured per-step thumbnails for the storyline view
|
|
468
|
+
(r.actions || []).forEach((a, idx) => {
|
|
469
|
+
if (a.autoScreenshot) {
|
|
470
|
+
try {
|
|
471
|
+
insertHash.run(computeScreenshotHash(a.autoScreenshot), a.autoScreenshot, projectId, runDbId, r.name, idx, null, 'step');
|
|
472
|
+
} catch { /* best effort */ }
|
|
473
|
+
}
|
|
474
|
+
});
|
|
419
475
|
if (r.errorScreenshot) {
|
|
420
476
|
insertHash.run(computeScreenshotHash(r.errorScreenshot), r.errorScreenshot, projectId, runDbId, r.name, null, null, 'error');
|
|
421
477
|
}
|
|
@@ -425,6 +481,9 @@ export function saveRun(projectId, report, runId, suiteName, triggeredBy) {
|
|
|
425
481
|
if (r.baselineScreenshot) {
|
|
426
482
|
insertHash.run(computeScreenshotHash(r.baselineScreenshot), r.baselineScreenshot, projectId, runDbId, r.name, null, null, 'baseline');
|
|
427
483
|
}
|
|
484
|
+
if (r.diffScreenshot) {
|
|
485
|
+
insertHash.run(computeScreenshotHash(r.diffScreenshot), r.diffScreenshot, projectId, runDbId, r.name, null, null, 'diff');
|
|
486
|
+
}
|
|
428
487
|
}
|
|
429
488
|
|
|
430
489
|
return runDbId;
|
|
@@ -480,7 +539,7 @@ export function listProjects() {
|
|
|
480
539
|
export function getProjectRuns(projectId, limit = 50, offset = 0) {
|
|
481
540
|
const d = getDb();
|
|
482
541
|
return d.prepare(`
|
|
483
|
-
SELECT id, run_id, total, passed, failed, pass_rate, duration, generated_at, suite_name, triggered_by
|
|
542
|
+
SELECT id, run_id, total, passed, failed, pass_rate, duration, generated_at, suite_name, triggered_by, pool_driver
|
|
484
543
|
FROM runs
|
|
485
544
|
WHERE project_id = ?
|
|
486
545
|
ORDER BY generated_at DESC
|
|
@@ -503,6 +562,9 @@ export function getRunDetail(runDbId) {
|
|
|
503
562
|
const ss = t.screenshots ? JSON.parse(t.screenshots) : [];
|
|
504
563
|
allPaths.push(...ss);
|
|
505
564
|
if (t.error_screenshot) allPaths.push(t.error_screenshot);
|
|
565
|
+
if (t.baseline_screenshot) allPaths.push(t.baseline_screenshot);
|
|
566
|
+
if (t.verification_screenshot) allPaths.push(t.verification_screenshot);
|
|
567
|
+
if (t.diff_screenshot) allPaths.push(t.diff_screenshot);
|
|
506
568
|
}
|
|
507
569
|
const hashMap = getScreenshotHashes(allPaths);
|
|
508
570
|
|
|
@@ -518,6 +580,7 @@ export function getRunDetail(runDbId) {
|
|
|
518
580
|
generatedAt: run.generated_at,
|
|
519
581
|
suiteName: run.suite_name,
|
|
520
582
|
triggeredBy: run.triggered_by || null,
|
|
583
|
+
poolDriver: run.pool_driver || null,
|
|
521
584
|
results: tests.map(t => {
|
|
522
585
|
const screenshots = t.screenshots ? JSON.parse(t.screenshots) : [];
|
|
523
586
|
const testPaths = [...screenshots];
|
|
@@ -543,6 +606,10 @@ export function getRunDetail(runDbId) {
|
|
|
543
606
|
actions: t.actions_json ? JSON.parse(t.actions_json) : [],
|
|
544
607
|
screenshotHashes,
|
|
545
608
|
poolUrl: t.pool_url || null,
|
|
609
|
+
baselineScreenshot: t.baseline_screenshot || null,
|
|
610
|
+
verificationScreenshot: t.verification_screenshot || null,
|
|
611
|
+
diffScreenshot: t.diff_screenshot || null,
|
|
612
|
+
visualDiff: t.visual_diff_json ? JSON.parse(t.visual_diff_json) : null,
|
|
546
613
|
};
|
|
547
614
|
}),
|
|
548
615
|
};
|
|
@@ -553,7 +620,7 @@ export function getAllRuns(limit = 50, offset = 0) {
|
|
|
553
620
|
const d = getDb();
|
|
554
621
|
return d.prepare(`
|
|
555
622
|
SELECT r.id, r.run_id, r.total, r.passed, r.failed, r.pass_rate, r.duration,
|
|
556
|
-
r.generated_at, r.suite_name, r.triggered_by, p.name AS project_name, p.id AS project_id
|
|
623
|
+
r.generated_at, r.suite_name, r.triggered_by, r.pool_driver, p.name AS project_name, p.id AS project_id
|
|
557
624
|
FROM runs r
|
|
558
625
|
JOIN projects p ON p.id = r.project_id
|
|
559
626
|
ORDER BY r.generated_at DESC
|
package/src/index.js
CHANGED
|
@@ -8,21 +8,23 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
export { loadConfig } from './config.js';
|
|
11
|
-
export { waitForPool, connectToPool, startPool, stopPool, restartPool, getPoolStatus } from './pool.js';
|
|
11
|
+
export { waitForPool, connectToPool, disconnectFromPool, startPool, stopPool, restartPool, getPoolStatus, clearDriverCache, getCachedDriver, trackCdpSession, releaseCdpSession, releaseSteelSession } from './pool.js';
|
|
12
12
|
export { getPoolUrls, getAllPoolStatuses, getAggregatedPoolStatus, waitForAnyPool, selectPool, selectAndConnect } from './pool-manager.js';
|
|
13
13
|
export { executeAction } from './actions.js';
|
|
14
|
-
export { runTest, runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites } from './runner.js';
|
|
14
|
+
export { runTest, runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites, fetchAuthToken } from './runner.js';
|
|
15
15
|
export { generateReport, generateJUnitXML, saveReport, printReport, saveHistory, loadHistory, loadHistoryRun } from './reporter.js';
|
|
16
16
|
export { startDashboard, stopDashboard } from './dashboard.js';
|
|
17
17
|
export { fetchIssue, parseIssueUrl, detectProvider, checkCliAuth } from './issues.js';
|
|
18
18
|
export { buildPrompt, generateTests, hasApiKey } from './ai-generate.js';
|
|
19
19
|
export { verifyIssue } from './verify.js';
|
|
20
20
|
export { resolveTestData, loadModuleRegistry, listModules } from './module-resolver.js';
|
|
21
|
-
export { learnFromRun, categorizeError } from './learner.js';
|
|
22
|
-
export { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights, getTestCreationContext, generateImprovements } from './learner-sqlite.js';
|
|
21
|
+
export { learnFromRun, categorizeError, isInfraError, INFRA_CATEGORIES } from './learner.js';
|
|
22
|
+
export { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights, getTestCreationContext, generateImprovements, getActionHealthScores } from './learner-sqlite.js';
|
|
23
23
|
export { generateLearningsMarkdown } from './learner-markdown.js';
|
|
24
24
|
export { writeToGraph, queryGraph, closeNeo4j } from './learner-neo4j.js';
|
|
25
25
|
export { startNeo4j, stopNeo4j, getNeo4jStatus } from './neo4j-pool.js';
|
|
26
|
+
export { forkAppInstance, destroyFork, destroyAllForks, getAppPoolStatus, isAppPoolEnabled } from './app-pool.js';
|
|
27
|
+
export { compareImages, assertVisualMatch } from './visual-diff.js';
|
|
26
28
|
|
|
27
29
|
import { loadConfig } from './config.js';
|
|
28
30
|
import { waitForAnyPool, getPoolUrls } from './pool-manager.js';
|