@matware/e2e-runner 1.1.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +9 -0
- package/.mcp.json +9 -0
- package/README.md +475 -307
- package/agents/test-analyzer.md +81 -0
- package/agents/test-creator.md +102 -0
- package/agents/test-improver.md +140 -0
- package/bin/cli.js +194 -6
- package/commands/create-test.md +50 -0
- package/commands/run.md +49 -0
- package/commands/verify-issue.md +63 -0
- package/package.json +10 -2
- package/skills/e2e-testing/SKILL.md +166 -0
- package/skills/e2e-testing/references/action-types.md +100 -0
- package/skills/e2e-testing/references/test-json-format.md +159 -0
- package/skills/e2e-testing/references/troubleshooting.md +182 -0
- package/src/actions.js +273 -18
- package/src/ai-generate.js +87 -7
- package/src/config.js +28 -0
- package/src/dashboard.js +156 -6
- package/src/db.js +207 -13
- package/src/index.js +9 -3
- package/src/learner-markdown.js +177 -0
- package/src/learner-neo4j.js +255 -0
- package/src/learner-sqlite.js +354 -0
- package/src/learner.js +413 -0
- package/src/mcp-tools.js +448 -18
- package/src/module-resolver.js +273 -0
- package/src/narrate.js +225 -0
- package/src/neo4j-pool.js +124 -0
- package/src/reporter.js +35 -2
- package/src/runner.js +120 -46
- package/src/verify.js +5 -3
- package/templates/build-dashboard.js +28 -0
- package/templates/dashboard/app.js +1152 -0
- package/templates/dashboard/styles.css +413 -0
- package/templates/dashboard/template.html +201 -0
- package/templates/dashboard.html +964 -378
- package/templates/docker-compose-neo4j.yml +19 -0
- package/templates/e2e.config.js +3 -0
package/src/db.js
CHANGED
|
@@ -107,6 +107,13 @@ function migrate(db) {
|
|
|
107
107
|
db.exec('ALTER TABLE test_results ADD COLUMN network_logs TEXT');
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
+
// Add actions_json column if upgrading from older schema
|
|
111
|
+
try {
|
|
112
|
+
db.prepare('SELECT actions_json FROM test_results LIMIT 0').run();
|
|
113
|
+
} catch {
|
|
114
|
+
db.exec('ALTER TABLE test_results ADD COLUMN actions_json TEXT');
|
|
115
|
+
}
|
|
116
|
+
|
|
110
117
|
// Add triggered_by column if upgrading from older schema
|
|
111
118
|
try {
|
|
112
119
|
db.prepare('SELECT triggered_by FROM runs LIMIT 0').run();
|
|
@@ -125,6 +132,113 @@ function migrate(db) {
|
|
|
125
132
|
);
|
|
126
133
|
CREATE INDEX IF NOT EXISTS idx_ss_path ON screenshot_hashes(file_path);
|
|
127
134
|
`);
|
|
135
|
+
|
|
136
|
+
// ── Learning system tables ──────────────────────────────────────────────────
|
|
137
|
+
|
|
138
|
+
db.exec(`
|
|
139
|
+
CREATE TABLE IF NOT EXISTS test_learnings (
|
|
140
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
141
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
142
|
+
run_id INTEGER REFERENCES runs(id) ON DELETE CASCADE,
|
|
143
|
+
test_name TEXT NOT NULL,
|
|
144
|
+
success INTEGER NOT NULL,
|
|
145
|
+
duration_ms INTEGER,
|
|
146
|
+
flaky INTEGER DEFAULT 0,
|
|
147
|
+
attempt INTEGER DEFAULT 1,
|
|
148
|
+
max_attempts INTEGER DEFAULT 1,
|
|
149
|
+
error_pattern TEXT,
|
|
150
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
151
|
+
);
|
|
152
|
+
CREATE INDEX IF NOT EXISTS idx_tl_project ON test_learnings(project_id);
|
|
153
|
+
CREATE INDEX IF NOT EXISTS idx_tl_test ON test_learnings(test_name);
|
|
154
|
+
CREATE INDEX IF NOT EXISTS idx_tl_created ON test_learnings(created_at);
|
|
155
|
+
|
|
156
|
+
CREATE TABLE IF NOT EXISTS selector_learnings (
|
|
157
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
158
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
159
|
+
run_id INTEGER REFERENCES runs(id) ON DELETE CASCADE,
|
|
160
|
+
selector TEXT NOT NULL,
|
|
161
|
+
action_type TEXT NOT NULL,
|
|
162
|
+
success INTEGER NOT NULL,
|
|
163
|
+
page_url TEXT,
|
|
164
|
+
test_name TEXT,
|
|
165
|
+
error TEXT,
|
|
166
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
167
|
+
);
|
|
168
|
+
CREATE INDEX IF NOT EXISTS idx_sl_project ON selector_learnings(project_id);
|
|
169
|
+
CREATE INDEX IF NOT EXISTS idx_sl_selector ON selector_learnings(selector);
|
|
170
|
+
|
|
171
|
+
CREATE TABLE IF NOT EXISTS page_learnings (
|
|
172
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
173
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
174
|
+
run_id INTEGER REFERENCES runs(id) ON DELETE CASCADE,
|
|
175
|
+
url_path TEXT NOT NULL,
|
|
176
|
+
load_time_ms INTEGER,
|
|
177
|
+
console_errors INTEGER DEFAULT 0,
|
|
178
|
+
console_warns INTEGER DEFAULT 0,
|
|
179
|
+
network_errors INTEGER DEFAULT 0,
|
|
180
|
+
test_name TEXT,
|
|
181
|
+
success INTEGER NOT NULL,
|
|
182
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
183
|
+
);
|
|
184
|
+
CREATE INDEX IF NOT EXISTS idx_pl_project ON page_learnings(project_id);
|
|
185
|
+
CREATE INDEX IF NOT EXISTS idx_pl_url ON page_learnings(url_path);
|
|
186
|
+
|
|
187
|
+
CREATE TABLE IF NOT EXISTS api_learnings (
|
|
188
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
189
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
190
|
+
run_id INTEGER REFERENCES runs(id) ON DELETE CASCADE,
|
|
191
|
+
endpoint TEXT NOT NULL,
|
|
192
|
+
method TEXT NOT NULL,
|
|
193
|
+
status INTEGER,
|
|
194
|
+
duration_ms INTEGER,
|
|
195
|
+
is_error INTEGER DEFAULT 0,
|
|
196
|
+
test_name TEXT,
|
|
197
|
+
created_at TEXT DEFAULT (datetime('now'))
|
|
198
|
+
);
|
|
199
|
+
CREATE INDEX IF NOT EXISTS idx_al_project ON api_learnings(project_id);
|
|
200
|
+
CREATE INDEX IF NOT EXISTS idx_al_endpoint ON api_learnings(endpoint);
|
|
201
|
+
|
|
202
|
+
CREATE TABLE IF NOT EXISTS error_patterns (
|
|
203
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
204
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
205
|
+
pattern TEXT NOT NULL,
|
|
206
|
+
category TEXT NOT NULL,
|
|
207
|
+
occurrence_count INTEGER DEFAULT 1,
|
|
208
|
+
first_seen TEXT DEFAULT (datetime('now')),
|
|
209
|
+
last_seen TEXT DEFAULT (datetime('now')),
|
|
210
|
+
example_error TEXT,
|
|
211
|
+
example_test TEXT,
|
|
212
|
+
UNIQUE(project_id, pattern)
|
|
213
|
+
);
|
|
214
|
+
CREATE INDEX IF NOT EXISTS idx_ep_project ON error_patterns(project_id);
|
|
215
|
+
CREATE INDEX IF NOT EXISTS idx_ep_cat ON error_patterns(category);
|
|
216
|
+
|
|
217
|
+
CREATE TABLE IF NOT EXISTS learning_summary (
|
|
218
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
219
|
+
project_id INTEGER NOT NULL UNIQUE REFERENCES projects(id),
|
|
220
|
+
total_runs INTEGER DEFAULT 0,
|
|
221
|
+
total_tests INTEGER DEFAULT 0,
|
|
222
|
+
overall_pass_rate REAL DEFAULT 0,
|
|
223
|
+
avg_duration_ms REAL DEFAULT 0,
|
|
224
|
+
flaky_tests TEXT,
|
|
225
|
+
slow_tests TEXT,
|
|
226
|
+
unstable_selectors TEXT,
|
|
227
|
+
failing_pages TEXT,
|
|
228
|
+
api_issues TEXT,
|
|
229
|
+
top_errors TEXT,
|
|
230
|
+
updated_at TEXT DEFAULT (datetime('now'))
|
|
231
|
+
);
|
|
232
|
+
`);
|
|
233
|
+
|
|
234
|
+
// Migrations: add metadata columns to screenshot_hashes
|
|
235
|
+
const ssColumns = db.pragma('table_info(screenshot_hashes)').map(c => c.name);
|
|
236
|
+
if (!ssColumns.includes('test_name')) {
|
|
237
|
+
db.exec('ALTER TABLE screenshot_hashes ADD COLUMN test_name TEXT');
|
|
238
|
+
db.exec('ALTER TABLE screenshot_hashes ADD COLUMN step_index INTEGER');
|
|
239
|
+
db.exec('ALTER TABLE screenshot_hashes ADD COLUMN page_url TEXT');
|
|
240
|
+
db.exec('ALTER TABLE screenshot_hashes ADD COLUMN screenshot_type TEXT');
|
|
241
|
+
}
|
|
128
242
|
}
|
|
129
243
|
|
|
130
244
|
/** Upsert a project row. Returns the project id. */
|
|
@@ -169,17 +283,19 @@ export function computeScreenshotHash(filePath) {
|
|
|
169
283
|
return crypto.createHash('sha256').update(filePath).digest('hex').slice(0, 8);
|
|
170
284
|
}
|
|
171
285
|
|
|
172
|
-
/** Register a screenshot hash. INSERT OR IGNORE avoids duplicates. */
|
|
173
|
-
export function registerScreenshotHash(hash, filePath, projectId, runDbId) {
|
|
286
|
+
/** Register a screenshot hash. INSERT OR IGNORE avoids duplicates. Optional metadata: testName, stepIndex, pageUrl, screenshotType. */
|
|
287
|
+
export function registerScreenshotHash(hash, filePath, projectId, runDbId, meta = {}) {
|
|
174
288
|
const d = getDb();
|
|
175
|
-
d.prepare(
|
|
289
|
+
d.prepare(
|
|
290
|
+
'INSERT OR IGNORE INTO screenshot_hashes (hash, file_path, project_id, run_id, test_name, step_index, page_url, screenshot_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
|
|
291
|
+
).run(hash, filePath, projectId || null, runDbId || null, meta.testName || null, meta.stepIndex ?? null, meta.pageUrl || null, meta.screenshotType || null);
|
|
176
292
|
}
|
|
177
293
|
|
|
178
|
-
/** Look up a screenshot by hash. Strips optional "ss:" prefix. Returns { hash, file_path, project_id } or null. */
|
|
294
|
+
/** Look up a screenshot by hash. Strips optional "ss:" prefix. Returns { hash, file_path, project_id, test_name, step_index, page_url, screenshot_type } or null. */
|
|
179
295
|
export function lookupScreenshotHash(rawHash) {
|
|
180
296
|
const d = getDb();
|
|
181
297
|
const hash = rawHash.replace(/^ss:/, '');
|
|
182
|
-
return d.prepare('SELECT hash, file_path, project_id FROM screenshot_hashes WHERE hash = ?').get(hash) || null;
|
|
298
|
+
return d.prepare('SELECT hash, file_path, project_id, test_name, step_index, page_url, screenshot_type FROM screenshot_hashes WHERE hash = ?').get(hash) || null;
|
|
183
299
|
}
|
|
184
300
|
|
|
185
301
|
/** Batch lookup: given an array of file paths, returns { [path]: hash } map. */
|
|
@@ -206,11 +322,11 @@ export function saveRun(projectId, report, runId, suiteName, triggeredBy) {
|
|
|
206
322
|
`);
|
|
207
323
|
|
|
208
324
|
const insertTest = d.prepare(`
|
|
209
|
-
INSERT INTO test_results (run_id, name, success, error, start_time, end_time, duration_ms, attempt, max_attempts, error_screenshot, console_logs, network_errors, screenshots, network_logs)
|
|
210
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
325
|
+
INSERT INTO test_results (run_id, name, success, error, start_time, end_time, duration_ms, attempt, max_attempts, error_screenshot, console_logs, network_errors, screenshots, network_logs, actions_json)
|
|
326
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
211
327
|
`);
|
|
212
328
|
|
|
213
|
-
const insertHash = d.prepare('INSERT OR IGNORE INTO screenshot_hashes (hash, file_path, project_id, run_id) VALUES (?, ?, ?, ?)');
|
|
329
|
+
const insertHash = d.prepare('INSERT OR IGNORE INTO screenshot_hashes (hash, file_path, project_id, run_id, test_name, step_index, page_url, screenshot_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)');
|
|
214
330
|
|
|
215
331
|
const tx = d.transaction(() => {
|
|
216
332
|
const runInfo = insertRun.run(
|
|
@@ -237,6 +353,19 @@ export function saveRun(projectId, report, runId, suiteName, triggeredBy) {
|
|
|
237
353
|
.filter(a => a.type === 'screenshot' && a.result?.screenshot)
|
|
238
354
|
.map(a => a.result.screenshot);
|
|
239
355
|
|
|
356
|
+
// Condensed actions for narrative display
|
|
357
|
+
const actionsCondensed = (r.actions || []).map(a => ({
|
|
358
|
+
type: a.type,
|
|
359
|
+
selector: a.selector || undefined,
|
|
360
|
+
value: a.value || undefined,
|
|
361
|
+
text: a.text || undefined,
|
|
362
|
+
success: a.success,
|
|
363
|
+
duration: a.duration,
|
|
364
|
+
narrative: a.narrative || undefined,
|
|
365
|
+
error: a.error || undefined,
|
|
366
|
+
actionRetries: a.actionRetries || undefined,
|
|
367
|
+
}));
|
|
368
|
+
|
|
240
369
|
insertTest.run(
|
|
241
370
|
runDbId,
|
|
242
371
|
r.name,
|
|
@@ -252,17 +381,21 @@ export function saveRun(projectId, report, runId, suiteName, triggeredBy) {
|
|
|
252
381
|
r.networkErrors ? JSON.stringify(r.networkErrors) : null,
|
|
253
382
|
screenshots.length ? JSON.stringify(screenshots) : null,
|
|
254
383
|
r.networkLogs?.length ? JSON.stringify(r.networkLogs) : null,
|
|
384
|
+
actionsCondensed.length ? JSON.stringify(actionsCondensed) : null,
|
|
255
385
|
);
|
|
256
386
|
|
|
257
|
-
// Register screenshot hashes
|
|
258
|
-
|
|
259
|
-
|
|
387
|
+
// Register screenshot hashes with metadata
|
|
388
|
+
const ssActions = (r.actions || []).filter(a => a.type === 'screenshot' && a.result?.screenshot);
|
|
389
|
+
for (let si = 0; si < ssActions.length; si++) {
|
|
390
|
+
const a = ssActions[si];
|
|
391
|
+
const actionIdx = r.actions.indexOf(a);
|
|
392
|
+
insertHash.run(computeScreenshotHash(a.result.screenshot), a.result.screenshot, projectId, runDbId, r.name, actionIdx, null, 'action');
|
|
260
393
|
}
|
|
261
394
|
if (r.errorScreenshot) {
|
|
262
|
-
insertHash.run(computeScreenshotHash(r.errorScreenshot), r.errorScreenshot, projectId, runDbId);
|
|
395
|
+
insertHash.run(computeScreenshotHash(r.errorScreenshot), r.errorScreenshot, projectId, runDbId, r.name, null, null, 'error');
|
|
263
396
|
}
|
|
264
397
|
if (r.verificationScreenshot) {
|
|
265
|
-
insertHash.run(computeScreenshotHash(r.verificationScreenshot), r.verificationScreenshot, projectId, runDbId);
|
|
398
|
+
insertHash.run(computeScreenshotHash(r.verificationScreenshot), r.verificationScreenshot, projectId, runDbId, r.name, null, null, 'verification');
|
|
266
399
|
}
|
|
267
400
|
}
|
|
268
401
|
|
|
@@ -352,6 +485,7 @@ export function getRunDetail(runDbId) {
|
|
|
352
485
|
consoleLogs: t.console_logs ? JSON.parse(t.console_logs) : [],
|
|
353
486
|
networkErrors: t.network_errors ? JSON.parse(t.network_errors) : [],
|
|
354
487
|
networkLogs: t.network_logs ? JSON.parse(t.network_logs) : [],
|
|
488
|
+
actions: t.actions_json ? JSON.parse(t.actions_json) : [],
|
|
355
489
|
screenshotHashes,
|
|
356
490
|
};
|
|
357
491
|
}),
|
|
@@ -378,6 +512,66 @@ export function getRunCount() {
|
|
|
378
512
|
return row.cnt;
|
|
379
513
|
}
|
|
380
514
|
|
|
515
|
+
/** Query network logs for a run with optional filters.
|
|
516
|
+
* Filters: testName, method, statusMin, statusMax, urlPattern, errorsOnly, includeHeaders, includeBodies.
|
|
517
|
+
* By default returns only: url, method, status, statusText, duration.
|
|
518
|
+
*/
|
|
519
|
+
export function getNetworkLogs(runDbId, filters = {}) {
|
|
520
|
+
const d = getDb();
|
|
521
|
+
|
|
522
|
+
let query = 'SELECT name, network_logs FROM test_results WHERE run_id = ?';
|
|
523
|
+
const params = [runDbId];
|
|
524
|
+
|
|
525
|
+
if (filters.testName) {
|
|
526
|
+
query += ' AND name = ?';
|
|
527
|
+
params.push(filters.testName);
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
const rows = d.prepare(query).all(...params);
|
|
531
|
+
const results = [];
|
|
532
|
+
|
|
533
|
+
for (const row of rows) {
|
|
534
|
+
if (!row.network_logs) continue;
|
|
535
|
+
let logs = JSON.parse(row.network_logs);
|
|
536
|
+
|
|
537
|
+
if (filters.method) {
|
|
538
|
+
logs = logs.filter(l => l.method === filters.method.toUpperCase());
|
|
539
|
+
}
|
|
540
|
+
if (filters.statusMin !== undefined) {
|
|
541
|
+
logs = logs.filter(l => l.status >= filters.statusMin);
|
|
542
|
+
}
|
|
543
|
+
if (filters.statusMax !== undefined) {
|
|
544
|
+
logs = logs.filter(l => l.status <= filters.statusMax);
|
|
545
|
+
}
|
|
546
|
+
if (filters.urlPattern) {
|
|
547
|
+
const re = new RegExp(filters.urlPattern, 'i');
|
|
548
|
+
logs = logs.filter(l => re.test(l.url));
|
|
549
|
+
}
|
|
550
|
+
if (filters.errorsOnly) {
|
|
551
|
+
logs = logs.filter(l => l.status >= 400);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
const mapped = logs.map(l => {
|
|
555
|
+
const entry = { url: l.url, method: l.method, status: l.status, statusText: l.statusText, duration: l.duration };
|
|
556
|
+
if (filters.includeHeaders || filters.includeBodies) {
|
|
557
|
+
entry.requestHeaders = l.requestHeaders;
|
|
558
|
+
entry.responseHeaders = l.responseHeaders;
|
|
559
|
+
}
|
|
560
|
+
if (filters.includeBodies) {
|
|
561
|
+
entry.requestBody = l.requestBody;
|
|
562
|
+
entry.responseBody = l.responseBody;
|
|
563
|
+
}
|
|
564
|
+
return entry;
|
|
565
|
+
});
|
|
566
|
+
|
|
567
|
+
if (mapped.length > 0) {
|
|
568
|
+
results.push({ testName: row.name, logs: mapped });
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
return results;
|
|
573
|
+
}
|
|
574
|
+
|
|
381
575
|
/** Close the database connection. */
|
|
382
576
|
export function closeDb() {
|
|
383
577
|
if (db) {
|
package/src/index.js
CHANGED
|
@@ -16,6 +16,12 @@ export { startDashboard, stopDashboard } from './dashboard.js';
|
|
|
16
16
|
export { fetchIssue, parseIssueUrl, detectProvider, checkCliAuth } from './issues.js';
|
|
17
17
|
export { buildPrompt, generateTests, hasApiKey } from './ai-generate.js';
|
|
18
18
|
export { verifyIssue } from './verify.js';
|
|
19
|
+
export { resolveTestData, loadModuleRegistry, listModules } from './module-resolver.js';
|
|
20
|
+
export { learnFromRun, categorizeError } from './learner.js';
|
|
21
|
+
export { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights } from './learner-sqlite.js';
|
|
22
|
+
export { generateLearningsMarkdown } from './learner-markdown.js';
|
|
23
|
+
export { writeToGraph, queryGraph, closeNeo4j } from './learner-neo4j.js';
|
|
24
|
+
export { startNeo4j, stopNeo4j, getNeo4jStatus } from './neo4j-pool.js';
|
|
19
25
|
|
|
20
26
|
import { loadConfig } from './config.js';
|
|
21
27
|
import { waitForPool } from './pool.js';
|
|
@@ -36,7 +42,7 @@ export async function createRunner(userConfig = {}) {
|
|
|
36
42
|
/** Runs all test suites from the tests directory */
|
|
37
43
|
async runAll() {
|
|
38
44
|
await waitForPool(config.poolUrl);
|
|
39
|
-
const { tests, hooks } = loadAllSuites(config.testsDir);
|
|
45
|
+
const { tests, hooks } = loadAllSuites(config.testsDir, config.modulesDir, config.exclude);
|
|
40
46
|
const results = await runTestsParallel(tests, config, hooks);
|
|
41
47
|
const report = generateReport(results);
|
|
42
48
|
saveReport(report, config.screenshotsDir, config);
|
|
@@ -47,7 +53,7 @@ export async function createRunner(userConfig = {}) {
|
|
|
47
53
|
/** Runs a single suite by name */
|
|
48
54
|
async runSuite(name) {
|
|
49
55
|
await waitForPool(config.poolUrl);
|
|
50
|
-
const { tests, hooks } = loadTestSuite(name, config.testsDir);
|
|
56
|
+
const { tests, hooks } = loadTestSuite(name, config.testsDir, config.modulesDir);
|
|
51
57
|
const results = await runTestsParallel(tests, config, hooks);
|
|
52
58
|
const report = generateReport(results);
|
|
53
59
|
saveReport(report, config.screenshotsDir, config);
|
|
@@ -68,7 +74,7 @@ export async function createRunner(userConfig = {}) {
|
|
|
68
74
|
/** Runs tests from a JSON file path */
|
|
69
75
|
async runFile(filePath) {
|
|
70
76
|
await waitForPool(config.poolUrl);
|
|
71
|
-
const { tests, hooks } = loadTestFile(filePath);
|
|
77
|
+
const { tests, hooks } = loadTestFile(filePath, config.modulesDir);
|
|
72
78
|
const results = await runTestsParallel(tests, config, hooks);
|
|
73
79
|
const report = generateReport(results);
|
|
74
80
|
saveReport(report, config.screenshotsDir, config);
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Learnings markdown generator.
|
|
3
|
+
*
|
|
4
|
+
* Generates {cwd}/e2e/learnings.md after each run, reading from SQLite.
|
|
5
|
+
* The file is designed to be portable, versionable in git, and human-readable.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import fs from 'fs';
|
|
9
|
+
import path from 'path';
|
|
10
|
+
import {
|
|
11
|
+
getLearningsSummary,
|
|
12
|
+
getFlakySummary,
|
|
13
|
+
getSelectorStability,
|
|
14
|
+
getPageHealth,
|
|
15
|
+
getApiHealth,
|
|
16
|
+
getErrorPatterns,
|
|
17
|
+
getTestTrends,
|
|
18
|
+
} from './learner-sqlite.js';
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Generates the learnings.md file for a project.
|
|
22
|
+
* Reads from SQLite and writes to {cwd}/e2e/learnings.md.
|
|
23
|
+
*/
|
|
24
|
+
export function generateLearningsMarkdown(projectId, config) {
|
|
25
|
+
const days = config?.learningsDays || 30;
|
|
26
|
+
const summary = getLearningsSummary(projectId);
|
|
27
|
+
const flaky = getFlakySummary(projectId, days);
|
|
28
|
+
const selectors = getSelectorStability(projectId, days);
|
|
29
|
+
const pages = getPageHealth(projectId, days);
|
|
30
|
+
const apis = getApiHealth(projectId, days);
|
|
31
|
+
const errors = getErrorPatterns(projectId);
|
|
32
|
+
const trendsResult = getTestTrends(projectId, 7);
|
|
33
|
+
const trends = trendsResult.data || trendsResult;
|
|
34
|
+
const trendsGranularity = trendsResult.granularity || 'daily';
|
|
35
|
+
|
|
36
|
+
const lines = [];
|
|
37
|
+
|
|
38
|
+
lines.push('# E2E Test Learnings');
|
|
39
|
+
lines.push('');
|
|
40
|
+
lines.push(`> Auto-generated after each test run. Analysis window: **${days} days**.`);
|
|
41
|
+
lines.push(`> Last updated: ${summary.updatedAt || 'never'}`);
|
|
42
|
+
lines.push('');
|
|
43
|
+
|
|
44
|
+
// ── Health Overview ─────────────────────────────────────────────────────────
|
|
45
|
+
lines.push('## Health Overview');
|
|
46
|
+
lines.push('');
|
|
47
|
+
lines.push('| Metric | Value |');
|
|
48
|
+
lines.push('|--------|-------|');
|
|
49
|
+
lines.push(`| Total Runs | ${summary.totalRuns} |`);
|
|
50
|
+
lines.push(`| Total Tests | ${summary.totalTests} |`);
|
|
51
|
+
lines.push(`| Pass Rate | ${summary.overallPassRate}% |`);
|
|
52
|
+
lines.push(`| Avg Duration | ${formatDuration(summary.avgDurationMs)} |`);
|
|
53
|
+
lines.push(`| Flaky Tests | ${flaky.length} |`);
|
|
54
|
+
lines.push(`| Unstable Selectors | ${selectors.length} |`);
|
|
55
|
+
|
|
56
|
+
// Trend arrow (compare last 2 days)
|
|
57
|
+
if (trends.length >= 2) {
|
|
58
|
+
const latest = trends[trends.length - 1];
|
|
59
|
+
const prev = trends[trends.length - 2];
|
|
60
|
+
const diff = latest.pass_rate - prev.pass_rate;
|
|
61
|
+
const arrow = diff > 0 ? 'improving' : diff < 0 ? 'declining' : 'stable';
|
|
62
|
+
lines.push(`| 7-Day Trend | ${arrow} (${diff > 0 ? '+' : ''}${diff.toFixed(1)}%) |`);
|
|
63
|
+
}
|
|
64
|
+
lines.push('');
|
|
65
|
+
|
|
66
|
+
// ── Flaky Tests ─────────────────────────────────────────────────────────────
|
|
67
|
+
if (flaky.length > 0) {
|
|
68
|
+
lines.push('## Flaky Tests');
|
|
69
|
+
lines.push('');
|
|
70
|
+
lines.push('Tests that pass only after retries — potential stability issues.');
|
|
71
|
+
lines.push('');
|
|
72
|
+
lines.push('| Test | Flaky Rate | Occurrences | Total Runs | Last Flaky | Avg Attempts |');
|
|
73
|
+
lines.push('|------|-----------|-------------|------------|------------|-------------|');
|
|
74
|
+
for (const f of flaky) {
|
|
75
|
+
lines.push(`| ${f.test_name} | ${f.flaky_rate}% | ${f.flaky_count} | ${f.total_runs} | ${formatDate(f.last_flaky)} | ${f.avg_attempts} |`);
|
|
76
|
+
}
|
|
77
|
+
lines.push('');
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// ── Unstable Selectors ──────────────────────────────────────────────────────
|
|
81
|
+
if (selectors.length > 0) {
|
|
82
|
+
lines.push('## Unstable Selectors');
|
|
83
|
+
lines.push('');
|
|
84
|
+
lines.push('CSS selectors that fail intermittently — candidates for improvement.');
|
|
85
|
+
lines.push('');
|
|
86
|
+
lines.push('| Selector | Action | Fail Rate | Uses | Tests | Page | Error |');
|
|
87
|
+
lines.push('|----------|--------|-----------|------|-------|------|-------|');
|
|
88
|
+
for (const s of selectors.slice(0, 20)) {
|
|
89
|
+
const selector = truncate(s.selector, 40);
|
|
90
|
+
const error = truncate(s.last_error || '-', 30);
|
|
91
|
+
lines.push(`| \`${selector}\` | ${s.action_type} | ${s.fail_rate}% | ${s.total_uses} | ${s.used_by_tests} | ${s.page_url || '-'} | ${error} |`);
|
|
92
|
+
}
|
|
93
|
+
lines.push('');
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// ── Failing Pages ───────────────────────────────────────────────────────────
|
|
97
|
+
const failingPages = pages.filter(p => p.fail_rate > 0);
|
|
98
|
+
if (failingPages.length > 0) {
|
|
99
|
+
lines.push('## Failing Pages');
|
|
100
|
+
lines.push('');
|
|
101
|
+
lines.push('| Page | Fail Rate | Visits | Tests | Console Errors | Network Errors | Avg Load |');
|
|
102
|
+
lines.push('|------|-----------|--------|-------|---------------|----------------|----------|');
|
|
103
|
+
for (const p of failingPages.slice(0, 20)) {
|
|
104
|
+
lines.push(`| ${p.url_path} | ${p.fail_rate}% | ${p.total_visits} | ${p.tested_by} | ${p.console_errors} | ${p.network_errors} | ${formatDuration(p.avg_load_ms)} |`);
|
|
105
|
+
}
|
|
106
|
+
lines.push('');
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// ── API Issues ──────────────────────────────────────────────────────────────
|
|
110
|
+
const apiIssues = apis.filter(a => a.error_rate > 0);
|
|
111
|
+
if (apiIssues.length > 0) {
|
|
112
|
+
lines.push('## API Issues');
|
|
113
|
+
lines.push('');
|
|
114
|
+
lines.push('| Endpoint | Error Rate | Calls | Avg Duration | Max Duration | Status Codes |');
|
|
115
|
+
lines.push('|----------|-----------|-------|-------------|-------------|-------------|');
|
|
116
|
+
for (const a of apiIssues.slice(0, 20)) {
|
|
117
|
+
lines.push(`| ${truncate(a.endpoint, 40)} | ${a.error_rate}% | ${a.total_calls} | ${formatDuration(a.avg_duration_ms)} | ${formatDuration(a.max_duration_ms)} | ${a.status_codes || '-'} |`);
|
|
118
|
+
}
|
|
119
|
+
lines.push('');
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// ── Error Patterns ──────────────────────────────────────────────────────────
|
|
123
|
+
if (errors.length > 0) {
|
|
124
|
+
lines.push('## Error Patterns');
|
|
125
|
+
lines.push('');
|
|
126
|
+
lines.push('| Pattern | Category | Count | First Seen | Last Seen | Example Test |');
|
|
127
|
+
lines.push('|---------|----------|-------|------------|-----------|-------------|');
|
|
128
|
+
for (const e of errors.slice(0, 20)) {
|
|
129
|
+
lines.push(`| ${truncate(e.pattern, 50)} | ${e.category} | ${e.occurrence_count} | ${formatDate(e.first_seen)} | ${formatDate(e.last_seen)} | ${e.example_test || '-'} |`);
|
|
130
|
+
}
|
|
131
|
+
lines.push('');
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// ── Recent Trend ────────────────────────────────────────────────────────────
|
|
135
|
+
if (trends.length > 0) {
|
|
136
|
+
const label = trendsGranularity === 'hourly' ? 'Recent Trend (hourly)' : 'Recent Trend (7 days)';
|
|
137
|
+
const col1 = trendsGranularity === 'hourly' ? 'Hour' : 'Date';
|
|
138
|
+
lines.push(`## ${label}`);
|
|
139
|
+
lines.push('');
|
|
140
|
+
lines.push(`| ${col1} | Pass Rate | Tests | Passed | Failed | Flaky | Avg Duration |`);
|
|
141
|
+
lines.push('|------|-----------|-------|--------|--------|-------|-------------|');
|
|
142
|
+
for (const t of trends) {
|
|
143
|
+
lines.push(`| ${t.date} | ${t.pass_rate}% | ${t.total_tests} | ${t.passed} | ${t.failed} | ${t.flaky_count} | ${formatDuration(t.avg_duration_ms)} |`);
|
|
144
|
+
}
|
|
145
|
+
lines.push('');
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Write the file
|
|
149
|
+
const cwd = config?._cwd || process.cwd();
|
|
150
|
+
const e2eDir = path.join(cwd, 'e2e');
|
|
151
|
+
if (!fs.existsSync(e2eDir)) {
|
|
152
|
+
fs.mkdirSync(e2eDir, { recursive: true });
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const mdPath = path.join(e2eDir, 'learnings.md');
|
|
156
|
+
fs.writeFileSync(mdPath, lines.join('\n') + '\n');
|
|
157
|
+
|
|
158
|
+
return mdPath;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// ── Helpers ───────────────────────────────────────────────────────────────────
|
|
162
|
+
|
|
163
|
+
function formatDuration(ms) {
|
|
164
|
+
if (ms == null || isNaN(ms)) return '-';
|
|
165
|
+
if (ms < 1000) return `${Math.round(ms)}ms`;
|
|
166
|
+
return `${(ms / 1000).toFixed(1)}s`;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function formatDate(dateStr) {
|
|
170
|
+
if (!dateStr) return '-';
|
|
171
|
+
return dateStr.split('T')[0] || dateStr.slice(0, 10);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function truncate(str, max) {
|
|
175
|
+
if (!str) return '-';
|
|
176
|
+
return str.length > max ? str.slice(0, max - 3) + '...' : str;
|
|
177
|
+
}
|