@matware/e2e-runner 1.3.1 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +4 -4
- package/.claude-plugin/plugin.json +2 -2
- package/LICENSE +1 -1
- package/README.md +491 -225
- package/agents/test-creator.md +4 -2
- package/agents/test-improver.md +7 -4
- package/bin/cli.js +93 -19
- package/package.json +4 -3
- package/skills/e2e-testing/SKILL.md +5 -3
- package/skills/e2e-testing/references/action-types.md +35 -18
- package/skills/e2e-testing/references/test-json-format.md +23 -0
- package/skills/e2e-testing/references/troubleshooting.md +2 -26
- package/src/actions.js +181 -15
- package/src/config.js +6 -0
- package/src/dashboard.js +185 -9
- package/src/db.js +26 -0
- package/src/mcp-tools.js +238 -69
- package/src/module-analysis.js +247 -0
- package/src/module-resolver.js +35 -2
- package/src/narrate.js +33 -1
- package/src/pool-manager.js +46 -1
- package/src/pool.js +177 -20
- package/src/runner.js +144 -19
- package/src/visual-diff.js +74 -4
- package/src/websocket.js +14 -3
- package/src/wizard.js +184 -0
- package/templates/build-dashboard.js +3 -0
- package/templates/dashboard/js/api.js +60 -3
- package/templates/dashboard/js/init.js +46 -0
- package/templates/dashboard/js/keyboard.js +8 -7
- package/templates/dashboard/js/quicksearch.js +277 -0
- package/templates/dashboard/js/state.js +61 -7
- package/templates/dashboard/js/toast.js +1 -1
- package/templates/dashboard/js/utils.js +23 -2
- package/templates/dashboard/js/view-live.js +235 -42
- package/templates/dashboard/js/view-runs.js +469 -42
- package/templates/dashboard/js/view-tests.js +157 -16
- package/templates/dashboard/js/view-tools.js +234 -0
- package/templates/dashboard/js/view-watch.js +2 -2
- package/templates/dashboard/js/websocket.js +33 -3
- package/templates/dashboard/styles/base.css +489 -53
- package/templates/dashboard/styles/components.css +736 -84
- package/templates/dashboard/styles/view-live.css +459 -78
- package/templates/dashboard/styles/view-runs.css +826 -177
- package/templates/dashboard/styles/view-tests.css +440 -77
- package/templates/dashboard/styles/view-tools.css +206 -0
- package/templates/dashboard/styles/view-watch.css +198 -41
- package/templates/dashboard/template.html +356 -58
- package/templates/dashboard.html +5354 -722
- package/templates/docker-compose-lightpanda.yml +7 -0
package/src/mcp-tools.js
CHANGED
|
@@ -23,11 +23,12 @@ import { lookupScreenshotHash, ensureProject, computeScreenshotHash, registerScr
|
|
|
23
23
|
import { fetchIssue, checkCliAuth, detectProvider } from './issues.js';
|
|
24
24
|
import { buildPrompt, hasApiKey, generateHindsightHint } from './ai-generate.js';
|
|
25
25
|
import { verifyIssue } from './verify.js';
|
|
26
|
-
import { listModules } from './module-resolver.js';
|
|
26
|
+
import { listModules, loadModuleRegistry } from './module-resolver.js';
|
|
27
27
|
import { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights, getTestHistory, getPageHistory, getSelectorHistory, getHealthSnapshot, getTestCreationContext, generateImprovements, getActionHealthScores } from './learner-sqlite.js';
|
|
28
28
|
import { queryGraph } from './learner-neo4j.js';
|
|
29
29
|
import { startNeo4j, stopNeo4j, getNeo4jStatus } from './neo4j-pool.js';
|
|
30
30
|
import { getAppPoolStatus, isAppPoolEnabled } from './app-pool.js';
|
|
31
|
+
import { looksLikeBlankCapture } from './actions.js';
|
|
31
32
|
|
|
32
33
|
/**
|
|
33
34
|
* Resolves auth token from config: uses static authToken if set,
|
|
@@ -103,7 +104,7 @@ export const TOOLS = [
|
|
|
103
104
|
},
|
|
104
105
|
cwd: {
|
|
105
106
|
type: 'string',
|
|
106
|
-
description: '
|
|
107
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
107
108
|
},
|
|
108
109
|
},
|
|
109
110
|
},
|
|
@@ -117,7 +118,7 @@ export const TOOLS = [
|
|
|
117
118
|
properties: {
|
|
118
119
|
cwd: {
|
|
119
120
|
type: 'string',
|
|
120
|
-
description: '
|
|
121
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
121
122
|
},
|
|
122
123
|
},
|
|
123
124
|
},
|
|
@@ -125,60 +126,19 @@ export const TOOLS = [
|
|
|
125
126
|
{
|
|
126
127
|
name: 'e2e_create_test',
|
|
127
128
|
description:
|
|
128
|
-
`Create a new E2E test JSON file.
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
**Asserting text presence/absence** — DON'T write evaluate with body.includes():
|
|
142
|
-
assert_text: { type: "assert_text", text: "Welcome" } — text IS on page (case-sensitive). Uses: text
|
|
143
|
-
assert_no_text: { type: "assert_no_text", text: "Error" } — text is NOT on page. Uses: text
|
|
144
|
-
assert_text_in: { type: "assert_text_in", selector: "[class*='Drawer']", text: "profesional|doctor" }
|
|
145
|
-
— scoped regex in container (case-insensitive default). Uses: selector + text (+ value:"exact")
|
|
146
|
-
|
|
147
|
-
**Asserting elements** — DON'T write evaluate to count or check visibility:
|
|
148
|
-
assert_visible: { type: "assert_visible", selector: ".modal" } — Uses: selector (NOT text)
|
|
149
|
-
assert_not_visible: { type: "assert_not_visible", selector: ".loader" } — Uses: selector (NOT text)
|
|
150
|
-
assert_count: { type: "assert_count", selector: "input", value: ">= 2" } — Uses: selector + value
|
|
151
|
-
assert_element_text: { type: "assert_element_text", selector: "h1", text: "Dashboard" } — Uses: selector + text
|
|
152
|
-
assert_matches: { type: "assert_matches", selector: ".date", value: "\\\\d{2}/\\\\d{2}" } — Uses: selector + value (regex)
|
|
153
|
-
assert_attribute: { type: "assert_attribute", selector: "button", value: "disabled" } — Uses: selector + value
|
|
154
|
-
assert_url: { type: "assert_url", value: "/dashboard" } — Uses: value
|
|
155
|
-
assert_input_value: { type: "assert_input_value", selector: "#email", value: "@" } — Uses: selector + value
|
|
156
|
-
|
|
157
|
-
IMPORTANT field rules:
|
|
158
|
-
- assert_text / assert_no_text: use "text" field only (checks full page body)
|
|
159
|
-
- assert_visible / assert_not_visible: use "selector" field only (CSS selector, NOT text)
|
|
160
|
-
- To verify text absence: use assert_no_text (NOT assert_not_visible with text)
|
|
161
|
-
|
|
162
|
-
**Navigation & waiting** — DON'T write evaluate with setTimeout polling:
|
|
163
|
-
goto: { type: "goto", value: "/login" } — full page navigation
|
|
164
|
-
navigate: { type: "navigate", value: "/settings" } — SPA-friendly (won't fail if no page load)
|
|
165
|
-
wait: { type: "wait", text: "Loading complete" } — wait for text to appear in body
|
|
166
|
-
wait: { type: "wait", selector: ".results" } — wait for element to appear
|
|
167
|
-
wait: { type: "wait", value: "2000" } — fixed delay (avoid when possible)
|
|
168
|
-
wait_network_idle: { type: "wait_network_idle", value: "500" } — wait until no network for N ms
|
|
169
|
-
|
|
170
|
-
**Form interaction** — DON'T write evaluate with native value setters (unless React):
|
|
171
|
-
type: { type: "type", selector: "#email", value: "a@b.com" } — clears + types
|
|
172
|
-
type_react: { type: "type_react", selector: "#email", value: "a@b.com" } — for React controlled inputs
|
|
173
|
-
select: { type: "select", selector: "select#country", value: "US" }
|
|
174
|
-
clear: { type: "clear", selector: "#search" }
|
|
175
|
-
press: { type: "press", value: "Enter" }
|
|
176
|
-
focus_autocomplete: { type: "focus_autocomplete", text: "City" } — focus MUI Autocomplete by label
|
|
177
|
-
|
|
178
|
-
**When evaluate IS appropriate**: computed styles, complex conditional logic, GraphQL via window.__e2eGql, math calculations, reading window/app state.
|
|
179
|
-
|
|
180
|
-
## Modules
|
|
181
|
-
Use { "$use": "module-name", "params": {...} } to reference reusable modules from e2e/modules/. Modules compose — a module can $use other modules. Check e2e_list to see available modules for the project.`,
|
|
129
|
+
`Create a new E2E test JSON file. Prefer built-in actions over evaluate — more robust and readable. Full catalog: the e2e-testing skill / references/action-types.md.
|
|
130
|
+
|
|
131
|
+
Action cheat-sheet:
|
|
132
|
+
- Click: click (by text), click_regex, click_menu_item, click_option, click_chip, click_icon, click_in_context; in a dialog use click with scope:"dialog" (+ last/visible).
|
|
133
|
+
- Select (MUI): select_combobox (open+optional filter+pick), select, focus_autocomplete.
|
|
134
|
+
- Assert text: assert_text (present), assert_no_text (absent), assert_text_in (scoped regex), assert_element_text, assert_matches.
|
|
135
|
+
- Assert elements (selector, NOT text): assert_visible, assert_not_visible, assert_count, assert_attribute, assert_input_value, assert_url.
|
|
136
|
+
- Nav/wait: goto, navigate (SPA), wait {text|selector|gone|value(ms)}, wait_network_idle.
|
|
137
|
+
- Form: type, type_react (React inputs; optional blur/waitAfter), clear, press.
|
|
138
|
+
|
|
139
|
+
Field rules: assert_text/assert_no_text use "text" (whole page); assert_visible/assert_not_visible use "selector"; for text absence use assert_no_text. Use evaluate only for computed styles, complex logic, GraphQL (window.__e2eGql), or app state.
|
|
140
|
+
|
|
141
|
+
Modules: { "$use": "module-name", "params": {...} } references reusable modules in e2e/modules/ (they compose). Run e2e_list to see available modules.`,
|
|
182
142
|
inputSchema: {
|
|
183
143
|
type: 'object',
|
|
184
144
|
properties: {
|
|
@@ -233,7 +193,7 @@ Use { "$use": "module-name", "params": {...} } to reference reusable modules fro
|
|
|
233
193
|
},
|
|
234
194
|
cwd: {
|
|
235
195
|
type: 'string',
|
|
236
|
-
description: '
|
|
196
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
237
197
|
},
|
|
238
198
|
},
|
|
239
199
|
required: ['name', 'tests'],
|
|
@@ -248,7 +208,7 @@ Use { "$use": "module-name", "params": {...} } to reference reusable modules fro
|
|
|
248
208
|
properties: {
|
|
249
209
|
cwd: {
|
|
250
210
|
type: 'string',
|
|
251
|
-
description: '
|
|
211
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
252
212
|
},
|
|
253
213
|
},
|
|
254
214
|
},
|
|
@@ -295,7 +255,7 @@ Use { "$use": "module-name", "params": {...} } to reference reusable modules fro
|
|
|
295
255
|
},
|
|
296
256
|
cwd: {
|
|
297
257
|
type: 'string',
|
|
298
|
-
description: '
|
|
258
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
299
259
|
},
|
|
300
260
|
},
|
|
301
261
|
},
|
|
@@ -321,7 +281,7 @@ Use { "$use": "module-name", "params": {...} } to reference reusable modules fro
|
|
|
321
281
|
},
|
|
322
282
|
cwd: {
|
|
323
283
|
type: 'string',
|
|
324
|
-
description: '
|
|
284
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
325
285
|
},
|
|
326
286
|
},
|
|
327
287
|
},
|
|
@@ -358,7 +318,7 @@ Use { "$use": "module-name", "params": {...} } to reference reusable modules fro
|
|
|
358
318
|
},
|
|
359
319
|
cwd: {
|
|
360
320
|
type: 'string',
|
|
361
|
-
description: '
|
|
321
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
362
322
|
},
|
|
363
323
|
},
|
|
364
324
|
required: ['url'],
|
|
@@ -406,7 +366,7 @@ Use { "$use": "module-name", "params": {...} } to reference reusable modules fro
|
|
|
406
366
|
},
|
|
407
367
|
cwd: {
|
|
408
368
|
type: 'string',
|
|
409
|
-
description: '
|
|
369
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
410
370
|
},
|
|
411
371
|
},
|
|
412
372
|
required: ['url'],
|
|
@@ -458,7 +418,7 @@ Use { "$use": "module-name", "params": {...} } to reference reusable modules fro
|
|
|
458
418
|
},
|
|
459
419
|
cwd: {
|
|
460
420
|
type: 'string',
|
|
461
|
-
description: '
|
|
421
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
462
422
|
},
|
|
463
423
|
},
|
|
464
424
|
required: ['url'],
|
|
@@ -533,7 +493,7 @@ Good module candidates: auth setup, page navigation, tab clicking, opening sideb
|
|
|
533
493
|
},
|
|
534
494
|
cwd: {
|
|
535
495
|
type: 'string',
|
|
536
|
-
description: '
|
|
496
|
+
description: 'Project root directory (defaults to the current working directory).',
|
|
537
497
|
},
|
|
538
498
|
},
|
|
539
499
|
required: ['query'],
|
|
@@ -1004,6 +964,65 @@ async function handleCreateTest(args) {
|
|
|
1004
964
|
}
|
|
1005
965
|
} catch { /* modules dir may not exist */ }
|
|
1006
966
|
|
|
967
|
+
// ── #2/#3: nudge module reuse and flag extractable duplication ──
|
|
968
|
+
try {
|
|
969
|
+
const fullModules = [...loadModuleRegistry(config.modulesDir).values()];
|
|
970
|
+
|
|
971
|
+
// #2 — submitted actions already match an existing module verbatim
|
|
972
|
+
const matches = detectModuleMatches(args.tests, fullModules);
|
|
973
|
+
for (const h of matches) {
|
|
974
|
+
const req = h.params ? Object.entries(h.params).filter(([, d]) => d?.required).map(([n]) => n) : [];
|
|
975
|
+
const paramHint = req.length ? `, "params": { ${req.map(n => `"${n}": ...`).join(', ')} }` : '';
|
|
976
|
+
warnings.push(`♻️ Test "${h.test}" repeats the ${h.len} actions of existing module "${h.module}" inline. ` +
|
|
977
|
+
`Replace them with { "$use": "${h.module}"${paramHint} }.`);
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
// #3 — action sequences duplicated across tests that aren't a module yet
|
|
981
|
+
const matchedTests = new Set(matches.map(m => m.test));
|
|
982
|
+
const testsActions = args.tests
|
|
983
|
+
.filter(t => t.actions && !t.actions.some(a => a && a.$use) && !matchedTests.has(t.name))
|
|
984
|
+
.map(t => ({ name: t.name, sigs: actionsSigList(t.actions) }));
|
|
985
|
+
|
|
986
|
+
// include existing test files (excluding the one just written) for cross-file duplication
|
|
987
|
+
try {
|
|
988
|
+
if (fs.existsSync(config.testsDir)) {
|
|
989
|
+
for (const f of fs.readdirSync(config.testsDir).filter(x => x.endsWith('.json'))) {
|
|
990
|
+
const fp = path.join(config.testsDir, f);
|
|
991
|
+
if (fp === filePath) continue;
|
|
992
|
+
let parsed;
|
|
993
|
+
try { parsed = JSON.parse(fs.readFileSync(fp, 'utf-8')); } catch { continue; }
|
|
994
|
+
const list = Array.isArray(parsed) ? parsed : (parsed.tests || []);
|
|
995
|
+
for (const t of list) {
|
|
996
|
+
if (t && t.actions && !t.actions.some(a => a && a.$use)) {
|
|
997
|
+
testsActions.push({ name: `${f}:${t.name}`, sigs: actionsSigList(t.actions) });
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
} catch { /* ignore unreadable test files */ }
|
|
1003
|
+
|
|
1004
|
+
// windows already covered by an existing module → don't re-suggest extracting them
|
|
1005
|
+
const moduleWindowSet = new Set();
|
|
1006
|
+
for (const m of fullModules) {
|
|
1007
|
+
const sigs = actionsSigList((m.actions || []).filter(a => a && !a.$use));
|
|
1008
|
+
for (const w of sigWindows(sigs, 2, 6)) moduleWindowSet.add(w);
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
const extractable = detectExtractableSequences(testsActions, moduleWindowSet);
|
|
1012
|
+
for (const e of extractable) {
|
|
1013
|
+
const sample = e.tests.slice(0, 3).join(', ') + (e.tests.length > 3 ? '…' : '');
|
|
1014
|
+
warnings.push(`🧩 A sequence of ${e.len} actions appears in ${e.count} tests (${sample}) but isn't a module yet. ` +
|
|
1015
|
+
`Consider extracting it with e2e_create_module and referencing it via $use.`);
|
|
1016
|
+
}
|
|
1017
|
+
} catch { /* never fail test creation */ }
|
|
1018
|
+
|
|
1019
|
+
// ── Verification coverage: tests whose outcome is never checked ──
|
|
1020
|
+
try {
|
|
1021
|
+
let registry;
|
|
1022
|
+
try { registry = loadModuleRegistry(config.modulesDir); } catch { registry = new Map(); }
|
|
1023
|
+
warnings.push(...detectUnverifiedTests(args.tests, registry));
|
|
1024
|
+
} catch { /* never fail test creation */ }
|
|
1025
|
+
|
|
1007
1026
|
const warningBlock = warnings.length > 0 ? '\n\n' + warnings.join('\n\n') : '';
|
|
1008
1027
|
|
|
1009
1028
|
// Enrich with learnings context for smarter test authoring
|
|
@@ -1259,6 +1278,135 @@ function analyzeActionPatterns(tests) {
|
|
|
1259
1278
|
return warnings;
|
|
1260
1279
|
}
|
|
1261
1280
|
|
|
1281
|
+
// ── Module-reuse detection (#2 exact match, #3 extractable duplication) ──
|
|
1282
|
+
|
|
1283
|
+
// Normalized signature of an action for sequence comparison.
|
|
1284
|
+
function actionSig(a) {
|
|
1285
|
+
if (!a || typeof a !== 'object') return '∅';
|
|
1286
|
+
if (a.$use) return `$use:${a.$use}`;
|
|
1287
|
+
return `${a.type || '?'}|${a.selector || ''}|${a.text || ''}`;
|
|
1288
|
+
}
|
|
1289
|
+
function actionsSigList(actions) {
|
|
1290
|
+
return (actions || []).map(actionSig);
|
|
1291
|
+
}
|
|
1292
|
+
// All contiguous sig windows of length minLen..maxLen, each joined with '»'.
|
|
1293
|
+
function sigWindows(sigs, minLen, maxLen) {
|
|
1294
|
+
const out = [];
|
|
1295
|
+
const top = Math.min(maxLen, sigs.length);
|
|
1296
|
+
for (let len = minLen; len <= top; len++) {
|
|
1297
|
+
for (let i = 0; i + len <= sigs.length; i++) {
|
|
1298
|
+
out.push(sigs.slice(i, i + len).join('»'));
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
return out;
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
// #2 — a module field matches a concrete test field; {{param}} placeholders are wildcards.
|
|
1305
|
+
function moduleFieldMatch(modVal, testVal) {
|
|
1306
|
+
if (modVal == null || modVal === '') return true; // module doesn't constrain it
|
|
1307
|
+
if (typeof modVal === 'string' && modVal.includes('{{')) return true; // placeholder → wildcard
|
|
1308
|
+
return modVal === testVal;
|
|
1309
|
+
}
|
|
1310
|
+
function moduleActionMatch(modA, testA) {
|
|
1311
|
+
if (!modA || !testA || modA.$use || testA.$use) return false;
|
|
1312
|
+
return modA.type === testA.type
|
|
1313
|
+
&& moduleFieldMatch(modA.selector, testA.selector)
|
|
1314
|
+
&& moduleFieldMatch(modA.text, testA.text);
|
|
1315
|
+
}
|
|
1316
|
+
// Find modules whose full leaf-action sequence appears as a contiguous run inside a test
|
|
1317
|
+
// that doesn't already use modules. Returns [{ module, params, test, len }].
|
|
1318
|
+
function detectModuleMatches(tests, fullModules) {
|
|
1319
|
+
const hits = [];
|
|
1320
|
+
for (const mod of fullModules) {
|
|
1321
|
+
const ma = (mod.actions || []).filter(a => a && !a.$use);
|
|
1322
|
+
if (ma.length < 2) continue;
|
|
1323
|
+
for (const test of tests) {
|
|
1324
|
+
const ta = test.actions || [];
|
|
1325
|
+
if (ta.some(a => a && a.$use)) continue; // already modular
|
|
1326
|
+
for (let i = 0; i + ma.length <= ta.length; i++) {
|
|
1327
|
+
let ok = true;
|
|
1328
|
+
for (let j = 0; j < ma.length; j++) {
|
|
1329
|
+
if (!moduleActionMatch(ma[j], ta[i + j])) { ok = false; break; }
|
|
1330
|
+
}
|
|
1331
|
+
if (ok) { hits.push({ module: mod.$module, params: mod.params, test: test.name, len: ma.length }); break; }
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
return hits;
|
|
1336
|
+
}
|
|
1337
|
+
// #3 — contiguous sig windows (len>=minLen) shared by >=2 distinct tests and not already a module.
|
|
1338
|
+
function detectExtractableSequences(testsActions, moduleWindowSet, { minLen = 3, maxLen = 6, cap = 2 } = {}) {
|
|
1339
|
+
const owners = new Map(); // windowKey -> Set(testName)
|
|
1340
|
+
const lenOf = new Map(); // windowKey -> action count
|
|
1341
|
+
for (const t of testsActions) {
|
|
1342
|
+
const seen = new Set();
|
|
1343
|
+
for (const key of sigWindows(t.sigs, minLen, maxLen)) {
|
|
1344
|
+
if (key.includes('$use:') || moduleWindowSet.has(key) || seen.has(key)) continue;
|
|
1345
|
+
seen.add(key);
|
|
1346
|
+
if (!owners.has(key)) { owners.set(key, new Set()); lenOf.set(key, key.split('»').length); }
|
|
1347
|
+
owners.get(key).add(t.name);
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
const cands = [];
|
|
1351
|
+
for (const [key, set] of owners) {
|
|
1352
|
+
if (set.size >= 2) cands.push({ key, len: lenOf.get(key), count: set.size, tests: [...set] });
|
|
1353
|
+
}
|
|
1354
|
+
cands.sort((a, b) => b.len - a.len || b.count - a.count); // longest, then most frequent
|
|
1355
|
+
const kept = [];
|
|
1356
|
+
for (const c of cands) {
|
|
1357
|
+
if (kept.some(k => k.key.includes(c.key))) continue; // subsumed by a longer kept window
|
|
1358
|
+
kept.push(c);
|
|
1359
|
+
if (kept.length >= cap) break;
|
|
1360
|
+
}
|
|
1361
|
+
return kept;
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
// Verification coverage — a test whose interactions are never followed by a check
|
|
1365
|
+
// can go green even when the flow silently breaks.
|
|
1366
|
+
function isVerifyingAction(a) {
|
|
1367
|
+
if (/^assert_/.test(a.type || '')) return true;
|
|
1368
|
+
if (a.type === 'evaluate' || a.type === 'gql') return true; // strict semantics / inline assertions
|
|
1369
|
+
if (a.type === 'wait' && (a.selector || a.text || a.gone)) return true; // condition waits fail if unmet
|
|
1370
|
+
return false;
|
|
1371
|
+
}
|
|
1372
|
+
function isInteractingAction(a) {
|
|
1373
|
+
return /^(click|select|type|fill|clear|press|hover|scroll|set_storage|focus_autocomplete|goto|navigate)/.test(a.type || '');
|
|
1374
|
+
}
|
|
1375
|
+
function detectUnverifiedTests(tests, registry) {
|
|
1376
|
+
const out = [];
|
|
1377
|
+
for (const t of tests || []) {
|
|
1378
|
+
if (!t || !Array.isArray(t.actions) || t.expect) continue; // "expect" verifies the end state visually
|
|
1379
|
+
// expand $use so module-provided assertions count
|
|
1380
|
+
const leaf = [];
|
|
1381
|
+
let resolvable = true;
|
|
1382
|
+
for (const a of t.actions) {
|
|
1383
|
+
if (a && a.$use) {
|
|
1384
|
+
const mod = registry.get(a.$use);
|
|
1385
|
+
if (mod?.actions) leaf.push(...mod.actions.filter(x => x && !x.$use));
|
|
1386
|
+
else { resolvable = false; break; }
|
|
1387
|
+
} else if (a) leaf.push(a);
|
|
1388
|
+
}
|
|
1389
|
+
if (!resolvable) continue;
|
|
1390
|
+
let lastInteract = -1;
|
|
1391
|
+
leaf.forEach((a, i) => { if (isInteractingAction(a)) lastInteract = i; });
|
|
1392
|
+
if (lastInteract === -1) continue; // nothing happens — nothing to verify
|
|
1393
|
+
if (!leaf.some(isVerifyingAction)) {
|
|
1394
|
+
out.push(`🔎 Test "${t.name}" has no assertions and no "expect" field — it can pass without verifying anything. ` +
|
|
1395
|
+
`Close with assert_* actions (assert_url, assert_text, assert_visible) or add an "expect" for visual verification.`);
|
|
1396
|
+
} else if (!leaf.slice(lastInteract + 1).some(isVerifyingAction)) {
|
|
1397
|
+
out.push(`🔎 Test "${t.name}" keeps interacting after its last check — the final steps are unverified. ` +
|
|
1398
|
+
`Close with an assert_* action so the end state is what passes the test.`);
|
|
1399
|
+
} else {
|
|
1400
|
+
const tail = leaf.slice(lastInteract + 1).filter(isVerifyingAction);
|
|
1401
|
+
if (tail.length && tail.every(a => a.type === 'assert_text')) {
|
|
1402
|
+
out.push(`📌 Test "${t.name}" closes with page-wide assert_text only — it matches anywhere on the page. ` +
|
|
1403
|
+
`Scope the final check with assert_element_text or assert_text_in.`);
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
return out;
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1262
1410
|
async function handlePoolStatus(args) {
|
|
1263
1411
|
const config = await loadConfig({}, args.cwd);
|
|
1264
1412
|
const poolUrls = getPoolUrls(config);
|
|
@@ -1955,7 +2103,19 @@ async function handleAnalyze(args) {
|
|
|
1955
2103
|
screenshotBase64 = data.toString('base64');
|
|
1956
2104
|
}
|
|
1957
2105
|
|
|
1958
|
-
|
|
2106
|
+
// Surface reusable modules at the decision point, so scaffolds can $use them
|
|
2107
|
+
// instead of duplicating action sequences.
|
|
2108
|
+
let availableModules = [];
|
|
2109
|
+
try {
|
|
2110
|
+
availableModules = listModules(config.modulesDir).map(m => ({
|
|
2111
|
+
name: m.name,
|
|
2112
|
+
params: m.params.map(p => (p.required ? p.name : `${p.name}?`)),
|
|
2113
|
+
description: m.description || undefined,
|
|
2114
|
+
hint: `{ "$use": "${m.name}"${m.params.some(p => p.required) ? ', "params": { ... }' : ''} }`,
|
|
2115
|
+
}));
|
|
2116
|
+
} catch { /* modules dir may not exist */ }
|
|
2117
|
+
|
|
2118
|
+
const result = { meta, ...structure, suggestedTests, availableModules };
|
|
1959
2119
|
const content = [{ type: 'text', text: JSON.stringify(result, null, 2) }];
|
|
1960
2120
|
|
|
1961
2121
|
if (screenshotBase64) {
|
|
@@ -2011,7 +2171,18 @@ async function handleCapture(args) {
|
|
|
2011
2171
|
}
|
|
2012
2172
|
|
|
2013
2173
|
const screenshotPath = path.join(config.screenshotsDir, filename);
|
|
2014
|
-
await page.screenshot({
|
|
2174
|
+
const data = await page.screenshot({ fullPage: !!args.fullPage });
|
|
2175
|
+
|
|
2176
|
+
// Blank frame (uniform color — page never rendered): don't save it,
|
|
2177
|
+
// report what happened instead of returning a useless white PNG.
|
|
2178
|
+
if (looksLikeBlankCapture(data, 'png')) {
|
|
2179
|
+
return {
|
|
2180
|
+
content: [
|
|
2181
|
+
{ type: 'text', text: `Capture skipped: ${args.url} rendered a blank (uniform-color) frame — nothing saved. The page likely failed to render (auth redirect, JS error, or slow load); try a longer delay or a selector to wait for.` },
|
|
2182
|
+
],
|
|
2183
|
+
};
|
|
2184
|
+
}
|
|
2185
|
+
fs.writeFileSync(screenshotPath, data);
|
|
2015
2186
|
|
|
2016
2187
|
// Register hash in SQLite
|
|
2017
2188
|
const cwd = args.cwd || process.cwd();
|
|
@@ -2020,8 +2191,6 @@ async function handleCapture(args) {
|
|
|
2020
2191
|
const hash = computeScreenshotHash(screenshotPath);
|
|
2021
2192
|
registerScreenshotHash(hash, screenshotPath, projectId, null);
|
|
2022
2193
|
|
|
2023
|
-
// Read image for response
|
|
2024
|
-
const data = fs.readFileSync(screenshotPath);
|
|
2025
2194
|
const base64 = data.toString('base64');
|
|
2026
2195
|
|
|
2027
2196
|
return {
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Module Analysis — deterministic duplication detector.
|
|
3
|
+
*
|
|
4
|
+
* Scans every test JSON in testsDir, normalizes action sequences to
|
|
5
|
+
* signatures (literal values become placeholders, but selectors/text
|
|
6
|
+
* stay so two unrelated clicks don't collide), and reports sequences of
|
|
7
|
+
* length 3-8 that appear in 2+ different tests. These are the canonical
|
|
8
|
+
* candidates the test-improver agent would extract into a `$use` module.
|
|
9
|
+
*
|
|
10
|
+
* Also enumerates current modules and counts how often each is referenced
|
|
11
|
+
* via `$use` across the project so users can see adoption.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import fs from 'fs';
|
|
15
|
+
import path from 'path';
|
|
16
|
+
|
|
17
|
+
const MIN_SEQ_LEN = 3;
|
|
18
|
+
const MAX_SEQ_LEN = 8;
|
|
19
|
+
|
|
20
|
+
/** Stable signature for one action — literals → '*', identifiers kept. */
|
|
21
|
+
function signatureOf(action) {
|
|
22
|
+
if (!action || typeof action !== 'object') return '?';
|
|
23
|
+
if (action.$use) return `$use:${action.$use}`;
|
|
24
|
+
const type = action.type || '?';
|
|
25
|
+
// Keep selector + text (semantic identifiers); replace `value` with `*`
|
|
26
|
+
// since values are usually parameterizable.
|
|
27
|
+
const parts = [type];
|
|
28
|
+
if (action.selector) parts.push(`@${action.selector}`);
|
|
29
|
+
if (action.text != null) parts.push(`"${String(action.text).slice(0, 40)}"`);
|
|
30
|
+
if (action.value != null) parts.push('*');
|
|
31
|
+
return parts.join('|');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function walkTests(testsDir) {
|
|
35
|
+
const out = [];
|
|
36
|
+
let files = [];
|
|
37
|
+
try {
|
|
38
|
+
files = fs.readdirSync(testsDir).filter(f => f.endsWith('.json')).sort();
|
|
39
|
+
} catch { return out; }
|
|
40
|
+
for (const file of files) {
|
|
41
|
+
const fp = path.join(testsDir, file);
|
|
42
|
+
let suite;
|
|
43
|
+
try { suite = JSON.parse(fs.readFileSync(fp, 'utf-8')); } catch { continue; }
|
|
44
|
+
const tests = Array.isArray(suite) ? suite : suite.tests || [];
|
|
45
|
+
for (const t of tests) {
|
|
46
|
+
if (!t || !Array.isArray(t.actions)) continue;
|
|
47
|
+
out.push({
|
|
48
|
+
file,
|
|
49
|
+
suite: file.replace(/\.json$/, ''),
|
|
50
|
+
test: t.name || '(unnamed)',
|
|
51
|
+
actions: t.actions,
|
|
52
|
+
signatures: t.actions.map(signatureOf),
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return out;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function findCandidates(tests) {
|
|
60
|
+
// Map signature-sequence-string → [{testIdx, start}]
|
|
61
|
+
const seen = new Map();
|
|
62
|
+
for (let ti = 0; ti < tests.length; ti++) {
|
|
63
|
+
const sig = tests[ti].signatures;
|
|
64
|
+
for (let len = MIN_SEQ_LEN; len <= MAX_SEQ_LEN; len++) {
|
|
65
|
+
for (let i = 0; i + len <= sig.length; i++) {
|
|
66
|
+
const key = sig.slice(i, i + len).join(' >> ');
|
|
67
|
+
if (!seen.has(key)) seen.set(key, []);
|
|
68
|
+
seen.get(key).push({ testIdx: ti, start: i, len });
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const candidates = [];
|
|
74
|
+
for (const [key, hits] of seen) {
|
|
75
|
+
// Distinct tests, not just same test repeated
|
|
76
|
+
const distinct = new Map();
|
|
77
|
+
for (const h of hits) {
|
|
78
|
+
const t = tests[h.testIdx];
|
|
79
|
+
const id = t.suite + '::' + t.test;
|
|
80
|
+
if (!distinct.has(id)) distinct.set(id, { test: t, hits: [] });
|
|
81
|
+
distinct.get(id).hits.push(h);
|
|
82
|
+
}
|
|
83
|
+
if (distinct.size < 2) continue;
|
|
84
|
+
candidates.push({
|
|
85
|
+
signature: key,
|
|
86
|
+
length: hits[0].len,
|
|
87
|
+
occurrenceCount: hits.length,
|
|
88
|
+
testCount: distinct.size,
|
|
89
|
+
// Best representative — first hit's actions, lifted from the test
|
|
90
|
+
sample: tests[hits[0].testIdx].actions.slice(hits[0].start, hits[0].start + hits[0].len),
|
|
91
|
+
usedBy: [...distinct.values()].map(d => ({
|
|
92
|
+
suite: d.test.suite,
|
|
93
|
+
test: d.test.test,
|
|
94
|
+
occurrences: d.hits.length,
|
|
95
|
+
})),
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Rank: maximize (savings ≈ length * (testCount - 1))
|
|
100
|
+
// Then prefer longer sequences over shorter ones.
|
|
101
|
+
candidates.sort((a, b) => {
|
|
102
|
+
const savingsA = a.length * (a.testCount - 1);
|
|
103
|
+
const savingsB = b.length * (b.testCount - 1);
|
|
104
|
+
if (savingsA !== savingsB) return savingsB - savingsA;
|
|
105
|
+
return b.length - a.length;
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
// Prune: drop sequences that are strict substrings of a higher-scored one
|
|
109
|
+
// covering the same set of tests (the shorter one is redundant once the
|
|
110
|
+
// longer one is extracted).
|
|
111
|
+
const kept = [];
|
|
112
|
+
for (const c of candidates) {
|
|
113
|
+
const covered = kept.find(k =>
|
|
114
|
+
k.signature.includes(c.signature) &&
|
|
115
|
+
JSON.stringify(k.usedBy.map(u => u.suite+'::'+u.test).sort()) ===
|
|
116
|
+
JSON.stringify(c.usedBy.map(u => u.suite+'::'+u.test).sort())
|
|
117
|
+
);
|
|
118
|
+
if (!covered) kept.push(c);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Suggest a name from the dominant action types
|
|
122
|
+
for (const c of kept) {
|
|
123
|
+
c.suggestedName = suggestModuleName(c.sample);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return kept.slice(0, 30);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function suggestModuleName(actions) {
|
|
130
|
+
if (!actions || !actions.length) return 'module';
|
|
131
|
+
const types = actions.map(a => a?.type).filter(Boolean);
|
|
132
|
+
// Heuristics for common patterns
|
|
133
|
+
const hasGoto = types.includes('goto');
|
|
134
|
+
const hasType = types.includes('type') || types.includes('fill') || types.includes('type_react');
|
|
135
|
+
const hasClick = types.some(t => t && t.startsWith('click'));
|
|
136
|
+
const hasAssert = types.some(t => t && t.startsWith('assert'));
|
|
137
|
+
// Pull a noun-y hint from selector or text of first non-goto action
|
|
138
|
+
const hint = (function () {
|
|
139
|
+
for (const a of actions) {
|
|
140
|
+
const s = a?.text || a?.selector || a?.value;
|
|
141
|
+
if (typeof s === 'string' && s.length > 0 && s.length < 30) {
|
|
142
|
+
return s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 24);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return '';
|
|
146
|
+
})();
|
|
147
|
+
if (hasGoto && hasType && hasClick) return 'navigate-and-submit' + (hint ? '-' + hint : '');
|
|
148
|
+
if (hasType && hasClick) return 'fill-form' + (hint ? '-' + hint : '');
|
|
149
|
+
if (hasGoto && hasAssert) return 'open-and-verify' + (hint ? '-' + hint : '');
|
|
150
|
+
if (hasGoto) return 'navigate' + (hint ? '-' + hint : '');
|
|
151
|
+
if (hint) return hint;
|
|
152
|
+
return 'extracted-module';
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function loadModules(modulesDir) {
|
|
156
|
+
if (!modulesDir || !fs.existsSync(modulesDir)) return [];
|
|
157
|
+
const files = fs.readdirSync(modulesDir).filter(f => f.endsWith('.json')).sort();
|
|
158
|
+
return files.map(f => {
|
|
159
|
+
const fp = path.join(modulesDir, f);
|
|
160
|
+
let data = {};
|
|
161
|
+
try { data = JSON.parse(fs.readFileSync(fp, 'utf-8')); } catch { /* */ }
|
|
162
|
+
return {
|
|
163
|
+
name: f.replace(/\.json$/, ''),
|
|
164
|
+
file: f,
|
|
165
|
+
description: data.description || null,
|
|
166
|
+
params: data.params || [],
|
|
167
|
+
actionCount: Array.isArray(data.actions) ? data.actions.length : 0,
|
|
168
|
+
};
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function countModuleUsage(tests, modules) {
|
|
173
|
+
const usage = new Map(modules.map(m => [m.name, { count: 0, usedBy: new Set() }]));
|
|
174
|
+
function walk(actions, testInfo) {
|
|
175
|
+
if (!Array.isArray(actions)) return;
|
|
176
|
+
for (const a of actions) {
|
|
177
|
+
if (a && a.$use) {
|
|
178
|
+
const u = usage.get(a.$use);
|
|
179
|
+
if (u) { u.count++; u.usedBy.add(testInfo); }
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
for (const t of tests) {
|
|
184
|
+
walk(t.actions, t.suite + '::' + t.test);
|
|
185
|
+
}
|
|
186
|
+
return modules.map(m => {
|
|
187
|
+
const u = usage.get(m.name) || { count: 0, usedBy: new Set() };
|
|
188
|
+
return { ...m, usageCount: u.count, usedBy: [...u.usedBy] };
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
export function runModuleAnalysis(testsDir, modulesDir) {
|
|
193
|
+
const tests = walkTests(testsDir);
|
|
194
|
+
const modules = loadModules(modulesDir);
|
|
195
|
+
const candidates = findCandidates(tests);
|
|
196
|
+
const modulesWithUsage = countModuleUsage(tests, modules);
|
|
197
|
+
|
|
198
|
+
// Build a Claude Code prompt the user can copy verbatim.
|
|
199
|
+
const prompt = buildAgentPrompt(testsDir, modulesDir, candidates, modulesWithUsage);
|
|
200
|
+
|
|
201
|
+
return {
|
|
202
|
+
testsDir,
|
|
203
|
+
modulesDir,
|
|
204
|
+
summary: {
|
|
205
|
+
testCount: tests.length,
|
|
206
|
+
moduleCount: modulesWithUsage.length,
|
|
207
|
+
candidateCount: candidates.length,
|
|
208
|
+
unusedModules: modulesWithUsage.filter(m => m.usageCount === 0).length,
|
|
209
|
+
},
|
|
210
|
+
modules: modulesWithUsage,
|
|
211
|
+
candidates,
|
|
212
|
+
agentPrompt: prompt,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function buildAgentPrompt(testsDir, modulesDir, candidates, modules) {
|
|
217
|
+
const topCandidates = candidates.slice(0, 10);
|
|
218
|
+
return [
|
|
219
|
+
'Analyze E2E test modules and recommend changes.',
|
|
220
|
+
'',
|
|
221
|
+
`Tests directory: ${testsDir}`,
|
|
222
|
+
`Modules directory: ${modulesDir}`,
|
|
223
|
+
'',
|
|
224
|
+
'Use the test-improver capabilities to:',
|
|
225
|
+
'1. Review the candidate sequences below and decide which should be extracted into reusable modules via `e2e_create_module`.',
|
|
226
|
+
'2. For each extracted module, suggest the parameters needed (selectors/text that vary between usages).',
|
|
227
|
+
'3. Check the current modules list for any that are unused or could be consolidated.',
|
|
228
|
+
'4. After creating modules, Edit the affected test files to replace inline action sequences with `{ "$use": "<module-name>", "params": {...} }`.',
|
|
229
|
+
'',
|
|
230
|
+
`## Top ${topCandidates.length} extraction candidates`,
|
|
231
|
+
'',
|
|
232
|
+
...topCandidates.map((c, i) =>
|
|
233
|
+
`${i + 1}. **${c.suggestedName}** (${c.length} actions, used in ${c.testCount} tests, ${c.occurrenceCount} total occurrences)\n` +
|
|
234
|
+
` Signature: \`${c.signature}\`\n` +
|
|
235
|
+
` Used by: ${c.usedBy.map(u => `${u.suite}::${u.test}`).join(', ')}`
|
|
236
|
+
),
|
|
237
|
+
'',
|
|
238
|
+
'## Current modules',
|
|
239
|
+
'',
|
|
240
|
+
...modules.map(m =>
|
|
241
|
+
`- **${m.name}** — ${m.actionCount} actions, ${m.params.length} params, used ${m.usageCount}x` +
|
|
242
|
+
(m.description ? `\n > ${m.description}` : '')
|
|
243
|
+
),
|
|
244
|
+
'',
|
|
245
|
+
'After making changes, run the affected tests to confirm nothing broke.',
|
|
246
|
+
].join('\n');
|
|
247
|
+
}
|