@matware/e2e-runner 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +52 -0
- package/.claude-plugin/plugin.json +17 -3
- package/.mcp.json +2 -2
- package/.opencode/commands/create-test.md +63 -0
- package/.opencode/commands/run.md +50 -0
- package/.opencode/commands/verify-issue.md +62 -0
- package/.opencode/skills/e2e-testing/SKILL.md +181 -0
- package/.opencode/skills/e2e-testing/references/action-types.md +143 -0
- package/.opencode/skills/e2e-testing/references/auth-strategies.md +91 -0
- package/.opencode/skills/e2e-testing/references/graphql.md +59 -0
- package/.opencode/skills/e2e-testing/references/issue-verification.md +59 -0
- package/.opencode/skills/e2e-testing/references/multi-pool.md +60 -0
- package/.opencode/skills/e2e-testing/references/network-debugging.md +62 -0
- package/.opencode/skills/e2e-testing/references/test-json-format.md +163 -0
- package/.opencode/skills/e2e-testing/references/troubleshooting.md +224 -0
- package/.opencode/skills/e2e-testing/references/variables.md +41 -0
- package/.opencode/skills/e2e-testing/references/visual-verification.md +89 -0
- package/LICENSE +190 -0
- package/OPENCODE.md +166 -0
- package/README.md +165 -104
- package/agents/test-creator.md +54 -1
- package/agents/test-improver.md +37 -0
- package/bin/cli.js +409 -16
- package/commands/capture.md +45 -0
- package/commands/create-test.md +16 -1
- package/opencode.json +11 -0
- package/package.json +7 -2
- package/scripts/setup-opencode.sh +113 -0
- package/skills/e2e-testing/SKILL.md +10 -3
- package/skills/e2e-testing/references/action-types.md +48 -5
- package/skills/e2e-testing/references/auth-strategies.md +91 -0
- package/skills/e2e-testing/references/graphql.md +59 -0
- package/skills/e2e-testing/references/issue-verification.md +59 -0
- package/skills/e2e-testing/references/multi-pool.md +60 -0
- package/skills/e2e-testing/references/network-debugging.md +62 -0
- package/skills/e2e-testing/references/test-json-format.md +4 -0
- package/skills/e2e-testing/references/troubleshooting.md +44 -2
- package/skills/e2e-testing/references/variables.md +41 -0
- package/skills/e2e-testing/references/visual-verification.md +89 -0
- package/src/actions.js +475 -2
- package/src/ai-generate.js +139 -8
- package/src/app-pool.js +339 -0
- package/src/config.js +266 -5
- package/src/dashboard.js +216 -17
- package/src/db.js +191 -7
- package/src/index.js +12 -9
- package/src/learner-sqlite.js +458 -0
- package/src/learner.js +78 -6
- package/src/mcp-tools.js +1348 -51
- package/src/module-resolver.js +37 -0
- package/src/narrate.js +65 -0
- package/src/pool-manager.js +229 -0
- package/src/pool.js +301 -31
- package/src/reporter.js +86 -2
- package/src/runner.js +480 -71
- package/src/sync/auth.js +354 -0
- package/src/sync/client.js +572 -0
- package/src/sync/hub-routes.js +816 -0
- package/src/sync/index.js +68 -0
- package/src/sync/middleware.js +347 -0
- package/src/sync/queue.js +209 -0
- package/src/sync/schema.js +540 -0
- package/src/verify.js +10 -7
- package/src/visual-diff.js +446 -0
- package/src/watch.js +384 -0
- package/templates/build-dashboard.js +47 -6
- package/templates/dashboard/js/api.js +62 -0
- package/templates/dashboard/js/init.js +13 -0
- package/templates/dashboard/js/keyboard.js +46 -0
- package/templates/dashboard/js/state.js +40 -0
- package/templates/dashboard/js/toast.js +41 -0
- package/templates/dashboard/js/utils.js +216 -0
- package/templates/dashboard/js/view-live.js +181 -0
- package/templates/dashboard/js/view-runs.js +676 -0
- package/templates/dashboard/js/view-tests.js +294 -0
- package/templates/dashboard/js/view-watch.js +242 -0
- package/templates/dashboard/js/websocket.js +116 -0
- package/templates/dashboard/styles/base.css +69 -0
- package/templates/dashboard/styles/components.css +117 -0
- package/templates/dashboard/styles/view-live.css +97 -0
- package/templates/dashboard/styles/view-runs.css +243 -0
- package/templates/dashboard/styles/view-tests.css +96 -0
- package/templates/dashboard/styles/view-watch.css +53 -0
- package/templates/dashboard/template.html +181 -100
- package/templates/dashboard.html +1614 -547
- package/templates/sample-test.json +0 -8
- package/templates/dashboard/app.js +0 -1152
- package/templates/dashboard/styles.css +0 -413
package/src/mcp-tools.js
CHANGED
|
@@ -13,19 +13,50 @@ import path from 'path';
|
|
|
13
13
|
import http from 'http';
|
|
14
14
|
|
|
15
15
|
import { loadConfig } from './config.js';
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
16
|
+
import { connectToPool } from './pool.js';
|
|
17
|
+
import { waitForAnyPool, getPoolUrls, getAggregatedPoolStatus, selectPool } from './pool-manager.js';
|
|
18
|
+
import { runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites, fetchAuthToken } from './runner.js';
|
|
18
19
|
import { generateReport, saveReport, persistRun } from './reporter.js';
|
|
19
20
|
import { narrateTest } from './narrate.js';
|
|
20
21
|
import { startDashboard, stopDashboard } from './dashboard.js';
|
|
21
|
-
import { lookupScreenshotHash, ensureProject, computeScreenshotHash, registerScreenshotHash, getNetworkLogs } from './db.js';
|
|
22
|
+
import { lookupScreenshotHash, ensureProject, computeScreenshotHash, registerScreenshotHash, getNetworkLogs, setVariable, getVariables, deleteVariable, listVariables } from './db.js';
|
|
22
23
|
import { fetchIssue, checkCliAuth, detectProvider } from './issues.js';
|
|
23
|
-
import { buildPrompt, hasApiKey } from './ai-generate.js';
|
|
24
|
+
import { buildPrompt, hasApiKey, generateHindsightHint } from './ai-generate.js';
|
|
24
25
|
import { verifyIssue } from './verify.js';
|
|
25
26
|
import { listModules } from './module-resolver.js';
|
|
26
|
-
import { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights, getTestHistory, getPageHistory, getSelectorHistory } from './learner-sqlite.js';
|
|
27
|
+
import { getLearningsSummary, getFlakySummary, getSelectorStability, getPageHealth, getApiHealth, getErrorPatterns, getTestTrends, getRunInsights, getTestHistory, getPageHistory, getSelectorHistory, getHealthSnapshot, getTestCreationContext, generateImprovements, getActionHealthScores } from './learner-sqlite.js';
|
|
27
28
|
import { queryGraph } from './learner-neo4j.js';
|
|
28
29
|
import { startNeo4j, stopNeo4j, getNeo4jStatus } from './neo4j-pool.js';
|
|
30
|
+
import { getAppPoolStatus, isAppPoolEnabled } from './app-pool.js';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Resolves auth token from config: uses static authToken if set,
|
|
34
|
+
* otherwise auto-logs in via authLoginEndpoint + authCredentials.
|
|
35
|
+
* If the endpoint is a Docker-internal hostname (e.g. "nginx", "api")
|
|
36
|
+
* and fails with ENOTFOUND, retries with localhost.
|
|
37
|
+
* Returns the token string or null if no auth is configured.
|
|
38
|
+
*/
|
|
39
|
+
async function resolveAuthToken(config) {
|
|
40
|
+
if (config.authToken) return config.authToken;
|
|
41
|
+
if (config.authLoginEndpoint && config.authCredentials) {
|
|
42
|
+
const tokenPath = config.authTokenPath || 'token';
|
|
43
|
+
try {
|
|
44
|
+
return await fetchAuthToken(config.authLoginEndpoint, config.authCredentials, tokenPath);
|
|
45
|
+
} catch (err) {
|
|
46
|
+
// Docker-internal hostname? Retry with localhost from host machine
|
|
47
|
+
if (err.message && err.message.includes('ENOTFOUND')) {
|
|
48
|
+
const url = new URL(config.authLoginEndpoint);
|
|
49
|
+
if (!url.hostname.includes('.')) {
|
|
50
|
+
// Simple hostname (nginx, api, etc.) → likely Docker service name
|
|
51
|
+
const localhostUrl = `http://localhost${url.port && url.port !== '80' ? ':' + url.port : ''}${url.pathname}${url.search}`;
|
|
52
|
+
return await fetchAuthToken(localhostUrl, config.authCredentials, tokenPath);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
throw err;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
29
60
|
|
|
30
61
|
// ── Tool definitions ──────────────────────────────────────────────────────────
|
|
31
62
|
|
|
@@ -65,6 +96,11 @@ export const TOOLS = [
|
|
|
65
96
|
type: 'boolean',
|
|
66
97
|
description: 'Fail tests when network requests fail (e.g. ERR_CONNECTION_REFUSED). Default: false.',
|
|
67
98
|
},
|
|
99
|
+
verificationStrictness: {
|
|
100
|
+
type: 'string',
|
|
101
|
+
enum: ['strict', 'moderate', 'lenient'],
|
|
102
|
+
description: 'Visual verification strictness. strict: no ambiguity allowed, any doubt = FAIL. moderate: reasonable judgment (default). lenient: only fail on clear contradictions.',
|
|
103
|
+
},
|
|
68
104
|
cwd: {
|
|
69
105
|
type: 'string',
|
|
70
106
|
description: 'Absolute path to the project root directory. Claude Code should pass its current working directory.',
|
|
@@ -89,35 +125,94 @@ export const TOOLS = [
|
|
|
89
125
|
{
|
|
90
126
|
name: 'e2e_create_test',
|
|
91
127
|
description:
|
|
92
|
-
|
|
128
|
+
`Create a new E2E test JSON file. IMPORTANT: prefer built-in actions over evaluate blocks.
|
|
129
|
+
|
|
130
|
+
## Action selection guide (use instead of evaluate)
|
|
131
|
+
|
|
132
|
+
**Clicking elements by text** — DON'T write evaluate to find+click elements:
|
|
133
|
+
click: { type: "click", text: "Submit" } — searches button, a, [role=tab], span, etc.
|
|
134
|
+
click_regex: { type: "click_regex", text: "save|guardar" } — regex match, case-insensitive
|
|
135
|
+
click_menu_item: { type: "click_menu_item", text: "Delete" } — [role=menuitem], .MenuItem, etc.
|
|
136
|
+
click_option: { type: "click_option", text: "Option A" } — [role=option] in dropdowns
|
|
137
|
+
click_chip: { type: "click_chip", text: "Active" } — MUI Chip / tag elements
|
|
138
|
+
click_icon: { type: "click_icon", value: "edit" } — SVG/icon by data-testid, aria-label, class
|
|
139
|
+
click_in_context:{ type: "click_in_context", text: "Row text", selector: "button" } — child within container
|
|
140
|
+
|
|
141
|
+
**Asserting text presence/absence** — DON'T write evaluate with body.includes():
|
|
142
|
+
assert_text: { type: "assert_text", text: "Welcome" } — text IS on page (case-sensitive). Uses: text
|
|
143
|
+
assert_no_text: { type: "assert_no_text", text: "Error" } — text is NOT on page. Uses: text
|
|
144
|
+
assert_text_in: { type: "assert_text_in", selector: "[class*='Drawer']", text: "profesional|doctor" }
|
|
145
|
+
— scoped regex in container (case-insensitive default). Uses: selector + text (+ value:"exact")
|
|
146
|
+
|
|
147
|
+
**Asserting elements** — DON'T write evaluate to count or check visibility:
|
|
148
|
+
assert_visible: { type: "assert_visible", selector: ".modal" } — Uses: selector (NOT text)
|
|
149
|
+
assert_not_visible: { type: "assert_not_visible", selector: ".loader" } — Uses: selector (NOT text)
|
|
150
|
+
assert_count: { type: "assert_count", selector: "input", value: ">= 2" } — Uses: selector + value
|
|
151
|
+
assert_element_text: { type: "assert_element_text", selector: "h1", text: "Dashboard" } — Uses: selector + text
|
|
152
|
+
assert_matches: { type: "assert_matches", selector: ".date", value: "\\\\d{2}/\\\\d{2}" } — Uses: selector + value (regex)
|
|
153
|
+
assert_attribute: { type: "assert_attribute", selector: "button", value: "disabled" } — Uses: selector + value
|
|
154
|
+
assert_url: { type: "assert_url", value: "/dashboard" } — Uses: value
|
|
155
|
+
assert_input_value: { type: "assert_input_value", selector: "#email", value: "@" } — Uses: selector + value
|
|
156
|
+
|
|
157
|
+
IMPORTANT field rules:
|
|
158
|
+
- assert_text / assert_no_text: use "text" field only (checks full page body)
|
|
159
|
+
- assert_visible / assert_not_visible: use "selector" field only (CSS selector, NOT text)
|
|
160
|
+
- To verify text absence: use assert_no_text (NOT assert_not_visible with text)
|
|
161
|
+
|
|
162
|
+
**Navigation & waiting** — DON'T write evaluate with setTimeout polling:
|
|
163
|
+
goto: { type: "goto", value: "/login" } — full page navigation
|
|
164
|
+
navigate: { type: "navigate", value: "/settings" } — SPA-friendly (won't fail if no page load)
|
|
165
|
+
wait: { type: "wait", text: "Loading complete" } — wait for text to appear in body
|
|
166
|
+
wait: { type: "wait", selector: ".results" } — wait for element to appear
|
|
167
|
+
wait: { type: "wait", value: "2000" } — fixed delay (avoid when possible)
|
|
168
|
+
wait_network_idle: { type: "wait_network_idle", value: "500" } — wait until no network for N ms
|
|
169
|
+
|
|
170
|
+
**Form interaction** — DON'T write evaluate with native value setters (unless React):
|
|
171
|
+
type: { type: "type", selector: "#email", value: "a@b.com" } — clears + types
|
|
172
|
+
type_react: { type: "type_react", selector: "#email", value: "a@b.com" } — for React controlled inputs
|
|
173
|
+
select: { type: "select", selector: "select#country", value: "US" }
|
|
174
|
+
clear: { type: "clear", selector: "#search" }
|
|
175
|
+
press: { type: "press", value: "Enter" }
|
|
176
|
+
focus_autocomplete: { type: "focus_autocomplete", text: "City" } — focus MUI Autocomplete by label
|
|
177
|
+
|
|
178
|
+
**When evaluate IS appropriate**: computed styles, complex conditional logic, GraphQL via window.__e2eGql, math calculations, reading window/app state.
|
|
179
|
+
|
|
180
|
+
## Modules
|
|
181
|
+
Use { "$use": "module-name", "params": {...} } to reference reusable modules from e2e/modules/. Modules compose — a module can $use other modules. Check e2e_list to see available modules for the project.`,
|
|
93
182
|
inputSchema: {
|
|
94
183
|
type: 'object',
|
|
95
184
|
properties: {
|
|
96
185
|
name: {
|
|
97
186
|
type: 'string',
|
|
98
|
-
description: 'Suite file name without .json extension (e.g. "login", "
|
|
187
|
+
description: 'Suite file name without .json extension (e.g. "login-flow", "issue-1743-sidebar")',
|
|
99
188
|
},
|
|
100
189
|
tests: {
|
|
101
190
|
type: 'array',
|
|
102
|
-
description: 'Array of test objects with { name, actions }',
|
|
191
|
+
description: 'Array of test objects with { name, actions, expect }',
|
|
103
192
|
items: {
|
|
104
193
|
type: 'object',
|
|
105
194
|
properties: {
|
|
106
|
-
name: { type: 'string', description: 'Test name' },
|
|
107
|
-
expect: {
|
|
195
|
+
name: { type: 'string', description: 'Test name — descriptive of what is being verified' },
|
|
196
|
+
expect: {
|
|
197
|
+
oneOf: [
|
|
198
|
+
{ type: 'string', description: 'Single description of expected visual outcome.' },
|
|
199
|
+
{ type: 'array', items: { type: 'string' }, description: 'Checklist of criteria — each evaluated independently as PASS/FAIL.' },
|
|
200
|
+
],
|
|
201
|
+
description: 'Expected visual outcome. String for free-form, array for per-criterion checklist.',
|
|
202
|
+
},
|
|
108
203
|
actions: {
|
|
109
204
|
type: 'array',
|
|
110
|
-
description: 'Sequential browser actions',
|
|
205
|
+
description: 'Sequential browser actions. Prefer built-in action types over evaluate — see tool description for the full guide.',
|
|
111
206
|
items: {
|
|
112
207
|
type: 'object',
|
|
113
208
|
properties: {
|
|
114
209
|
type: {
|
|
115
210
|
type: 'string',
|
|
116
|
-
description: 'Action type
|
|
211
|
+
description: 'Action type. Prefer declarative actions (assert_text, assert_no_text, click, assert_visible, assert_count, assert_text_in, click_menu_item, etc.) over evaluate.',
|
|
117
212
|
},
|
|
118
|
-
selector: { type: 'string', description: 'CSS selector' },
|
|
119
|
-
value: { type: 'string', description: 'Value for
|
|
120
|
-
text: { type: 'string', description: 'Text
|
|
213
|
+
selector: { type: 'string', description: 'CSS selector (supports compound selectors like "[class*=\'Drawer\'], [role=\'presentation\']")' },
|
|
214
|
+
value: { type: 'string', description: 'Value — varies by action type (URL for goto, ms for wait, regex for assert_matches, ">= N" for assert_count)' },
|
|
215
|
+
text: { type: 'string', description: 'Text to match — used by click (substring), assert_text/assert_no_text (substring on body), assert_text_in (regex), click_regex (regex). NOT used by assert_visible/assert_not_visible (use selector instead).' },
|
|
121
216
|
},
|
|
122
217
|
required: ['type'],
|
|
123
218
|
},
|
|
@@ -128,7 +223,7 @@ export const TOOLS = [
|
|
|
128
223
|
},
|
|
129
224
|
hooks: {
|
|
130
225
|
type: 'object',
|
|
131
|
-
description: 'Optional hooks: beforeAll, afterAll, beforeEach, afterEach (each an array of actions)',
|
|
226
|
+
description: 'Optional hooks: beforeAll, afterAll, beforeEach, afterEach (each an array of actions). Note: beforeAll runs on a SEPARATE page that is closed before tests — use beforeEach for auth/setup.',
|
|
132
227
|
properties: {
|
|
133
228
|
beforeAll: { type: 'array', items: { type: 'object' } },
|
|
134
229
|
afterAll: { type: 'array', items: { type: 'object' } },
|
|
@@ -158,6 +253,20 @@ export const TOOLS = [
|
|
|
158
253
|
},
|
|
159
254
|
},
|
|
160
255
|
},
|
|
256
|
+
{
|
|
257
|
+
name: 'e2e_app_pool_status',
|
|
258
|
+
description:
|
|
259
|
+
'Get the status of the app environment pool. Shows active forks, allocated ports, per-fork details (driver, baseUrl, test name, fork time). Only relevant when appPool is enabled in config.',
|
|
260
|
+
inputSchema: {
|
|
261
|
+
type: 'object',
|
|
262
|
+
properties: {
|
|
263
|
+
cwd: {
|
|
264
|
+
type: 'string',
|
|
265
|
+
description: 'Absolute path to the project root directory.',
|
|
266
|
+
},
|
|
267
|
+
},
|
|
268
|
+
},
|
|
269
|
+
},
|
|
161
270
|
{
|
|
162
271
|
name: 'e2e_screenshot',
|
|
163
272
|
description:
|
|
@@ -199,6 +308,24 @@ export const TOOLS = [
|
|
|
199
308
|
properties: {},
|
|
200
309
|
},
|
|
201
310
|
},
|
|
311
|
+
{
|
|
312
|
+
name: 'e2e_dashboard_restart',
|
|
313
|
+
description:
|
|
314
|
+
'Restart the E2E Runner web dashboard. Stops the current instance and starts a new one, optionally with a new cwd or port. Useful when switching projects or when the dashboard was started from another session.',
|
|
315
|
+
inputSchema: {
|
|
316
|
+
type: 'object',
|
|
317
|
+
properties: {
|
|
318
|
+
port: {
|
|
319
|
+
type: 'number',
|
|
320
|
+
description: 'Dashboard port (default: same port or 8484)',
|
|
321
|
+
},
|
|
322
|
+
cwd: {
|
|
323
|
+
type: 'string',
|
|
324
|
+
description: 'Absolute path to the project root directory. Claude Code should pass its current working directory.',
|
|
325
|
+
},
|
|
326
|
+
},
|
|
327
|
+
},
|
|
328
|
+
},
|
|
202
329
|
{
|
|
203
330
|
name: 'e2e_issue',
|
|
204
331
|
description:
|
|
@@ -272,6 +399,63 @@ export const TOOLS = [
|
|
|
272
399
|
type: 'string',
|
|
273
400
|
description: 'localStorage key name for the auth token (default: "accessToken")',
|
|
274
401
|
},
|
|
402
|
+
waitUntil: {
|
|
403
|
+
type: 'string',
|
|
404
|
+
enum: ['networkidle2', 'domcontentloaded', 'load', 'auto'],
|
|
405
|
+
description: 'Navigation wait strategy. "auto" (default) tries networkidle2 first with a short timeout, then falls back to domcontentloaded + delay for SPA/WebSocket apps. Use "domcontentloaded" for apps with persistent connections (WebSocket, SSE).',
|
|
406
|
+
},
|
|
407
|
+
cwd: {
|
|
408
|
+
type: 'string',
|
|
409
|
+
description: 'Absolute path to the project root directory. Claude Code should pass its current working directory.',
|
|
410
|
+
},
|
|
411
|
+
},
|
|
412
|
+
required: ['url'],
|
|
413
|
+
},
|
|
414
|
+
},
|
|
415
|
+
{
|
|
416
|
+
name: 'e2e_analyze',
|
|
417
|
+
description:
|
|
418
|
+
'Analyze a page\'s structure and return all interactive elements (forms, buttons, links, navigation, tables, modals, etc.) with their CSS selectors, plus suggested test scaffolds. One call replaces the entire screenshot→guess-selectors→retry cycle.',
|
|
419
|
+
inputSchema: {
|
|
420
|
+
type: 'object',
|
|
421
|
+
properties: {
|
|
422
|
+
url: {
|
|
423
|
+
type: 'string',
|
|
424
|
+
description: 'Full URL to analyze (e.g. "https://example.com" or "http://host.docker.internal:3000/dashboard")',
|
|
425
|
+
},
|
|
426
|
+
scope: {
|
|
427
|
+
type: 'string',
|
|
428
|
+
description: 'CSS selector to limit analysis to a section (e.g. "#sidebar", ".modal-content")',
|
|
429
|
+
},
|
|
430
|
+
maxElements: {
|
|
431
|
+
type: 'number',
|
|
432
|
+
description: 'Max elements per category (default: 50). Lower values produce smaller responses.',
|
|
433
|
+
},
|
|
434
|
+
includeScreenshot: {
|
|
435
|
+
type: 'boolean',
|
|
436
|
+
description: 'Include a screenshot alongside the JSON analysis (default: true)',
|
|
437
|
+
},
|
|
438
|
+
selector: {
|
|
439
|
+
type: 'string',
|
|
440
|
+
description: 'Wait for this CSS selector before analyzing',
|
|
441
|
+
},
|
|
442
|
+
delay: {
|
|
443
|
+
type: 'number',
|
|
444
|
+
description: 'Wait N milliseconds after page load before analyzing (default: 0)',
|
|
445
|
+
},
|
|
446
|
+
authToken: {
|
|
447
|
+
type: 'string',
|
|
448
|
+
description: 'JWT or auth token to inject into localStorage before navigating (for authenticated pages)',
|
|
449
|
+
},
|
|
450
|
+
authStorageKey: {
|
|
451
|
+
type: 'string',
|
|
452
|
+
description: 'localStorage key name for the auth token (default: "accessToken")',
|
|
453
|
+
},
|
|
454
|
+
waitUntil: {
|
|
455
|
+
type: 'string',
|
|
456
|
+
enum: ['networkidle2', 'domcontentloaded', 'load', 'auto'],
|
|
457
|
+
description: 'Navigation wait strategy. "auto" (default) tries networkidle2 first with a short timeout, then falls back to domcontentloaded + delay for SPA/WebSocket apps. Use "domcontentloaded" for apps with persistent connections (WebSocket, SSE).',
|
|
458
|
+
},
|
|
275
459
|
cwd: {
|
|
276
460
|
type: 'string',
|
|
277
461
|
description: 'Absolute path to the project root directory. Claude Code should pass its current working directory.',
|
|
@@ -283,7 +467,9 @@ export const TOOLS = [
|
|
|
283
467
|
{
|
|
284
468
|
name: 'e2e_create_module',
|
|
285
469
|
description:
|
|
286
|
-
|
|
470
|
+
`Create a reusable module for E2E tests. Modules encapsulate repeated action sequences referenced via { "$use": "module-name", "params": {...} }.
|
|
471
|
+
|
|
472
|
+
Good module candidates: auth setup, page navigation, tab clicking, opening sidebars/drawers, form fill sequences, cleanup routines. Modules can compose — a module can $use other modules. Params use {{paramName}} mustache syntax in action fields. Extract a module when you see the same 2+ action sequence in multiple tests.`,
|
|
287
473
|
inputSchema: {
|
|
288
474
|
type: 'object',
|
|
289
475
|
properties: {
|
|
@@ -420,11 +606,43 @@ export const TOOLS = [
|
|
|
420
606
|
required: ['runDbId'],
|
|
421
607
|
},
|
|
422
608
|
},
|
|
609
|
+
{
|
|
610
|
+
name: 'e2e_vars',
|
|
611
|
+
description:
|
|
612
|
+
'Manage project variables stored in SQLite. Variables can be referenced in test JSON as {{var.KEY}}. Supports project-wide and per-suite scoping.',
|
|
613
|
+
inputSchema: {
|
|
614
|
+
type: 'object',
|
|
615
|
+
properties: {
|
|
616
|
+
action: {
|
|
617
|
+
type: 'string',
|
|
618
|
+
enum: ['set', 'get', 'list', 'delete'],
|
|
619
|
+
description: 'Action to perform: set (upsert), get (one key), list (all), delete',
|
|
620
|
+
},
|
|
621
|
+
key: {
|
|
622
|
+
type: 'string',
|
|
623
|
+
description: 'Variable name (required for set, get, delete)',
|
|
624
|
+
},
|
|
625
|
+
value: {
|
|
626
|
+
type: 'string',
|
|
627
|
+
description: 'Variable value (required for set)',
|
|
628
|
+
},
|
|
629
|
+
scope: {
|
|
630
|
+
type: 'string',
|
|
631
|
+
description: 'Scope: "project" (default) or a suite name for suite-specific override',
|
|
632
|
+
},
|
|
633
|
+
cwd: {
|
|
634
|
+
type: 'string',
|
|
635
|
+
description: 'Absolute path to the project root directory.',
|
|
636
|
+
},
|
|
637
|
+
},
|
|
638
|
+
required: ['action'],
|
|
639
|
+
},
|
|
640
|
+
},
|
|
423
641
|
];
|
|
424
642
|
|
|
425
|
-
/** Tools exposed on the dashboard — excludes dashboard start/stop (already running). */
|
|
643
|
+
/** Tools exposed on the dashboard — excludes dashboard start/stop/restart (already running). */
|
|
426
644
|
export const DASHBOARD_TOOLS = TOOLS.filter(
|
|
427
|
-
t => t.name !== 'e2e_dashboard_start' && t.name !== 'e2e_dashboard_stop'
|
|
645
|
+
t => t.name !== 'e2e_dashboard_start' && t.name !== 'e2e_dashboard_stop' && t.name !== 'e2e_dashboard_restart'
|
|
428
646
|
);
|
|
429
647
|
|
|
430
648
|
// ── Dashboard broadcast helper ────────────────────────────────────────────────
|
|
@@ -469,11 +687,13 @@ async function handleRun(args) {
|
|
|
469
687
|
if (args.baseUrl) configOverrides.baseUrl = args.baseUrl;
|
|
470
688
|
if (args.retries !== undefined) configOverrides.retries = args.retries;
|
|
471
689
|
if (args.failOnNetworkError !== undefined) configOverrides.failOnNetworkError = args.failOnNetworkError;
|
|
690
|
+
if (args.verificationStrictness) configOverrides.verificationStrictness = args.verificationStrictness;
|
|
472
691
|
|
|
473
692
|
const config = await loadConfig(configOverrides, args.cwd);
|
|
474
693
|
config.triggeredBy = 'mcp';
|
|
475
694
|
|
|
476
|
-
|
|
695
|
+
const driverOpts = { poolDriver: config.poolDriver, maxSessions: config.maxSessions };
|
|
696
|
+
await waitForAnyPool(getPoolUrls(config), 30000, driverOpts);
|
|
477
697
|
|
|
478
698
|
let tests, hooks;
|
|
479
699
|
|
|
@@ -506,7 +726,12 @@ async function handleRun(args) {
|
|
|
506
726
|
|
|
507
727
|
const report = generateReport(results);
|
|
508
728
|
saveReport(report, config.screenshotsDir, config);
|
|
509
|
-
|
|
729
|
+
// Derive suite name: explicit suite > file basename > null (for "all")
|
|
730
|
+
let suiteName = args.suite || null;
|
|
731
|
+
if (!suiteName && args.file) {
|
|
732
|
+
suiteName = path.basename(args.file, '.json');
|
|
733
|
+
}
|
|
734
|
+
const { runDbId } = await persistRun(report, config, suiteName);
|
|
510
735
|
|
|
511
736
|
const failures = report.results
|
|
512
737
|
.filter(r => !r.success)
|
|
@@ -517,8 +742,13 @@ async function handleRun(args) {
|
|
|
517
742
|
}));
|
|
518
743
|
|
|
519
744
|
const flaky = report.results
|
|
520
|
-
.filter(r => r.success && r.attempt > 1)
|
|
521
|
-
.map(r =>
|
|
745
|
+
.filter(r => r.success && (r.attempt > 1 || r.flaky))
|
|
746
|
+
.map(r => {
|
|
747
|
+
const entry = { name: r.name };
|
|
748
|
+
if (r.voting) entry.voting = r.voting;
|
|
749
|
+
else entry.attempts = r.attempt;
|
|
750
|
+
return entry;
|
|
751
|
+
});
|
|
522
752
|
|
|
523
753
|
const summary = {
|
|
524
754
|
...report.summary,
|
|
@@ -563,12 +793,21 @@ async function handleRun(args) {
|
|
|
563
793
|
|
|
564
794
|
const verifications = report.results
|
|
565
795
|
.filter(r => r.expect && r.verificationScreenshot)
|
|
566
|
-
.map(r =>
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
796
|
+
.map(r => {
|
|
797
|
+
const entry = {
|
|
798
|
+
name: r.name,
|
|
799
|
+
expect: r.expect,
|
|
800
|
+
success: r.success,
|
|
801
|
+
screenshotHash: 'ss:' + computeScreenshotHash(r.verificationScreenshot),
|
|
802
|
+
};
|
|
803
|
+
if (r.baselineScreenshot) {
|
|
804
|
+
entry.baselineScreenshotHash = 'ss:' + computeScreenshotHash(r.baselineScreenshot);
|
|
805
|
+
}
|
|
806
|
+
if (Array.isArray(r.expect)) {
|
|
807
|
+
entry.isChecklist = true;
|
|
808
|
+
}
|
|
809
|
+
return entry;
|
|
810
|
+
});
|
|
572
811
|
|
|
573
812
|
if (flaky.length > 0) summary.flaky = flaky;
|
|
574
813
|
if (failures.length > 0) summary.failures = failures;
|
|
@@ -590,7 +829,9 @@ async function handleRun(args) {
|
|
|
590
829
|
}
|
|
591
830
|
if (verifications.length > 0) {
|
|
592
831
|
summary.verifications = verifications;
|
|
593
|
-
|
|
832
|
+
const hasBaselines = verifications.some(v => v.baselineScreenshotHash);
|
|
833
|
+
const hasChecklists = verifications.some(v => v.isChecklist);
|
|
834
|
+
summary.verificationInstructions = buildVerificationInstructions(config.verificationStrictness || 'moderate', hasBaselines, hasChecklists);
|
|
594
835
|
}
|
|
595
836
|
|
|
596
837
|
// Build per-test narrative: a step-by-step human-readable story of what happened
|
|
@@ -601,10 +842,19 @@ async function handleRun(args) {
|
|
|
601
842
|
}));
|
|
602
843
|
if (narratives.length > 0) summary.narratives = narratives;
|
|
603
844
|
|
|
604
|
-
// Enrich with learning insights (fire-and-forget — never fails the response)
|
|
845
|
+
// Enrich with learning insights + health snapshot (fire-and-forget — never fails the response)
|
|
605
846
|
if (config.learningsEnabled !== false) {
|
|
606
847
|
try {
|
|
607
848
|
const projectId = ensureProject(config._cwd, config.projectName, config.screenshotsDir, config.testsDir);
|
|
849
|
+
|
|
850
|
+
// Always include health snapshot (~200 bytes) for project context
|
|
851
|
+
const health = getHealthSnapshot(projectId);
|
|
852
|
+
if (health) {
|
|
853
|
+
summary.healthSnapshot = health;
|
|
854
|
+
summary.learningsHint = "Use e2e_learnings tool with query 'summary' for full analysis.";
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
// Contextual insights for this specific run
|
|
608
858
|
const insights = getRunInsights(projectId, report);
|
|
609
859
|
if (insights.length > 0) {
|
|
610
860
|
summary.learnings = {
|
|
@@ -618,6 +868,25 @@ async function handleRun(args) {
|
|
|
618
868
|
: null,
|
|
619
869
|
};
|
|
620
870
|
}
|
|
871
|
+
|
|
872
|
+
// Actionable improvements from cross-referencing this run with historical data
|
|
873
|
+
const improvements = generateImprovements(projectId, report);
|
|
874
|
+
if (improvements.length > 0) {
|
|
875
|
+
summary.improvements = improvements;
|
|
876
|
+
}
|
|
877
|
+
} catch { /* never fail the run response */ }
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
// Hindsight hints — LLM-powered fix suggestions for failures (async, never blocks)
|
|
881
|
+
if (hasApiKey(config) && failures.length > 0) {
|
|
882
|
+
try {
|
|
883
|
+
const maxHints = config.hintsMaxFailures ?? 3;
|
|
884
|
+
const hintTargets = failures.slice(0, maxHints);
|
|
885
|
+
const failedResults = hintTargets.map(f => report.results.find(r => r.name === f.name)).filter(Boolean);
|
|
886
|
+
const hints = (await Promise.all(failedResults.map(r => generateHindsightHint(r, config)))).filter(Boolean);
|
|
887
|
+
if (hints.length > 0) {
|
|
888
|
+
summary.hindsightHints = hints;
|
|
889
|
+
}
|
|
621
890
|
} catch { /* never fail the run response */ }
|
|
622
891
|
}
|
|
623
892
|
|
|
@@ -660,6 +929,14 @@ async function handleCreateTest(args) {
|
|
|
660
929
|
}
|
|
661
930
|
|
|
662
931
|
const safeName = path.basename(args.name);
|
|
932
|
+
|
|
933
|
+
// Reject generic/ambiguous suite names
|
|
934
|
+
const baseName = safeName.replace(/\.json$/, '').replace(/^\d+-/, '');
|
|
935
|
+
const FORBIDDEN_NAMES = ['all', 'test', 'tests', 'debug', 'new', 'temp', 'tmp', 'main', 'suite', 'run', 'e2e', 'default', 'untitled'];
|
|
936
|
+
if (FORBIDDEN_NAMES.includes(baseName.toLowerCase())) {
|
|
937
|
+
return errorResult(`Suite name "${baseName}" is too generic. Use a descriptive name specific to the feature or issue being tested (e.g. "login-valid-credentials", "issue-1743-auth-redirect").`);
|
|
938
|
+
}
|
|
939
|
+
|
|
663
940
|
const filename = safeName.endsWith('.json') ? safeName : `${safeName}.json`;
|
|
664
941
|
const filePath = path.join(config.testsDir, filename);
|
|
665
942
|
|
|
@@ -676,36 +953,361 @@ async function handleCreateTest(args) {
|
|
|
676
953
|
|
|
677
954
|
fs.writeFileSync(filePath, JSON.stringify(content, null, 2) + '\n');
|
|
678
955
|
|
|
679
|
-
//
|
|
680
|
-
|
|
956
|
+
// ── Collect all actions (tests + hooks) for analysis ──
|
|
957
|
+
const allActions = [];
|
|
958
|
+
for (const test of args.tests) {
|
|
959
|
+
if (test.actions) allActions.push(...test.actions);
|
|
960
|
+
}
|
|
961
|
+
if (args.hooks) {
|
|
962
|
+
for (const hookActions of Object.values(args.hooks)) {
|
|
963
|
+
if (Array.isArray(hookActions)) allActions.push(...hookActions);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
const warnings = [];
|
|
968
|
+
|
|
969
|
+
// ── Warn about beforeAll pitfall ──
|
|
681
970
|
const beforeAll = args.hooks?.beforeAll;
|
|
682
971
|
if (beforeAll?.length) {
|
|
683
972
|
const stateActions = beforeAll.filter(a =>
|
|
684
973
|
['evaluate', 'goto', 'navigate', 'clear_cookies', 'type', 'click', 'select'].includes(a.type)
|
|
685
974
|
);
|
|
686
975
|
if (stateActions.length > 0) {
|
|
687
|
-
|
|
688
|
-
'Actions that set browser state (evaluate, goto, cookies, etc.) will NOT carry over
|
|
689
|
-
'Use beforeEach instead if tests need this setup.';
|
|
976
|
+
warnings.push('⚠️ beforeAll runs on a separate browser page that is closed before tests start. ' +
|
|
977
|
+
'Actions that set browser state (evaluate, goto, cookies, etc.) will NOT carry over. ' +
|
|
978
|
+
'Use beforeEach instead if tests need this setup.');
|
|
690
979
|
}
|
|
691
980
|
}
|
|
692
981
|
|
|
693
|
-
|
|
982
|
+
// ── Detect evaluate blocks that could use built-in actions ──
|
|
983
|
+
const suggestions = analyzeEvaluateUsage(allActions);
|
|
984
|
+
if (suggestions.length > 0) {
|
|
985
|
+
warnings.push(`💡 ${suggestions.length} evaluate action(s) could potentially use built-in actions instead:\n` +
|
|
986
|
+
suggestions.map(s => ` • ${s}`).join('\n'));
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
// ── Detect suite-level issues: fixed waits, cross-test dependencies ──
|
|
990
|
+
const actionWarnings = analyzeActionPatterns(args.tests);
|
|
991
|
+
if (actionWarnings.length > 0) {
|
|
992
|
+
warnings.push(...actionWarnings);
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
// ── List available modules ──
|
|
996
|
+
let modulesInfo = '';
|
|
997
|
+
try {
|
|
998
|
+
const modules = listModules(config.modulesDir);
|
|
999
|
+
if (modules.length > 0) {
|
|
1000
|
+
modulesInfo = '\n\n📦 Available modules: ' + modules.map(m => {
|
|
1001
|
+
const params = m.params.filter(p => p.required).map(p => p.name);
|
|
1002
|
+
return m.name + (params.length ? `(${params.join(', ')})` : '');
|
|
1003
|
+
}).join(', ');
|
|
1004
|
+
}
|
|
1005
|
+
} catch { /* modules dir may not exist */ }
|
|
1006
|
+
|
|
1007
|
+
const warningBlock = warnings.length > 0 ? '\n\n' + warnings.join('\n\n') : '';
|
|
1008
|
+
|
|
1009
|
+
// Enrich with learnings context for smarter test authoring
|
|
1010
|
+
let learningsBlock = '';
|
|
1011
|
+
try {
|
|
1012
|
+
const projectId = ensureProject(config._cwd, config.projectName, config.screenshotsDir, config.testsDir);
|
|
1013
|
+
const ctx = getTestCreationContext(projectId);
|
|
1014
|
+
if (ctx) {
|
|
1015
|
+
const lines = ['\n\n⚠ LEARNINGS FROM PREVIOUS RUNS:'];
|
|
1016
|
+
|
|
1017
|
+
if (ctx.unstableSelectors?.length) {
|
|
1018
|
+
lines.push(' Unstable selectors (avoid these):');
|
|
1019
|
+
for (const s of ctx.unstableSelectors) {
|
|
1020
|
+
lines.push(` - ${s.selector} (${s.failRate}% fail rate) → ${s.suggestion}`);
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
if (ctx.errorPatterns?.length) {
|
|
1025
|
+
lines.push(' Common errors:');
|
|
1026
|
+
for (const e of ctx.errorPatterns) {
|
|
1027
|
+
lines.push(` - ${e.category || 'unknown'} (${e.count}x) — ${e.pattern}`);
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
if (ctx.slowPages?.length) {
|
|
1032
|
+
lines.push(' Slow pages (add extra waits):');
|
|
1033
|
+
for (const p of ctx.slowPages) {
|
|
1034
|
+
lines.push(` - ${p.page} (avg ${(p.avgLoadMs / 1000).toFixed(1)}s load)`);
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
if (ctx.stableSelectors?.length) {
|
|
1039
|
+
lines.push(' Reliable selectors (safe to use):');
|
|
1040
|
+
for (const s of ctx.stableSelectors) {
|
|
1041
|
+
lines.push(` - ${s.selector} (100% success, ${s.uses} uses)`);
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
if (ctx.flakyTests?.length) {
|
|
1046
|
+
lines.push(' Flaky tests (consider retries):');
|
|
1047
|
+
for (const f of ctx.flakyTests) {
|
|
1048
|
+
lines.push(` - ${f.name} (${f.flakyCount} flaky runs out of ${f.totalRuns})`);
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
if (ctx.apiIssues?.length) {
|
|
1053
|
+
lines.push(' Unreliable API endpoints:');
|
|
1054
|
+
for (const a of ctx.apiIssues) {
|
|
1055
|
+
lines.push(` - ${a.endpoint} (${a.errorRate}% error rate)`);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
if (ctx.passRate !== undefined) {
|
|
1060
|
+
lines.push(` Overall project pass rate: ${ctx.passRate}%`);
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
learningsBlock = lines.join('\n');
|
|
1064
|
+
}
|
|
1065
|
+
} catch { /* never fail test creation */ }
|
|
1066
|
+
|
|
1067
|
+
return textResult(`Created test file: ${filePath}\n\n${args.tests.length} test(s) defined.${warningBlock}${modulesInfo}${learningsBlock}`);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
/**
|
|
1071
|
+
* Analyze evaluate actions and suggest built-in replacements.
|
|
1072
|
+
* Returns an array of human-readable suggestion strings.
|
|
1073
|
+
*/
|
|
1074
|
+
function analyzeEvaluateUsage(actions) {
|
|
1075
|
+
const suggestions = [];
|
|
1076
|
+
|
|
1077
|
+
for (const action of actions) {
|
|
1078
|
+
if (action.type !== 'evaluate' || !action.value) continue;
|
|
1079
|
+
const code = action.value;
|
|
1080
|
+
|
|
1081
|
+
// Pattern: clicking elements by text — .click() after finding by textContent
|
|
1082
|
+
if (/\.textContent[^]*\.click\(\)/s.test(code) || /\.find\([^)]*textContent[^)]*\)[^]*\.click/s.test(code)) {
|
|
1083
|
+
if (/tab/i.test(code)) {
|
|
1084
|
+
suggestions.push('Tab click via evaluate → use { type: "click", text: "Tab Name" } (click searches [role="tab"] natively)');
|
|
1085
|
+
} else if (/menu/i.test(code)) {
|
|
1086
|
+
suggestions.push('Menu item click via evaluate → use { type: "click_menu_item", text: "Item Name" }');
|
|
1087
|
+
} else {
|
|
1088
|
+
suggestions.push('Element click via evaluate → use { type: "click", text: "..." } or click_regex/click_in_context');
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
// Pattern: body.innerText.includes() for text presence
|
|
1093
|
+
if (/document\.body\.innerText[^]*\.includes\(/s.test(code) || /body\.includes\(/s.test(code)) {
|
|
1094
|
+
// Detect negation patterns (!includes) that should use assert_no_text
|
|
1095
|
+
const hasNegation = /!\s*body\.includes\(|!\s*\w+\.includes\(|!body\.includes\(/s.test(code)
|
|
1096
|
+
|| /=\s*!.*\.includes\(/s.test(code);
|
|
1097
|
+
const includeCount = (code.match(/\.includes\(/g) || []).length;
|
|
1098
|
+
|
|
1099
|
+
if (hasNegation) {
|
|
1100
|
+
suggestions.push(`🚨 Text negation check (!includes) → use { type: "assert_no_text", text: "..." } for absent text, and { type: "assert_text", text: "..." } for present text`);
|
|
1101
|
+
} else if (includeCount <= 3) {
|
|
1102
|
+
suggestions.push(`Text presence check (${includeCount} includes) → use ${includeCount}x { type: "assert_text", text: "..." }`);
|
|
1103
|
+
} else {
|
|
1104
|
+
suggestions.push(`Text presence check (${includeCount} includes) → use assert_text for each, or assert_text_in with regex: { type: "assert_text_in", selector: "body", text: "word1|word2" }`);
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
// Pattern: querySelectorAll(...).length checks
|
|
1109
|
+
if (/querySelectorAll\([^)]+\)\.length/s.test(code) && !/getComputedStyle/.test(code)) {
|
|
1110
|
+
suggestions.push('Element counting via evaluate → use { type: "assert_count", selector: "...", value: ">= N" }');
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
// Pattern: checking element visibility/existence without computed styles
|
|
1114
|
+
if (/querySelector\([^)]+\)\s*;?\s*(if\s*\(!\s*\w+\)|===?\s*null)/s.test(code) && !/getComputedStyle/.test(code)) {
|
|
1115
|
+
suggestions.push('Element existence check via evaluate → use { type: "assert_visible", selector: "..." }');
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
// Pattern: return JSON.stringify for debug info (no throw/Error)
|
|
1119
|
+
if (/return\s+JSON\.stringify/s.test(code) && !/throw\s+new\s+Error/s.test(code) && !/FAIL/s.test(code)) {
|
|
1120
|
+
suggestions.push('Informational evaluate (returns JSON, never throws) → remove or replace with specific assertions');
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
// Pattern: setTimeout polling loop
|
|
1124
|
+
if (/setTimeout|setInterval/s.test(code) && /while|Date\.now/s.test(code)) {
|
|
1125
|
+
suggestions.push('Polling loop in evaluate → use { type: "wait", text: "..." } or { type: "wait", selector: "..." } with timeout');
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
// Pattern: return static string with no checks
|
|
1129
|
+
if (/^\(\(\)\s*=>\s*\{\s*return\s+['"`][^]*['"`];\s*\}\)\(\)$/.test(code.trim())) {
|
|
1130
|
+
suggestions.push('No-op evaluate (returns static string) → remove entirely');
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
// 🚨 Pattern: evaluate that NEVER fails — no throw, no FAIL:/ERROR:, no return false
|
|
1134
|
+
const canFail = /throw\s+new\s+Error/s.test(code) || /\bFAIL[:\s]/s.test(code) || /\bERROR[:\s]/s.test(code)
|
|
1135
|
+
|| /return\s+false\b/s.test(code) || /return\s+'FAIL/s.test(code) || /return\s+`FAIL/s.test(code);
|
|
1136
|
+
|
|
1137
|
+
if (!canFail) {
|
|
1138
|
+
// Any template string return → always truthy, test always passes
|
|
1139
|
+
if (/return\s+`[^`]*\$\{[^}]+\}[^`]*`/s.test(code)) {
|
|
1140
|
+
suggestions.push(
|
|
1141
|
+
'🚨 Evaluate returns template string but NEVER throws or returns false — ' +
|
|
1142
|
+
'this action will ALWAYS PASS regardless of results. Either throw new Error("FAIL: ...") when conditions fail, or use built-in assert actions'
|
|
1143
|
+
);
|
|
1144
|
+
}
|
|
1145
|
+
// Returns a plain string (not template) that isn't FAIL/ERROR
|
|
1146
|
+
else if (/return\s+['"][^'"]*['"]/s.test(code) && code.length > 60) {
|
|
1147
|
+
suggestions.push(
|
|
1148
|
+
'⚠️ Evaluate returns a plain string but never fails — informational-only. Add failure conditions or replace with assert actions'
|
|
1149
|
+
);
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
// 🚨 Pattern: .click() inside evaluate — should use built-in click action
|
|
1154
|
+
if (/\.click\(\)/s.test(code) && !(/\.textContent[^]*\.click\(\)/s.test(code))) {
|
|
1155
|
+
// Only flag if not already caught by the textContent click pattern above
|
|
1156
|
+
if (/\.filter\([^)]*text/s.test(code) || /querySelectorAll[^)]*\)[^]*\.click/s.test(code) || /querySelector[^)]*\)[^]*\.click/s.test(code)) {
|
|
1157
|
+
suggestions.push(
|
|
1158
|
+
'🚨 Element click via evaluate → use { type: "click", text: "..." } or { type: "click", selector: "..." }. ' +
|
|
1159
|
+
'Built-in click has retries, waits, and better error reporting'
|
|
1160
|
+
);
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
// 🚨 Pattern: MUI/framework selectors inside evaluate — fragile
|
|
1165
|
+
const muiMatches = code.match(/\.Mui[\w-]+/g) || [];
|
|
1166
|
+
if (muiMatches.length > 0) {
|
|
1167
|
+
suggestions.push(
|
|
1168
|
+
`⚠️ MUI class selectors (${muiMatches.slice(0, 3).join(', ')}) are auto-generated and change between versions. ` +
|
|
1169
|
+
`Prefer [data-testid="..."], [role="..."], or text-based selectors`
|
|
1170
|
+
);
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
// 🚨 Pattern: sets window.__e2e_* globals for cross-test state sharing
|
|
1174
|
+
if (/window\.__e2e_\w+\s*=/.test(code) && !/window\.__e2e\./.test(code.replace(/window\.__e2e_\w+\s*=/g, ''))) {
|
|
1175
|
+
suggestions.push(
|
|
1176
|
+
'⚠️ Cross-test state via window.__e2e_* — if test retries are enabled, retried tests get a fresh page and lose this state. ' +
|
|
1177
|
+
'Make each test self-contained by re-querying data, or disable retries for this suite'
|
|
1178
|
+
);
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
return suggestions;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
/**
|
|
1186
|
+
* Analyze all actions in a suite for non-evaluate issues:
|
|
1187
|
+
* fixed numeric waits, cross-test dependencies, etc.
|
|
1188
|
+
*/
|
|
1189
|
+
function analyzeActionPatterns(tests) {
|
|
1190
|
+
const warnings = [];
|
|
1191
|
+
|
|
1192
|
+
// Detect fixed numeric waits (could be text/selector-based)
|
|
1193
|
+
for (const test of tests) {
|
|
1194
|
+
if (!test.actions) continue;
|
|
1195
|
+
for (const action of test.actions) {
|
|
1196
|
+
if (action.type === 'wait' && /^\d+$/.test(String(action.value))) {
|
|
1197
|
+
const ms = parseInt(action.value, 10);
|
|
1198
|
+
if (ms >= 3000) {
|
|
1199
|
+
warnings.push(
|
|
1200
|
+
`⏱️ Fixed ${ms}ms wait in "${test.name}" — prefer { type: "wait", text: "..." } or { type: "wait", selector: "..." } ` +
|
|
1201
|
+
`which retries until the condition is met. Fixed waits are either too short (flaky) or too long (slow).`
|
|
1202
|
+
);
|
|
1203
|
+
break; // one warning per test is enough
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
// Detect MUI/framework selectors in action selectors
|
|
1210
|
+
for (const test of tests) {
|
|
1211
|
+
if (!test.actions) continue;
|
|
1212
|
+
for (const action of test.actions) {
|
|
1213
|
+
const sel = action.selector || '';
|
|
1214
|
+
if (/\.Mui[\w-]+/.test(sel) || /\.ant-[\w-]+/.test(sel) || /\.v-[\w-]+/.test(sel)) {
|
|
1215
|
+
const match = sel.match(/\.(Mui[\w-]+|ant-[\w-]+|v-[\w-]+)/);
|
|
1216
|
+
warnings.push(
|
|
1217
|
+
`⚠️ Framework selector ".${match[1]}" in "${test.name}" (${action.type}) — ` +
|
|
1218
|
+
`these class names are auto-generated and break on version upgrades. ` +
|
|
1219
|
+
`Prefer [data-testid="..."], [role="..."], or text-based actions`
|
|
1220
|
+
);
|
|
1221
|
+
break;
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
// Detect cross-test state: test N writes window.__e2e_*, test M reads it
|
|
1227
|
+
const writers = new Map(); // varName → test name
|
|
1228
|
+
const readers = new Map(); // varName → [test names]
|
|
1229
|
+
for (const test of tests) {
|
|
1230
|
+
if (!test.actions) continue;
|
|
1231
|
+
for (const action of test.actions) {
|
|
1232
|
+
if (action.type !== 'evaluate' || !action.value) continue;
|
|
1233
|
+
const code = action.value;
|
|
1234
|
+
// Find writes: window.__e2e_foo = ...
|
|
1235
|
+
const writeMatches = code.matchAll(/window\.(__e2e_\w+)\s*=/g);
|
|
1236
|
+
for (const m of writeMatches) {
|
|
1237
|
+
if (!writers.has(m[1])) writers.set(m[1], test.name);
|
|
1238
|
+
}
|
|
1239
|
+
// Find reads: window.__e2e_foo (not followed by =)
|
|
1240
|
+
const readMatches = code.matchAll(/window\.(__e2e_\w+)(?!\s*=)/g);
|
|
1241
|
+
for (const m of readMatches) {
|
|
1242
|
+
if (!readers.has(m[1])) readers.set(m[1], []);
|
|
1243
|
+
if (!readers.get(m[1]).includes(test.name)) readers.get(m[1]).push(test.name);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
for (const [varName, writerTest] of writers) {
|
|
1249
|
+
const readerTests = (readers.get(varName) || []).filter(t => t !== writerTest);
|
|
1250
|
+
if (readerTests.length > 0) {
|
|
1251
|
+
warnings.push(
|
|
1252
|
+
`🔗 Cross-test dependency: "${writerTest}" sets ${varName}, read by: ${readerTests.map(t => `"${t}"`).join(', ')}. ` +
|
|
1253
|
+
`If "${writerTest}" fails, dependent tests will cascade-fail with confusing errors. ` +
|
|
1254
|
+
`Consider re-querying data in each test or combining them into a single test.`
|
|
1255
|
+
);
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
return warnings;
|
|
694
1260
|
}
|
|
695
1261
|
|
|
696
1262
|
async function handlePoolStatus(args) {
|
|
697
1263
|
const config = await loadConfig({}, args.cwd);
|
|
698
|
-
const
|
|
1264
|
+
const poolUrls = getPoolUrls(config);
|
|
1265
|
+
const aggregated = await getAggregatedPoolStatus(poolUrls, { poolDriver: config.poolDriver, maxSessions: config.maxSessions });
|
|
1266
|
+
|
|
1267
|
+
const lines = [];
|
|
699
1268
|
|
|
1269
|
+
if (poolUrls.length > 1) {
|
|
1270
|
+
lines.push(`Pools: ${aggregated.totalPools} (${aggregated.availableCount} available)`);
|
|
1271
|
+
lines.push(`Running: ${aggregated.totalRunning}/${aggregated.totalMaxConcurrent}`);
|
|
1272
|
+
lines.push(`Queued: ${aggregated.totalQueued}`);
|
|
1273
|
+
lines.push('');
|
|
1274
|
+
for (const pool of aggregated.pools) {
|
|
1275
|
+
const status = pool.available ? 'available' : pool.error ? `offline (${pool.error})` : 'busy';
|
|
1276
|
+
lines.push(` ${pool.url}: ${status} (${pool.running}/${pool.maxConcurrent}, ${pool.queued} queued)`);
|
|
1277
|
+
}
|
|
1278
|
+
} else {
|
|
1279
|
+
const pool = aggregated.pools[0];
|
|
1280
|
+
lines.push(`Available: ${pool.available ? 'yes' : 'no'}`);
|
|
1281
|
+
lines.push(`Running: ${pool.running}/${pool.maxConcurrent}`);
|
|
1282
|
+
lines.push(`Queued: ${pool.queued}`);
|
|
1283
|
+
lines.push(`Sessions: ${pool.sessions?.length ?? 0}`);
|
|
1284
|
+
if (pool.error) {
|
|
1285
|
+
lines.push(`Error: ${pool.error}`);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
return textResult(lines.join('\n'));
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
async function handleAppPoolStatus(args) {
|
|
1293
|
+
const config = await loadConfig({}, args.cwd);
|
|
1294
|
+
if (!isAppPoolEnabled(config)) {
|
|
1295
|
+
return textResult('App pool is not enabled. Set appPool.enabled = true in e2e.config.js to use isolated app environments per test.');
|
|
1296
|
+
}
|
|
1297
|
+
|
|
1298
|
+
const status = getAppPoolStatus();
|
|
700
1299
|
const lines = [
|
|
701
|
-
`
|
|
702
|
-
`
|
|
703
|
-
`
|
|
704
|
-
`
|
|
1300
|
+
`Driver: ${config.appPool.driver}`,
|
|
1301
|
+
`Active forks: ${status.activeForks}/${config.appPool.maxForks}`,
|
|
1302
|
+
`Port range: ${config.appPool.forkBasePort}-${config.appPool.forkBasePort + config.appPool.maxForks - 1}`,
|
|
1303
|
+
`Allocated: ${status.allocatedPorts.length ? status.allocatedPorts.join(', ') : 'none'}`,
|
|
705
1304
|
];
|
|
706
1305
|
|
|
707
|
-
if (status.
|
|
708
|
-
lines.push(
|
|
1306
|
+
if (status.forks.length > 0) {
|
|
1307
|
+
lines.push('');
|
|
1308
|
+
for (const fork of status.forks) {
|
|
1309
|
+
lines.push(` ${fork.forkId}: port ${fork.port}, ${fork.baseUrl} (${fork.testName || 'unnamed'}, ${fork.forkTimeMs}ms)`);
|
|
1310
|
+
}
|
|
709
1311
|
}
|
|
710
1312
|
|
|
711
1313
|
return textResult(lines.join('\n'));
|
|
@@ -831,21 +1433,556 @@ async function handleCreateModule(args) {
|
|
|
831
1433
|
return textResult(`Created module: ${filePath}\n\nName: ${args.name}\nParams: ${paramNames.length ? paramNames.join(', ') : 'none'}\nActions: ${args.actions.length}\n\nUsage in tests: { "$use": "${args.name}", "params": { ... } }`);
|
|
832
1434
|
}
|
|
833
1435
|
|
|
834
|
-
|
|
1436
|
+
// ── Page analysis helpers ─────────────────────────────────────────────────────
|
|
1437
|
+
|
|
1438
|
+
/**
|
|
1439
|
+
* Browser-side function passed to page.evaluate().
|
|
1440
|
+
* Extracts the complete interactive structure of a page in a single DOM pass.
|
|
1441
|
+
*/
|
|
1442
|
+
function extractPageStructure(scopeSelector, maxElements) {
|
|
1443
|
+
const MAX = maxElements || 50;
|
|
1444
|
+
const root = scopeSelector ? document.querySelector(scopeSelector) : document.body;
|
|
1445
|
+
if (!root) return { error: `Scope selector not found: ${scopeSelector}` };
|
|
1446
|
+
|
|
1447
|
+
// ── bestSelector: generate the most reliable CSS selector for an element ──
|
|
1448
|
+
const FRAMEWORK_CLASS_RE = /^(css-|sc-|jss\d|Mui|emotion-|chakra-|ant-|el-|v-|ng-|_|svelte-|tw-)/;
|
|
1449
|
+
|
|
1450
|
+
function bestSelector(el) {
|
|
1451
|
+
// 1. ID (if unique)
|
|
1452
|
+
if (el.id && document.querySelectorAll(`#${CSS.escape(el.id)}`).length === 1) {
|
|
1453
|
+
return `#${CSS.escape(el.id)}`;
|
|
1454
|
+
}
|
|
1455
|
+
// 2. data-testid
|
|
1456
|
+
const testId = el.getAttribute('data-testid');
|
|
1457
|
+
if (testId) return `[data-testid="${testId}"]`;
|
|
1458
|
+
// 3. aria-label
|
|
1459
|
+
const ariaLabel = el.getAttribute('aria-label');
|
|
1460
|
+
if (ariaLabel && document.querySelectorAll(`[aria-label="${CSS.escape(ariaLabel)}"]`).length === 1) {
|
|
1461
|
+
return `[aria-label="${CSS.escape(ariaLabel)}"]`;
|
|
1462
|
+
}
|
|
1463
|
+
// 4. name attribute
|
|
1464
|
+
const name = el.getAttribute('name');
|
|
1465
|
+
if (name && document.querySelectorAll(`[name="${CSS.escape(name)}"]`).length === 1) {
|
|
1466
|
+
return `[name="${CSS.escape(name)}"]`;
|
|
1467
|
+
}
|
|
1468
|
+
// 5. Unique CSS class (filter framework-generated)
|
|
1469
|
+
const tag = el.tagName.toLowerCase();
|
|
1470
|
+
const classes = [...el.classList].filter(c => !FRAMEWORK_CLASS_RE.test(c));
|
|
1471
|
+
for (const cls of classes) {
|
|
1472
|
+
const sel = `${tag}.${CSS.escape(cls)}`;
|
|
1473
|
+
if (document.querySelectorAll(sel).length === 1) return sel;
|
|
1474
|
+
}
|
|
1475
|
+
// 6. Two-class combination
|
|
1476
|
+
for (let i = 0; i < classes.length; i++) {
|
|
1477
|
+
for (let j = i + 1; j < classes.length; j++) {
|
|
1478
|
+
const sel = `${tag}.${CSS.escape(classes[i])}.${CSS.escape(classes[j])}`;
|
|
1479
|
+
if (document.querySelectorAll(sel).length === 1) return sel;
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
// 7. Parent with ID + tag:nth-of-type
|
|
1483
|
+
let parent = el.parentElement;
|
|
1484
|
+
while (parent && parent !== document.body) {
|
|
1485
|
+
if (parent.id) {
|
|
1486
|
+
const siblings = [...parent.querySelectorAll(`:scope > ${tag}`)];
|
|
1487
|
+
const idx = siblings.indexOf(el);
|
|
1488
|
+
if (idx !== -1) {
|
|
1489
|
+
const sel = `#${CSS.escape(parent.id)} > ${tag}:nth-of-type(${idx + 1})`;
|
|
1490
|
+
if (document.querySelectorAll(sel).length === 1) return sel;
|
|
1491
|
+
}
|
|
1492
|
+
break;
|
|
1493
|
+
}
|
|
1494
|
+
parent = parent.parentElement;
|
|
1495
|
+
}
|
|
1496
|
+
// 8. Fallback: tag:nth-of-type within parent
|
|
1497
|
+
if (el.parentElement) {
|
|
1498
|
+
const siblings = [...el.parentElement.querySelectorAll(`:scope > ${tag}`)];
|
|
1499
|
+
const idx = siblings.indexOf(el);
|
|
1500
|
+
if (idx !== -1) return `${tag}:nth-of-type(${idx + 1})`;
|
|
1501
|
+
}
|
|
1502
|
+
return tag;
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
function getLabel(el) {
|
|
1506
|
+
// Check for associated label
|
|
1507
|
+
if (el.id) {
|
|
1508
|
+
const label = root.querySelector(`label[for="${CSS.escape(el.id)}"]`);
|
|
1509
|
+
if (label) return label.textContent.trim();
|
|
1510
|
+
}
|
|
1511
|
+
// Check for wrapping label
|
|
1512
|
+
const parentLabel = el.closest('label');
|
|
1513
|
+
if (parentLabel) return parentLabel.textContent.trim();
|
|
1514
|
+
// aria-label
|
|
1515
|
+
if (el.getAttribute('aria-label')) return el.getAttribute('aria-label');
|
|
1516
|
+
// placeholder
|
|
1517
|
+
if (el.placeholder) return el.placeholder;
|
|
1518
|
+
return '';
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
function isVisible(el) {
|
|
1522
|
+
const style = getComputedStyle(el);
|
|
1523
|
+
return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0';
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
function truncate(arr) {
|
|
1527
|
+
return arr.slice(0, MAX);
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
// ── Extract forms ──
|
|
1531
|
+
const forms = [];
|
|
1532
|
+
for (const form of root.querySelectorAll('form')) {
|
|
1533
|
+
if (!isVisible(form)) continue;
|
|
1534
|
+
const fields = [];
|
|
1535
|
+
for (const input of form.querySelectorAll('input, select, textarea')) {
|
|
1536
|
+
if (!isVisible(input) || input.type === 'hidden') continue;
|
|
1537
|
+
fields.push({
|
|
1538
|
+
selector: bestSelector(input),
|
|
1539
|
+
tag: input.tagName.toLowerCase(),
|
|
1540
|
+
type: input.type || input.tagName.toLowerCase(),
|
|
1541
|
+
name: input.name || undefined,
|
|
1542
|
+
label: getLabel(input) || undefined,
|
|
1543
|
+
required: input.required || undefined,
|
|
1544
|
+
placeholder: input.placeholder || undefined,
|
|
1545
|
+
});
|
|
1546
|
+
}
|
|
1547
|
+
const submitBtn = form.querySelector('button[type="submit"], input[type="submit"]');
|
|
1548
|
+
forms.push({
|
|
1549
|
+
selector: bestSelector(form),
|
|
1550
|
+
action: form.action || undefined,
|
|
1551
|
+
method: form.method || undefined,
|
|
1552
|
+
fields: truncate(fields),
|
|
1553
|
+
submitButton: submitBtn ? { selector: bestSelector(submitBtn), text: submitBtn.textContent?.trim() || submitBtn.value } : undefined,
|
|
1554
|
+
});
|
|
1555
|
+
if (forms.length >= MAX) break;
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
// ── Standalone inputs (outside forms) ──
|
|
1559
|
+
const standaloneInputs = [];
|
|
1560
|
+
for (const input of root.querySelectorAll('input, select, textarea')) {
|
|
1561
|
+
if (!isVisible(input) || input.type === 'hidden' || input.closest('form')) continue;
|
|
1562
|
+
standaloneInputs.push({
|
|
1563
|
+
selector: bestSelector(input),
|
|
1564
|
+
tag: input.tagName.toLowerCase(),
|
|
1565
|
+
type: input.type || input.tagName.toLowerCase(),
|
|
1566
|
+
name: input.name || undefined,
|
|
1567
|
+
label: getLabel(input) || undefined,
|
|
1568
|
+
placeholder: input.placeholder || undefined,
|
|
1569
|
+
});
|
|
1570
|
+
if (standaloneInputs.length >= MAX) break;
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
// ── Buttons ──
|
|
1574
|
+
const buttons = [];
|
|
1575
|
+
for (const btn of root.querySelectorAll('button, [role="button"], input[type="button"], input[type="submit"]')) {
|
|
1576
|
+
if (!isVisible(btn)) continue;
|
|
1577
|
+
buttons.push({
|
|
1578
|
+
selector: bestSelector(btn),
|
|
1579
|
+
text: btn.textContent?.trim() || btn.value || '',
|
|
1580
|
+
type: btn.type || undefined,
|
|
1581
|
+
disabled: btn.disabled || undefined,
|
|
1582
|
+
ariaLabel: btn.getAttribute('aria-label') || undefined,
|
|
1583
|
+
});
|
|
1584
|
+
if (buttons.length >= MAX) break;
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
// ── Links ──
|
|
1588
|
+
const links = [];
|
|
1589
|
+
for (const a of root.querySelectorAll('a[href]')) {
|
|
1590
|
+
if (!isVisible(a)) continue;
|
|
1591
|
+
links.push({
|
|
1592
|
+
selector: bestSelector(a),
|
|
1593
|
+
text: a.textContent?.trim() || '',
|
|
1594
|
+
href: a.getAttribute('href'),
|
|
1595
|
+
});
|
|
1596
|
+
if (links.length >= MAX) break;
|
|
1597
|
+
}
|
|
1598
|
+
|
|
1599
|
+
// ── Navigation regions ──
|
|
1600
|
+
const navigation = [];
|
|
1601
|
+
for (const nav of root.querySelectorAll('nav, [role="navigation"]')) {
|
|
1602
|
+
if (!isVisible(nav)) continue;
|
|
1603
|
+
const items = [];
|
|
1604
|
+
for (const link of nav.querySelectorAll('a, button, [role="tab"], [role="menuitem"]')) {
|
|
1605
|
+
if (!isVisible(link)) continue;
|
|
1606
|
+
items.push({
|
|
1607
|
+
selector: bestSelector(link),
|
|
1608
|
+
text: link.textContent?.trim() || '',
|
|
1609
|
+
href: link.getAttribute('href') || undefined,
|
|
1610
|
+
active: link.classList.contains('active') || link.getAttribute('aria-current') === 'page' || undefined,
|
|
1611
|
+
});
|
|
1612
|
+
}
|
|
1613
|
+
navigation.push({
|
|
1614
|
+
selector: bestSelector(nav),
|
|
1615
|
+
ariaLabel: nav.getAttribute('aria-label') || undefined,
|
|
1616
|
+
items: truncate(items),
|
|
1617
|
+
});
|
|
1618
|
+
if (navigation.length >= MAX) break;
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
// ── Tabs ──
|
|
1622
|
+
const tabs = [];
|
|
1623
|
+
for (const tab of root.querySelectorAll('[role="tab"]')) {
|
|
1624
|
+
if (!isVisible(tab)) continue;
|
|
1625
|
+
tabs.push({
|
|
1626
|
+
selector: bestSelector(tab),
|
|
1627
|
+
text: tab.textContent?.trim() || '',
|
|
1628
|
+
selected: tab.getAttribute('aria-selected') === 'true' || undefined,
|
|
1629
|
+
});
|
|
1630
|
+
if (tabs.length >= MAX) break;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
// ── Headings ──
|
|
1634
|
+
const headings = [];
|
|
1635
|
+
for (const h of root.querySelectorAll('h1, h2, h3, h4, h5, h6')) {
|
|
1636
|
+
if (!isVisible(h)) continue;
|
|
1637
|
+
headings.push({
|
|
1638
|
+
level: parseInt(h.tagName[1]),
|
|
1639
|
+
text: h.textContent?.trim() || '',
|
|
1640
|
+
selector: bestSelector(h),
|
|
1641
|
+
});
|
|
1642
|
+
if (headings.length >= MAX) break;
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
// ── Tables ──
|
|
1646
|
+
const tables = [];
|
|
1647
|
+
for (const table of root.querySelectorAll('table')) {
|
|
1648
|
+
if (!isVisible(table)) continue;
|
|
1649
|
+
const headers = [...table.querySelectorAll('th')].map(th => th.textContent?.trim());
|
|
1650
|
+
tables.push({
|
|
1651
|
+
selector: bestSelector(table),
|
|
1652
|
+
headers: truncate(headers),
|
|
1653
|
+
rowCount: table.querySelectorAll('tbody tr, tr').length,
|
|
1654
|
+
hasHeader: headers.length > 0,
|
|
1655
|
+
});
|
|
1656
|
+
if (tables.length >= MAX) break;
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
// ── Modals/Dialogs ──
|
|
1660
|
+
const modals = [];
|
|
1661
|
+
for (const modal of root.querySelectorAll('[role="dialog"], dialog, .modal, [class*="modal"], [class*="Modal"]')) {
|
|
1662
|
+
if (!isVisible(modal)) continue;
|
|
1663
|
+
const title = modal.querySelector('[class*="title"], [class*="Title"], h1, h2, h3, [role="heading"]');
|
|
1664
|
+
const closeBtn = modal.querySelector('[aria-label="close"], [aria-label="Close"], button.close, [class*="close"]');
|
|
1665
|
+
modals.push({
|
|
1666
|
+
selector: bestSelector(modal),
|
|
1667
|
+
title: title?.textContent?.trim() || undefined,
|
|
1668
|
+
hasCloseButton: !!closeBtn,
|
|
1669
|
+
closeSelector: closeBtn ? bestSelector(closeBtn) : undefined,
|
|
1670
|
+
});
|
|
1671
|
+
if (modals.length >= MAX) break;
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
// ── Menus/Dropdowns ──
|
|
1675
|
+
const menus = [];
|
|
1676
|
+
for (const menu of root.querySelectorAll('[role="menu"], .dropdown-menu, [class*="dropdown"]')) {
|
|
1677
|
+
if (!isVisible(menu)) continue;
|
|
1678
|
+
const items = [];
|
|
1679
|
+
for (const item of menu.querySelectorAll('[role="menuitem"], [role="menuitemradio"], [role="menuitemcheckbox"], .dropdown-item, [class*="MenuItem"]')) {
|
|
1680
|
+
if (!isVisible(item)) continue;
|
|
1681
|
+
items.push({ text: item.textContent?.trim() || '', selector: bestSelector(item) });
|
|
1682
|
+
}
|
|
1683
|
+
menus.push({
|
|
1684
|
+
selector: bestSelector(menu),
|
|
1685
|
+
items: truncate(items),
|
|
1686
|
+
});
|
|
1687
|
+
if (menus.length >= MAX) break;
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
// ── Alerts/Banners ──
|
|
1691
|
+
const alerts = [];
|
|
1692
|
+
for (const alert of root.querySelectorAll('[role="alert"], [role="status"], .alert, [class*="banner"], [class*="Banner"], [class*="toast"], [class*="Toast"], [class*="notification"], [class*="Notification"]')) {
|
|
1693
|
+
if (!isVisible(alert)) continue;
|
|
1694
|
+
alerts.push({
|
|
1695
|
+
selector: bestSelector(alert),
|
|
1696
|
+
text: alert.textContent?.trim().slice(0, 200) || '',
|
|
1697
|
+
role: alert.getAttribute('role') || undefined,
|
|
1698
|
+
});
|
|
1699
|
+
if (alerts.length >= MAX) break;
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
// ── Significant images (>50px) ──
|
|
1703
|
+
const images = [];
|
|
1704
|
+
for (const img of root.querySelectorAll('img, svg[role="img"], [role="img"]')) {
|
|
1705
|
+
if (!isVisible(img)) continue;
|
|
1706
|
+
const rect = img.getBoundingClientRect();
|
|
1707
|
+
if (rect.width < 50 && rect.height < 50) continue;
|
|
1708
|
+
images.push({
|
|
1709
|
+
selector: bestSelector(img),
|
|
1710
|
+
alt: img.alt || img.getAttribute('aria-label') || undefined,
|
|
1711
|
+
width: Math.round(rect.width),
|
|
1712
|
+
height: Math.round(rect.height),
|
|
1713
|
+
src: img.src ? img.src.slice(0, 200) : undefined,
|
|
1714
|
+
});
|
|
1715
|
+
if (images.length >= MAX) break;
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
return {
|
|
1719
|
+
forms,
|
|
1720
|
+
standaloneInputs: standaloneInputs.length > 0 ? standaloneInputs : undefined,
|
|
1721
|
+
buttons,
|
|
1722
|
+
links,
|
|
1723
|
+
navigation: navigation.length > 0 ? navigation : undefined,
|
|
1724
|
+
tabs: tabs.length > 0 ? tabs : undefined,
|
|
1725
|
+
headings,
|
|
1726
|
+
tables: tables.length > 0 ? tables : undefined,
|
|
1727
|
+
modals: modals.length > 0 ? modals : undefined,
|
|
1728
|
+
menus: menus.length > 0 ? menus : undefined,
|
|
1729
|
+
alerts: alerts.length > 0 ? alerts : undefined,
|
|
1730
|
+
images: images.length > 0 ? images : undefined,
|
|
1731
|
+
stats: {
|
|
1732
|
+
totalForms: forms.length,
|
|
1733
|
+
totalButtons: buttons.length,
|
|
1734
|
+
totalLinks: links.length,
|
|
1735
|
+
totalInputs: forms.reduce((n, f) => n + f.fields.length, 0) + standaloneInputs.length,
|
|
1736
|
+
totalHeadings: headings.length,
|
|
1737
|
+
totalTables: tables.length,
|
|
1738
|
+
totalNavRegions: navigation.length,
|
|
1739
|
+
totalTabs: tabs.length,
|
|
1740
|
+
totalModals: modals.length,
|
|
1741
|
+
totalImages: images.length,
|
|
1742
|
+
},
|
|
1743
|
+
};
|
|
1744
|
+
}
|
|
1745
|
+
|
|
1746
|
+
/**
|
|
1747
|
+
* Analyzes extracted page structure and generates ready-to-use test scaffolds.
|
|
1748
|
+
* Runs on the Node.js side after page.evaluate returns.
|
|
1749
|
+
*/
|
|
1750
|
+
function buildSuggestedTests(structure, pageUrl) {
|
|
1751
|
+
const tests = [];
|
|
1752
|
+
const urlPath = (() => { try { return new URL(pageUrl).pathname; } catch { return '/'; } })();
|
|
1753
|
+
|
|
1754
|
+
// Login form detection
|
|
1755
|
+
for (const form of structure.forms || []) {
|
|
1756
|
+
const fields = form.fields || [];
|
|
1757
|
+
const hasPassword = fields.some(f => f.type === 'password');
|
|
1758
|
+
const hasEmail = fields.some(f => f.type === 'email' || f.name === 'email' || (f.label || '').toLowerCase().includes('email'));
|
|
1759
|
+
const hasUsername = fields.some(f => f.name === 'username' || (f.label || '').toLowerCase().includes('user'));
|
|
1760
|
+
|
|
1761
|
+
if (hasPassword && (hasEmail || hasUsername)) {
|
|
1762
|
+
const actions = [{ type: 'goto', value: urlPath }];
|
|
1763
|
+
const emailField = fields.find(f => f.type === 'email' || f.name === 'email' || (f.label || '').toLowerCase().includes('email'));
|
|
1764
|
+
const usernameField = fields.find(f => f.name === 'username' || (f.label || '').toLowerCase().includes('user'));
|
|
1765
|
+
const passwordField = fields.find(f => f.type === 'password');
|
|
1766
|
+
const credential = emailField || usernameField;
|
|
1767
|
+
if (credential) actions.push({ type: 'type', selector: credential.selector, value: 'test@example.com' });
|
|
1768
|
+
if (passwordField) actions.push({ type: 'type', selector: passwordField.selector, value: 'password123' });
|
|
1769
|
+
if (form.submitButton) actions.push({ type: 'click', selector: form.submitButton.selector });
|
|
1770
|
+
actions.push({ type: 'wait', value: '2000' });
|
|
1771
|
+
tests.push({ name: 'login-form-submission', actions });
|
|
1772
|
+
continue;
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
// Generic form fill + submit
|
|
1776
|
+
if (fields.length > 0) {
|
|
1777
|
+
const actions = [{ type: 'goto', value: urlPath }];
|
|
1778
|
+
for (const field of fields.slice(0, 10)) {
|
|
1779
|
+
const val = field.type === 'email' ? 'test@example.com'
|
|
1780
|
+
: field.type === 'number' ? '42'
|
|
1781
|
+
: field.type === 'tel' ? '555-0100'
|
|
1782
|
+
: field.type === 'date' ? '2025-01-15'
|
|
1783
|
+
: field.tag === 'select' ? undefined
|
|
1784
|
+
: field.tag === 'textarea' ? 'Sample text input'
|
|
1785
|
+
: 'Test value';
|
|
1786
|
+
if (val && field.tag !== 'select') {
|
|
1787
|
+
actions.push({ type: 'type', selector: field.selector, value: val });
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
if (form.submitButton) actions.push({ type: 'click', selector: form.submitButton.selector });
|
|
1791
|
+
actions.push({ type: 'wait', value: '1000' });
|
|
1792
|
+
tests.push({ name: `form-submission-${tests.length + 1}`, actions });
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
// Navigation test
|
|
1797
|
+
const navItems = (structure.navigation || []).flatMap(n => n.items || []).filter(i => i.href && i.text);
|
|
1798
|
+
if (navItems.length > 0) {
|
|
1799
|
+
const actions = [{ type: 'goto', value: urlPath }];
|
|
1800
|
+
for (const item of navItems.slice(0, 5)) {
|
|
1801
|
+
actions.push({ type: 'click', selector: item.selector });
|
|
1802
|
+
actions.push({ type: 'wait', value: '1000' });
|
|
1803
|
+
if (item.href && item.href !== '#' && !item.href.startsWith('javascript:')) {
|
|
1804
|
+
actions.push({ type: 'assert_url', value: item.href });
|
|
1805
|
+
}
|
|
1806
|
+
actions.push({ type: 'goto', value: urlPath });
|
|
1807
|
+
}
|
|
1808
|
+
tests.push({ name: 'navigation-links', actions });
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
// Table data assertion
|
|
1812
|
+
for (const table of structure.tables || []) {
|
|
1813
|
+
if (table.rowCount > 0) {
|
|
1814
|
+
tests.push({
|
|
1815
|
+
name: `table-has-data`,
|
|
1816
|
+
actions: [
|
|
1817
|
+
{ type: 'goto', value: urlPath },
|
|
1818
|
+
{ type: 'wait', selector: table.selector },
|
|
1819
|
+
{ type: 'assert_count', selector: `${table.selector} tbody tr`, value: '>=1' },
|
|
1820
|
+
],
|
|
1821
|
+
});
|
|
1822
|
+
break;
|
|
1823
|
+
}
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
// Tab switching test
|
|
1827
|
+
if ((structure.tabs || []).length >= 2) {
|
|
1828
|
+
const actions = [{ type: 'goto', value: urlPath }];
|
|
1829
|
+
for (const tab of structure.tabs.slice(0, 5)) {
|
|
1830
|
+
actions.push({ type: 'click', selector: tab.selector });
|
|
1831
|
+
actions.push({ type: 'wait', value: '500' });
|
|
1832
|
+
}
|
|
1833
|
+
tests.push({ name: 'tab-switching', actions });
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
// Page structure verification (always generated)
|
|
1837
|
+
const verifyActions = [{ type: 'goto', value: urlPath }];
|
|
1838
|
+
for (const h of (structure.headings || []).filter(h => h.level <= 2).slice(0, 3)) {
|
|
1839
|
+
verifyActions.push({ type: 'assert_text', text: h.text });
|
|
1840
|
+
}
|
|
1841
|
+
if (structure.stats.totalButtons > 0) {
|
|
1842
|
+
const visibleBtns = (structure.buttons || []).filter(b => b.text);
|
|
1843
|
+
for (const btn of visibleBtns.slice(0, 3)) {
|
|
1844
|
+
verifyActions.push({ type: 'assert_visible', selector: btn.selector });
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
tests.push({ name: 'page-structure-verification', actions: verifyActions });
|
|
1848
|
+
|
|
1849
|
+
return tests;
|
|
1850
|
+
}
|
|
1851
|
+
|
|
1852
|
+
/**
|
|
1853
|
+
* Smart page navigation with fallback for SPA/WebSocket apps.
|
|
1854
|
+
* - "auto" (default): tries networkidle2 with 10s timeout, falls back to domcontentloaded + 2s delay
|
|
1855
|
+
* - "networkidle2"/"load"/"domcontentloaded": uses that strategy directly with 30s timeout
|
|
1856
|
+
*/
|
|
1857
|
+
async function smartNavigate(page, url, waitUntil) {
|
|
1858
|
+
const strategy = waitUntil || 'auto';
|
|
1859
|
+
|
|
1860
|
+
if (strategy === 'auto') {
|
|
1861
|
+
try {
|
|
1862
|
+
await page.goto(url, { waitUntil: 'networkidle2', timeout: 10000 });
|
|
1863
|
+
} catch (err) {
|
|
1864
|
+
if (err.name === 'TimeoutError' || (err.message && err.message.includes('timeout'))) {
|
|
1865
|
+
// networkidle2 timed out — likely a SPA with WebSocket/SSE/polling
|
|
1866
|
+
// Fall back to domcontentloaded + wait for hydration
|
|
1867
|
+
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
|
|
1868
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
1869
|
+
} else {
|
|
1870
|
+
throw err;
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
} else {
|
|
1874
|
+
await page.goto(url, { waitUntil: strategy, timeout: 30000 });
|
|
1875
|
+
// For domcontentloaded, add a small hydration delay for SPAs
|
|
1876
|
+
if (strategy === 'domcontentloaded') {
|
|
1877
|
+
await new Promise(r => setTimeout(r, 1500));
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
|
|
1882
|
+
async function handleAnalyze(args) {
|
|
835
1883
|
if (!args.url) return errorResult('Missing required parameter: url');
|
|
836
1884
|
|
|
837
1885
|
const config = await loadConfig({}, args.cwd);
|
|
1886
|
+
const poolUrls = getPoolUrls(config);
|
|
1887
|
+
const chosenPool = await selectPool(poolUrls, 2000, 60000, { poolDriver: config.poolDriver, maxSessions: config.maxSessions });
|
|
838
1888
|
|
|
839
|
-
|
|
1889
|
+
let browser;
|
|
1890
|
+
try {
|
|
1891
|
+
browser = await connectToPool(chosenPool);
|
|
1892
|
+
const page = await browser.newPage();
|
|
1893
|
+
await page.setViewport(config.viewport);
|
|
1894
|
+
|
|
1895
|
+
// Resolve auth token: explicit arg > config static > auto-login
|
|
1896
|
+
const authToken = args.authToken || await resolveAuthToken(config);
|
|
1897
|
+
if (authToken) {
|
|
1898
|
+
const storageKey = args.authStorageKey || config.authStorageKey || 'accessToken';
|
|
1899
|
+
const origin = new URL(args.url).origin;
|
|
1900
|
+
await page.goto(origin, { waitUntil: 'domcontentloaded', timeout: 15000 });
|
|
1901
|
+
await page.evaluate((key, token) => { localStorage.setItem(key, token); }, storageKey, authToken);
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
await smartNavigate(page, args.url, args.waitUntil);
|
|
1905
|
+
|
|
1906
|
+
if (args.selector) {
|
|
1907
|
+
await page.waitForSelector(args.selector, { timeout: 10000 });
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
if (args.delay && args.delay > 0) {
|
|
1911
|
+
await new Promise(r => setTimeout(r, args.delay));
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1914
|
+
// Extract page structure
|
|
1915
|
+
const structure = await page.evaluate(extractPageStructure, args.scope || null, args.maxElements || 50);
|
|
1916
|
+
|
|
1917
|
+
if (structure.error) {
|
|
1918
|
+
return errorResult(structure.error);
|
|
1919
|
+
}
|
|
1920
|
+
|
|
1921
|
+
// Build meta
|
|
1922
|
+
const title = await page.title();
|
|
1923
|
+
const meta = {
|
|
1924
|
+
url: args.url,
|
|
1925
|
+
title,
|
|
1926
|
+
viewport: config.viewport,
|
|
1927
|
+
scope: args.scope || undefined,
|
|
1928
|
+
};
|
|
1929
|
+
|
|
1930
|
+
// Build suggested tests
|
|
1931
|
+
const suggestedTests = buildSuggestedTests(structure, args.url);
|
|
1932
|
+
|
|
1933
|
+
// Optional screenshot (default: true)
|
|
1934
|
+
const includeScreenshot = args.includeScreenshot !== false;
|
|
1935
|
+
let screenshotHash;
|
|
1936
|
+
let screenshotBase64;
|
|
1937
|
+
|
|
1938
|
+
if (includeScreenshot) {
|
|
1939
|
+
const filename = `analyze-${Date.now()}.png`;
|
|
1940
|
+
if (!fs.existsSync(config.screenshotsDir)) {
|
|
1941
|
+
fs.mkdirSync(config.screenshotsDir, { recursive: true });
|
|
1942
|
+
}
|
|
1943
|
+
const screenshotPath = path.join(config.screenshotsDir, filename);
|
|
1944
|
+
await page.screenshot({ path: screenshotPath, fullPage: false });
|
|
1945
|
+
|
|
1946
|
+
const cwd = args.cwd || process.cwd();
|
|
1947
|
+
const projectName = config.projectName || path.basename(cwd);
|
|
1948
|
+
const projectId = ensureProject(cwd, projectName, config.screenshotsDir, config.testsDir);
|
|
1949
|
+
const hash = computeScreenshotHash(screenshotPath);
|
|
1950
|
+
registerScreenshotHash(hash, screenshotPath, projectId, null);
|
|
1951
|
+
screenshotHash = `ss:${hash}`;
|
|
1952
|
+
meta.screenshotHash = screenshotHash;
|
|
1953
|
+
|
|
1954
|
+
const data = fs.readFileSync(screenshotPath);
|
|
1955
|
+
screenshotBase64 = data.toString('base64');
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1958
|
+
const result = { meta, ...structure, suggestedTests };
|
|
1959
|
+
const content = [{ type: 'text', text: JSON.stringify(result, null, 2) }];
|
|
1960
|
+
|
|
1961
|
+
if (screenshotBase64) {
|
|
1962
|
+
content.push({ type: 'image', data: screenshotBase64, mimeType: 'image/png' });
|
|
1963
|
+
}
|
|
1964
|
+
|
|
1965
|
+
return { content };
|
|
1966
|
+
} finally {
|
|
1967
|
+
if (browser) browser.disconnect();
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
async function handleCapture(args) {
|
|
1972
|
+
if (!args.url) return errorResult('Missing required parameter: url');
|
|
1973
|
+
|
|
1974
|
+
const config = await loadConfig({}, args.cwd);
|
|
1975
|
+
const capturePoolUrls = getPoolUrls(config);
|
|
1976
|
+
const capturePool = await selectPool(capturePoolUrls, 2000, 60000, { poolDriver: config.poolDriver, maxSessions: config.maxSessions });
|
|
840
1977
|
|
|
841
1978
|
let browser;
|
|
842
1979
|
try {
|
|
843
|
-
browser = await connectToPool(
|
|
1980
|
+
browser = await connectToPool(capturePool);
|
|
844
1981
|
const page = await browser.newPage();
|
|
845
1982
|
await page.setViewport(config.viewport);
|
|
846
1983
|
|
|
847
|
-
//
|
|
848
|
-
const authToken = args.authToken || config
|
|
1984
|
+
// Resolve auth token: explicit arg > config static > auto-login
|
|
1985
|
+
const authToken = args.authToken || await resolveAuthToken(config);
|
|
849
1986
|
if (authToken) {
|
|
850
1987
|
const storageKey = args.authStorageKey || config.authStorageKey || 'accessToken';
|
|
851
1988
|
// Navigate to origin first so localStorage is accessible
|
|
@@ -854,7 +1991,7 @@ async function handleCapture(args) {
|
|
|
854
1991
|
await page.evaluate((key, token) => { localStorage.setItem(key, token); }, storageKey, authToken);
|
|
855
1992
|
}
|
|
856
1993
|
|
|
857
|
-
await page
|
|
1994
|
+
await smartNavigate(page, args.url, args.waitUntil);
|
|
858
1995
|
|
|
859
1996
|
if (args.selector) {
|
|
860
1997
|
await page.waitForSelector(args.selector, { timeout: 10000 });
|
|
@@ -921,6 +2058,35 @@ async function handleDashboardStop() {
|
|
|
921
2058
|
return textResult('Dashboard stopped');
|
|
922
2059
|
}
|
|
923
2060
|
|
|
2061
|
+
async function handleDashboardRestart(args) {
|
|
2062
|
+
const port = args.port || (dashboardHandle ? dashboardHandle.port : 8484);
|
|
2063
|
+
|
|
2064
|
+
// Stop current instance if we own it
|
|
2065
|
+
if (dashboardHandle) {
|
|
2066
|
+
stopDashboard(dashboardHandle);
|
|
2067
|
+
dashboardHandle = null;
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
// Kill any process occupying the target port (e.g. from another session)
|
|
2071
|
+
try {
|
|
2072
|
+
const { execFileSync } = await import('child_process');
|
|
2073
|
+
const lsof = execFileSync('lsof', ['-ti', `:${port}`], { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim();
|
|
2074
|
+
if (lsof) {
|
|
2075
|
+
for (const pid of lsof.split('\n').filter(Boolean)) {
|
|
2076
|
+
try { process.kill(Number(pid), 'SIGTERM'); } catch {}
|
|
2077
|
+
}
|
|
2078
|
+
// Brief wait for port to free up
|
|
2079
|
+
await new Promise(r => setTimeout(r, 500));
|
|
2080
|
+
}
|
|
2081
|
+
} catch {}
|
|
2082
|
+
|
|
2083
|
+
// Start fresh
|
|
2084
|
+
const overrides = { dashboardPort: port };
|
|
2085
|
+
const config = await loadConfig(overrides, args.cwd);
|
|
2086
|
+
dashboardHandle = await startDashboard(config);
|
|
2087
|
+
return textResult(`Dashboard restarted at http://localhost:${dashboardHandle.port}`);
|
|
2088
|
+
}
|
|
2089
|
+
|
|
924
2090
|
async function handleNeo4j(args) {
|
|
925
2091
|
if (!args.action) return errorResult('Missing required parameter: action');
|
|
926
2092
|
|
|
@@ -1012,8 +2178,10 @@ async function handleLearnings(args) {
|
|
|
1012
2178
|
return textResult(JSON.stringify(getErrorPatterns(projectId), null, 2));
|
|
1013
2179
|
case 'trends':
|
|
1014
2180
|
return textResult(JSON.stringify(getTestTrends(projectId, days), null, 2));
|
|
2181
|
+
case 'actions':
|
|
2182
|
+
return textResult(JSON.stringify(getActionHealthScores(projectId, days), null, 2));
|
|
1015
2183
|
default:
|
|
1016
|
-
return errorResult(`Unknown query: "${args.query}". Use: summary, flaky, selectors, pages, apis, errors, trends, test:<name>, page:<path>, selector:<value>`);
|
|
2184
|
+
return errorResult(`Unknown query: "${args.query}". Use: summary, flaky, selectors, pages, apis, errors, trends, actions, test:<name>, page:<path>, selector:<value>`);
|
|
1017
2185
|
}
|
|
1018
2186
|
}
|
|
1019
2187
|
|
|
@@ -1039,6 +2207,127 @@ async function handleNetworkLogs(args) {
|
|
|
1039
2207
|
return textResult(JSON.stringify(results, null, 2));
|
|
1040
2208
|
}
|
|
1041
2209
|
|
|
2210
|
+
async function handleVars(args) {
|
|
2211
|
+
const action = args.action;
|
|
2212
|
+
if (!action) return errorResult('Missing required parameter: action');
|
|
2213
|
+
|
|
2214
|
+
const cwd = args.cwd || process.cwd();
|
|
2215
|
+
const config = await loadConfig({}, cwd);
|
|
2216
|
+
const projectName = config.projectName || cwd.split('/').pop() || 'default';
|
|
2217
|
+
const projectId = ensureProject(cwd, projectName, config.screenshotsDir, config.testsDir);
|
|
2218
|
+
const scope = args.scope || 'project';
|
|
2219
|
+
|
|
2220
|
+
switch (action) {
|
|
2221
|
+
case 'set': {
|
|
2222
|
+
if (!args.key) return errorResult('Missing required parameter: key');
|
|
2223
|
+
if (args.value === undefined) return errorResult('Missing required parameter: value');
|
|
2224
|
+
setVariable(projectId, scope, args.key, args.value);
|
|
2225
|
+
return textResult(`Variable set: ${args.key} (scope: ${scope})`);
|
|
2226
|
+
}
|
|
2227
|
+
case 'get': {
|
|
2228
|
+
if (!args.key) return errorResult('Missing required parameter: key');
|
|
2229
|
+
const vars = getVariables(projectId, scope);
|
|
2230
|
+
if (vars[args.key] !== undefined) {
|
|
2231
|
+
return textResult(JSON.stringify({ key: args.key, value: vars[args.key], scope }));
|
|
2232
|
+
}
|
|
2233
|
+
// Fall back to project scope if not found in specific scope
|
|
2234
|
+
if (scope !== 'project') {
|
|
2235
|
+
const projectVars = getVariables(projectId, 'project');
|
|
2236
|
+
if (projectVars[args.key] !== undefined) {
|
|
2237
|
+
return textResult(JSON.stringify({ key: args.key, value: projectVars[args.key], scope: 'project' }));
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
return errorResult(`Variable not found: ${args.key} (scope: ${scope})`);
|
|
2241
|
+
}
|
|
2242
|
+
case 'list': {
|
|
2243
|
+
const all = listVariables(projectId);
|
|
2244
|
+
if (Object.keys(all).length === 0) {
|
|
2245
|
+
return textResult('No variables set for this project.');
|
|
2246
|
+
}
|
|
2247
|
+
return textResult(JSON.stringify(all, null, 2));
|
|
2248
|
+
}
|
|
2249
|
+
case 'delete': {
|
|
2250
|
+
if (!args.key) return errorResult('Missing required parameter: key');
|
|
2251
|
+
const deleted = deleteVariable(projectId, scope, args.key);
|
|
2252
|
+
if (deleted) {
|
|
2253
|
+
return textResult(`Variable deleted: ${args.key} (scope: ${scope})`);
|
|
2254
|
+
}
|
|
2255
|
+
return errorResult(`Variable not found: ${args.key} (scope: ${scope})`);
|
|
2256
|
+
}
|
|
2257
|
+
default:
|
|
2258
|
+
return errorResult(`Unknown action: ${action}. Use set, get, list, or delete.`);
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
// ── Verification instructions builder ─────────────────────────────────────────
|
|
2263
|
+
|
|
2264
|
+
function buildVerificationInstructions(strictness, hasBaselines, hasChecklists) {
|
|
2265
|
+
const levels = {
|
|
2266
|
+
strict: 'STRICT — No ambiguity allowed. If ANY criterion is unclear, not fully visible, or doubtful, verdict is FAIL. Err on the side of failing.',
|
|
2267
|
+
moderate: 'MODERATE — Use reasonable judgment. Minor cosmetic differences are acceptable, but functional mismatches or missing elements are FAIL.',
|
|
2268
|
+
lenient: 'LENIENT — Only fail on clear, obvious contradictions. Partial matches and minor discrepancies are acceptable.',
|
|
2269
|
+
};
|
|
2270
|
+
|
|
2271
|
+
const lines = [
|
|
2272
|
+
`Verification strictness: ${levels[strictness] || levels.moderate}`,
|
|
2273
|
+
'',
|
|
2274
|
+
'For each entry in the verifications array:',
|
|
2275
|
+
'',
|
|
2276
|
+
'1. RETRIEVE SCREENSHOTS',
|
|
2277
|
+
' - Call e2e_screenshot with the screenshotHash (after-state).',
|
|
2278
|
+
];
|
|
2279
|
+
|
|
2280
|
+
if (hasBaselines) {
|
|
2281
|
+
lines.push(' - If baselineScreenshotHash is present, also call e2e_screenshot with it (before-state).');
|
|
2282
|
+
}
|
|
2283
|
+
|
|
2284
|
+
lines.push(
|
|
2285
|
+
'',
|
|
2286
|
+
'2. EVALUATE',
|
|
2287
|
+
);
|
|
2288
|
+
|
|
2289
|
+
if (hasChecklists) {
|
|
2290
|
+
lines.push(
|
|
2291
|
+
' - If isChecklist is true, evaluate EACH item in the expect array independently as PASS or FAIL.',
|
|
2292
|
+
' - If isChecklist is false (or absent), evaluate the single expect description as a whole.',
|
|
2293
|
+
);
|
|
2294
|
+
} else {
|
|
2295
|
+
lines.push(' - Compare the screenshot against the expect description.');
|
|
2296
|
+
}
|
|
2297
|
+
|
|
2298
|
+
if (hasBaselines) {
|
|
2299
|
+
lines.push(
|
|
2300
|
+
'',
|
|
2301
|
+
'3. COMPARE BEFORE/AFTER',
|
|
2302
|
+
' - If a baseline screenshot was retrieved, describe the state change between baseline and after screenshots.',
|
|
2303
|
+
' - Verify the state change is consistent with what the test actions intended.',
|
|
2304
|
+
);
|
|
2305
|
+
}
|
|
2306
|
+
|
|
2307
|
+
lines.push(
|
|
2308
|
+
'',
|
|
2309
|
+
`${hasBaselines ? '4' : '3'}. REPORT VERDICT — use this exact format for each test:`,
|
|
2310
|
+
'',
|
|
2311
|
+
' TEST: <test-name>',
|
|
2312
|
+
' VERDICT: PASS | FAIL',
|
|
2313
|
+
);
|
|
2314
|
+
|
|
2315
|
+
if (hasBaselines) {
|
|
2316
|
+
lines.push(' STATE CHANGE: <one-line description of what changed from baseline to after>');
|
|
2317
|
+
}
|
|
2318
|
+
|
|
2319
|
+
if (hasChecklists) {
|
|
2320
|
+
lines.push(
|
|
2321
|
+
' CRITERIA:',
|
|
2322
|
+
' - "<criterion text>": PASS | FAIL (reason if FAIL)',
|
|
2323
|
+
);
|
|
2324
|
+
}
|
|
2325
|
+
|
|
2326
|
+
lines.push(' REASON: <brief explanation of the verdict>');
|
|
2327
|
+
|
|
2328
|
+
return lines.join('\n');
|
|
2329
|
+
}
|
|
2330
|
+
|
|
1042
2331
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
|
1043
2332
|
|
|
1044
2333
|
export function textResult(text) {
|
|
@@ -1062,24 +2351,32 @@ export async function dispatchTool(name, args = {}) {
|
|
|
1062
2351
|
return await handleCreateTest(args);
|
|
1063
2352
|
case 'e2e_pool_status':
|
|
1064
2353
|
return await handlePoolStatus(args);
|
|
2354
|
+
case 'e2e_app_pool_status':
|
|
2355
|
+
return await handleAppPoolStatus(args);
|
|
1065
2356
|
case 'e2e_screenshot':
|
|
1066
2357
|
return await handleScreenshot(args);
|
|
1067
2358
|
case 'e2e_dashboard_start':
|
|
1068
2359
|
return await handleDashboardStart(args);
|
|
1069
2360
|
case 'e2e_dashboard_stop':
|
|
1070
2361
|
return await handleDashboardStop();
|
|
2362
|
+
case 'e2e_dashboard_restart':
|
|
2363
|
+
return await handleDashboardRestart(args);
|
|
1071
2364
|
case 'e2e_issue':
|
|
1072
2365
|
return await handleIssue(args);
|
|
1073
2366
|
case 'e2e_create_module':
|
|
1074
2367
|
return await handleCreateModule(args);
|
|
1075
2368
|
case 'e2e_capture':
|
|
1076
2369
|
return await handleCapture(args);
|
|
2370
|
+
case 'e2e_analyze':
|
|
2371
|
+
return await handleAnalyze(args);
|
|
1077
2372
|
case 'e2e_learnings':
|
|
1078
2373
|
return await handleLearnings(args);
|
|
1079
2374
|
case 'e2e_neo4j':
|
|
1080
2375
|
return await handleNeo4j(args);
|
|
1081
2376
|
case 'e2e_network_logs':
|
|
1082
2377
|
return await handleNetworkLogs(args);
|
|
2378
|
+
case 'e2e_vars':
|
|
2379
|
+
return await handleVars(args);
|
|
1083
2380
|
default:
|
|
1084
2381
|
return errorResult(`Unknown tool: ${name}`);
|
|
1085
2382
|
}
|