@matware/e2e-runner 1.1.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +21 -0
- package/.claude-plugin/plugin.json +9 -0
- package/.mcp.json +9 -0
- package/.opencode/commands/create-test.md +63 -0
- package/.opencode/commands/run.md +50 -0
- package/.opencode/commands/verify-issue.md +62 -0
- package/.opencode/skills/e2e-testing/SKILL.md +181 -0
- package/.opencode/skills/e2e-testing/references/action-types.md +143 -0
- package/.opencode/skills/e2e-testing/references/auth-strategies.md +91 -0
- package/.opencode/skills/e2e-testing/references/graphql.md +59 -0
- package/.opencode/skills/e2e-testing/references/issue-verification.md +59 -0
- package/.opencode/skills/e2e-testing/references/multi-pool.md +60 -0
- package/.opencode/skills/e2e-testing/references/network-debugging.md +62 -0
- package/.opencode/skills/e2e-testing/references/test-json-format.md +163 -0
- package/.opencode/skills/e2e-testing/references/troubleshooting.md +224 -0
- package/.opencode/skills/e2e-testing/references/variables.md +41 -0
- package/.opencode/skills/e2e-testing/references/visual-verification.md +89 -0
- package/OPENCODE.md +166 -0
- package/README.md +990 -296
- package/agents/test-analyzer.md +81 -0
- package/agents/test-creator.md +155 -0
- package/agents/test-improver.md +177 -0
- package/bin/cli.js +602 -22
- package/commands/create-test.md +65 -0
- package/commands/run.md +49 -0
- package/commands/verify-issue.md +63 -0
- package/opencode.json +11 -0
- package/package.json +15 -2
- package/scripts/setup-opencode.sh +113 -0
- package/skills/e2e-testing/SKILL.md +173 -0
- package/skills/e2e-testing/references/action-types.md +143 -0
- package/skills/e2e-testing/references/auth-strategies.md +91 -0
- package/skills/e2e-testing/references/graphql.md +59 -0
- package/skills/e2e-testing/references/issue-verification.md +59 -0
- package/skills/e2e-testing/references/multi-pool.md +60 -0
- package/skills/e2e-testing/references/network-debugging.md +62 -0
- package/skills/e2e-testing/references/test-json-format.md +163 -0
- package/skills/e2e-testing/references/troubleshooting.md +224 -0
- package/skills/e2e-testing/references/variables.md +41 -0
- package/skills/e2e-testing/references/visual-verification.md +89 -0
- package/src/actions.js +597 -20
- package/src/ai-generate.js +142 -12
- package/src/config.js +171 -0
- package/src/dashboard.js +299 -17
- package/src/db.js +335 -13
- package/src/index.js +15 -8
- package/src/learner-markdown.js +177 -0
- package/src/learner-neo4j.js +255 -0
- package/src/learner-sqlite.js +658 -0
- package/src/learner.js +418 -0
- package/src/mcp-tools.js +1558 -50
- package/src/module-resolver.js +310 -0
- package/src/narrate.js +262 -0
- package/src/neo4j-pool.js +124 -0
- package/src/pool-manager.js +223 -0
- package/src/reporter.js +117 -3
- package/src/runner.js +274 -71
- package/src/sync/auth.js +354 -0
- package/src/sync/client.js +572 -0
- package/src/sync/hub-routes.js +816 -0
- package/src/sync/index.js +68 -0
- package/src/sync/middleware.js +347 -0
- package/src/sync/queue.js +209 -0
- package/src/sync/schema.js +540 -0
- package/src/verify.js +14 -9
- package/src/watch.js +384 -0
- package/templates/build-dashboard.js +69 -0
- package/templates/dashboard/js/api.js +60 -0
- package/templates/dashboard/js/init.js +13 -0
- package/templates/dashboard/js/keyboard.js +46 -0
- package/templates/dashboard/js/state.js +40 -0
- package/templates/dashboard/js/toast.js +41 -0
- package/templates/dashboard/js/utils.js +196 -0
- package/templates/dashboard/js/view-live.js +143 -0
- package/templates/dashboard/js/view-runs.js +572 -0
- package/templates/dashboard/js/view-tests.js +294 -0
- package/templates/dashboard/js/view-watch.js +242 -0
- package/templates/dashboard/js/websocket.js +110 -0
- package/templates/dashboard/styles/base.css +69 -0
- package/templates/dashboard/styles/components.css +110 -0
- package/templates/dashboard/styles/view-live.css +74 -0
- package/templates/dashboard/styles/view-runs.css +207 -0
- package/templates/dashboard/styles/view-tests.css +96 -0
- package/templates/dashboard/styles/view-watch.css +53 -0
- package/templates/dashboard/template.html +267 -0
- package/templates/dashboard.html +2171 -530
- package/templates/docker-compose-neo4j.yml +19 -0
- package/templates/e2e.config.js +3 -0
- package/templates/sample-test.json +0 -8
package/README.md
CHANGED
|
@@ -1,13 +1,33 @@
|
|
|
1
|
+
<p align="right">
|
|
2
|
+
<strong>English</strong> · <a href="LEEME.md">Español</a>
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">@matware/e2e-runner</h1>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
<strong>The AI-native E2E test runner that writes, runs, and debugs tests for you.</strong>
|
|
9
|
+
</p>
|
|
10
|
+
|
|
1
11
|
<p align="center">
|
|
2
12
|
<img src="https://img.shields.io/npm/v/@matware/e2e-runner?color=blue" alt="npm version" />
|
|
3
13
|
<img src="https://img.shields.io/node/v/@matware/e2e-runner" alt="node version" />
|
|
4
14
|
<img src="https://img.shields.io/npm/l/@matware/e2e-runner" alt="license" />
|
|
5
15
|
<img src="https://img.shields.io/badge/MCP-compatible-green" alt="MCP compatible" />
|
|
16
|
+
<img src="https://img.shields.io/badge/AI--native-Claude%20Code-blueviolet" alt="AI native" />
|
|
17
|
+
<img src="https://img.shields.io/badge/AI--native-OpenCode-orange" alt="OpenCode compatible" />
|
|
18
|
+
</p>
|
|
19
|
+
|
|
20
|
+
<p align="center">
|
|
21
|
+
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-live-running.png" alt="E2E Runner Dashboard - Live Execution" width="800" />
|
|
6
22
|
</p>
|
|
7
23
|
|
|
8
|
-
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
**E2E Runner** is a zero-code browser testing framework where tests are plain JSON files — no Playwright scripts, no Cypress boilerplate, no test framework to learn. Define what to click, type, and assert, and the runner executes it in parallel against a shared Chrome pool.
|
|
27
|
+
|
|
28
|
+
But what makes it truly different is its **deep AI integration**. With a built-in [MCP server](https://modelcontextprotocol.io/), Claude Code can create tests from a conversation, run them, read the results, capture screenshots, and even visually verify that pages look correct — all without leaving the chat. Paste a GitHub issue URL and get a runnable test back. That's the workflow.
|
|
9
29
|
|
|
10
|
-
|
|
30
|
+
### This is a test
|
|
11
31
|
|
|
12
32
|
```json
|
|
13
33
|
[
|
|
@@ -25,62 +45,239 @@ JSON-driven E2E test runner. Define browser tests as simple JSON action arrays,
|
|
|
25
45
|
]
|
|
26
46
|
```
|
|
27
47
|
|
|
48
|
+
No imports. No `describe`/`it`. No compilation step. Just a JSON file that describes what a user does — and the runner makes it happen.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## Getting Started
|
|
53
|
+
|
|
54
|
+
### Prerequisites
|
|
55
|
+
|
|
56
|
+
- **Node.js** >= 20
|
|
57
|
+
- **Docker** running (for the Chrome pool)
|
|
58
|
+
- Your app running on a known port (e.g. `http://localhost:3000`)
|
|
59
|
+
|
|
60
|
+
> **Why `host.docker.internal`?**
|
|
61
|
+
>
|
|
62
|
+
> Chrome runs inside a Docker container. From inside the container, `localhost` refers to the container itself — not your machine. The special hostname `host.docker.internal` resolves to your host machine, so Chrome can reach your locally running app.
|
|
63
|
+
>
|
|
64
|
+
> The default `baseUrl` is `http://host.docker.internal:3000`. If your app runs on a different port, change it in `e2e.config.js` after init.
|
|
65
|
+
>
|
|
66
|
+
> **Linux note:** On Docker Engine (not Docker Desktop), you may need to add `--add-host=host.docker.internal:host-gateway` to the Docker run flags, or use your machine's LAN IP directly as the `baseUrl`.
|
|
67
|
+
|
|
28
68
|
---
|
|
29
69
|
|
|
30
|
-
|
|
70
|
+
### Path A: With Claude Code
|
|
31
71
|
|
|
32
|
-
|
|
33
|
-
- **Parallel** -- Run N tests simultaneously against a shared Chrome pool.
|
|
34
|
-
- **Portable** -- Chrome runs in Docker, tests run anywhere.
|
|
35
|
-
- **CI-ready** -- JUnit XML output, exit code 1 on failure, error screenshots.
|
|
36
|
-
- **AI-native** -- Built-in MCP server for Claude Code integration.
|
|
72
|
+
If you use [Claude Code](https://docs.anthropic.com/en/docs/claude-code), this is the fastest path — Claude handles test creation and debugging for you.
|
|
37
73
|
|
|
38
|
-
|
|
74
|
+
**1. Install the package**
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
npm install --save-dev @matware/e2e-runner
|
|
78
|
+
```
|
|
39
79
|
|
|
40
|
-
**
|
|
80
|
+
**2. Scaffold the project structure**
|
|
41
81
|
|
|
42
82
|
```bash
|
|
43
|
-
|
|
83
|
+
npx e2e-runner init
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
This creates `e2e/tests/` with a sample test and `e2e/screenshots/` for captures.
|
|
87
|
+
|
|
88
|
+
**3. Configure your base URL**
|
|
89
|
+
|
|
90
|
+
Edit `e2e.config.js` and set `baseUrl` to match your app's port:
|
|
91
|
+
|
|
92
|
+
```js
|
|
93
|
+
export default {
|
|
94
|
+
baseUrl: 'http://host.docker.internal:3000', // change 3000 to your port
|
|
95
|
+
};
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
**4. Start the Chrome pool**
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
npx e2e-runner pool start
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
You should see:
|
|
105
|
+
|
|
106
|
+
```
|
|
107
|
+
✓ Chrome pool started on port 3333 (max 3 sessions)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
**5. Install the Claude Code plugin**
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
# Add the marketplace (one-time)
|
|
114
|
+
claude plugin marketplace add fastslack/mtw-e2e-runner
|
|
115
|
+
|
|
116
|
+
# Install the plugin
|
|
117
|
+
claude plugin install e2e-runner@matware
|
|
44
118
|
```
|
|
45
119
|
|
|
46
|
-
|
|
120
|
+
The plugin gives Claude 13 MCP tools, a workflow skill, 3 slash commands, and 3 specialized agents.
|
|
47
121
|
|
|
48
|
-
**
|
|
122
|
+
**6. Ask Claude to run the sample test**
|
|
123
|
+
|
|
124
|
+
In Claude Code, just say:
|
|
125
|
+
|
|
126
|
+
> "Run all E2E tests"
|
|
127
|
+
|
|
128
|
+
Claude will check the pool, run the sample test, and report back:
|
|
129
|
+
|
|
130
|
+
```
|
|
131
|
+
==================================================
|
|
132
|
+
E2E RESULTS
|
|
133
|
+
==================================================
|
|
134
|
+
Total: 1
|
|
135
|
+
Passed: 1
|
|
136
|
+
Failed: 0
|
|
137
|
+
Rate: 100.00%
|
|
138
|
+
Duration: 1.23s
|
|
139
|
+
==================================================
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
From here, you can ask Claude to create new tests ("test the login flow"), debug failures, or verify GitHub issues.
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
### Path B: CLI Only
|
|
147
|
+
|
|
148
|
+
No AI required — use the runner directly from your terminal.
|
|
149
|
+
|
|
150
|
+
**1. Install the package**
|
|
49
151
|
|
|
50
152
|
```bash
|
|
51
|
-
# 1. Install
|
|
52
153
|
npm install --save-dev @matware/e2e-runner
|
|
154
|
+
```
|
|
53
155
|
|
|
54
|
-
|
|
156
|
+
**2. Scaffold the project structure**
|
|
157
|
+
|
|
158
|
+
```bash
|
|
55
159
|
npx e2e-runner init
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
This creates `e2e/tests/` with a sample test and `e2e/screenshots/` for captures.
|
|
163
|
+
|
|
164
|
+
**3. Configure your base URL**
|
|
165
|
+
|
|
166
|
+
Edit `e2e.config.js` and set `baseUrl` to match your app's port:
|
|
167
|
+
|
|
168
|
+
```js
|
|
169
|
+
export default {
|
|
170
|
+
baseUrl: 'http://host.docker.internal:3000', // change 3000 to your port
|
|
171
|
+
};
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
**4. Start the Chrome pool**
|
|
56
175
|
|
|
57
|
-
|
|
176
|
+
```bash
|
|
58
177
|
npx e2e-runner pool start
|
|
178
|
+
```
|
|
59
179
|
|
|
60
|
-
|
|
61
|
-
npx e2e-runner run --all
|
|
180
|
+
You should see:
|
|
62
181
|
|
|
63
|
-
|
|
64
|
-
|
|
182
|
+
```
|
|
183
|
+
✓ Chrome pool started on port 3333 (max 3 sessions)
|
|
65
184
|
```
|
|
66
185
|
|
|
67
|
-
**
|
|
186
|
+
**5. Run the sample test**
|
|
68
187
|
|
|
69
188
|
```bash
|
|
70
|
-
|
|
71
|
-
|
|
189
|
+
npx e2e-runner run --all
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Expected output:
|
|
193
|
+
|
|
194
|
+
```
|
|
195
|
+
==================================================
|
|
196
|
+
E2E RESULTS
|
|
197
|
+
==================================================
|
|
198
|
+
Total: 1
|
|
199
|
+
Passed: 1
|
|
200
|
+
Failed: 0
|
|
201
|
+
Rate: 100.00%
|
|
202
|
+
Duration: 1.23s
|
|
203
|
+
==================================================
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
A screenshot is saved at `e2e/screenshots/homepage.png`.
|
|
207
|
+
|
|
208
|
+
**6. Write your first real test**
|
|
209
|
+
|
|
210
|
+
Create `e2e/tests/my-first-test.json`:
|
|
211
|
+
|
|
212
|
+
```json
|
|
213
|
+
[
|
|
214
|
+
{
|
|
215
|
+
"name": "homepage-visible",
|
|
216
|
+
"actions": [
|
|
217
|
+
{ "type": "goto", "value": "/" },
|
|
218
|
+
{ "type": "assert_visible", "selector": "body" },
|
|
219
|
+
{ "type": "screenshot", "value": "my-first-test.png" }
|
|
220
|
+
]
|
|
221
|
+
}
|
|
222
|
+
]
|
|
72
223
|
```
|
|
73
224
|
|
|
74
|
-
|
|
225
|
+
Run it:
|
|
75
226
|
|
|
227
|
+
```bash
|
|
228
|
+
npx e2e-runner run --suite my-first-test
|
|
76
229
|
```
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
230
|
+
|
|
231
|
+
---
|
|
232
|
+
|
|
233
|
+
### One-liner quickstart
|
|
234
|
+
|
|
235
|
+
If you want to skip the step-by-step and get everything running in one command:
|
|
236
|
+
|
|
237
|
+
```bash
|
|
238
|
+
curl -fsSL https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/scripts/quickstart.sh | bash
|
|
82
239
|
```
|
|
83
240
|
|
|
241
|
+
> This installs the package, scaffolds the project, and starts the Chrome pool. You'll still need to configure your `baseUrl` afterwards.
|
|
242
|
+
|
|
243
|
+
### What's next?
|
|
244
|
+
|
|
245
|
+
- [Test Format](#test-format) — learn the full action vocabulary
|
|
246
|
+
- [Claude Code Integration](#claude-code-integration) — set up AI-powered testing
|
|
247
|
+
- [Visual Verification](#visual-verification) — describe expected pages in plain English
|
|
248
|
+
- [Issue-to-Test](#issue-to-test) — turn bug reports into executable tests
|
|
249
|
+
- [Web Dashboard](#web-dashboard) — monitor tests in real time
|
|
250
|
+
|
|
251
|
+
---
|
|
252
|
+
|
|
253
|
+
## What you get
|
|
254
|
+
|
|
255
|
+
🧪 **Zero-code tests** — JSON files that anyone on your team can read and write. No JavaScript, no compilation, no framework lock-in.
|
|
256
|
+
|
|
257
|
+
🤖 **AI-powered testing** — Claude Code creates, executes, and debugs tests natively through 13 MCP tools. Ask it to "test the checkout flow" and it builds the JSON, runs it, and reports back.
|
|
258
|
+
|
|
259
|
+
🐛 **Issue-to-Test pipeline** — Paste a GitHub or GitLab issue URL. The runner fetches it, generates E2E tests, runs them, and tells you: *bug confirmed* or *not reproducible*.
|
|
260
|
+
|
|
261
|
+
👁️ **Visual verification** — Describe what the page should look like in plain English. The AI captures a screenshot and judges pass/fail against your description. No pixel-diffing setup needed.
|
|
262
|
+
|
|
263
|
+
🧠 **Learning system** — Tracks test stability across runs. Detects flaky tests, unstable selectors, slow APIs, and error patterns — then surfaces actionable insights.
|
|
264
|
+
|
|
265
|
+
⚡ **Parallel execution** — Run N tests simultaneously against a shared Chrome pool (browserless/chrome). Serial mode available for tests that share state.
|
|
266
|
+
|
|
267
|
+
📊 **Real-time dashboard** — Live execution view, run history with pass-rate charts, screenshot gallery with hash-based search, expandable network request logs.
|
|
268
|
+
|
|
269
|
+
🔁 **Smart retries** — Test-level and action-level retries with configurable delays. Flaky tests are detected and flagged automatically.
|
|
270
|
+
|
|
271
|
+
📦 **Reusable modules** — Extract common flows (login, navigation, setup) into parameterized modules and reference them with `$use`.
|
|
272
|
+
|
|
273
|
+
🏗️ **CI-ready** — JUnit XML output, exit code 1 on failure, auto-captured error screenshots. Drop-in GitHub Actions example included.
|
|
274
|
+
|
|
275
|
+
🌐 **Multi-project** — One dashboard aggregates test results from all your projects. One Chrome pool serves them all.
|
|
276
|
+
|
|
277
|
+
🐳 **Portable** — Chrome runs in Docker, tests are JSON files in your repo. Works on any machine with Node.js and Docker.
|
|
278
|
+
|
|
279
|
+
---
|
|
280
|
+
|
|
84
281
|
## Test Format
|
|
85
282
|
|
|
86
283
|
Each `.json` file in `e2e/tests/` contains an array of tests. Each test has a `name` and sequential `actions`:
|
|
@@ -91,8 +288,7 @@ Each `.json` file in `e2e/tests/` contains an array of tests. Each test has a `n
|
|
|
91
288
|
"name": "homepage-loads",
|
|
92
289
|
"actions": [
|
|
93
290
|
{ "type": "goto", "value": "/" },
|
|
94
|
-
{ "type": "
|
|
95
|
-
{ "type": "assert_text", "text": "Welcome" },
|
|
291
|
+
{ "type": "assert_visible", "selector": "body" },
|
|
96
292
|
{ "type": "assert_url", "value": "/" },
|
|
97
293
|
{ "type": "screenshot", "value": "homepage.png" }
|
|
98
294
|
]
|
|
@@ -110,346 +306,536 @@ Suite files can have numeric prefixes for ordering (`01-auth.json`, `02-dashboar
|
|
|
110
306
|
| `click` | `selector` or `text` | Click by CSS selector or visible text content |
|
|
111
307
|
| `type` / `fill` | `selector`, `value` | Clear field and type text |
|
|
112
308
|
| `wait` | `selector`, `text`, or `value` (ms) | Wait for element, text, or fixed delay |
|
|
113
|
-
| `assert_text` | `text` | Assert text exists on the page |
|
|
114
|
-
| `assert_url` | `value` | Assert current URL contains value |
|
|
115
|
-
| `assert_visible` | `selector` | Assert element is visible |
|
|
116
|
-
| `assert_count` | `selector`, `value` | Assert element count matches |
|
|
117
309
|
| `screenshot` | `value` (filename) | Capture a screenshot |
|
|
118
310
|
| `select` | `selector`, `value` | Select a dropdown option |
|
|
119
311
|
| `clear` | `selector` | Clear an input field |
|
|
120
|
-
| `press` | `value` | Press a keyboard key (
|
|
312
|
+
| `press` | `value` | Press a keyboard key (`Enter`, `Tab`, etc.) |
|
|
121
313
|
| `scroll` | `selector` or `value` (px) | Scroll to element or by pixel amount |
|
|
122
314
|
| `hover` | `selector` | Hover over an element |
|
|
123
315
|
| `evaluate` | `value` | Execute JavaScript in the browser context |
|
|
316
|
+
| `navigate` | `value` | Browser navigation (`back`, `forward`, `reload`) |
|
|
317
|
+
| `clear_cookies` | — | Clear all cookies for the current page |
|
|
318
|
+
|
|
319
|
+
### Assertions
|
|
320
|
+
|
|
321
|
+
| Action | Fields | Description |
|
|
322
|
+
|--------|--------|-------------|
|
|
323
|
+
| `assert_text` | `text` | Assert text exists anywhere on the page (substring) |
|
|
324
|
+
| `assert_element_text` | `selector`, `text`, optional `value: "exact"` | Assert element's text contains (or exactly matches) the expected text |
|
|
325
|
+
| `assert_url` | `value` | Assert current URL path or full URL. Paths (`/dashboard`) compare against pathname only |
|
|
326
|
+
| `assert_visible` | `selector` | Assert element exists and is visible |
|
|
327
|
+
| `assert_not_visible` | `selector` | Assert element is hidden or doesn't exist |
|
|
328
|
+
| `assert_attribute` | `selector`, `value` | Check attribute: `"type=email"` for value, `"disabled"` for existence |
|
|
329
|
+
| `assert_class` | `selector`, `value` | Assert element has a CSS class |
|
|
330
|
+
| `assert_input_value` | `selector`, `value` | Assert input/select/textarea `.value` contains text |
|
|
331
|
+
| `assert_matches` | `selector`, `value` (regex) | Assert element text matches a regex pattern |
|
|
332
|
+
| `assert_count` | `selector`, `value` | Assert element count: exact (`"5"`), or operators (`">3"`, `">=1"`, `"<10"`) |
|
|
333
|
+
| `assert_no_network_errors` | — | Fail if any network requests failed (e.g. `ERR_CONNECTION_REFUSED`) |
|
|
334
|
+
| `get_text` | `selector` | Extract element text (non-assertion, never fails). Result: `{ value: "..." }` |
|
|
124
335
|
|
|
125
336
|
### Click by Text
|
|
126
337
|
|
|
127
|
-
When `click` uses `text` instead of `selector`, it searches across interactive elements:
|
|
338
|
+
When `click` uses `text` instead of `selector`, it searches across common interactive and content elements:
|
|
128
339
|
|
|
129
340
|
```
|
|
130
|
-
button, a, [role="button"], [role="tab"], [role="menuitem"],
|
|
341
|
+
button, a, [role="button"], [role="tab"], [role="menuitem"], [role="option"],
|
|
342
|
+
[role="listitem"], div[class*="cursor"], span, li, td, th, label, p, h1-h6
|
|
131
343
|
```
|
|
132
344
|
|
|
133
345
|
```json
|
|
134
346
|
{ "type": "click", "text": "Sign In" }
|
|
135
347
|
```
|
|
136
348
|
|
|
137
|
-
|
|
349
|
+
### Framework-Aware Actions
|
|
138
350
|
|
|
139
|
-
|
|
140
|
-
# Run tests
|
|
141
|
-
npx e2e-runner run --all # All suites
|
|
142
|
-
npx e2e-runner run --suite auth # Single suite
|
|
143
|
-
npx e2e-runner run --tests path/to.json # Specific file
|
|
144
|
-
npx e2e-runner run --inline '<json>' # Inline JSON
|
|
351
|
+
These actions handle common patterns in React/MUI apps that normally require verbose `evaluate` boilerplate:
|
|
145
352
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
353
|
+
| Action | Fields | Description |
|
|
354
|
+
|--------|--------|-------------|
|
|
355
|
+
| `type_react` | `selector`, `value` | Type into React controlled inputs using the native value setter. Dispatches `input` + `change` events so React state updates correctly. |
|
|
356
|
+
| `click_regex` | `text` (regex), optional `selector`, optional `value: "last"` | Click element whose textContent matches a regex (case-insensitive). Default: first match. Use `value: "last"` for last match. |
|
|
357
|
+
| `click_option` | `text` | Click a `[role="option"]` element by text — common in autocomplete/select dropdowns. |
|
|
358
|
+
| `focus_autocomplete` | `text` (label text) | Focus an autocomplete input by its label text. Supports MUI and generic `[role="combobox"]`. |
|
|
359
|
+
| `click_chip` | `text` | Click a chip/tag element by text. Searches `[class*="Chip"]`, `[class*="chip"]`, `[data-chip]`. |
|
|
150
360
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
npx e2e-runner issue <url> --verify # Generate + run + report
|
|
361
|
+
```json
|
|
362
|
+
// Before: 5 lines of evaluate boilerplate
|
|
363
|
+
{ "type": "evaluate", "value": "const input = document.querySelector('#search'); const nativeSet = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set; nativeSet.call(input, 'term'); input.dispatchEvent(new Event('input', {bubbles: true})); input.dispatchEvent(new Event('change', {bubbles: true}));" }
|
|
155
364
|
|
|
156
|
-
|
|
157
|
-
|
|
365
|
+
// After: 1 action
|
|
366
|
+
{ "type": "type_react", "selector": "#search", "value": "term" }
|
|
367
|
+
```
|
|
158
368
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
369
|
+
---
|
|
370
|
+
|
|
371
|
+
## Retries
|
|
372
|
+
|
|
373
|
+
### Test-Level Retry
|
|
374
|
+
|
|
375
|
+
Retry an entire test on failure. Set globally via config or per-test:
|
|
376
|
+
|
|
377
|
+
```json
|
|
378
|
+
{ "name": "flaky-test", "retries": 3, "timeout": 15000, "actions": [...] }
|
|
162
379
|
```
|
|
163
380
|
|
|
164
|
-
|
|
381
|
+
Tests that pass after retry are flagged as **flaky** in the report and learning system.
|
|
165
382
|
|
|
166
|
-
|
|
167
|
-
|------|---------|-------------|
|
|
168
|
-
| `--base-url <url>` | `http://host.docker.internal:3000` | Application base URL |
|
|
169
|
-
| `--pool-url <ws>` | `ws://localhost:3333` | Chrome pool WebSocket URL |
|
|
170
|
-
| `--tests-dir <dir>` | `e2e/tests` | Tests directory |
|
|
171
|
-
| `--screenshots-dir <dir>` | `e2e/screenshots` | Screenshots/reports directory |
|
|
172
|
-
| `--concurrency <n>` | `3` | Parallel test workers |
|
|
173
|
-
| `--timeout <ms>` | `10000` | Default action timeout |
|
|
174
|
-
| `--retries <n>` | `0` | Retry failed tests N times |
|
|
175
|
-
| `--retry-delay <ms>` | `1000` | Delay between retries |
|
|
176
|
-
| `--test-timeout <ms>` | `60000` | Per-test timeout |
|
|
177
|
-
| `--output <format>` | `json` | Report format: `json`, `junit`, `both` |
|
|
178
|
-
| `--env <name>` | `default` | Environment profile |
|
|
179
|
-
| `--pool-port <port>` | `3333` | Chrome pool port |
|
|
180
|
-
| `--max-sessions <n>` | `10` | Max concurrent Chrome sessions |
|
|
181
|
-
| `--project-name <name>` | dir name | Project display name for dashboard |
|
|
383
|
+
### Action-Level Retry
|
|
182
384
|
|
|
183
|
-
|
|
385
|
+
Retry a single action without rerunning the entire test. Useful for timing-sensitive clicks and waits:
|
|
184
386
|
|
|
185
|
-
|
|
387
|
+
```json
|
|
388
|
+
{ "type": "click", "selector": "#dynamic-btn", "retries": 3 }
|
|
389
|
+
{ "type": "wait", "selector": ".lazy-loaded", "retries": 2 }
|
|
390
|
+
```
|
|
186
391
|
|
|
187
|
-
|
|
188
|
-
export default {
|
|
189
|
-
baseUrl: 'http://host.docker.internal:3000',
|
|
190
|
-
concurrency: 4,
|
|
191
|
-
retries: 2,
|
|
192
|
-
testTimeout: 30000,
|
|
193
|
-
outputFormat: 'both',
|
|
392
|
+
Set globally: `actionRetries` in config, `--action-retries <n>` CLI, or `ACTION_RETRIES` env var. Delay between retries: `actionRetryDelay` (default 500ms).
|
|
194
393
|
|
|
195
|
-
|
|
196
|
-
beforeEach: [{ type: 'goto', value: '/' }],
|
|
197
|
-
afterEach: [{ type: 'screenshot', value: 'after-test.png' }],
|
|
198
|
-
},
|
|
394
|
+
---
|
|
199
395
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
396
|
+
## Serial Tests
|
|
397
|
+
|
|
398
|
+
Tests that share state (e.g., two tests modifying the same record) can race when running in parallel. Mark them as serial:
|
|
399
|
+
|
|
400
|
+
```json
|
|
401
|
+
{ "name": "create-patient", "serial": true, "actions": [...] }
|
|
402
|
+
{ "name": "verify-patient-list", "serial": true, "actions": [...] }
|
|
205
403
|
```
|
|
206
404
|
|
|
207
|
-
|
|
405
|
+
Serial tests run one at a time **after** all parallel tests finish — preventing interference without slowing down independent tests.
|
|
208
406
|
|
|
209
|
-
|
|
210
|
-
2. Environment variables (`BASE_URL`, `CONCURRENCY`, ...)
|
|
211
|
-
3. Config file (`e2e.config.js` or `e2e.config.json`)
|
|
212
|
-
4. Defaults
|
|
407
|
+
---
|
|
213
408
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
### Environment Variables
|
|
217
|
-
|
|
218
|
-
| Variable | Maps to |
|
|
219
|
-
|----------|---------|
|
|
220
|
-
| `BASE_URL` | `baseUrl` |
|
|
221
|
-
| `CHROME_POOL_URL` | `poolUrl` |
|
|
222
|
-
| `TESTS_DIR` | `testsDir` |
|
|
223
|
-
| `SCREENSHOTS_DIR` | `screenshotsDir` |
|
|
224
|
-
| `CONCURRENCY` | `concurrency` |
|
|
225
|
-
| `DEFAULT_TIMEOUT` | `defaultTimeout` |
|
|
226
|
-
| `POOL_PORT` | `poolPort` |
|
|
227
|
-
| `MAX_SESSIONS` | `maxSessions` |
|
|
228
|
-
| `RETRIES` | `retries` |
|
|
229
|
-
| `RETRY_DELAY` | `retryDelay` |
|
|
230
|
-
| `TEST_TIMEOUT` | `testTimeout` |
|
|
231
|
-
| `OUTPUT_FORMAT` | `outputFormat` |
|
|
232
|
-
| `E2E_ENV` | `env` |
|
|
233
|
-
| `PROJECT_NAME` | `projectName` |
|
|
234
|
-
| `ANTHROPIC_API_KEY` | `anthropicApiKey` |
|
|
235
|
-
| `ANTHROPIC_MODEL` | `anthropicModel` |
|
|
409
|
+
## Testing Authenticated Apps
|
|
236
410
|
|
|
237
|
-
|
|
411
|
+
Most real-world apps require login before tests can interact with protected pages. E2E Runner provides multiple strategies — choose the one that matches your app's auth mechanism.
|
|
412
|
+
|
|
413
|
+
### Strategy 1: UI Login Flow (any app)
|
|
238
414
|
|
|
239
|
-
|
|
415
|
+
The most universal approach — fill in the login form like a real user. Works with **any** authentication system (session cookies, JWT, OAuth redirect, etc.):
|
|
240
416
|
|
|
241
417
|
```json
|
|
242
418
|
{
|
|
243
419
|
"hooks": {
|
|
244
|
-
"
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
420
|
+
"beforeEach": [
|
|
421
|
+
{ "type": "goto", "value": "/login" },
|
|
422
|
+
{ "type": "type", "selector": "#email", "value": "test@example.com" },
|
|
423
|
+
{ "type": "type", "selector": "#password", "value": "test-password" },
|
|
424
|
+
{ "type": "click", "text": "Sign In" },
|
|
425
|
+
{ "type": "wait", "selector": ".dashboard" }
|
|
426
|
+
]
|
|
248
427
|
},
|
|
249
428
|
"tests": [
|
|
250
|
-
{
|
|
429
|
+
{
|
|
430
|
+
"name": "profile-page",
|
|
431
|
+
"actions": [
|
|
432
|
+
{ "type": "goto", "value": "/profile" },
|
|
433
|
+
{ "type": "assert_text", "text": "My Profile" }
|
|
434
|
+
]
|
|
435
|
+
}
|
|
251
436
|
]
|
|
252
437
|
}
|
|
253
438
|
```
|
|
254
439
|
|
|
255
|
-
|
|
440
|
+
> **When to use:** You don't know or care how auth works internally. The browser handles cookies/tokens automatically after login — just like a real user.
|
|
256
441
|
|
|
257
|
-
|
|
442
|
+
### Strategy 2: JWT Token Injection (SPAs)
|
|
258
443
|
|
|
259
|
-
|
|
444
|
+
For single-page apps that store JWT tokens in `localStorage` or `sessionStorage`. Skip the login form entirely by injecting the token directly:
|
|
260
445
|
|
|
261
446
|
```json
|
|
262
447
|
{
|
|
263
|
-
"
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
448
|
+
"hooks": {
|
|
449
|
+
"beforeEach": [
|
|
450
|
+
{ "type": "goto", "value": "/" },
|
|
451
|
+
{ "type": "set_storage", "value": "accessToken=eyJhbGciOiJIUzI1NiIs..." },
|
|
452
|
+
{ "type": "goto", "value": "/dashboard" },
|
|
453
|
+
{ "type": "wait", "selector": ".dashboard-loaded" }
|
|
454
|
+
]
|
|
455
|
+
},
|
|
456
|
+
"tests": [...]
|
|
267
457
|
}
|
|
268
458
|
```
|
|
269
459
|
|
|
270
|
-
|
|
271
|
-
- **Timeout**: Applied via `Promise.race()`. Defaults to 60s.
|
|
460
|
+
**Common storage key names** (depends on your app):
|
|
272
461
|
|
|
273
|
-
|
|
462
|
+
| Framework / Library | Typical key | Storage |
|
|
463
|
+
|---------------------|-------------|---------|
|
|
464
|
+
| Custom JWT | `accessToken`, `token`, `jwt` | localStorage |
|
|
465
|
+
| Auth0 SPA SDK | `@@auth0spajs@@::*` | localStorage |
|
|
466
|
+
| Firebase Auth | `firebase:authUser:*` | localStorage |
|
|
467
|
+
| AWS Amplify | `CognitoIdentityServiceProvider.*` | localStorage |
|
|
468
|
+
| Supabase | `sb-<ref>-auth-token` | localStorage |
|
|
469
|
+
| NextAuth (client) | `next-auth.session-token` | cookie (see Strategy 4) |
|
|
274
470
|
|
|
275
|
-
|
|
471
|
+
**Using `sessionStorage` instead:**
|
|
276
472
|
|
|
277
|
-
```
|
|
278
|
-
|
|
279
|
-
# or: --output both (JSON + XML)
|
|
473
|
+
```json
|
|
474
|
+
{ "type": "set_storage", "value": "token=eyJhbG...", "selector": "session" }
|
|
280
475
|
```
|
|
281
476
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
### GitHub Actions
|
|
477
|
+
**Asserting the token was stored correctly:**
|
|
285
478
|
|
|
286
|
-
```
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
runs-on: ubuntu-latest
|
|
290
|
-
steps:
|
|
291
|
-
- uses: actions/checkout@v4
|
|
292
|
-
- uses: actions/setup-node@v4
|
|
293
|
-
with:
|
|
294
|
-
node-version: 20
|
|
295
|
-
- run: npm ci
|
|
296
|
-
- run: npx e2e-runner pool start
|
|
297
|
-
- run: npx e2e-runner run --all --output junit
|
|
298
|
-
- uses: mikepenz/action-junit-report@v4
|
|
299
|
-
if: always()
|
|
300
|
-
with:
|
|
301
|
-
report_paths: e2e/screenshots/junit.xml
|
|
479
|
+
```json
|
|
480
|
+
{ "type": "assert_storage", "value": "accessToken" }
|
|
481
|
+
{ "type": "assert_storage", "value": "accessToken=eyJhbG..." }
|
|
302
482
|
```
|
|
303
483
|
|
|
304
|
-
|
|
484
|
+
> **When to use:** Your SPA reads auth tokens from browser storage. Fastest strategy — no network round-trip for login.
|
|
305
485
|
|
|
306
|
-
|
|
307
|
-
|------|---------|
|
|
308
|
-
| `0` | All tests passed |
|
|
309
|
-
| `1` | One or more tests failed |
|
|
486
|
+
### Strategy 3: Config-Level Auth Token
|
|
310
487
|
|
|
311
|
-
|
|
488
|
+
For apps where every test needs the same JWT token. Set it once in config — it's injected into `localStorage` before every `e2e_capture` and `e2e_issue --verify` run:
|
|
312
489
|
|
|
313
490
|
```js
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
// Run a specific suite
|
|
322
|
-
const report = await runner.runSuite('auth');
|
|
491
|
+
// e2e.config.js
|
|
492
|
+
export default {
|
|
493
|
+
authToken: 'eyJhbGciOiJIUzI1NiIs...',
|
|
494
|
+
authStorageKey: 'accessToken', // default
|
|
495
|
+
};
|
|
496
|
+
```
|
|
323
497
|
|
|
324
|
-
|
|
325
|
-
const report = await runner.runFile('e2e/tests/login.json');
|
|
498
|
+
Or via environment variables:
|
|
326
499
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
{
|
|
330
|
-
name: 'quick-check',
|
|
331
|
-
actions: [
|
|
332
|
-
{ type: 'goto', value: '/' },
|
|
333
|
-
{ type: 'assert_text', text: 'Hello' },
|
|
334
|
-
],
|
|
335
|
-
},
|
|
336
|
-
]);
|
|
500
|
+
```bash
|
|
501
|
+
AUTH_TOKEN="eyJhbGciOiJIUzI1NiIs..." npx e2e-runner run --all
|
|
337
502
|
```
|
|
338
503
|
|
|
339
|
-
|
|
504
|
+
Or via CLI:
|
|
340
505
|
|
|
341
|
-
```
|
|
342
|
-
|
|
343
|
-
loadConfig,
|
|
344
|
-
waitForPool, connectToPool, getPoolStatus, startPool, stopPool,
|
|
345
|
-
runTest, runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites,
|
|
346
|
-
generateReport, generateJUnitXML, saveReport, printReport,
|
|
347
|
-
executeAction,
|
|
348
|
-
} from '@matware/e2e-runner';
|
|
506
|
+
```bash
|
|
507
|
+
npx e2e-runner run --all --auth-token "eyJhbG..." --auth-storage-key "jwt"
|
|
349
508
|
```
|
|
350
509
|
|
|
351
|
-
|
|
510
|
+
MCP tools (`e2e_capture`, `e2e_issue`) also accept `authToken` and `authStorageKey` per call.
|
|
352
511
|
|
|
353
|
-
|
|
512
|
+
> **When to use:** All tests share the same user session and your app uses JWT in localStorage.
|
|
354
513
|
|
|
355
|
-
|
|
514
|
+
### Strategy 4: Cookie-Based Auth (server-rendered apps)
|
|
356
515
|
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
516
|
+
For apps that use HTTP cookies (Rails, Django, Laravel, Express sessions, NextAuth, etc.). Use `evaluate` to set cookies before navigating:
|
|
517
|
+
|
|
518
|
+
```json
|
|
519
|
+
{
|
|
520
|
+
"hooks": {
|
|
521
|
+
"beforeEach": [
|
|
522
|
+
{ "type": "goto", "value": "/" },
|
|
523
|
+
{ "type": "evaluate", "value": "document.cookie = 'session_id=abc123; path=/; SameSite=Lax'" },
|
|
524
|
+
{ "type": "goto", "value": "/dashboard" }
|
|
525
|
+
]
|
|
526
|
+
},
|
|
527
|
+
"tests": [...]
|
|
528
|
+
}
|
|
360
529
|
```
|
|
361
530
|
|
|
362
|
-
**
|
|
531
|
+
**Multiple cookies:**
|
|
363
532
|
|
|
364
|
-
```
|
|
365
|
-
|
|
366
|
-
-- docker run -i --rm fastslack/e2e-runner-mcp
|
|
533
|
+
```json
|
|
534
|
+
{ "type": "evaluate", "value": "document.cookie = 'session_id=abc123; path=/'; document.cookie = '_csrf_token=xyz789; path=/'" }
|
|
367
535
|
```
|
|
368
536
|
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
| Tool | Description |
|
|
372
|
-
|------|-------------|
|
|
373
|
-
| `e2e_run` | Run tests (all suites, by suite name, or by file path) |
|
|
374
|
-
| `e2e_list` | List available test suites with test names and counts |
|
|
375
|
-
| `e2e_create_test` | Create a new test JSON file |
|
|
376
|
-
| `e2e_pool_status` | Check Chrome pool availability and capacity |
|
|
377
|
-
| `e2e_screenshot` | Retrieve a screenshot by its hash (e.g. `ss:a3f2b1c9`) |
|
|
378
|
-
| `e2e_issue` | Fetch a GitHub/GitLab issue and generate E2E tests |
|
|
537
|
+
**For `HttpOnly` cookies** (can't be set via JavaScript), use the UI login strategy instead — the browser will store them automatically.
|
|
379
538
|
|
|
380
|
-
> **
|
|
539
|
+
> **When to use:** Traditional server-rendered apps, or any app that authenticates via cookies.
|
|
381
540
|
|
|
382
|
-
|
|
541
|
+
### Strategy 5: HTTP Header Auth (API tests)
|
|
383
542
|
|
|
384
|
-
|
|
543
|
+
For API testing where you need to send `Authorization` headers with every request. Use `evaluate` to override `fetch`/`XMLHttpRequest`:
|
|
385
544
|
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
545
|
+
```json
|
|
546
|
+
{
|
|
547
|
+
"hooks": {
|
|
548
|
+
"beforeEach": [
|
|
549
|
+
{ "type": "goto", "value": "/" },
|
|
550
|
+
{ "type": "evaluate", "value": "const origFetch = window.fetch; window.fetch = (url, opts = {}) => { opts.headers = { ...opts.headers, 'Authorization': 'Bearer eyJhbG...' }; return origFetch(url, opts); }" }
|
|
551
|
+
]
|
|
552
|
+
},
|
|
553
|
+
"tests": [
|
|
554
|
+
{
|
|
555
|
+
"name": "api-returns-user",
|
|
556
|
+
"actions": [
|
|
557
|
+
{ "type": "evaluate", "value": "const res = await fetch('/api/me'); const data = await res.json(); if (data.email !== 'test@example.com') throw new Error('Wrong user: ' + data.email)" }
|
|
558
|
+
]
|
|
559
|
+
}
|
|
560
|
+
]
|
|
561
|
+
}
|
|
562
|
+
```
|
|
563
|
+
|
|
564
|
+
> **When to use:** API-level tests (with `--test-type api`) that need auth headers.
|
|
389
565
|
|
|
390
|
-
###
|
|
566
|
+
### Strategy 6: OAuth / SSO (external provider)
|
|
567
|
+
|
|
568
|
+
OAuth flows redirect to external providers (Google, GitHub, Okta, etc.) which can't be automated reliably. Common workarounds:
|
|
569
|
+
|
|
570
|
+
**Option A — Test environment bypass:** Most apps have a direct login endpoint for testing that skips OAuth:
|
|
571
|
+
|
|
572
|
+
```json
|
|
573
|
+
{ "type": "goto", "value": "/auth/test-login?user=test@example.com" }
|
|
574
|
+
```
|
|
575
|
+
|
|
576
|
+
**Option B — Pre-authenticated token:** Get a token from your auth provider's API and inject it:
|
|
577
|
+
|
|
578
|
+
```json
|
|
579
|
+
{
|
|
580
|
+
"hooks": {
|
|
581
|
+
"beforeEach": [
|
|
582
|
+
{ "type": "goto", "value": "/" },
|
|
583
|
+
{ "type": "set_storage", "value": "oidc.user:https://auth.example.com:client_id={\"access_token\":\"...\"}" }
|
|
584
|
+
]
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
```
|
|
588
|
+
|
|
589
|
+
**Option C — Session cookie from CI:** If your CI can authenticate via API, pass the session cookie as an env var:
|
|
391
590
|
|
|
392
591
|
```bash
|
|
393
|
-
|
|
394
|
-
|
|
592
|
+
SESSION=$(curl -s -c - https://api.example.com/auth/login -d '{"email":"test@example.com","password":"secret"}' | grep session_id | awk '{print $NF}')
|
|
593
|
+
AUTH_TOKEN="$SESSION" AUTH_STORAGE_KEY="session_id" npx e2e-runner run --all
|
|
395
594
|
```
|
|
396
595
|
|
|
397
|
-
|
|
596
|
+
> **When to use:** Apps with Google/GitHub/Okta/Auth0 login. You almost always need a test-environment backdoor.
|
|
597
|
+
|
|
598
|
+
### Reusable Auth Modules
|
|
599
|
+
|
|
600
|
+
Extract your auth strategy into a module so every test can reference it without duplication:
|
|
601
|
+
|
|
602
|
+
```json
|
|
603
|
+
// e2e/modules/login.json — UI login (universal)
|
|
604
|
+
{
|
|
605
|
+
"$module": "login",
|
|
606
|
+
"description": "Log in via the UI login form",
|
|
607
|
+
"params": {
|
|
608
|
+
"email": { "required": true, "description": "User email" },
|
|
609
|
+
"password": { "required": true, "description": "User password" },
|
|
610
|
+
"redirectTo": { "default": "/dashboard", "description": "Page to land on after login" }
|
|
611
|
+
},
|
|
612
|
+
"actions": [
|
|
613
|
+
{ "type": "goto", "value": "/login" },
|
|
614
|
+
{ "type": "type", "selector": "#email", "value": "{{email}}" },
|
|
615
|
+
{ "type": "type", "selector": "#password", "value": "{{password}}" },
|
|
616
|
+
{ "type": "click", "text": "Sign In" },
|
|
617
|
+
{ "type": "wait", "selector": "{{redirectTo}}" }
|
|
618
|
+
]
|
|
619
|
+
}
|
|
620
|
+
```
|
|
621
|
+
|
|
622
|
+
```json
|
|
623
|
+
// e2e/modules/auth-token.json — JWT injection (SPAs)
|
|
624
|
+
{
|
|
625
|
+
"$module": "auth-token",
|
|
626
|
+
"description": "Inject an auth token into browser storage",
|
|
627
|
+
"params": {
|
|
628
|
+
"token": { "required": true, "description": "JWT or session token" },
|
|
629
|
+
"storageKey": { "default": "accessToken", "description": "Storage key name" },
|
|
630
|
+
"storage": { "default": "local", "description": "local or session" },
|
|
631
|
+
"redirectTo": { "default": "/dashboard", "description": "Page to navigate to after injection" }
|
|
632
|
+
},
|
|
633
|
+
"actions": [
|
|
634
|
+
{ "type": "goto", "value": "/" },
|
|
635
|
+
{ "type": "set_storage", "value": "{{storageKey}}={{token}}", "selector": "{{#storage}}{{storage}}{{/storage}}" },
|
|
636
|
+
{ "type": "goto", "value": "{{redirectTo}}" }
|
|
637
|
+
]
|
|
638
|
+
}
|
|
639
|
+
```
|
|
640
|
+
|
|
641
|
+
Use in tests:
|
|
642
|
+
|
|
643
|
+
```json
|
|
644
|
+
// UI login
|
|
645
|
+
{ "$use": "login", "params": { "email": "admin@test.com", "password": "secret" } }
|
|
646
|
+
|
|
647
|
+
// Token injection
|
|
648
|
+
{ "$use": "auth-token", "params": { "token": "eyJhbG..." } }
|
|
649
|
+
|
|
650
|
+
// Token in sessionStorage, redirect to /settings
|
|
651
|
+
{ "$use": "auth-token", "params": { "token": "eyJhbG...", "storage": "session", "redirectTo": "/settings" } }
|
|
652
|
+
```
|
|
653
|
+
|
|
654
|
+
### Testing Different User Roles
|
|
655
|
+
|
|
656
|
+
Use separate tests (or the same module with different credentials) to test role-based access:
|
|
657
|
+
|
|
658
|
+
```json
|
|
659
|
+
[
|
|
660
|
+
{
|
|
661
|
+
"name": "admin-sees-settings",
|
|
662
|
+
"actions": [
|
|
663
|
+
{ "$use": "login", "params": { "email": "admin@test.com", "password": "admin-pass" } },
|
|
664
|
+
{ "type": "goto", "value": "/settings" },
|
|
665
|
+
{ "type": "assert_visible", "selector": ".admin-panel" }
|
|
666
|
+
]
|
|
667
|
+
},
|
|
668
|
+
{
|
|
669
|
+
"name": "viewer-cannot-access-settings",
|
|
670
|
+
"actions": [
|
|
671
|
+
{ "$use": "login", "params": { "email": "viewer@test.com", "password": "viewer-pass" } },
|
|
672
|
+
{ "type": "goto", "value": "/settings" },
|
|
673
|
+
{ "type": "assert_text", "text": "Access Denied" }
|
|
674
|
+
]
|
|
675
|
+
}
|
|
676
|
+
]
|
|
677
|
+
```
|
|
678
|
+
|
|
679
|
+
### Clearing Auth State
|
|
680
|
+
|
|
681
|
+
Each test runs in a **fresh browser context** (new connection to the Chrome pool), so cookies and storage are automatically clean. If you need to explicitly clear state mid-test:
|
|
682
|
+
|
|
683
|
+
```json
|
|
684
|
+
{ "type": "clear_cookies" }
|
|
685
|
+
```
|
|
686
|
+
|
|
687
|
+
This clears cookies, localStorage, and sessionStorage for the current origin.
|
|
688
|
+
|
|
689
|
+
### Quick Reference
|
|
690
|
+
|
|
691
|
+
| Auth type | Strategy | Key actions |
|
|
692
|
+
|-----------|----------|-------------|
|
|
693
|
+
| Username/password form | UI Login | `goto` + `type` + `click` in `beforeEach` |
|
|
694
|
+
| JWT in localStorage | Token Injection | `set_storage` in `beforeEach` |
|
|
695
|
+
| JWT in sessionStorage | Token Injection | `set_storage` with `selector: "session"` |
|
|
696
|
+
| Session cookies | Cookie | `evaluate` to set `document.cookie` |
|
|
697
|
+
| HttpOnly cookies | UI Login | Must go through login form |
|
|
698
|
+
| OAuth / SSO | Test bypass | App-specific test login endpoint |
|
|
699
|
+
| API auth headers | Header Override | `evaluate` to patch `fetch` |
|
|
700
|
+
| Config-level token | Config | `authToken` + `authStorageKey` in config |
|
|
701
|
+
|
|
702
|
+
---
|
|
703
|
+
|
|
704
|
+
## Reusable Modules
|
|
705
|
+
|
|
706
|
+
Extract common flows into parameterized modules:
|
|
707
|
+
|
|
708
|
+
```json
|
|
709
|
+
// e2e/modules/login.json
|
|
710
|
+
{
|
|
711
|
+
"$module": "login",
|
|
712
|
+
"description": "Log in via the UI login form",
|
|
713
|
+
"params": {
|
|
714
|
+
"email": { "required": true, "description": "User email" },
|
|
715
|
+
"password": { "required": true, "description": "User password" }
|
|
716
|
+
},
|
|
717
|
+
"actions": [
|
|
718
|
+
{ "type": "goto", "value": "/login" },
|
|
719
|
+
{ "type": "type", "selector": "#email", "value": "{{email}}" },
|
|
720
|
+
{ "type": "type", "selector": "#password", "value": "{{password}}" },
|
|
721
|
+
{ "type": "click", "text": "Sign In" },
|
|
722
|
+
{ "type": "wait", "value": "2000" }
|
|
723
|
+
]
|
|
724
|
+
}
|
|
725
|
+
```
|
|
398
726
|
|
|
399
|
-
|
|
727
|
+
Use in tests:
|
|
400
728
|
|
|
401
|
-
|
|
729
|
+
```json
|
|
730
|
+
{
|
|
731
|
+
"name": "dashboard-loads",
|
|
732
|
+
"actions": [
|
|
733
|
+
{ "$use": "login", "params": { "email": "user@test.com", "password": "secret" } },
|
|
734
|
+
{ "type": "assert_text", "text": "Dashboard" }
|
|
735
|
+
]
|
|
736
|
+
}
|
|
737
|
+
```
|
|
402
738
|
|
|
403
|
-
|
|
404
|
-
2. **Generate** -- AI creates JSON test actions based on the issue description
|
|
405
|
-
3. **Run** -- Optionally executes the tests immediately to verify if a bug is reproducible
|
|
739
|
+
Modules support parameter validation (required params fail fast), conditional blocks (`{{#param}}...{{/param}}`), nested composition, and cycle detection.
|
|
406
740
|
|
|
407
|
-
|
|
741
|
+
---
|
|
408
742
|
|
|
409
|
-
|
|
743
|
+
## Exclude Patterns
|
|
410
744
|
|
|
411
|
-
|
|
745
|
+
Skip exploratory or draft tests from `--all` runs:
|
|
412
746
|
|
|
413
|
-
|
|
747
|
+
```js
|
|
748
|
+
// e2e.config.js
|
|
749
|
+
export default {
|
|
750
|
+
exclude: ['explore-*', 'debug-*', 'draft-*'],
|
|
751
|
+
};
|
|
752
|
+
```
|
|
753
|
+
|
|
754
|
+
Individual suite runs (`--suite`) are not affected by exclude patterns.
|
|
755
|
+
|
|
756
|
+
---
|
|
757
|
+
|
|
758
|
+
## Visual Verification
|
|
759
|
+
|
|
760
|
+
Describe what the page should look like — AI judges pass/fail from screenshots:
|
|
761
|
+
|
|
762
|
+
```json
|
|
763
|
+
{
|
|
764
|
+
"name": "dashboard-loads",
|
|
765
|
+
"expect": "Patient list with at least 3 rows, no error messages, sidebar with navigation links",
|
|
766
|
+
"actions": [
|
|
767
|
+
{ "type": "goto", "value": "/dashboard" },
|
|
768
|
+
{ "type": "wait", "selector": ".patient-list" }
|
|
769
|
+
]
|
|
770
|
+
}
|
|
771
|
+
```
|
|
772
|
+
|
|
773
|
+
After test actions complete, the runner auto-captures a verification screenshot. The MCP response includes the screenshot hash — Claude Code retrieves it and visually verifies against your `expect` description. No API key required.
|
|
774
|
+
|
|
775
|
+
---
|
|
776
|
+
|
|
777
|
+
## Issue-to-Test
|
|
778
|
+
|
|
779
|
+
Turn GitHub and GitLab issues into executable E2E tests. Paste an issue URL and get runnable tests — automatically.
|
|
780
|
+
|
|
781
|
+
**How it works:**
|
|
782
|
+
|
|
783
|
+
1. **Fetch** — Pulls issue details (title, body, labels) via `gh` or `glab` CLI
|
|
784
|
+
2. **Generate** — AI creates JSON test actions based on the issue description
|
|
785
|
+
3. **Run** — Optionally executes the tests immediately to verify if a bug is reproducible
|
|
414
786
|
|
|
415
787
|
```bash
|
|
416
|
-
# Fetch and display
|
|
788
|
+
# Fetch and display
|
|
417
789
|
e2e-runner issue https://github.com/owner/repo/issues/42
|
|
418
790
|
|
|
419
791
|
# Generate a test file via Claude API
|
|
420
792
|
e2e-runner issue https://github.com/owner/repo/issues/42 --generate
|
|
421
|
-
# -> Creates e2e/tests/issue-42.json
|
|
422
793
|
|
|
423
|
-
# Generate + run + report
|
|
794
|
+
# Generate + run + report
|
|
424
795
|
e2e-runner issue https://github.com/owner/repo/issues/42 --verify
|
|
425
796
|
# -> "BUG CONFIRMED" or "NOT REPRODUCIBLE"
|
|
426
|
-
|
|
427
|
-
# Output AI prompt as JSON (for piping)
|
|
428
|
-
e2e-runner issue https://github.com/owner/repo/issues/42 --prompt
|
|
429
797
|
```
|
|
430
798
|
|
|
431
|
-
|
|
799
|
+
In Claude Code, just ask:
|
|
800
|
+
> "Fetch issue #42 and create E2E tests for it"
|
|
801
|
+
|
|
802
|
+
**Bug verification logic:** Generated tests assert the **correct** behavior. Test failure = bug confirmed. All tests pass = not reproducible.
|
|
432
803
|
|
|
433
|
-
|
|
804
|
+
**Auth:** GitHub requires `gh` CLI, GitLab requires `glab` CLI. Self-hosted GitLab is supported.
|
|
434
805
|
|
|
435
|
-
|
|
806
|
+
---
|
|
436
807
|
|
|
437
|
-
|
|
808
|
+
## Learning System
|
|
438
809
|
|
|
439
|
-
|
|
810
|
+
The runner learns from every test run — building knowledge about your test suite over time.
|
|
440
811
|
|
|
441
|
-
|
|
442
|
-
- **GitLab**: `glab` CLI authenticated (`glab auth login`)
|
|
812
|
+
Query insights via the `e2e_learnings` MCP tool:
|
|
443
813
|
|
|
444
|
-
|
|
814
|
+
| Query | Returns |
|
|
815
|
+
|-------|---------|
|
|
816
|
+
| `summary` | Full health overview: pass rate, flaky tests, unstable selectors, API issues |
|
|
817
|
+
| `flaky` | Tests that pass only after retries |
|
|
818
|
+
| `selectors` | CSS selectors with high failure rates |
|
|
819
|
+
| `pages` | Pages with console errors, network failures, load time issues |
|
|
820
|
+
| `apis` | API endpoints with error rates and latency (auto-normalized: UUIDs, hashes, IDs) |
|
|
821
|
+
| `errors` | Most frequent error patterns, categorized |
|
|
822
|
+
| `trends` | Pass rate over time (auto-switches to hourly when all data is from one day) |
|
|
823
|
+
| `test:<name>` | Drill-down history for a specific test |
|
|
824
|
+
| `page:<path>` | Drill-down history for a specific page |
|
|
825
|
+
| `selector:<value>` | Drill-down history for a specific selector |
|
|
445
826
|
|
|
446
|
-
|
|
827
|
+
**Storage & export:**
|
|
828
|
+
- SQLite (`~/.e2e-runner/dashboard.db`) — default, zero setup
|
|
829
|
+
- Neo4j knowledge graph — optional, for relationship-based analysis. Manage via `e2e_neo4j` MCP tool or `docker compose`
|
|
830
|
+
- Markdown report (`e2e/learnings.md`) — auto-generated after each run
|
|
447
831
|
|
|
448
|
-
|
|
832
|
+
**Test narration:** Each test run generates a human-readable narrative of what happened step by step, visible in the CLI output and the dashboard.
|
|
833
|
+
|
|
834
|
+
---
|
|
449
835
|
|
|
450
836
|
## Web Dashboard
|
|
451
837
|
|
|
452
|
-
Real-time UI for running tests, viewing results, screenshots, and
|
|
838
|
+
Real-time UI for running tests, viewing results, screenshots, and network logs.
|
|
453
839
|
|
|
454
840
|
```bash
|
|
455
841
|
e2e-runner dashboard # Start on default port 8484
|
|
@@ -458,10 +844,10 @@ e2e-runner dashboard --port 9090 # Custom port
|
|
|
458
844
|
|
|
459
845
|
### Live Execution
|
|
460
846
|
|
|
461
|
-
Monitor tests in real-time
|
|
847
|
+
Monitor tests in real-time with step-by-step progress, durations, and active worker count.
|
|
462
848
|
|
|
463
849
|
<p align="center">
|
|
464
|
-
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-live-running.png" alt="Dashboard - Live test execution" width="
|
|
850
|
+
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-live-running.png" alt="Dashboard - Live test execution" width="800" />
|
|
465
851
|
</p>
|
|
466
852
|
|
|
467
853
|
### Test Suites
|
|
@@ -469,63 +855,381 @@ Monitor tests in real-time as they run. Each test shows its steps with individua
|
|
|
469
855
|
Browse all test suites across multiple projects. Run a single suite or all tests with one click.
|
|
470
856
|
|
|
471
857
|
<p align="center">
|
|
472
|
-
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-suites.png" alt="Dashboard - Test suites grid" width="
|
|
858
|
+
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-suites.png" alt="Dashboard - Test suites grid" width="800" />
|
|
473
859
|
</p>
|
|
474
860
|
|
|
475
861
|
### Run History
|
|
476
862
|
|
|
477
|
-
Track pass rate trends
|
|
863
|
+
Track pass rate trends with the built-in chart. Click any row to expand full detail with per-test results, screenshot hashes, and errors.
|
|
478
864
|
|
|
479
865
|
<p align="center">
|
|
480
|
-
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-runs.png" alt="Dashboard - Run history
|
|
866
|
+
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-runs.png" alt="Dashboard - Run history" width="800" />
|
|
481
867
|
</p>
|
|
482
868
|
|
|
483
869
|
### Run Detail
|
|
484
870
|
|
|
485
|
-
Expanded view
|
|
871
|
+
Expanded view with PASS/FAIL badges, screenshot thumbnails with copyable hashes (`ss:77c28b5a`), formatted console errors, and network request logs.
|
|
486
872
|
|
|
487
873
|
<p align="center">
|
|
488
|
-
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-run-detail.png" alt="Dashboard - Run detail
|
|
874
|
+
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-run-detail.png" alt="Dashboard - Run detail" width="800" />
|
|
489
875
|
</p>
|
|
490
876
|
|
|
491
877
|
### Screenshot Gallery
|
|
492
878
|
|
|
493
|
-
Browse all captured screenshots
|
|
879
|
+
Browse all captured screenshots with hash search. Includes action screenshots, error screenshots, and verification captures.
|
|
880
|
+
|
|
881
|
+
<p align="center">
|
|
882
|
+
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-screenshots-gallery.png" alt="Dashboard - Screenshot gallery" width="800" />
|
|
883
|
+
</p>
|
|
884
|
+
|
|
885
|
+
### Pool Status
|
|
886
|
+
|
|
887
|
+
Monitor Chrome pool health: available slots, running sessions, memory pressure.
|
|
494
888
|
|
|
495
889
|
<p align="center">
|
|
496
|
-
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-
|
|
890
|
+
<img src="https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/docs/screenshots/blog-dashboard-pool-status.png" alt="Dashboard - Pool status" width="800" />
|
|
497
891
|
</p>
|
|
498
892
|
|
|
499
|
-
|
|
893
|
+
---
|
|
894
|
+
|
|
895
|
+
## Screenshot Capture
|
|
896
|
+
|
|
897
|
+
Capture screenshots of any URL on demand — no test suite required:
|
|
500
898
|
|
|
899
|
+
```bash
|
|
900
|
+
e2e-runner capture https://example.com
|
|
901
|
+
e2e-runner capture https://example.com --full-page --selector ".loaded" --delay 2000
|
|
501
902
|
```
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
903
|
+
|
|
904
|
+
Via MCP, the `e2e_capture` tool supports `authToken` and `authStorageKey` for authenticated pages — it injects the token into localStorage before navigating.
|
|
905
|
+
|
|
906
|
+
Every screenshot gets a deterministic hash (`ss:a3f2b1c9`). Use `e2e_screenshot` to retrieve any screenshot by hash — it returns the image with metadata (test name, step, type).
|
|
907
|
+
|
|
908
|
+
---
|
|
909
|
+
|
|
910
|
+
## Claude Code Integration
|
|
911
|
+
|
|
912
|
+
The package ships as a **Claude Code plugin** — a single install that gives Claude native access to the test runner, teaches it the optimal workflow, and adds slash commands and specialized agents.
|
|
913
|
+
|
|
914
|
+
### Install as Plugin (recommended)
|
|
915
|
+
|
|
916
|
+
```bash
|
|
917
|
+
# 1. Add the marketplace (one-time)
|
|
918
|
+
claude plugin marketplace add fastslack/mtw-e2e-runner
|
|
919
|
+
|
|
920
|
+
# 2. Install the plugin
|
|
921
|
+
claude plugin install e2e-runner@matware
|
|
519
922
|
```
|
|
520
923
|
|
|
521
|
-
|
|
924
|
+
**What you get:**
|
|
925
|
+
|
|
926
|
+
| Component | Description |
|
|
927
|
+
|-----------|-------------|
|
|
928
|
+
| **13 MCP tools** | Run tests, create test files, capture screenshots, query network logs, manage dashboard, verify issues, query learnings |
|
|
929
|
+
| **Skill** | Teaches Claude the full e2e-runner workflow — how to combine tools, interpret results, debug failures, create tests |
|
|
930
|
+
| **3 Commands** | `/e2e-runner:run` — run & analyze tests<br>`/e2e-runner:create-test` — explore UI and create tests<br>`/e2e-runner:verify-issue <url>` — verify GitHub/GitLab bugs |
|
|
931
|
+
| **3 Agents** | **test-analyzer** — diagnoses failures, analyzes flaky tests, drills into network errors<br>**test-creator** — explores UI, discovers selectors, designs and validates tests<br>**test-improver** — refactors verbose evaluate actions, extracts modules, adds waits/retries, eliminates hardcoded delays |
|
|
932
|
+
|
|
933
|
+
### Install MCP-only (alternative)
|
|
934
|
+
|
|
935
|
+
If you only want the 13 MCP tools without skills, commands, or agents:
|
|
936
|
+
|
|
937
|
+
```bash
|
|
938
|
+
claude mcp add --transport stdio --scope user e2e-runner \
|
|
939
|
+
-- npx -y -p @matware/e2e-runner e2e-runner-mcp
|
|
940
|
+
```
|
|
941
|
+
|
|
942
|
+
### Slash Commands
|
|
943
|
+
|
|
944
|
+
| Command | Description |
|
|
945
|
+
|---------|-------------|
|
|
946
|
+
| `/e2e-runner:run` | Check pool, list suites, run tests, analyze results with screenshots and network drill-down |
|
|
947
|
+
| `/e2e-runner:create-test` | Explore the UI with screenshots, find selectors in source code, design test actions, create and validate |
|
|
948
|
+
| `/e2e-runner:verify-issue <url>` | Fetch a GitHub/GitLab issue, create tests that verify correct behavior, report bug confirmed or not reproducible |
|
|
949
|
+
|
|
950
|
+
### MCP Tools
|
|
951
|
+
|
|
952
|
+
| Tool | Description |
|
|
953
|
+
|------|-------------|
|
|
954
|
+
| `e2e_run` | Run tests: all suites, by name, or by file. Supports `concurrency`, `baseUrl`, `retries`, `failOnNetworkError` overrides. Returns verification results if tests have `expect`. |
|
|
955
|
+
| `e2e_list` | List available test suites with test names and counts |
|
|
956
|
+
| `e2e_create_test` | Create a new test JSON file with name, tests, and optional hooks |
|
|
957
|
+
| `e2e_create_module` | Create a reusable module with parameterized actions |
|
|
958
|
+
| `e2e_pool_status` | Check Chrome pool availability, running sessions, capacity |
|
|
959
|
+
| `e2e_screenshot` | Retrieve a screenshot by hash (`ss:a3f2b1c9`). Returns image + metadata |
|
|
960
|
+
| `e2e_capture` | Capture screenshot of any URL. Supports `authToken`, `fullPage`, `selector`, `delay` |
|
|
961
|
+
| `e2e_dashboard_start` | Start the web dashboard |
|
|
962
|
+
| `e2e_dashboard_stop` | Stop the web dashboard |
|
|
963
|
+
| `e2e_issue` | Fetch GitHub/GitLab issue and generate tests. `mode: "prompt"` or `mode: "verify"` |
|
|
964
|
+
| `e2e_network_logs` | Query network request/response logs by `runDbId`. Filter by test name, method, status, URL pattern. Supports headers and bodies |
|
|
965
|
+
| `e2e_learnings` | Query the learning system: `summary`, `flaky`, `selectors`, `pages`, `apis`, `errors`, `trends` |
|
|
966
|
+
| `e2e_neo4j` | Manage Neo4j knowledge graph container: `start`, `stop`, `status` |
|
|
967
|
+
|
|
968
|
+
> **Note:** Pool start/stop are CLI-only (`e2e-runner pool start|stop`) — not exposed via MCP to prevent killing active sessions.
|
|
969
|
+
|
|
970
|
+
### What You Can Ask Claude Code
|
|
971
|
+
|
|
972
|
+
> "Run all E2E tests"
|
|
973
|
+
> "Create a test that verifies the checkout flow"
|
|
974
|
+
> "What tests are flaky? Show me the learning summary"
|
|
975
|
+
> "Capture a screenshot of /dashboard with auth"
|
|
976
|
+
> "Fetch issue #42 and create tests for it"
|
|
977
|
+
> "What's the API error rate for the last 7 days?"
|
|
978
|
+
|
|
979
|
+
---
|
|
980
|
+
|
|
981
|
+
## OpenCode Integration
|
|
982
|
+
|
|
983
|
+
The package also supports [OpenCode](https://github.com/anomalyco/opencode) with native MCP server configuration, skills, and commands.
|
|
984
|
+
|
|
985
|
+
### Quick Setup
|
|
986
|
+
|
|
987
|
+
```bash
|
|
988
|
+
# 1. Install the package
|
|
989
|
+
npm install --save-dev @matware/e2e-runner
|
|
522
990
|
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
3. **Actions**: Each JSON action maps to a Puppeteer call (`page.goto`, `page.click`, `page.type`, etc.).
|
|
526
|
-
4. **Reports**: Results are collected, aggregated into a report, and saved as JSON and/or JUnit XML.
|
|
991
|
+
# 2. Copy OpenCode config to your project
|
|
992
|
+
cp node_modules/@matware/e2e-runner/opencode.json ./
|
|
527
993
|
|
|
528
|
-
|
|
994
|
+
# 3. Copy skills and commands (optional)
|
|
995
|
+
mkdir -p .opencode
|
|
996
|
+
cp -r node_modules/@matware/e2e-runner/.opencode/* .opencode/
|
|
997
|
+
|
|
998
|
+
# 4. Start the Chrome pool
|
|
999
|
+
npx e2e-runner pool start
|
|
1000
|
+
```
|
|
1001
|
+
|
|
1002
|
+
### What's Included
|
|
1003
|
+
|
|
1004
|
+
| Component | Description |
|
|
1005
|
+
|-----------|-------------|
|
|
1006
|
+
| **15 MCP tools** | Same tools as Claude Code — run tests, create files, screenshots, network logs, learnings, etc. |
|
|
1007
|
+
| **Skill** | `e2e-testing` — full workflow guidance with references |
|
|
1008
|
+
| **3 Commands** | `/run`, `/create-test`, `/verify-issue` |
|
|
1009
|
+
|
|
1010
|
+
### MCP Configuration
|
|
1011
|
+
|
|
1012
|
+
The `opencode.json` configures the MCP server as a local process:
|
|
1013
|
+
|
|
1014
|
+
```json
|
|
1015
|
+
{
|
|
1016
|
+
"mcp": {
|
|
1017
|
+
"e2e-runner": {
|
|
1018
|
+
"type": "local",
|
|
1019
|
+
"command": "node",
|
|
1020
|
+
"args": ["node_modules/@matware/e2e-runner/bin/mcp-server.js"],
|
|
1021
|
+
"cwd": "${workspaceFolder}"
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
```
|
|
1026
|
+
|
|
1027
|
+
For global installation, use the binary directly:
|
|
1028
|
+
|
|
1029
|
+
```json
|
|
1030
|
+
{
|
|
1031
|
+
"mcp": {
|
|
1032
|
+
"e2e-runner": {
|
|
1033
|
+
"type": "local",
|
|
1034
|
+
"command": "e2e-runner-mcp"
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
```
|
|
1039
|
+
|
|
1040
|
+
See [OPENCODE.md](OPENCODE.md) for full documentation on OpenCode integration.
|
|
1041
|
+
|
|
1042
|
+
---
|
|
1043
|
+
|
|
1044
|
+
## Network Error Handling
|
|
1045
|
+
|
|
1046
|
+
### Explicit Assertion
|
|
1047
|
+
|
|
1048
|
+
Place `assert_no_network_errors` after critical page loads:
|
|
1049
|
+
|
|
1050
|
+
```json
|
|
1051
|
+
{ "type": "goto", "value": "/dashboard" },
|
|
1052
|
+
{ "type": "wait", "selector": ".loaded" },
|
|
1053
|
+
{ "type": "assert_no_network_errors" }
|
|
1054
|
+
```
|
|
1055
|
+
|
|
1056
|
+
### Global Flag
|
|
1057
|
+
|
|
1058
|
+
Set `failOnNetworkError: true` to automatically fail any test with network errors:
|
|
1059
|
+
|
|
1060
|
+
```bash
|
|
1061
|
+
e2e-runner run --all --fail-on-network-error
|
|
1062
|
+
```
|
|
1063
|
+
|
|
1064
|
+
When disabled (default), the runner still collects and reports network errors — the MCP response includes a warning when tests pass but have network errors.
|
|
1065
|
+
|
|
1066
|
+
### Full Network Logging
|
|
1067
|
+
|
|
1068
|
+
All XHR/fetch requests are captured with: URL, method, status, duration, request/response headers, and response body (truncated at 50KB). Viewable in the dashboard with expandable request detail rows.
|
|
1069
|
+
|
|
1070
|
+
**MCP drill-down flow:**
|
|
1071
|
+
|
|
1072
|
+
```
|
|
1073
|
+
1. e2e_run → compact networkSummary + runDbId
|
|
1074
|
+
2. e2e_network_logs(runDbId) → all requests (url, method, status, duration)
|
|
1075
|
+
3. e2e_network_logs(runDbId, errorsOnly: true) → only failed requests
|
|
1076
|
+
4. e2e_network_logs(runDbId, includeHeaders: true) → with headers
|
|
1077
|
+
5. e2e_network_logs(runDbId, includeBodies: true) → full request/response bodies
|
|
1078
|
+
```
|
|
1079
|
+
|
|
1080
|
+
The `e2e_run` response stays compact (~5KB) regardless of how many requests were captured. Use `e2e_network_logs` with the returned `runDbId` to drill into details on demand.
|
|
1081
|
+
|
|
1082
|
+
---
|
|
1083
|
+
|
|
1084
|
+
## Hooks
|
|
1085
|
+
|
|
1086
|
+
Run actions at lifecycle points. Define globally in config or per-suite:
|
|
1087
|
+
|
|
1088
|
+
```json
|
|
1089
|
+
{
|
|
1090
|
+
"hooks": {
|
|
1091
|
+
"beforeAll": [{ "type": "goto", "value": "/setup" }],
|
|
1092
|
+
"beforeEach": [{ "type": "goto", "value": "/" }],
|
|
1093
|
+
"afterEach": [{ "type": "screenshot", "value": "after.png" }],
|
|
1094
|
+
"afterAll": []
|
|
1095
|
+
},
|
|
1096
|
+
"tests": [...]
|
|
1097
|
+
}
|
|
1098
|
+
```
|
|
1099
|
+
|
|
1100
|
+
> **Important:** `beforeAll` runs on a separate browser page that is closed before tests start. Use `beforeEach` for state that tests need (cookies, localStorage, auth tokens).
|
|
1101
|
+
|
|
1102
|
+
---
|
|
1103
|
+
|
|
1104
|
+
## CLI
|
|
1105
|
+
|
|
1106
|
+
```bash
|
|
1107
|
+
# Run tests
|
|
1108
|
+
e2e-runner run --all # All suites
|
|
1109
|
+
e2e-runner run --suite auth # Single suite
|
|
1110
|
+
e2e-runner run --tests path/to.json # Specific file
|
|
1111
|
+
e2e-runner run --inline '<json>' # Inline JSON
|
|
1112
|
+
|
|
1113
|
+
# Pool management (CLI only, not MCP)
|
|
1114
|
+
e2e-runner pool start # Start Chrome container
|
|
1115
|
+
e2e-runner pool stop # Stop Chrome container
|
|
1116
|
+
e2e-runner pool status # Check pool health
|
|
1117
|
+
|
|
1118
|
+
# Issue-to-test
|
|
1119
|
+
e2e-runner issue <url> # Fetch issue
|
|
1120
|
+
e2e-runner issue <url> --generate # Generate test via AI
|
|
1121
|
+
e2e-runner issue <url> --verify # Generate + run + report
|
|
1122
|
+
|
|
1123
|
+
# Dashboard
|
|
1124
|
+
e2e-runner dashboard # Start web dashboard
|
|
1125
|
+
|
|
1126
|
+
# Other
|
|
1127
|
+
e2e-runner list # List available suites
|
|
1128
|
+
e2e-runner capture <url> # On-demand screenshot
|
|
1129
|
+
e2e-runner init # Scaffold project
|
|
1130
|
+
```
|
|
1131
|
+
|
|
1132
|
+
### CLI Options
|
|
1133
|
+
|
|
1134
|
+
| Flag | Default | Description |
|
|
1135
|
+
|------|---------|-------------|
|
|
1136
|
+
| `--base-url <url>` | `http://host.docker.internal:3000` | Application base URL |
|
|
1137
|
+
| `--pool-url <ws>` | `ws://localhost:3333` | Chrome pool WebSocket URL |
|
|
1138
|
+
| `--concurrency <n>` | `3` | Parallel test workers |
|
|
1139
|
+
| `--retries <n>` | `0` | Retry failed tests N times |
|
|
1140
|
+
| `--action-retries <n>` | `0` | Retry failed actions N times |
|
|
1141
|
+
| `--test-timeout <ms>` | `60000` | Per-test timeout |
|
|
1142
|
+
| `--timeout <ms>` | `10000` | Default action timeout |
|
|
1143
|
+
| `--output <format>` | `json` | Report: `json`, `junit`, `both` |
|
|
1144
|
+
| `--env <name>` | `default` | Environment profile |
|
|
1145
|
+
| `--fail-on-network-error` | `false` | Fail tests with network errors |
|
|
1146
|
+
| `--project-name <name>` | dir name | Project display name |
|
|
1147
|
+
|
|
1148
|
+
---
|
|
1149
|
+
|
|
1150
|
+
## Configuration
|
|
1151
|
+
|
|
1152
|
+
Create `e2e.config.js` in your project root:
|
|
1153
|
+
|
|
1154
|
+
```js
|
|
1155
|
+
export default {
|
|
1156
|
+
baseUrl: 'http://host.docker.internal:3000',
|
|
1157
|
+
concurrency: 4,
|
|
1158
|
+
retries: 2,
|
|
1159
|
+
actionRetries: 1,
|
|
1160
|
+
testTimeout: 30000,
|
|
1161
|
+
outputFormat: 'both',
|
|
1162
|
+
failOnNetworkError: true,
|
|
1163
|
+
exclude: ['explore-*', 'debug-*'],
|
|
1164
|
+
|
|
1165
|
+
hooks: {
|
|
1166
|
+
beforeEach: [{ type: 'goto', value: '/' }],
|
|
1167
|
+
},
|
|
1168
|
+
|
|
1169
|
+
environments: {
|
|
1170
|
+
staging: { baseUrl: 'https://staging.example.com' },
|
|
1171
|
+
production: { baseUrl: 'https://example.com', concurrency: 5 },
|
|
1172
|
+
},
|
|
1173
|
+
};
|
|
1174
|
+
```
|
|
1175
|
+
|
|
1176
|
+
### Config Priority (highest wins)
|
|
1177
|
+
|
|
1178
|
+
1. CLI flags
|
|
1179
|
+
2. Environment variables
|
|
1180
|
+
3. Config file (`e2e.config.js` or `e2e.config.json`)
|
|
1181
|
+
4. Defaults
|
|
1182
|
+
|
|
1183
|
+
When `--env <name>` is set, the matching profile overrides everything.
|
|
1184
|
+
|
|
1185
|
+
---
|
|
1186
|
+
|
|
1187
|
+
## CI/CD
|
|
1188
|
+
|
|
1189
|
+
### JUnit XML
|
|
1190
|
+
|
|
1191
|
+
```bash
|
|
1192
|
+
e2e-runner run --all --output junit
|
|
1193
|
+
```
|
|
1194
|
+
|
|
1195
|
+
### GitHub Actions
|
|
1196
|
+
|
|
1197
|
+
```yaml
|
|
1198
|
+
jobs:
|
|
1199
|
+
e2e:
|
|
1200
|
+
runs-on: ubuntu-latest
|
|
1201
|
+
steps:
|
|
1202
|
+
- uses: actions/checkout@v4
|
|
1203
|
+
- uses: actions/setup-node@v4
|
|
1204
|
+
with:
|
|
1205
|
+
node-version: 20
|
|
1206
|
+
- run: npm ci
|
|
1207
|
+
- run: npx e2e-runner pool start
|
|
1208
|
+
- run: npx e2e-runner run --all --output junit
|
|
1209
|
+
- uses: mikepenz/action-junit-report@v4
|
|
1210
|
+
if: always()
|
|
1211
|
+
with:
|
|
1212
|
+
report_paths: e2e/screenshots/junit.xml
|
|
1213
|
+
```
|
|
1214
|
+
|
|
1215
|
+
---
|
|
1216
|
+
|
|
1217
|
+
## Programmatic API
|
|
1218
|
+
|
|
1219
|
+
```js
|
|
1220
|
+
import { createRunner } from '@matware/e2e-runner';
|
|
1221
|
+
|
|
1222
|
+
const runner = await createRunner({ baseUrl: 'http://localhost:3000' });
|
|
1223
|
+
|
|
1224
|
+
const report = await runner.runAll();
|
|
1225
|
+
const report = await runner.runSuite('auth');
|
|
1226
|
+
const report = await runner.runFile('e2e/tests/login.json');
|
|
1227
|
+
const report = await runner.runTests([
|
|
1228
|
+
{ name: 'quick-check', actions: [{ type: 'goto', value: '/' }] },
|
|
1229
|
+
]);
|
|
1230
|
+
```
|
|
1231
|
+
|
|
1232
|
+
---
|
|
529
1233
|
|
|
530
1234
|
## Requirements
|
|
531
1235
|
|
|
@@ -536,14 +1240,4 @@ The `baseUrl` defaults to `http://host.docker.internal:3000` because Chrome runs
|
|
|
536
1240
|
|
|
537
1241
|
Copyright 2025 Matias Aguirre (fastslack)
|
|
538
1242
|
|
|
539
|
-
Licensed under the Apache License, Version 2.0 (
|
|
540
|
-
you may not use this file except in compliance with the License.
|
|
541
|
-
You may obtain a copy of the License at
|
|
542
|
-
|
|
543
|
-
http://www.apache.org/licenses/LICENSE-2.0
|
|
544
|
-
|
|
545
|
-
Unless required by applicable law or agreed to in writing, software
|
|
546
|
-
distributed under the License is distributed on an "AS IS" BASIS,
|
|
547
|
-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
548
|
-
See the License for the specific language governing permissions and
|
|
549
|
-
limitations under the License.
|
|
1243
|
+
Licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for details.
|