@matware/e2e-runner 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +52 -0
- package/.claude-plugin/plugin.json +17 -3
- package/.mcp.json +2 -2
- package/.opencode/commands/create-test.md +63 -0
- package/.opencode/commands/run.md +50 -0
- package/.opencode/commands/verify-issue.md +62 -0
- package/.opencode/skills/e2e-testing/SKILL.md +181 -0
- package/.opencode/skills/e2e-testing/references/action-types.md +143 -0
- package/.opencode/skills/e2e-testing/references/auth-strategies.md +91 -0
- package/.opencode/skills/e2e-testing/references/graphql.md +59 -0
- package/.opencode/skills/e2e-testing/references/issue-verification.md +59 -0
- package/.opencode/skills/e2e-testing/references/multi-pool.md +60 -0
- package/.opencode/skills/e2e-testing/references/network-debugging.md +62 -0
- package/.opencode/skills/e2e-testing/references/test-json-format.md +163 -0
- package/.opencode/skills/e2e-testing/references/troubleshooting.md +224 -0
- package/.opencode/skills/e2e-testing/references/variables.md +41 -0
- package/.opencode/skills/e2e-testing/references/visual-verification.md +89 -0
- package/LICENSE +190 -0
- package/OPENCODE.md +166 -0
- package/README.md +165 -104
- package/agents/test-creator.md +54 -1
- package/agents/test-improver.md +37 -0
- package/bin/cli.js +409 -16
- package/commands/capture.md +45 -0
- package/commands/create-test.md +16 -1
- package/opencode.json +11 -0
- package/package.json +7 -2
- package/scripts/setup-opencode.sh +113 -0
- package/skills/e2e-testing/SKILL.md +10 -3
- package/skills/e2e-testing/references/action-types.md +48 -5
- package/skills/e2e-testing/references/auth-strategies.md +91 -0
- package/skills/e2e-testing/references/graphql.md +59 -0
- package/skills/e2e-testing/references/issue-verification.md +59 -0
- package/skills/e2e-testing/references/multi-pool.md +60 -0
- package/skills/e2e-testing/references/network-debugging.md +62 -0
- package/skills/e2e-testing/references/test-json-format.md +4 -0
- package/skills/e2e-testing/references/troubleshooting.md +44 -2
- package/skills/e2e-testing/references/variables.md +41 -0
- package/skills/e2e-testing/references/visual-verification.md +89 -0
- package/src/actions.js +475 -2
- package/src/ai-generate.js +139 -8
- package/src/app-pool.js +339 -0
- package/src/config.js +266 -5
- package/src/dashboard.js +216 -17
- package/src/db.js +191 -7
- package/src/index.js +12 -9
- package/src/learner-sqlite.js +458 -0
- package/src/learner.js +78 -6
- package/src/mcp-tools.js +1348 -51
- package/src/module-resolver.js +37 -0
- package/src/narrate.js +65 -0
- package/src/pool-manager.js +229 -0
- package/src/pool.js +301 -31
- package/src/reporter.js +86 -2
- package/src/runner.js +480 -71
- package/src/sync/auth.js +354 -0
- package/src/sync/client.js +572 -0
- package/src/sync/hub-routes.js +816 -0
- package/src/sync/index.js +68 -0
- package/src/sync/middleware.js +347 -0
- package/src/sync/queue.js +209 -0
- package/src/sync/schema.js +540 -0
- package/src/verify.js +10 -7
- package/src/visual-diff.js +446 -0
- package/src/watch.js +384 -0
- package/templates/build-dashboard.js +47 -6
- package/templates/dashboard/js/api.js +62 -0
- package/templates/dashboard/js/init.js +13 -0
- package/templates/dashboard/js/keyboard.js +46 -0
- package/templates/dashboard/js/state.js +40 -0
- package/templates/dashboard/js/toast.js +41 -0
- package/templates/dashboard/js/utils.js +216 -0
- package/templates/dashboard/js/view-live.js +181 -0
- package/templates/dashboard/js/view-runs.js +676 -0
- package/templates/dashboard/js/view-tests.js +294 -0
- package/templates/dashboard/js/view-watch.js +242 -0
- package/templates/dashboard/js/websocket.js +116 -0
- package/templates/dashboard/styles/base.css +69 -0
- package/templates/dashboard/styles/components.css +117 -0
- package/templates/dashboard/styles/view-live.css +97 -0
- package/templates/dashboard/styles/view-runs.css +243 -0
- package/templates/dashboard/styles/view-tests.css +96 -0
- package/templates/dashboard/styles/view-watch.css +53 -0
- package/templates/dashboard/template.html +181 -100
- package/templates/dashboard.html +1614 -547
- package/templates/sample-test.json +0 -8
- package/templates/dashboard/app.js +0 -1152
- package/templates/dashboard/styles.css +0 -413
package/OPENCODE.md
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# OpenCode Integration
|
|
2
|
+
|
|
3
|
+
This document describes how to use `@matware/e2e-runner` with [OpenCode](https://github.com/anomalyco/opencode).
|
|
4
|
+
|
|
5
|
+
## Quick Setup
|
|
6
|
+
|
|
7
|
+
1. **Install the package** (if not already installed):
|
|
8
|
+
```bash
|
|
9
|
+
npm install -g @matware/e2e-runner
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
2. **Copy configuration to your project**:
|
|
13
|
+
```bash
|
|
14
|
+
# Copy opencode.json to your project root
|
|
15
|
+
cp node_modules/@matware/e2e-runner/opencode.json ./opencode.json
|
|
16
|
+
|
|
17
|
+
# Or merge with existing opencode.json
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
3. **Copy skills and commands** (optional, for skill/command support):
|
|
21
|
+
```bash
|
|
22
|
+
mkdir -p .opencode
|
|
23
|
+
cp -r node_modules/@matware/e2e-runner/.opencode/* .opencode/
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Configuration
|
|
27
|
+
|
|
28
|
+
### opencode.json
|
|
29
|
+
|
|
30
|
+
The MCP server is configured as a `local` type:
|
|
31
|
+
|
|
32
|
+
```json
|
|
33
|
+
{
|
|
34
|
+
"mcp": {
|
|
35
|
+
"e2e-runner": {
|
|
36
|
+
"type": "local",
|
|
37
|
+
"command": "node",
|
|
38
|
+
"args": ["node_modules/@matware/e2e-runner/bin/mcp-server.js"],
|
|
39
|
+
"cwd": "${workspaceFolder}"
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
If installed globally, use the binary name directly:
|
|
46
|
+
```json
|
|
47
|
+
{
|
|
48
|
+
"mcp": {
|
|
49
|
+
"e2e-runner": {
|
|
50
|
+
"type": "local",
|
|
51
|
+
"command": "e2e-runner-mcp"
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Available MCP Tools
|
|
58
|
+
|
|
59
|
+
All tools are prefixed with `e2e_`:
|
|
60
|
+
|
|
61
|
+
| Tool | Description |
|
|
62
|
+
|------|-------------|
|
|
63
|
+
| `e2e_pool_status` | Check Chrome pool availability |
|
|
64
|
+
| `e2e_list` | List test suites and modules |
|
|
65
|
+
| `e2e_run` | Execute tests (all, suite, or file) |
|
|
66
|
+
| `e2e_create_test` | Create a new test JSON file |
|
|
67
|
+
| `e2e_create_module` | Create a reusable module |
|
|
68
|
+
| `e2e_screenshot` | Retrieve screenshot by hash |
|
|
69
|
+
| `e2e_capture` | Capture screenshot of any URL |
|
|
70
|
+
| `e2e_network_logs` | Inspect network requests from a run |
|
|
71
|
+
| `e2e_learnings` | Query the learning system |
|
|
72
|
+
| `e2e_issue` | Fetch GitHub/GitLab issue details |
|
|
73
|
+
| `e2e_variables` | Manage test variables |
|
|
74
|
+
| `e2e_dashboard_start` | Start the web dashboard |
|
|
75
|
+
| `e2e_dashboard_stop` | Stop the web dashboard |
|
|
76
|
+
|
|
77
|
+
## Differences from Claude Code
|
|
78
|
+
|
|
79
|
+
| Feature | Claude Code | OpenCode |
|
|
80
|
+
|---------|-------------|----------|
|
|
81
|
+
| MCP Config | `.mcp.json` with `mcpServers` | `opencode.json` with `mcp` |
|
|
82
|
+
| MCP Type | `stdio` | `local` or `remote` |
|
|
83
|
+
| Skills Location | `skills/<name>/SKILL.md` | `.opencode/skills/<name>/SKILL.md` |
|
|
84
|
+
| Commands Location | `commands/*.md` | `.opencode/commands/*.md` |
|
|
85
|
+
| Frontmatter | `allowed_tools` array | No `allowed_tools` (tools are auto-detected) |
|
|
86
|
+
| Skill Triggers | Implicit from description | Explicit `triggers` array in frontmatter |
|
|
87
|
+
| Variable Substitution | `${CLAUDE_PLUGIN_ROOT}` | `${workspaceFolder}` |
|
|
88
|
+
|
|
89
|
+
### Key Differences Explained
|
|
90
|
+
|
|
91
|
+
1. **MCP Server Configuration**
|
|
92
|
+
- Claude Code: Uses `mcpServers` key with `type: "stdio"`
|
|
93
|
+
- OpenCode: Uses `mcp` key with `type: "local"` or `type: "remote"`
|
|
94
|
+
|
|
95
|
+
2. **Skills**
|
|
96
|
+
- Both use `SKILL.md` files with YAML frontmatter
|
|
97
|
+
- OpenCode supports an explicit `triggers` array to activate the skill
|
|
98
|
+
- Location differs: `.opencode/skills/` vs `skills/`
|
|
99
|
+
|
|
100
|
+
3. **Commands**
|
|
101
|
+
- Claude Code: Supports `allowed_tools` to restrict tool access
|
|
102
|
+
- OpenCode: Tools are auto-detected from the MCP server
|
|
103
|
+
- Location differs: `.opencode/commands/` vs `commands/`
|
|
104
|
+
|
|
105
|
+
4. **Variable Expansion**
|
|
106
|
+
- Claude Code: `${CLAUDE_PLUGIN_ROOT}` points to the package root
|
|
107
|
+
- OpenCode: `${workspaceFolder}` points to the current workspace
|
|
108
|
+
|
|
109
|
+
## Directory Structure
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
your-project/
|
|
113
|
+
├── opencode.json # OpenCode configuration
|
|
114
|
+
├── .opencode/
|
|
115
|
+
│ ├── skills/
|
|
116
|
+
│ │ └── e2e-testing/
|
|
117
|
+
│ │ ├── SKILL.md
|
|
118
|
+
│ │ └── references/
|
|
119
|
+
│ │ ├── action-types.md
|
|
120
|
+
│ │ ├── auth-strategies.md
|
|
121
|
+
│ │ └── ...
|
|
122
|
+
│ └── commands/
|
|
123
|
+
│ ├── run.md
|
|
124
|
+
│ ├── create-test.md
|
|
125
|
+
│ └── verify-issue.md
|
|
126
|
+
└── e2e/
|
|
127
|
+
├── e2e.config.json # Test configuration
|
|
128
|
+
├── tests/ # Test JSON files
|
|
129
|
+
└── modules/ # Reusable modules
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Global Installation
|
|
133
|
+
|
|
134
|
+
To make the skill and commands available to all projects:
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
# Copy to global OpenCode config
|
|
138
|
+
mkdir -p ~/.config/opencode/skills
|
|
139
|
+
mkdir -p ~/.config/opencode/commands
|
|
140
|
+
|
|
141
|
+
cp -r node_modules/@matware/e2e-runner/.opencode/skills/* ~/.config/opencode/skills/
|
|
142
|
+
cp -r node_modules/@matware/e2e-runner/.opencode/commands/* ~/.config/opencode/commands/
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Troubleshooting
|
|
146
|
+
|
|
147
|
+
### MCP Server Not Starting
|
|
148
|
+
|
|
149
|
+
1. Check that Node.js >= 20 is installed
|
|
150
|
+
2. Verify the path in `opencode.json` is correct
|
|
151
|
+
3. Try running the server manually:
|
|
152
|
+
```bash
|
|
153
|
+
node node_modules/@matware/e2e-runner/bin/mcp-server.js
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
### Tools Not Available
|
|
157
|
+
|
|
158
|
+
1. Restart OpenCode after changing `opencode.json`
|
|
159
|
+
2. Check the MCP server logs for errors
|
|
160
|
+
3. Verify the Chrome pool is running: `npx e2e-runner pool status`
|
|
161
|
+
|
|
162
|
+
### Skill Not Loading
|
|
163
|
+
|
|
164
|
+
1. Ensure the skill is in `.opencode/skills/e2e-testing/SKILL.md`
|
|
165
|
+
2. Check the frontmatter has `name` and `description`
|
|
166
|
+
3. Try using a trigger word from the `triggers` array
|
package/README.md
CHANGED
|
@@ -9,11 +9,16 @@
|
|
|
9
9
|
</p>
|
|
10
10
|
|
|
11
11
|
<p align="center">
|
|
12
|
-
<img src="https://img.shields.io/npm/v/@matware/e2e-runner?color=blue" alt="npm version"
|
|
12
|
+
<a href="https://www.npmjs.com/package/@matware/e2e-runner"><img src="https://img.shields.io/npm/v/@matware/e2e-runner?color=blue" alt="npm version" /></a>
|
|
13
13
|
<img src="https://img.shields.io/node/v/@matware/e2e-runner" alt="node version" />
|
|
14
|
-
<img src="https://img.shields.io/npm/
|
|
14
|
+
<a href="https://www.npmjs.com/package/@matware/e2e-runner"><img src="https://img.shields.io/npm/dm/@matware/e2e-runner" alt="npm downloads" /></a>
|
|
15
|
+
<a href="https://hub.docker.com/r/fastslack/e2e-runner-mcp"><img src="https://img.shields.io/docker/pulls/fastslack/e2e-runner-mcp" alt="Docker pulls" /></a>
|
|
16
|
+
<a href="https://github.com/fastslack/mtw-e2e-runner/stargazers"><img src="https://img.shields.io/github/stars/fastslack/mtw-e2e-runner" alt="GitHub stars" /></a>
|
|
17
|
+
<a href="LICENSE"><img src="https://img.shields.io/npm/l/@matware/e2e-runner" alt="license" /></a>
|
|
15
18
|
<img src="https://img.shields.io/badge/MCP-compatible-green" alt="MCP compatible" />
|
|
16
19
|
<img src="https://img.shields.io/badge/AI--native-Claude%20Code-blueviolet" alt="AI native" />
|
|
20
|
+
<img src="https://img.shields.io/badge/AI--native-OpenCode-orange" alt="OpenCode compatible" />
|
|
21
|
+
<a href="https://skills.sh"><img src="https://img.shields.io/badge/skills.sh-e2e--testing-ff6600" alt="Agent Skills" /></a>
|
|
17
22
|
</p>
|
|
18
23
|
|
|
19
24
|
<p align="center">
|
|
@@ -26,32 +31,6 @@
|
|
|
26
31
|
|
|
27
32
|
But what makes it truly different is its **deep AI integration**. With a built-in [MCP server](https://modelcontextprotocol.io/), Claude Code can create tests from a conversation, run them, read the results, capture screenshots, and even visually verify that pages look correct — all without leaving the chat. Paste a GitHub issue URL and get a runnable test back. That's the workflow.
|
|
28
33
|
|
|
29
|
-
### What you get
|
|
30
|
-
|
|
31
|
-
🧪 **Zero-code tests** — JSON files that anyone on your team can read and write. No JavaScript, no compilation, no framework lock-in.
|
|
32
|
-
|
|
33
|
-
🤖 **AI-powered testing** — Claude Code creates, executes, and debugs tests natively through 13 MCP tools. Ask it to "test the checkout flow" and it builds the JSON, runs it, and reports back.
|
|
34
|
-
|
|
35
|
-
🐛 **Issue-to-Test pipeline** — Paste a GitHub or GitLab issue URL. The runner fetches it, generates E2E tests, runs them, and tells you: *bug confirmed* or *not reproducible*.
|
|
36
|
-
|
|
37
|
-
👁️ **Visual verification** — Describe what the page should look like in plain English. The AI captures a screenshot and judges pass/fail against your description. No pixel-diffing setup needed.
|
|
38
|
-
|
|
39
|
-
🧠 **Learning system** — Tracks test stability across runs. Detects flaky tests, unstable selectors, slow APIs, and error patterns — then surfaces actionable insights.
|
|
40
|
-
|
|
41
|
-
⚡ **Parallel execution** — Run N tests simultaneously against a shared Chrome pool (browserless/chrome). Serial mode available for tests that share state.
|
|
42
|
-
|
|
43
|
-
📊 **Real-time dashboard** — Live execution view, run history with pass-rate charts, screenshot gallery with hash-based search, expandable network request logs.
|
|
44
|
-
|
|
45
|
-
🔁 **Smart retries** — Test-level and action-level retries with configurable delays. Flaky tests are detected and flagged automatically.
|
|
46
|
-
|
|
47
|
-
📦 **Reusable modules** — Extract common flows (login, navigation, setup) into parameterized modules and reference them with `$use`.
|
|
48
|
-
|
|
49
|
-
🏗️ **CI-ready** — JUnit XML output, exit code 1 on failure, auto-captured error screenshots. Drop-in GitHub Actions example included.
|
|
50
|
-
|
|
51
|
-
🌐 **Multi-project** — One dashboard aggregates test results from all your projects. One Chrome pool serves them all.
|
|
52
|
-
|
|
53
|
-
🐳 **Portable** — Chrome runs in Docker, tests are JSON files in your repo. Works on any machine with Node.js and Docker.
|
|
54
|
-
|
|
55
34
|
### This is a test
|
|
56
35
|
|
|
57
36
|
```json
|
|
@@ -74,45 +53,102 @@ No imports. No `describe`/`it`. No compilation step. Just a JSON file that descr
|
|
|
74
53
|
|
|
75
54
|
---
|
|
76
55
|
|
|
77
|
-
##
|
|
56
|
+
## Agent Skills
|
|
78
57
|
|
|
79
|
-
|
|
58
|
+
Install E2E testing skills for any coding agent (Claude Code, Cursor, Codex, Copilot, and [40+ more](https://github.com/vercel-labs/skills#supported-agents)):
|
|
80
59
|
|
|
81
60
|
```bash
|
|
82
|
-
|
|
61
|
+
npx skills add fastslack/mtw-e2e-runner
|
|
83
62
|
```
|
|
84
63
|
|
|
85
|
-
|
|
64
|
+
This gives your agent the knowledge to create, run, and debug JSON-driven E2E tests — no documentation reading required.
|
|
65
|
+
|
|
66
|
+
> Browse all available skills at [skills.sh](https://skills.sh)
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## Getting Started
|
|
71
|
+
|
|
72
|
+
**Prerequisites:** Node.js >= 20, Docker running, your app on a known port.
|
|
73
|
+
|
|
74
|
+
### Quickstart
|
|
86
75
|
|
|
87
76
|
```bash
|
|
88
|
-
# 1. Install
|
|
89
77
|
npm install --save-dev @matware/e2e-runner
|
|
78
|
+
npx e2e-runner init # creates e2e/tests/ with a sample test
|
|
79
|
+
npx e2e-runner pool start # starts Chrome in Docker
|
|
80
|
+
npx e2e-runner run --all # runs the sample test
|
|
81
|
+
```
|
|
90
82
|
|
|
91
|
-
|
|
92
|
-
npx e2e-runner init
|
|
83
|
+
Or do it all in one command:
|
|
93
84
|
|
|
94
|
-
|
|
95
|
-
|
|
85
|
+
```bash
|
|
86
|
+
curl -fsSL https://raw.githubusercontent.com/fastslack/mtw-e2e-runner/main/scripts/quickstart.sh | bash
|
|
87
|
+
```
|
|
96
88
|
|
|
97
|
-
|
|
98
|
-
npx e2e-runner run --all
|
|
89
|
+
After setup, edit `e2e.config.js` to set your app's port:
|
|
99
90
|
|
|
100
|
-
|
|
101
|
-
|
|
91
|
+
```js
|
|
92
|
+
export default {
|
|
93
|
+
baseUrl: 'http://host.docker.internal:3000', // change 3000 to your port
|
|
94
|
+
};
|
|
102
95
|
```
|
|
103
96
|
|
|
104
|
-
**
|
|
97
|
+
> **Why `host.docker.internal`?** Chrome runs inside Docker and can't reach `localhost` on your machine. This hostname bridges the gap. On Linux (Docker Engine, not Desktop), you may need `--add-host=host.docker.internal:host-gateway` or use your LAN IP directly.
|
|
98
|
+
|
|
99
|
+
### Add Claude Code (optional)
|
|
105
100
|
|
|
106
101
|
```bash
|
|
107
|
-
|
|
108
|
-
claude plugin install
|
|
102
|
+
claude plugin marketplace add fastslack/mtw-e2e-runner
|
|
103
|
+
claude plugin install e2e-runner@matware
|
|
104
|
+
```
|
|
109
105
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
106
|
+
This gives Claude 13 MCP tools, slash commands, and specialized agents. Just say *"Run all E2E tests"* or *"Create a test for the login flow"*.
|
|
107
|
+
|
|
108
|
+
### Add OpenCode (optional)
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
cp node_modules/@matware/e2e-runner/opencode.json ./
|
|
112
|
+
mkdir -p .opencode && cp -r node_modules/@matware/e2e-runner/.opencode/* .opencode/
|
|
113
113
|
```
|
|
114
114
|
|
|
115
|
-
|
|
115
|
+
See [OPENCODE.md](OPENCODE.md) for details.
|
|
116
|
+
|
|
117
|
+
### What's next?
|
|
118
|
+
|
|
119
|
+
- [Test Format](#test-format) — learn the full action vocabulary
|
|
120
|
+
- [Claude Code Integration](#claude-code-integration) — set up AI-powered testing
|
|
121
|
+
- [Visual Verification](#visual-verification) — describe expected pages in plain English
|
|
122
|
+
- [Issue-to-Test](#issue-to-test) — turn bug reports into executable tests
|
|
123
|
+
- [Web Dashboard](#web-dashboard) — monitor tests in real time
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
## What you get
|
|
128
|
+
|
|
129
|
+
🧪 **Zero-code tests** — JSON files that anyone on your team can read and write. No JavaScript, no compilation, no framework lock-in.
|
|
130
|
+
|
|
131
|
+
🤖 **AI-powered testing** — Claude Code creates, executes, and debugs tests natively through 13 MCP tools. Ask it to "test the checkout flow" and it builds the JSON, runs it, and reports back.
|
|
132
|
+
|
|
133
|
+
🐛 **Issue-to-Test pipeline** — Paste a GitHub or GitLab issue URL. The runner fetches it, generates E2E tests, runs them, and tells you: *bug confirmed* or *not reproducible*.
|
|
134
|
+
|
|
135
|
+
👁️ **Visual verification** — Describe what the page should look like in plain English. The AI captures a screenshot and judges pass/fail against your description. No pixel-diffing setup needed.
|
|
136
|
+
|
|
137
|
+
🧠 **Learning system** — Tracks test stability across runs. Detects flaky tests, unstable selectors, slow APIs, and error patterns — then surfaces actionable insights.
|
|
138
|
+
|
|
139
|
+
⚡ **Parallel execution** — Run N tests simultaneously against a shared Chrome pool (browserless/chrome). Serial mode available for tests that share state.
|
|
140
|
+
|
|
141
|
+
📊 **Real-time dashboard** — Live execution view, run history with pass-rate charts, screenshot gallery with hash-based search, expandable network request logs.
|
|
142
|
+
|
|
143
|
+
🔁 **Smart retries** — Test-level and action-level retries with configurable delays. Flaky tests are detected and flagged automatically.
|
|
144
|
+
|
|
145
|
+
📦 **Reusable modules** — Extract common flows (login, navigation, setup) into parameterized modules and reference them with `$use`.
|
|
146
|
+
|
|
147
|
+
🏗️ **CI-ready** — JUnit XML output, exit code 1 on failure, auto-captured error screenshots. Drop-in GitHub Actions example included.
|
|
148
|
+
|
|
149
|
+
🌐 **Multi-project** — One dashboard aggregates test results from all your projects. One Chrome pool serves them all.
|
|
150
|
+
|
|
151
|
+
🐳 **Portable** — Chrome runs in Docker, tests are JSON files in your repo. Works on any machine with Node.js and Docker.
|
|
116
152
|
|
|
117
153
|
---
|
|
118
154
|
|
|
@@ -126,8 +162,7 @@ Each `.json` file in `e2e/tests/` contains an array of tests. Each test has a `n
|
|
|
126
162
|
"name": "homepage-loads",
|
|
127
163
|
"actions": [
|
|
128
164
|
{ "type": "goto", "value": "/" },
|
|
129
|
-
{ "type": "
|
|
130
|
-
{ "type": "assert_text", "text": "Welcome" },
|
|
165
|
+
{ "type": "assert_visible", "selector": "body" },
|
|
131
166
|
{ "type": "assert_url", "value": "/" },
|
|
132
167
|
{ "type": "screenshot", "value": "homepage.png" }
|
|
133
168
|
]
|
|
@@ -245,22 +280,66 @@ Serial tests run one at a time **after** all parallel tests finish — preventin
|
|
|
245
280
|
|
|
246
281
|
---
|
|
247
282
|
|
|
283
|
+
## Testing Authenticated Apps
|
|
284
|
+
|
|
285
|
+
The simplest approach — log in via the UI like a real user:
|
|
286
|
+
|
|
287
|
+
```json
|
|
288
|
+
{
|
|
289
|
+
"hooks": {
|
|
290
|
+
"beforeEach": [
|
|
291
|
+
{ "type": "goto", "value": "/login" },
|
|
292
|
+
{ "type": "type", "selector": "#email", "value": "test@example.com" },
|
|
293
|
+
{ "type": "type", "selector": "#password", "value": "test-password" },
|
|
294
|
+
{ "type": "click", "text": "Sign In" },
|
|
295
|
+
{ "type": "wait", "selector": ".dashboard" }
|
|
296
|
+
]
|
|
297
|
+
},
|
|
298
|
+
"tests": [...]
|
|
299
|
+
}
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
For SPAs with JWT, skip the login form by injecting the token directly:
|
|
303
|
+
|
|
304
|
+
```json
|
|
305
|
+
{ "type": "set_storage", "value": "accessToken=eyJhbGciOiJIUzI1NiIs..." }
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Or set it globally in config:
|
|
309
|
+
|
|
310
|
+
```js
|
|
311
|
+
// e2e.config.js
|
|
312
|
+
export default {
|
|
313
|
+
authToken: 'eyJhbGciOiJIUzI1NiIs...',
|
|
314
|
+
authStorageKey: 'accessToken',
|
|
315
|
+
};
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
Each test runs in a **fresh browser context**, so auth state is automatically clean between tests.
|
|
319
|
+
|
|
320
|
+
> **More strategies:** Cookie-based auth, HTTP header injection, OAuth/SSO bypasses, reusable auth modules, and role-based testing — see [docs/authentication.md](docs/authentication.md)
|
|
321
|
+
|
|
322
|
+
---
|
|
323
|
+
|
|
248
324
|
## Reusable Modules
|
|
249
325
|
|
|
250
326
|
Extract common flows into parameterized modules:
|
|
251
327
|
|
|
252
328
|
```json
|
|
253
|
-
// e2e/modules/
|
|
329
|
+
// e2e/modules/login.json
|
|
254
330
|
{
|
|
255
|
-
"$module": "
|
|
256
|
-
"description": "
|
|
331
|
+
"$module": "login",
|
|
332
|
+
"description": "Log in via the UI login form",
|
|
257
333
|
"params": {
|
|
258
|
-
"
|
|
259
|
-
"
|
|
334
|
+
"email": { "required": true, "description": "User email" },
|
|
335
|
+
"password": { "required": true, "description": "User password" }
|
|
260
336
|
},
|
|
261
337
|
"actions": [
|
|
262
|
-
{ "type": "
|
|
263
|
-
{ "type": "
|
|
338
|
+
{ "type": "goto", "value": "/login" },
|
|
339
|
+
{ "type": "type", "selector": "#email", "value": "{{email}}" },
|
|
340
|
+
{ "type": "type", "selector": "#password", "value": "{{password}}" },
|
|
341
|
+
{ "type": "click", "text": "Sign In" },
|
|
342
|
+
{ "type": "wait", "value": "2000" }
|
|
264
343
|
]
|
|
265
344
|
}
|
|
266
345
|
```
|
|
@@ -271,7 +350,7 @@ Use in tests:
|
|
|
271
350
|
{
|
|
272
351
|
"name": "dashboard-loads",
|
|
273
352
|
"actions": [
|
|
274
|
-
{ "$use": "
|
|
353
|
+
{ "$use": "login", "params": { "email": "user@test.com", "password": "secret" } },
|
|
275
354
|
{ "type": "assert_text", "text": "Dashboard" }
|
|
276
355
|
]
|
|
277
356
|
}
|
|
@@ -448,70 +527,52 @@ Every screenshot gets a deterministic hash (`ss:a3f2b1c9`). Use `e2e_screenshot`
|
|
|
448
527
|
|
|
449
528
|
---
|
|
450
529
|
|
|
451
|
-
##
|
|
530
|
+
## AI Integration
|
|
452
531
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
### Install as Plugin (recommended)
|
|
532
|
+
### Claude Code
|
|
456
533
|
|
|
457
534
|
```bash
|
|
458
|
-
claude plugin
|
|
535
|
+
claude plugin marketplace add fastslack/mtw-e2e-runner
|
|
536
|
+
claude plugin install e2e-runner@matware
|
|
459
537
|
```
|
|
460
538
|
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
| Component | Description |
|
|
464
|
-
|-----------|-------------|
|
|
465
|
-
| **13 MCP tools** | Run tests, create test files, capture screenshots, query network logs, manage dashboard, verify issues, query learnings |
|
|
466
|
-
| **Skill** | Teaches Claude the full e2e-runner workflow — how to combine tools, interpret results, debug failures, create tests |
|
|
467
|
-
| **3 Commands** | `/e2e-runner:run` — run & analyze tests<br>`/e2e-runner:create-test` — explore UI and create tests<br>`/e2e-runner:verify-issue <url>` — verify GitHub/GitLab bugs |
|
|
468
|
-
| **2 Agents** | **test-analyzer** — diagnoses failures, analyzes flaky tests, drills into network errors<br>**test-creator** — explores UI, discovers selectors, designs and validates tests |
|
|
469
|
-
|
|
470
|
-
### Install MCP-only (alternative)
|
|
539
|
+
This gives Claude 13 MCP tools, a workflow skill, 3 slash commands (`/e2e-runner:run`, `/e2e-runner:create-test`, `/e2e-runner:verify-issue`), and 3 specialized agents (test-analyzer, test-creator, test-improver).
|
|
471
540
|
|
|
472
|
-
|
|
541
|
+
**MCP-only install** (tools only, no skill/commands/agents):
|
|
473
542
|
|
|
474
543
|
```bash
|
|
475
544
|
claude mcp add --transport stdio --scope user e2e-runner \
|
|
476
545
|
-- npx -y -p @matware/e2e-runner e2e-runner-mcp
|
|
477
546
|
```
|
|
478
547
|
|
|
479
|
-
###
|
|
548
|
+
### OpenCode
|
|
549
|
+
|
|
550
|
+
```bash
|
|
551
|
+
cp node_modules/@matware/e2e-runner/opencode.json ./
|
|
552
|
+
mkdir -p .opencode && cp -r node_modules/@matware/e2e-runner/.opencode/* .opencode/
|
|
553
|
+
```
|
|
480
554
|
|
|
481
|
-
|
|
482
|
-
|---------|-------------|
|
|
483
|
-
| `/e2e-runner:run` | Check pool, list suites, run tests, analyze results with screenshots and network drill-down |
|
|
484
|
-
| `/e2e-runner:create-test` | Explore the UI with screenshots, find selectors in source code, design test actions, create and validate |
|
|
485
|
-
| `/e2e-runner:verify-issue <url>` | Fetch a GitHub/GitLab issue, create tests that verify correct behavior, report bug confirmed or not reproducible |
|
|
555
|
+
See [OPENCODE.md](OPENCODE.md) for details.
|
|
486
556
|
|
|
487
557
|
### MCP Tools
|
|
488
558
|
|
|
489
559
|
| Tool | Description |
|
|
490
560
|
|------|-------------|
|
|
491
|
-
| `e2e_run` | Run tests
|
|
492
|
-
| `e2e_list` | List available test suites
|
|
493
|
-
| `e2e_create_test` | Create a new test JSON file
|
|
494
|
-
| `e2e_create_module` | Create a reusable module
|
|
495
|
-
| `e2e_pool_status` | Check Chrome pool
|
|
496
|
-
| `e2e_screenshot` | Retrieve a screenshot by hash
|
|
497
|
-
| `e2e_capture` | Capture screenshot of any URL
|
|
498
|
-
| `e2e_dashboard_start` | Start
|
|
499
|
-
| `e2e_dashboard_stop` | Stop
|
|
500
|
-
| `e2e_issue` | Fetch
|
|
501
|
-
| `e2e_network_logs` | Query network
|
|
502
|
-
| `e2e_learnings` | Query
|
|
503
|
-
| `e2e_neo4j` | Manage Neo4j knowledge graph
|
|
504
|
-
|
|
505
|
-
>
|
|
506
|
-
|
|
507
|
-
### What You Can Ask Claude Code
|
|
508
|
-
|
|
509
|
-
> "Run all E2E tests"
|
|
510
|
-
> "Create a test that verifies the checkout flow"
|
|
511
|
-
> "What tests are flaky? Show me the learning summary"
|
|
512
|
-
> "Capture a screenshot of /dashboard with auth"
|
|
513
|
-
> "Fetch issue #42 and create tests for it"
|
|
514
|
-
> "What's the API error rate for the last 7 days?"
|
|
561
|
+
| `e2e_run` | Run tests (all, by suite, or by file) |
|
|
562
|
+
| `e2e_list` | List available test suites |
|
|
563
|
+
| `e2e_create_test` | Create a new test JSON file |
|
|
564
|
+
| `e2e_create_module` | Create a reusable module |
|
|
565
|
+
| `e2e_pool_status` | Check Chrome pool health |
|
|
566
|
+
| `e2e_screenshot` | Retrieve a screenshot by hash |
|
|
567
|
+
| `e2e_capture` | Capture screenshot of any URL |
|
|
568
|
+
| `e2e_dashboard_start` | Start web dashboard |
|
|
569
|
+
| `e2e_dashboard_stop` | Stop web dashboard |
|
|
570
|
+
| `e2e_issue` | Fetch issue and generate tests |
|
|
571
|
+
| `e2e_network_logs` | Query network logs for a run |
|
|
572
|
+
| `e2e_learnings` | Query stability insights |
|
|
573
|
+
| `e2e_neo4j` | Manage Neo4j knowledge graph |
|
|
574
|
+
|
|
575
|
+
> Pool start/stop are CLI-only — not exposed via MCP.
|
|
515
576
|
|
|
516
577
|
---
|
|
517
578
|
|
package/agents/test-creator.md
CHANGED
|
@@ -69,6 +69,17 @@ You are a specialist in creating robust E2E tests for web applications. You expl
|
|
|
69
69
|
- Clear field → `clear`
|
|
70
70
|
- Submit → `click` on submit button or `press` Enter
|
|
71
71
|
|
|
72
|
+
### Storage
|
|
73
|
+
- Set localStorage key → `set_storage` with `value: "key=val"`
|
|
74
|
+
- Set sessionStorage key → `set_storage` with `value: "key=val"`, `selector: "session"`
|
|
75
|
+
- Assert storage key exists → `assert_storage` with `value: "key"`
|
|
76
|
+
- Assert storage value → `assert_storage` with `value: "key=expected"`
|
|
77
|
+
|
|
78
|
+
### Smart Clicks
|
|
79
|
+
- Click icon button → `click_icon` with `value` (icon identifier like "edit", "delete")
|
|
80
|
+
- Click menu item → `click_menu_item` with `text` (after opening the menu)
|
|
81
|
+
- Click element in a specific row/card → `click_in_context` with `text` (row text) + `selector` (child to click)
|
|
82
|
+
|
|
72
83
|
### Waiting
|
|
73
84
|
- Element appears → `wait` with `selector`
|
|
74
85
|
- Text appears → `wait` with `text`
|
|
@@ -86,12 +97,54 @@ You are a specialist in creating robust E2E tests for web applications. You expl
|
|
|
86
97
|
- CSS class → `assert_class`
|
|
87
98
|
- URL → `assert_url`
|
|
88
99
|
|
|
100
|
+
### Naming Rules (CRITICAL)
|
|
101
|
+
- **Suite file names MUST be unique and specific** to the feature, issue, or user flow being tested
|
|
102
|
+
- NEVER use generic names like `all`, `test`, `tests`, `debug`, `new`, `temp`, `main`, `suite`
|
|
103
|
+
- Include the feature or issue context: `login-valid-credentials`, `issue-1743-auth-redirect`, `checkout-payment-flow`
|
|
104
|
+
- If testing a GitHub/GitLab issue, include the issue number: `issue-1743-auth-timeout`, `bug-502-duplicate-submit`
|
|
105
|
+
- Before creating a test, call `e2e_list` and verify your chosen name doesn't already exist
|
|
106
|
+
- Individual test names within a suite must also be unique and descriptive
|
|
107
|
+
|
|
108
|
+
### Variables
|
|
109
|
+
- Use `{{var.KEY}}` to reference project variables instead of hardcoding sensitive values (tokens, IDs, secrets)
|
|
110
|
+
- Use `{{env.KEY}}` to reference environment variables from `process.env`
|
|
111
|
+
- Variables are stored in SQLite and managed via `e2e_vars` MCP tool or the dashboard UI
|
|
112
|
+
- Suite-scoped variables override project-scoped variables with the same key
|
|
113
|
+
- Example: `{ "type": "set_storage", "value": "accessToken={{var.JWT_TOKEN}}" }`
|
|
114
|
+
- Example: `{ "type": "goto", "value": "/patient/{{var.PATIENT_ID}}" }`
|
|
115
|
+
|
|
116
|
+
### DRY Patterns (CRITICAL)
|
|
117
|
+
|
|
118
|
+
Before creating tests, **always check existing modules** with `Glob` on `e2e/modules/*.json`. Reuse existing modules instead of duplicating actions.
|
|
119
|
+
|
|
120
|
+
**Use `beforeEach` when auth or setup is repeated across tests:**
|
|
121
|
+
When multiple tests in a suite share the same setup (e.g., same auth-jwt call), use the object format with `beforeEach` instead of repeating it in every test:
|
|
122
|
+
|
|
123
|
+
```json
|
|
124
|
+
{
|
|
125
|
+
"beforeEach": [
|
|
126
|
+
{ "$use": "auth-jwt", "params": { "token": "{{var.JWT_TOKEN}}", "institutionId": "{{var.INST_ID}}" } }
|
|
127
|
+
],
|
|
128
|
+
"tests": [
|
|
129
|
+
{ "name": "test-one", "actions": [...] },
|
|
130
|
+
{ "name": "test-two", "actions": [...] }
|
|
131
|
+
]
|
|
132
|
+
}
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
**Create modules for repeated action patterns:**
|
|
136
|
+
When 3+ tests repeat the same sequence (e.g., goto → wait → screenshot), create a module first with `e2e_create_module`, then use `$use` in the tests. This reduces test size by 70-80%.
|
|
137
|
+
|
|
138
|
+
**Use the object format (not array) when hooks are needed:**
|
|
139
|
+
- Array format: `[{ "name": ..., "actions": [...] }]` — no hooks
|
|
140
|
+
- Object format: `{ "beforeEach": [...], "tests": [...] }` — with hooks
|
|
141
|
+
|
|
89
142
|
### Best Practices
|
|
90
143
|
- Never use `evaluate` when a built-in action exists
|
|
144
|
+
- **Never hardcode tokens, passwords, or IDs in test files** — use `{{var.KEY}}` variables instead
|
|
91
145
|
- Add `retries` to actions on dynamically loaded elements
|
|
92
146
|
- Mark state-sharing tests as `serial: true`
|
|
93
147
|
- Use `screenshot` actions at key points for debugging
|
|
94
|
-
- Keep test names descriptive and kebab-case (`login-valid-credentials`)
|
|
95
148
|
|
|
96
149
|
## Output
|
|
97
150
|
|
package/agents/test-improver.md
CHANGED
|
@@ -73,6 +73,11 @@ When you find an `evaluate` action, check if it matches one of these patterns
|
|
|
73
73
|
| `MuiAutocomplete-root...input.focus()` | `focus_autocomplete` with `text` |
|
|
74
74
|
| `querySelectorAll('button').filter(regex)...click()` | `click_regex` with `text` + optional `selector` + `value` |
|
|
75
75
|
| `querySelectorAll('[class*="Chip"]')...click()` | `click_chip` with `text` |
|
|
76
|
+
| `localStorage.setItem(key, val)` or `sessionStorage.setItem(...)` | `set_storage` with `value: "key=val"`, `selector: "session"` for session |
|
|
77
|
+
| `localStorage.getItem(key)` check or `sessionStorage.getItem(...)` | `assert_storage` with `value: "key"` or `"key=expected"`, `selector: "session"` for session |
|
|
78
|
+
| `querySelector('svg[data-testid]').closest('button').click()` | `click_icon` with `value` (icon identifier) + optional `selector` (scope) |
|
|
79
|
+
| `querySelectorAll('[role="menuitem"]')...click()` | `click_menu_item` with `text` + optional `selector` (scope) |
|
|
80
|
+
| Container-by-text then child click: `rows.find(r => r.textContent.includes(text))...querySelector(child).click()` | `click_in_context` with `text` (container) + `selector` (child) |
|
|
76
81
|
| `document.title` or simple property read | `get_text` or `evaluate` (keep if no built-in equivalent) |
|
|
77
82
|
|
|
78
83
|
### Replacement Examples
|
|
@@ -101,6 +106,38 @@ When you find an `evaluate` action, check if it matches one of these patterns
|
|
|
101
106
|
{ "type": "click_option", "text": "Cefalea" }
|
|
102
107
|
```
|
|
103
108
|
|
|
109
|
+
```json
|
|
110
|
+
// BEFORE: evaluate for localStorage
|
|
111
|
+
{ "type": "evaluate", "value": "localStorage.setItem('authToken', 'abc123')" }
|
|
112
|
+
|
|
113
|
+
// AFTER: one action
|
|
114
|
+
{ "type": "set_storage", "value": "authToken=abc123" }
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
```json
|
|
118
|
+
// BEFORE: evaluate for icon click
|
|
119
|
+
{ "type": "evaluate", "value": "document.querySelector('svg[data-testid=\"EditIcon\"]').closest('button').click()" }
|
|
120
|
+
|
|
121
|
+
// AFTER: one action
|
|
122
|
+
{ "type": "click_icon", "value": "Edit" }
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
```json
|
|
126
|
+
// BEFORE: evaluate for menu item click
|
|
127
|
+
{ "type": "evaluate", "value": "const items = [...document.querySelectorAll('[role=\"menuitem\"]')]; items.find(el => el.textContent.includes('Delete')).click();" }
|
|
128
|
+
|
|
129
|
+
// AFTER: one action
|
|
130
|
+
{ "type": "click_menu_item", "text": "Delete" }
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
```json
|
|
134
|
+
// BEFORE: evaluate for contextual click
|
|
135
|
+
{ "type": "evaluate", "value": "const rows = [...document.querySelectorAll('tr')]; const row = rows.find(r => r.textContent.includes('John Doe')); row.querySelector('button.edit').click();" }
|
|
136
|
+
|
|
137
|
+
// AFTER: one action
|
|
138
|
+
{ "type": "click_in_context", "text": "John Doe", "selector": "button.edit" }
|
|
139
|
+
```
|
|
140
|
+
|
|
104
141
|
## Duplication Detection
|
|
105
142
|
|
|
106
143
|
Look for these common duplication patterns:
|