@matware/e2e-runner 1.1.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +21 -0
- package/.claude-plugin/plugin.json +9 -0
- package/.mcp.json +9 -0
- package/.opencode/commands/create-test.md +63 -0
- package/.opencode/commands/run.md +50 -0
- package/.opencode/commands/verify-issue.md +62 -0
- package/.opencode/skills/e2e-testing/SKILL.md +181 -0
- package/.opencode/skills/e2e-testing/references/action-types.md +143 -0
- package/.opencode/skills/e2e-testing/references/auth-strategies.md +91 -0
- package/.opencode/skills/e2e-testing/references/graphql.md +59 -0
- package/.opencode/skills/e2e-testing/references/issue-verification.md +59 -0
- package/.opencode/skills/e2e-testing/references/multi-pool.md +60 -0
- package/.opencode/skills/e2e-testing/references/network-debugging.md +62 -0
- package/.opencode/skills/e2e-testing/references/test-json-format.md +163 -0
- package/.opencode/skills/e2e-testing/references/troubleshooting.md +224 -0
- package/.opencode/skills/e2e-testing/references/variables.md +41 -0
- package/.opencode/skills/e2e-testing/references/visual-verification.md +89 -0
- package/OPENCODE.md +166 -0
- package/README.md +990 -296
- package/agents/test-analyzer.md +81 -0
- package/agents/test-creator.md +155 -0
- package/agents/test-improver.md +177 -0
- package/bin/cli.js +602 -22
- package/commands/create-test.md +65 -0
- package/commands/run.md +49 -0
- package/commands/verify-issue.md +63 -0
- package/opencode.json +11 -0
- package/package.json +15 -2
- package/scripts/setup-opencode.sh +113 -0
- package/skills/e2e-testing/SKILL.md +173 -0
- package/skills/e2e-testing/references/action-types.md +143 -0
- package/skills/e2e-testing/references/auth-strategies.md +91 -0
- package/skills/e2e-testing/references/graphql.md +59 -0
- package/skills/e2e-testing/references/issue-verification.md +59 -0
- package/skills/e2e-testing/references/multi-pool.md +60 -0
- package/skills/e2e-testing/references/network-debugging.md +62 -0
- package/skills/e2e-testing/references/test-json-format.md +163 -0
- package/skills/e2e-testing/references/troubleshooting.md +224 -0
- package/skills/e2e-testing/references/variables.md +41 -0
- package/skills/e2e-testing/references/visual-verification.md +89 -0
- package/src/actions.js +597 -20
- package/src/ai-generate.js +142 -12
- package/src/config.js +171 -0
- package/src/dashboard.js +299 -17
- package/src/db.js +335 -13
- package/src/index.js +15 -8
- package/src/learner-markdown.js +177 -0
- package/src/learner-neo4j.js +255 -0
- package/src/learner-sqlite.js +658 -0
- package/src/learner.js +418 -0
- package/src/mcp-tools.js +1558 -50
- package/src/module-resolver.js +310 -0
- package/src/narrate.js +262 -0
- package/src/neo4j-pool.js +124 -0
- package/src/pool-manager.js +223 -0
- package/src/reporter.js +117 -3
- package/src/runner.js +274 -71
- package/src/sync/auth.js +354 -0
- package/src/sync/client.js +572 -0
- package/src/sync/hub-routes.js +816 -0
- package/src/sync/index.js +68 -0
- package/src/sync/middleware.js +347 -0
- package/src/sync/queue.js +209 -0
- package/src/sync/schema.js +540 -0
- package/src/verify.js +14 -9
- package/src/watch.js +384 -0
- package/templates/build-dashboard.js +69 -0
- package/templates/dashboard/js/api.js +60 -0
- package/templates/dashboard/js/init.js +13 -0
- package/templates/dashboard/js/keyboard.js +46 -0
- package/templates/dashboard/js/state.js +40 -0
- package/templates/dashboard/js/toast.js +41 -0
- package/templates/dashboard/js/utils.js +196 -0
- package/templates/dashboard/js/view-live.js +143 -0
- package/templates/dashboard/js/view-runs.js +572 -0
- package/templates/dashboard/js/view-tests.js +294 -0
- package/templates/dashboard/js/view-watch.js +242 -0
- package/templates/dashboard/js/websocket.js +110 -0
- package/templates/dashboard/styles/base.css +69 -0
- package/templates/dashboard/styles/components.css +110 -0
- package/templates/dashboard/styles/view-live.css +74 -0
- package/templates/dashboard/styles/view-runs.css +207 -0
- package/templates/dashboard/styles/view-tests.css +96 -0
- package/templates/dashboard/styles/view-watch.css +53 -0
- package/templates/dashboard/template.html +267 -0
- package/templates/dashboard.html +2171 -530
- package/templates/docker-compose-neo4j.yml +19 -0
- package/templates/e2e.config.js +3 -0
- package/templates/sample-test.json +0 -8
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "matware",
|
|
3
|
+
"owner": {
|
|
4
|
+
"name": "Matware"
|
|
5
|
+
},
|
|
6
|
+
"metadata": {
|
|
7
|
+
"description": "E2E testing tools by Matware"
|
|
8
|
+
},
|
|
9
|
+
"plugins": [
|
|
10
|
+
{
|
|
11
|
+
"name": "e2e-runner",
|
|
12
|
+
"source": "./",
|
|
13
|
+
"description": "JSON-driven E2E browser test runner with Chrome pool, visual verification, and learning system",
|
|
14
|
+
"version": "1.2.1",
|
|
15
|
+
"author": { "name": "Matware" },
|
|
16
|
+
"repository": "https://github.com/fastslack/mtw-e2e-runner",
|
|
17
|
+
"license": "Apache-2.0",
|
|
18
|
+
"keywords": ["e2e", "testing", "chrome", "puppeteer", "mcp"]
|
|
19
|
+
}
|
|
20
|
+
]
|
|
21
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "e2e-runner",
|
|
3
|
+
"version": "1.2.0",
|
|
4
|
+
"description": "JSON-driven E2E browser test runner with Chrome pool, visual verification, and learning system",
|
|
5
|
+
"author": { "name": "Matware" },
|
|
6
|
+
"repository": "https://github.com/fastslack/mtw-e2e-runner",
|
|
7
|
+
"license": "Apache-2.0",
|
|
8
|
+
"keywords": ["e2e", "testing", "chrome", "puppeteer", "mcp"]
|
|
9
|
+
}
|
package/.mcp.json
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Create a new E2E test by exploring the UI and designing test actions
|
|
3
|
+
---
|
|
4
|
+
|
|
5
|
+
# Create E2E Test
|
|
6
|
+
|
|
7
|
+
Help the user create a new E2E test file by exploring the application and designing appropriate test actions.
|
|
8
|
+
|
|
9
|
+
## Workflow
|
|
10
|
+
|
|
11
|
+
1. **Understand the goal** — Ask the user what they want to test if not already specified. Identify the page(s), user flow, and expected outcomes.
|
|
12
|
+
|
|
13
|
+
2. **Check pool** — Call `e2e_pool_status` to ensure the Chrome pool is available.
|
|
14
|
+
|
|
15
|
+
3. **Explore the UI** — Use `e2e_capture` to screenshot the target page(s). This helps understand the current state of the UI, available elements, and layout.
|
|
16
|
+
|
|
17
|
+
4. **Check existing tests** — Call `e2e_list` to see what test suites already exist. Read relevant existing test files with `Read` to follow conventions and avoid duplication.
|
|
18
|
+
|
|
19
|
+
5. **Explore source code** (optional) — If needed, use `Grep` and `Read` to find selectors, form field IDs, API endpoints, or component structure in the application source code.
|
|
20
|
+
|
|
21
|
+
6. **Design the test** — Based on UI exploration and source code analysis, design the test actions:
|
|
22
|
+
- Use the most specific selectors available (data-testid > id > class > text)
|
|
23
|
+
- Prefer granular assertion actions over `evaluate`
|
|
24
|
+
- Use framework-aware actions for React/MUI (`type_react`, `click_option`, `focus_autocomplete`)
|
|
25
|
+
- Add `wait` actions before assertions on dynamic content
|
|
26
|
+
- Add `assert_no_network_errors` after critical page loads
|
|
27
|
+
- Consider adding an `expect` field for visual verification
|
|
28
|
+
- **DRY**: If auth/setup is repeated across tests, use `beforeEach` hook (object format) instead of repeating per test
|
|
29
|
+
- **DRY**: If 3+ tests repeat the same action pattern, create a module with `e2e_create_module` first
|
|
30
|
+
|
|
31
|
+
7. **Create the test** — Call `e2e_create_test` with the designed test structure.
|
|
32
|
+
- Check existing modules (`e2e/modules/`) — reuse them via `$use` instead of duplicating actions
|
|
33
|
+
- Create new modules with `e2e_create_module` for repeated sequences (auth, navigation, screenshot patterns)
|
|
34
|
+
- Use object format `{ "beforeEach": [...], "tests": [...] }` when hooks are needed
|
|
35
|
+
|
|
36
|
+
8. **Validate** — Run the newly created test with `e2e_run` using the `suite` parameter. Analyze results and iterate if needed.
|
|
37
|
+
|
|
38
|
+
## Naming Rules (CRITICAL)
|
|
39
|
+
|
|
40
|
+
Suite names MUST be unique and specific to the feature, issue, or user flow:
|
|
41
|
+
- GOOD: `login-valid-credentials`, `issue-1743-auth-redirect`, `checkout-payment-flow`
|
|
42
|
+
- BAD: `all`, `test`, `debug`, `new`, `temp`, `main`, `suite`
|
|
43
|
+
|
|
44
|
+
If testing a GitHub/GitLab issue, include the issue number: `issue-1743-auth-timeout`, `bug-502-duplicate-submit`
|
|
45
|
+
|
|
46
|
+
Before creating, always call `e2e_list` to verify the name doesn't already exist.
|
|
47
|
+
|
|
48
|
+
## Arguments
|
|
49
|
+
|
|
50
|
+
The user may provide:
|
|
51
|
+
- A test name: `/create-test login-flow`
|
|
52
|
+
- A description of what to test: `/create-test test the checkout process`
|
|
53
|
+
- A URL to start from: `/create-test http://localhost:3000/checkout`
|
|
54
|
+
|
|
55
|
+
## MCP Tools Used
|
|
56
|
+
|
|
57
|
+
- `e2e_pool_status` — Check Chrome pool availability
|
|
58
|
+
- `e2e_capture` — Capture screenshots of pages
|
|
59
|
+
- `e2e_list` — List existing test suites
|
|
60
|
+
- `e2e_create_test` — Write test JSON file
|
|
61
|
+
- `e2e_create_module` — Write reusable module
|
|
62
|
+
- `e2e_run` — Validate the new test
|
|
63
|
+
- `e2e_screenshot` — View test screenshots
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Run E2E tests and analyze results with screenshots and network drill-down
|
|
3
|
+
---
|
|
4
|
+
|
|
5
|
+
# Run E2E Tests
|
|
6
|
+
|
|
7
|
+
Execute E2E tests and provide a complete analysis of results.
|
|
8
|
+
|
|
9
|
+
## Workflow
|
|
10
|
+
|
|
11
|
+
1. **Check pool availability** — Call `e2e_pool_status` to confirm the Chrome pool is running. If not available, tell the user to run `npx e2e-runner pool start` via CLI.
|
|
12
|
+
|
|
13
|
+
2. **List available suites** — Call `e2e_list` to show the user what test suites are available.
|
|
14
|
+
|
|
15
|
+
3. **Run tests** — Call `e2e_run` based on user input:
|
|
16
|
+
- If user specified a suite name: use `suite` parameter
|
|
17
|
+
- If user specified a file: use `file` parameter
|
|
18
|
+
- If user said "all" or didn't specify: use `all: true`
|
|
19
|
+
- Always pass `cwd` with the current working directory
|
|
20
|
+
- Pass any user-specified overrides: `baseUrl`, `concurrency`, `retries`, `failOnNetworkError`
|
|
21
|
+
|
|
22
|
+
4. **Analyze results** — Parse the run response:
|
|
23
|
+
- Report pass/fail summary and duration
|
|
24
|
+
- For failures: show error messages and retrieve error screenshots with `e2e_screenshot`
|
|
25
|
+
- For verifications (tests with `expect`): retrieve verification screenshots and judge against descriptions
|
|
26
|
+
- Highlight flaky tests if any
|
|
27
|
+
- Summarize network activity (failed requests, slow requests)
|
|
28
|
+
|
|
29
|
+
5. **Drill down if needed** — For failed tests:
|
|
30
|
+
- Use `e2e_network_logs` with `runDbId` to investigate network failures
|
|
31
|
+
- Use `e2e_learnings` to check if this is a known pattern or new failure
|
|
32
|
+
|
|
33
|
+
6. **Report** — Provide a clear summary to the user with actionable next steps.
|
|
34
|
+
|
|
35
|
+
## Arguments
|
|
36
|
+
|
|
37
|
+
The user may pass arguments after the command:
|
|
38
|
+
- Suite name: `/run auth` → run the auth suite
|
|
39
|
+
- `--all`: run all suites
|
|
40
|
+
- `--base-url <url>`: override base URL
|
|
41
|
+
- `--retries <n>`: set retry count
|
|
42
|
+
|
|
43
|
+
## MCP Tools Used
|
|
44
|
+
|
|
45
|
+
- `e2e_pool_status` — Check Chrome pool availability
|
|
46
|
+
- `e2e_list` — List test suites and modules
|
|
47
|
+
- `e2e_run` — Execute tests
|
|
48
|
+
- `e2e_screenshot` — Retrieve screenshots by hash
|
|
49
|
+
- `e2e_network_logs` — Inspect network requests
|
|
50
|
+
- `e2e_learnings` — Query learning system
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Verify a GitHub/GitLab issue by creating and running E2E tests
|
|
3
|
+
---
|
|
4
|
+
|
|
5
|
+
# Verify Issue
|
|
6
|
+
|
|
7
|
+
Turn a GitHub or GitLab bug report into executable E2E tests to confirm or dismiss the bug.
|
|
8
|
+
|
|
9
|
+
## Workflow
|
|
10
|
+
|
|
11
|
+
1. **Check pool** — Call `e2e_pool_status` to ensure the Chrome pool is available.
|
|
12
|
+
|
|
13
|
+
2. **Fetch the issue** — Call `e2e_issue` with the issue URL. Default `mode: "prompt"` returns issue details + a structured prompt for test creation.
|
|
14
|
+
|
|
15
|
+
3. **Analyze the issue** — Parse the issue details:
|
|
16
|
+
- Understand the reported bug or expected behavior
|
|
17
|
+
- Identify affected pages/flows
|
|
18
|
+
- Note any reproduction steps provided
|
|
19
|
+
|
|
20
|
+
4. **Explore the app** — Use `e2e_capture` to screenshot relevant pages. Use `Read` and `Grep` to check source code for related components, API endpoints, or selectors.
|
|
21
|
+
|
|
22
|
+
5. **Design tests** — Create tests that assert the **correct behavior**:
|
|
23
|
+
- If tests **fail** → bug is confirmed (correct behavior is not working)
|
|
24
|
+
- If tests **pass** → bug is not reproducible
|
|
25
|
+
|
|
26
|
+
6. **Create and run** — Use `e2e_create_test` to write the test file, then `e2e_run` to execute it.
|
|
27
|
+
|
|
28
|
+
7. **Analyze results** — For failures:
|
|
29
|
+
- Retrieve error screenshots with `e2e_screenshot`
|
|
30
|
+
- Check network logs with `e2e_network_logs` for API-related issues
|
|
31
|
+
- Determine if the failure confirms the bug
|
|
32
|
+
|
|
33
|
+
8. **Report verdict** — Clearly state:
|
|
34
|
+
- **BUG CONFIRMED**: tests failed, reproducing the issue
|
|
35
|
+
- **NOT REPRODUCIBLE**: tests passed, correct behavior works as expected
|
|
36
|
+
- Include evidence (screenshots, error messages, network details)
|
|
37
|
+
|
|
38
|
+
## Alternative: Verify Mode
|
|
39
|
+
|
|
40
|
+
If `ANTHROPIC_API_KEY` is set, use `e2e_issue` with `mode: "verify"` for a fully automated flow — it generates tests via Claude API, runs them, and reports the result.
|
|
41
|
+
|
|
42
|
+
## Arguments
|
|
43
|
+
|
|
44
|
+
**Required**: GitHub or GitLab issue URL
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
/verify-issue https://github.com/org/repo/issues/123
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Optional flags:
|
|
51
|
+
- `--test-type api` — generate API tests instead of UI tests
|
|
52
|
+
- `--verify` — use verify mode (requires ANTHROPIC_API_KEY)
|
|
53
|
+
|
|
54
|
+
## MCP Tools Used
|
|
55
|
+
|
|
56
|
+
- `e2e_pool_status` — Check Chrome pool availability
|
|
57
|
+
- `e2e_issue` — Fetch and parse issue details
|
|
58
|
+
- `e2e_create_test` — Write test file
|
|
59
|
+
- `e2e_run` — Execute tests
|
|
60
|
+
- `e2e_screenshot` — View screenshots
|
|
61
|
+
- `e2e_network_logs` — Inspect network requests
|
|
62
|
+
- `e2e_capture` — Screenshot pages
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: e2e-testing
|
|
3
|
+
description: Create, run, and debug JSON-driven E2E browser tests with Chrome pool integration
|
|
4
|
+
triggers:
|
|
5
|
+
- e2e
|
|
6
|
+
- end to end
|
|
7
|
+
- browser test
|
|
8
|
+
- chrome pool
|
|
9
|
+
- puppeteer
|
|
10
|
+
- visual verification
|
|
11
|
+
- issue verification
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
# E2E Testing with @matware/e2e-runner
|
|
15
|
+
|
|
16
|
+
## Overview
|
|
17
|
+
|
|
18
|
+
`@matware/e2e-runner` is a JSON-driven E2E test runner. Tests are defined as JSON files with sequential browser actions — no JavaScript test code. Tests run in parallel against a Chrome pool (browserless/chrome via Docker) using Puppeteer.
|
|
19
|
+
|
|
20
|
+
**Key capabilities:** 13 MCP tools for running tests, creating test files, capturing screenshots, analyzing network traffic, verifying GitHub/GitLab issues, and querying a learning system for stability insights.
|
|
21
|
+
|
|
22
|
+
## Prerequisites
|
|
23
|
+
|
|
24
|
+
Before running any tests, verify the Chrome pool is available:
|
|
25
|
+
|
|
26
|
+
```
|
|
27
|
+
e2e_pool_status → check "Available: yes" and session capacity
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
If the pool is not running, the user must start it via CLI (not available via MCP):
|
|
31
|
+
```bash
|
|
32
|
+
npx e2e-runner pool start
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Core Workflow
|
|
36
|
+
|
|
37
|
+
The standard test execution flow:
|
|
38
|
+
|
|
39
|
+
1. **Check pool** → `e2e_pool_status` — confirm Chrome pool is ready
|
|
40
|
+
2. **List suites** → `e2e_list` — discover available test files and modules
|
|
41
|
+
3. **Run tests** → `e2e_run` — execute with `all`, `suite`, or `file` parameter
|
|
42
|
+
4. **Interpret results** — check `summary`, `failures`, `narratives`, `networkSummary`
|
|
43
|
+
5. **View screenshots** → `e2e_screenshot` — retrieve error/verification screenshots by `ss:HASH`
|
|
44
|
+
6. **Drill into network** → `e2e_network_logs` — use `runDbId` to inspect requests/responses
|
|
45
|
+
7. **Check learnings** → `e2e_learnings` — query stability trends, flaky tests, error patterns
|
|
46
|
+
|
|
47
|
+
### Interpreting Run Results
|
|
48
|
+
|
|
49
|
+
The `e2e_run` response includes:
|
|
50
|
+
|
|
51
|
+
- **summary**: pass/fail counts, duration, `runDbId` for drill-down
|
|
52
|
+
- **failures**: failed test names with error messages and error screenshot hashes
|
|
53
|
+
- **narratives**: step-by-step human-readable story of each test execution
|
|
54
|
+
- **networkSummary**: per-test request stats (status distribution, slow/failed requests)
|
|
55
|
+
- **verifications**: tests with `expect` field — call `e2e_screenshot` to visually verify
|
|
56
|
+
- **learnings**: stability insights from the learning system (new failures, flaky patterns)
|
|
57
|
+
|
|
58
|
+
## Creating Tests
|
|
59
|
+
|
|
60
|
+
### Basic Structure
|
|
61
|
+
|
|
62
|
+
```json
|
|
63
|
+
[
|
|
64
|
+
{
|
|
65
|
+
"name": "login-flow",
|
|
66
|
+
"actions": [
|
|
67
|
+
{ "type": "goto", "value": "/login" },
|
|
68
|
+
{ "type": "type", "selector": "#email", "value": "user@example.com" },
|
|
69
|
+
{ "type": "type", "selector": "#password", "value": "secret" },
|
|
70
|
+
{ "type": "click", "text": "Sign In" },
|
|
71
|
+
{ "type": "wait", "selector": ".dashboard" },
|
|
72
|
+
{ "type": "assert_url", "value": "/dashboard" }
|
|
73
|
+
]
|
|
74
|
+
}
|
|
75
|
+
]
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Use `e2e_create_test` to write test files. Use `e2e_create_module` for reusable action sequences.
|
|
79
|
+
|
|
80
|
+
### Key Action Patterns
|
|
81
|
+
|
|
82
|
+
- **Navigation**: `goto` (full page load), `navigate` (SPA-friendly, non-blocking)
|
|
83
|
+
- **Interaction**: `click` (selector or text), `type`/`fill`, `select`, `press`, `hover`, `scroll`
|
|
84
|
+
- **React/MUI**: `type_react` (controlled inputs), `click_option`, `focus_autocomplete`, `click_chip`, `click_regex`
|
|
85
|
+
- **Assertions**: `assert_text` (page-wide), `assert_element_text` (scoped), `assert_url`, `assert_visible`, `assert_not_visible`, `assert_count`, `assert_attribute`, `assert_class`, `assert_input_value`, `assert_matches`
|
|
86
|
+
- **Extraction**: `get_text` (non-assertion, returns element text), `screenshot`
|
|
87
|
+
- **Advanced**: `evaluate` (run JS in browser), `assert_no_network_errors`, `clear_cookies`
|
|
88
|
+
|
|
89
|
+
### Visual Verification
|
|
90
|
+
|
|
91
|
+
Add an `expect` field to any test for AI-powered visual verification:
|
|
92
|
+
|
|
93
|
+
```json
|
|
94
|
+
{
|
|
95
|
+
"name": "dashboard-loads",
|
|
96
|
+
"expect": "Should show data table with at least 3 rows and no error messages",
|
|
97
|
+
"actions": [...]
|
|
98
|
+
}
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
After running, call `e2e_screenshot` with each verification hash and judge the screenshot against the description.
|
|
102
|
+
|
|
103
|
+
### Reusable Modules
|
|
104
|
+
|
|
105
|
+
Create modules with `e2e_create_module`, reference them in tests:
|
|
106
|
+
|
|
107
|
+
```json
|
|
108
|
+
{ "$use": "auth-jwt", "params": { "email": "admin@test.com" } }
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
For complete action type reference, see [action-types.md](references/action-types.md).
|
|
112
|
+
For JSON format details (hooks, serial, retries, modules), see [test-json-format.md](references/test-json-format.md).
|
|
113
|
+
|
|
114
|
+
## Issue Verification
|
|
115
|
+
|
|
116
|
+
Turn GitHub/GitLab bug reports into executable tests:
|
|
117
|
+
|
|
118
|
+
### Prompt Mode (default, no API key needed)
|
|
119
|
+
|
|
120
|
+
1. `e2e_issue` with issue URL → returns structured prompt with issue details
|
|
121
|
+
2. Analyze the issue and design test actions
|
|
122
|
+
3. `e2e_create_test` → create the test file
|
|
123
|
+
4. `e2e_run` → execute and verify
|
|
124
|
+
|
|
125
|
+
### Verify Mode (requires ANTHROPIC_API_KEY)
|
|
126
|
+
|
|
127
|
+
1. `e2e_issue` with `mode: "verify"` → auto-generates tests via Claude API, runs them, reports result
|
|
128
|
+
2. Test failure = bug confirmed, all pass = not reproducible
|
|
129
|
+
|
|
130
|
+
Supports both UI tests (`testType: "e2e"`) and API tests (`testType: "api"`).
|
|
131
|
+
|
|
132
|
+
## Debugging & Analysis
|
|
133
|
+
|
|
134
|
+
### Network Inspection
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
e2e_network_logs(runDbId) → all requests
|
|
138
|
+
e2e_network_logs(runDbId, errorsOnly: true) → failed requests only
|
|
139
|
+
e2e_network_logs(runDbId, includeBodies: true) → full request/response bodies
|
|
140
|
+
e2e_network_logs(runDbId, urlPattern: "/api/") → filter by URL pattern
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### Learning System
|
|
144
|
+
|
|
145
|
+
```
|
|
146
|
+
e2e_learnings("summary") → full project overview
|
|
147
|
+
e2e_learnings("flaky") → flaky test analysis
|
|
148
|
+
e2e_learnings("selectors") → selector stability
|
|
149
|
+
e2e_learnings("errors") → recurring error patterns
|
|
150
|
+
e2e_learnings("test:name") → drill into specific test history
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### On-Demand Capture
|
|
154
|
+
|
|
155
|
+
Use `e2e_capture` to screenshot any URL without running a full test suite. Useful for visual exploration or verifying current state.
|
|
156
|
+
|
|
157
|
+
### Dashboard
|
|
158
|
+
|
|
159
|
+
Start/stop the web dashboard with `e2e_dashboard_start` / `e2e_dashboard_stop` for a visual UI at `http://localhost:8484`.
|
|
160
|
+
|
|
161
|
+
## Important Rules
|
|
162
|
+
|
|
163
|
+
1. **Always pass `cwd`** — All MCP tools accept `cwd` (the project root). Always pass it so config files and test directories resolve correctly.
|
|
164
|
+
2. **`baseUrl` default is `http://host.docker.internal:3000`** — Chrome runs inside Docker, so it uses `host.docker.internal` to reach the host machine. Override with `baseUrl` if the app runs on a different port.
|
|
165
|
+
3. **Pool management is CLI-only** — `pool start` and `pool stop` are not available via MCP. Only `e2e_pool_status` is an MCP tool.
|
|
166
|
+
4. **`evaluate` is strict** — Returns starting with `FAIL:`/`ERROR:` or returning `false` will fail the test. Prefer granular assertion actions over `evaluate` with inline JS.
|
|
167
|
+
5. **Serial tests** — Mark tests with `"serial": true` if they share mutable state. They run after all parallel tests.
|
|
168
|
+
6. **Action retries** — Use `"retries": N` on individual actions for flaky selectors, or globally via config.
|
|
169
|
+
|
|
170
|
+
## References
|
|
171
|
+
|
|
172
|
+
- [Action Types Reference](references/action-types.md) — Complete catalog of 28+ action types with fields and examples
|
|
173
|
+
- [Test JSON Format](references/test-json-format.md) — JSON structure, hooks, serial, retries, modules, exclude patterns, environment profiles, CI output
|
|
174
|
+
- [GraphQL Action](references/graphql.md) — GQL action config, variables, inline assertions, __e2eGql helper
|
|
175
|
+
- [Authentication Strategies](references/auth-strategies.md) — 6 auth methods + auto-login + reusable auth modules
|
|
176
|
+
- [Network Debugging](references/network-debugging.md) — Error handling, request logging, drill-down pattern
|
|
177
|
+
- [Visual Verification](references/visual-verification.md) — Expect field, double screenshots, strictness levels, verdict format
|
|
178
|
+
- [Multi-Pool Support](references/multi-pool.md) — Config, selection algorithm, failover, pool-aware queue
|
|
179
|
+
- [Variables](references/variables.md) — SQLite-backed variables, syntax, MCP tool, dashboard UI, REST API
|
|
180
|
+
- [Issue Verification](references/issue-verification.md) — GitHub/GitLab, AI modes, test categories, GitLab limitations
|
|
181
|
+
- [Troubleshooting](references/troubleshooting.md) — Common problems, pre-validation, screenshot hashes, dashboard
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# Action Types Reference
|
|
2
|
+
|
|
3
|
+
Complete catalog of all action types supported by @matware/e2e-runner.
|
|
4
|
+
|
|
5
|
+
## Navigation
|
|
6
|
+
|
|
7
|
+
| Action | Fields | Description |
|
|
8
|
+
|--------|--------|-------------|
|
|
9
|
+
| `goto` | `value` (URL or path) | Full page navigation. Relative paths are prefixed with `baseUrl`. Waits for `domcontentloaded`. |
|
|
10
|
+
| `navigate` | `value` (URL or path) | SPA-friendly navigation. Uses `load` event with a 5s race timeout — won't block if client-side routing doesn't fire `load`. |
|
|
11
|
+
|
|
12
|
+
## Interaction
|
|
13
|
+
|
|
14
|
+
| Action | Fields | Description |
|
|
15
|
+
|--------|--------|-------------|
|
|
16
|
+
| `click` | `selector` OR `text` | Click by CSS selector or by visible text content. Text search covers: `button, a, [role="button"], [role="tab"], [role="menuitem"], [role="option"], [role="listitem"], div[class*="cursor"], span, li, td, th, label, p, h1-h6, dd, dt`. |
|
|
17
|
+
| `type` / `fill` | `selector`, `value` | Triple-clicks to select all, then Backspace to clear, then types with 20ms delay per character. |
|
|
18
|
+
| `select` | `selector`, `value` | Select an `<option>` value in a `<select>` element. |
|
|
19
|
+
| `clear` | `selector` | Triple-click + Backspace to clear an input field. |
|
|
20
|
+
| `press` | `value` (key name) | Press a keyboard key (e.g. `"Enter"`, `"Tab"`, `"Escape"`, `"ArrowDown"`). |
|
|
21
|
+
| `scroll` | `selector` (optional), `value` (optional) | Scroll element into view, or scroll window by Y pixels (default 300). |
|
|
22
|
+
| `hover` | `selector` | Hover over an element. |
|
|
23
|
+
|
|
24
|
+
## Framework-Aware (React/MUI)
|
|
25
|
+
|
|
26
|
+
| Action | Fields | Description |
|
|
27
|
+
|--------|--------|-------------|
|
|
28
|
+
| `type_react` | `selector`, `value` | Types into React controlled inputs using native value setter. Dispatches `input` + `change` events so React state updates. Supports `<input>` and `<textarea>`. |
|
|
29
|
+
| `click_regex` | `text` (regex), `selector` (optional), `value` (`"last"` optional) | Click element whose textContent matches regex (case-insensitive). Default: first match. `value: "last"` for last match. `selector` scopes the search. |
|
|
30
|
+
| `click_option` | `text` | Click a `[role="option"]` element by text — for autocomplete/select dropdowns. Waits for option to appear. |
|
|
31
|
+
| `focus_autocomplete` | `text` (label text) | Focus an autocomplete input by label. Supports MUI `.MuiAutocomplete-root` and `[role="combobox"]`. |
|
|
32
|
+
| `click_chip` | `text` | Click a chip/tag element by text. Searches `[class*="Chip"]`, `[class*="chip"]`, `[data-chip]`. |
|
|
33
|
+
|
|
34
|
+
## Storage
|
|
35
|
+
|
|
36
|
+
| Action | Fields | Description |
|
|
37
|
+
|--------|--------|-------------|
|
|
38
|
+
| `set_storage` | `value` (`"key=val"`), `selector` (`"session"` optional) | Set a `localStorage` key (default) or `sessionStorage` key (with `selector: "session"`). |
|
|
39
|
+
| `assert_storage` | `value` (`"key"` or `"key=expected"`), `selector` (`"session"` optional) | Without `=`: checks key exists. With `=`: checks exact value match. Uses `localStorage` by default, `sessionStorage` with `selector: "session"`. |
|
|
40
|
+
|
|
41
|
+
## Smart Interaction
|
|
42
|
+
|
|
43
|
+
| Action | Fields | Description |
|
|
44
|
+
|--------|--------|-------------|
|
|
45
|
+
| `click_icon` | `value` (icon identifier), `selector` (scope, optional) | Click an icon by `data-testid`, `data-icon`, `aria-label`, CSS class, or SVG title. Walks up to nearest clickable ancestor (`button`, `a`, `[role="button"]`). Works with MUI, FontAwesome, Heroicons, Bootstrap Icons, Lucide. |
|
|
46
|
+
| `click_menu_item` | `text` (menu item text), `selector` (scope, optional) | Click a menu item by text. Searches `[role="menuitem"]`, `[role="menuitemradio"]`, `[role="menuitemcheckbox"]`, `.dropdown-item`, `.menu-item`, `[class*="MenuItem"]`, `[role="menu"] > li`. Waits for element to appear. |
|
|
47
|
+
| `click_in_context` | `text` (container text), `selector` (child to click) | Find the smallest container whose text includes `text`, then click the `selector` child within it. Containers: `section`, `article`, `[class*="card"]`, `li`, `tr`, `div[class]`, etc. Both fields required. |
|
|
48
|
+
|
|
49
|
+
## Assertions
|
|
50
|
+
|
|
51
|
+
| Action | Fields | Description |
|
|
52
|
+
|--------|--------|-------------|
|
|
53
|
+
| `assert_text` | `text` | Check entire page body contains text (substring match). |
|
|
54
|
+
| `assert_element_text` | `selector`, `text`, `value` (`"exact"` optional) | Check specific element's `textContent`. Default: substring match. With `value: "exact"`: strict `trim() ===` comparison. |
|
|
55
|
+
| `assert_url` | `value` | Check current URL. Path-only (`/dashboard`) compares pathname. Full URL does substring match. |
|
|
56
|
+
| `assert_visible` | `selector` | Element exists and is visible (`display`, `visibility`, `opacity` checks). |
|
|
57
|
+
| `assert_not_visible` | `selector` | Passes if element doesn't exist OR is hidden. |
|
|
58
|
+
| `assert_count` | `selector`, `value` | Count matching elements. Supports exact (`"5"`) and operators (`">3"`, `">=1"`, `"<10"`, `"<=5"`). |
|
|
59
|
+
| `assert_attribute` | `selector`, `value` (`"attr=expected"` or `"attr"`) | With `=`: checks attribute value. Without: checks attribute existence. |
|
|
60
|
+
| `assert_class` | `selector`, `value` | Checks `classList.contains(value)`. |
|
|
61
|
+
| `assert_input_value` | `selector`, `value` | Checks `element.value.includes(value)` on input/select/textarea. |
|
|
62
|
+
| `assert_matches` | `selector`, `value` (regex) | Tests element's `textContent` against `new RegExp(value)`. |
|
|
63
|
+
| `assert_no_network_errors` | — | Checks accumulated `requestfailed` events during the test. Fails with error details if any exist. |
|
|
64
|
+
|
|
65
|
+
### Assertion Disambiguation
|
|
66
|
+
|
|
67
|
+
- **`assert_text`** → searches the **entire page body** (substring)
|
|
68
|
+
- **`assert_element_text`** → checks a **specific element** (substring, or exact with `value: "exact"`)
|
|
69
|
+
- **`assert_matches`** → checks a specific element against a **regex** pattern
|
|
70
|
+
- **`assert_input_value`** → reads the `.value` property (for form fields)
|
|
71
|
+
|
|
72
|
+
## Extraction & Utility
|
|
73
|
+
|
|
74
|
+
| Action | Fields | Description |
|
|
75
|
+
|--------|--------|-------------|
|
|
76
|
+
| `get_text` | `selector` | Returns `{ value: textContent.trim() }`. Non-assertion — never fails. |
|
|
77
|
+
| `screenshot` | `value` (filename, optional) | Captures screenshot. Filename gets timestamp suffix for uniqueness. |
|
|
78
|
+
| `wait` | `selector` OR `text` OR `value` (ms) | Wait for selector, text on page, or fixed delay. |
|
|
79
|
+
| `wait_network_idle` | `value` (idle ms, default 500), `timeout` (max wait ms, default 30000) | Waits for all network requests to complete. Uses Puppeteer's `page.waitForNetworkIdle()`. Useful after SPA page transitions or data loading. |
|
|
80
|
+
| `evaluate` | `value` (JS code) | Run JavaScript in browser context. See **Strict Evaluate** below. |
|
|
81
|
+
| `clear_cookies` | `value` (origin, optional) | Clears cookies, localStorage, sessionStorage for origin. |
|
|
82
|
+
|
|
83
|
+
### Strict Evaluate Semantics
|
|
84
|
+
|
|
85
|
+
The `evaluate` action checks the return value:
|
|
86
|
+
|
|
87
|
+
- If the JS returns a string starting with `FAIL:`, `ERROR:`, or `FAILED:` → the test **fails** with that message.
|
|
88
|
+
- If the JS returns `false` → the test **fails** (`evaluate returned false`).
|
|
89
|
+
- If the JS returns any other non-null value → stored as `{ value: result }` for visibility.
|
|
90
|
+
- If the JS throws → the test **fails** (standard Puppeteer error).
|
|
91
|
+
|
|
92
|
+
This prevents false PASSes where evaluate actions return error strings that were previously silently ignored.
|
|
93
|
+
|
|
94
|
+
## Action-Level Retry
|
|
95
|
+
|
|
96
|
+
Any action can have `"retries": N` for per-action retry on failure:
|
|
97
|
+
|
|
98
|
+
```json
|
|
99
|
+
{ "type": "click", "selector": "#dynamic-btn", "retries": 3 }
|
|
100
|
+
{ "type": "wait", "selector": ".lazy-loaded", "retries": 2 }
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Delay between retries: `actionRetryDelay` config (default 500ms).
|
|
104
|
+
|
|
105
|
+
## Examples
|
|
106
|
+
|
|
107
|
+
### React input + autocomplete flow
|
|
108
|
+
```json
|
|
109
|
+
{ "type": "focus_autocomplete", "text": "Category" },
|
|
110
|
+
{ "type": "type_react", "selector": "#category-input", "value": "Electr" },
|
|
111
|
+
{ "type": "click_option", "text": "Electronics" }
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Regex click (last match)
|
|
115
|
+
```json
|
|
116
|
+
{ "type": "click_regex", "text": "add to cart", "selector": "button", "value": "last" }
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
### Form validation assertions
|
|
120
|
+
```json
|
|
121
|
+
{ "type": "assert_attribute", "selector": "input#email", "value": "type=email" },
|
|
122
|
+
{ "type": "assert_attribute", "selector": "button.submit", "value": "disabled" },
|
|
123
|
+
{ "type": "assert_class", "selector": ".nav-item:first-child", "value": "active" },
|
|
124
|
+
{ "type": "assert_input_value", "selector": "#email", "value": "user@example.com" },
|
|
125
|
+
{ "type": "assert_matches", "selector": ".phone", "value": "\\d{3}-\\d{3}-\\d{4}" },
|
|
126
|
+
{ "type": "assert_count", "selector": ".table-row", "value": ">3" }
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### Storage operations
|
|
130
|
+
```json
|
|
131
|
+
{ "type": "set_storage", "value": "authToken=eyJhbGciOiJIUzI1NiJ9..." },
|
|
132
|
+
{ "type": "assert_storage", "value": "authToken" },
|
|
133
|
+
{ "type": "set_storage", "value": "theme=dark", "selector": "session" },
|
|
134
|
+
{ "type": "assert_storage", "value": "theme=dark", "selector": "session" }
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### Icon, menu, and contextual clicks
|
|
138
|
+
```json
|
|
139
|
+
{ "type": "click_icon", "value": "edit" },
|
|
140
|
+
{ "type": "click_icon", "value": "delete", "selector": ".user-card" },
|
|
141
|
+
{ "type": "click_menu_item", "text": "Export as PDF" },
|
|
142
|
+
{ "type": "click_in_context", "text": "John Doe", "selector": "button.edit" }
|
|
143
|
+
```
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# Authentication Strategies
|
|
2
|
+
|
|
3
|
+
Tests can authenticate using multiple strategies depending on the app's auth mechanism.
|
|
4
|
+
|
|
5
|
+
## 1. UI Login (universal)
|
|
6
|
+
|
|
7
|
+
Fill the login form in `beforeEach`. Works with any auth system — the browser stores cookies/tokens automatically.
|
|
8
|
+
|
|
9
|
+
```json
|
|
10
|
+
{ "hooks": { "beforeEach": [
|
|
11
|
+
{ "type": "goto", "value": "/login" },
|
|
12
|
+
{ "type": "type", "selector": "#email", "value": "test@example.com" },
|
|
13
|
+
{ "type": "type", "selector": "#password", "value": "secret" },
|
|
14
|
+
{ "type": "click", "text": "Sign In" },
|
|
15
|
+
{ "type": "wait", "selector": ".dashboard" }
|
|
16
|
+
]}}
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## 2. JWT Injection (SPAs)
|
|
20
|
+
|
|
21
|
+
Skip the login form by injecting a token into `localStorage` or `sessionStorage`:
|
|
22
|
+
|
|
23
|
+
```json
|
|
24
|
+
{ "type": "set_storage", "value": "accessToken=eyJhbG..." }
|
|
25
|
+
{ "type": "set_storage", "value": "token=eyJhbG...", "selector": "session" }
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## 3. Config-Level Token
|
|
29
|
+
|
|
30
|
+
Set `authToken` + `authStorageKey` in config, env vars (`AUTH_TOKEN`, `AUTH_STORAGE_KEY`), or CLI flags (`--auth-token`, `--auth-storage-key`). Used by `e2e_capture` and `e2e_issue --verify`.
|
|
31
|
+
|
|
32
|
+
## 4. Cookie-Based (server-rendered)
|
|
33
|
+
|
|
34
|
+
Set cookies via `evaluate`:
|
|
35
|
+
```json
|
|
36
|
+
{ "type": "evaluate", "value": "document.cookie = 'session_id=abc123; path=/; SameSite=Lax'" }
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## 5. API Headers
|
|
40
|
+
|
|
41
|
+
Override `fetch` to inject Authorization headers (useful for `--test-type api`):
|
|
42
|
+
```json
|
|
43
|
+
{ "type": "evaluate", "value": "const orig = window.fetch; window.fetch = (url, opts = {}) => { opts.headers = { ...opts.headers, 'Authorization': 'Bearer eyJhbG...' }; return orig(url, opts); }" }
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## 6. OAuth/SSO
|
|
47
|
+
|
|
48
|
+
Use a test-environment bypass endpoint, pre-authenticated token injection, or CI-obtained session cookie. External OAuth providers (Google, GitHub, Okta) can't be automated directly.
|
|
49
|
+
|
|
50
|
+
## Auth Auto-Login
|
|
51
|
+
|
|
52
|
+
Automatically fetch an auth token from a login API endpoint before tests run. Avoids repeating login flows in every test.
|
|
53
|
+
|
|
54
|
+
### Config fields
|
|
55
|
+
|
|
56
|
+
| Field | Description | Env / CLI |
|
|
57
|
+
|-------|-------------|-----------|
|
|
58
|
+
| `authLoginEndpoint` | Full URL to POST credentials to | `AUTH_LOGIN_ENDPOINT` / `--auth-login-endpoint` |
|
|
59
|
+
| `authCredentials` | Object with login credentials (config file only — never in env vars) | — |
|
|
60
|
+
| `authTokenPath` | Dot-path to extract token from response JSON (default: `'token'`) | `AUTH_TOKEN_PATH` / `--auth-token-path` |
|
|
61
|
+
|
|
62
|
+
### Flow
|
|
63
|
+
|
|
64
|
+
1. Before workers start, if `authLoginEndpoint` is set and `authToken` is NOT already set, `fetchAuthToken()` POSTs `authCredentials` as JSON.
|
|
65
|
+
2. The token is extracted from the response using `authTokenPath` (supports dot notation, e.g. `'data.access_token'`).
|
|
66
|
+
3. The extracted token is stored in `config.authToken`.
|
|
67
|
+
4. Each test worker navigates to `baseUrl` and injects `localStorage[authStorageKey] = authToken` before `beforeEach` hooks run.
|
|
68
|
+
|
|
69
|
+
### Example config
|
|
70
|
+
|
|
71
|
+
```js
|
|
72
|
+
export default {
|
|
73
|
+
authLoginEndpoint: 'https://api.example.com/auth/login',
|
|
74
|
+
authCredentials: { email: 'test@example.com', password: 'secret123' },
|
|
75
|
+
authTokenPath: 'data.access_token',
|
|
76
|
+
authStorageKey: 'accessToken',
|
|
77
|
+
};
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
If `authToken` is already set (via config, env var, or CLI), the auto-login step is skipped.
|
|
81
|
+
|
|
82
|
+
## Clearing State
|
|
83
|
+
|
|
84
|
+
Each test runs in a fresh browser context (auto-clean). Use `clear_cookies` to explicitly clear cookies + localStorage + sessionStorage mid-test.
|
|
85
|
+
|
|
86
|
+
## Reusable Auth Modules
|
|
87
|
+
|
|
88
|
+
Create `e2e/modules/login.json` or `e2e/modules/auth-token.json` and reference via:
|
|
89
|
+
```json
|
|
90
|
+
{ "$use": "login", "params": { "email": "...", "password": "..." } }
|
|
91
|
+
```
|