e2e 0.21.1 → 0.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +2 -0
  2. package/dist/.build.json +1 -1
  3. package/dist/agent/act-validation.js +1 -0
  4. package/dist/agent/act.js +7 -3
  5. package/dist/agent/action-dispatcher.js +30 -8
  6. package/dist/agent/actions.d.ts +2 -0
  7. package/dist/agent/actions.js +5 -2
  8. package/dist/agent/error.js +1 -0
  9. package/dist/agent/executor.d.ts +14 -6
  10. package/dist/agent/index.js +5 -1
  11. package/dist/agent/invocation.d.ts +3 -0
  12. package/dist/agent/model/sdk.d.ts +5 -0
  13. package/dist/agent/model/sdk.js +18 -1
  14. package/dist/agent/observation.js +1 -0
  15. package/dist/agent/primitives.js +14 -3
  16. package/dist/agent/replay.js +9 -3
  17. package/dist/agent/step-cache.d.ts +3 -2
  18. package/dist/agent/step-cache.js +7 -4
  19. package/dist/agent/tool-loop.js +4 -1
  20. package/dist/cache/context.js +1 -1
  21. package/dist/cache/recorder.d.ts +2 -1
  22. package/dist/cache/recorder.js +11 -2
  23. package/dist/cache/trace-fields.d.ts +10 -0
  24. package/dist/cache/trace-fields.js +17 -0
  25. package/dist/cache/trace.d.ts +9 -1
  26. package/dist/cache/trace.js +26 -7
  27. package/dist/cli/feedback.js +2 -1
  28. package/dist/cli/init/sibling-versions.json +1 -1
  29. package/dist/collect/select.js +12 -2
  30. package/dist/config/agent.d.ts +1 -1
  31. package/dist/config/diagnose.js +9 -3
  32. package/dist/config/esm-hooks.js +3 -2
  33. package/dist/config/resolve.js +3 -2
  34. package/dist/config/typescript.js +52 -2
  35. package/dist/engine/contract.d.ts +3 -2
  36. package/dist/engine/manifest.js +2 -2
  37. package/dist/internal/errors.js +5 -2
  38. package/dist/internal/source.js +2 -0
  39. package/dist/internal/unserved-model.d.ts +5 -0
  40. package/dist/internal/unserved-model.js +22 -0
  41. package/dist/locator/screen.js +1 -2
  42. package/dist/oauth/copilot.js +23 -5
  43. package/dist/oauth/device-code.js +35 -19
  44. package/dist/oauth/errors.d.ts +3 -0
  45. package/dist/oauth/errors.js +26 -11
  46. package/dist/oauth/providers/github-copilot.d.ts +4 -0
  47. package/dist/oauth/providers/github-copilot.js +64 -16
  48. package/dist/oauth/providers/openai.d.ts +2 -2
  49. package/dist/oauth/providers/openai.js +47 -19
  50. package/dist/oauth/providers/opencode-console.js +36 -15
  51. package/dist/oauth/providers/xai.js +18 -9
  52. package/dist/oauth/token-endpoint.d.ts +15 -6
  53. package/dist/oauth/token-endpoint.js +18 -6
  54. package/dist/report/code-frame.js +9 -3
  55. package/dist/report/failure-text.d.ts +10 -2
  56. package/dist/report/failure-text.js +24 -8
  57. package/dist/report/list-model.d.ts +13 -0
  58. package/dist/report/list-steps.d.ts +6 -1
  59. package/dist/report/list-steps.js +29 -0
  60. package/dist/report/list.d.ts +1 -0
  61. package/dist/report/list.js +16 -7
  62. package/dist/report/live-window.js +2 -1
  63. package/dist/report/markdown.js +1 -1
  64. package/dist/run/fixtures.js +2 -0
  65. package/dist/run/host-trace.d.ts +3 -2
  66. package/dist/run/host-trace.js +2 -1
  67. package/dist/run/last-run.d.ts +9 -0
  68. package/dist/run/last-run.js +28 -0
  69. package/dist/run/open-session.d.ts +3 -1
  70. package/dist/run/open-session.js +1 -1
  71. package/dist/run/runner.js +6 -2
  72. package/dist/run/sessions.js +3 -3
  73. package/dist/run/standalone-attempt.d.ts +2 -1
  74. package/dist/run/standalone-attempt.js +3 -1
  75. package/dist/run/steps.d.ts +3 -1
  76. package/dist/run/units.d.ts +8 -0
  77. package/dist/run/units.js +14 -0
  78. package/dist/run/worker-models.d.ts +2 -0
  79. package/dist/run/worker-models.js +7 -0
  80. package/dist/types.d.ts +2 -2
  81. package/docs/authentication.mdx +26 -23
  82. package/docs/browser.mdx +7 -4
  83. package/docs/bug-bash.mdx +2 -2
  84. package/docs/cache.mdx +6 -1
  85. package/docs/ci/bitrise.mdx +1 -1
  86. package/docs/ci/codemagic.mdx +2 -2
  87. package/docs/ci/eas-workflows.mdx +1 -1
  88. package/docs/coding-agents.mdx +7 -35
  89. package/docs/debugging.mdx +24 -36
  90. package/docs/executors.mdx +1 -1
  91. package/docs/github.mdx +2 -1
  92. package/docs/integrations/eas.mdx +1 -1
  93. package/docs/integrations/index.mdx +6 -0
  94. package/docs/integrations/lightpanda.mdx +192 -0
  95. package/docs/integrations/testmu.mdx +152 -0
  96. package/docs/mcp.mdx +123 -0
  97. package/docs/migrate/detox.mdx +1 -1
  98. package/docs/migrate/maestro.mdx +101 -105
  99. package/docs/migrate/playwright.mdx +65 -70
  100. package/docs/mobile.mdx +18 -3
  101. package/docs/models.mdx +4 -2
  102. package/docs/quickstart.mdx +7 -0
  103. package/docs/reference/agent.mdx +369 -277
  104. package/docs/reference/app.mdx +141 -32
  105. package/docs/reference/cli.mdx +674 -558
  106. package/docs/reference/config.mdx +1406 -752
  107. package/docs/reference/engine.mdx +1062 -256
  108. package/docs/reference/environment.mdx +156 -84
  109. package/docs/reference/errors.mdx +106 -72
  110. package/docs/reference/expect.mdx +752 -372
  111. package/docs/reference/mcp.mdx +366 -141
  112. package/docs/reference/mobile.mdx +844 -403
  113. package/docs/reference/reporters.mdx +312 -106
  114. package/docs/reference/runner.mdx +373 -174
  115. package/docs/reference/screen.mdx +931 -398
  116. package/docs/reference/security.mdx +106 -40
  117. package/docs/reference/test.mdx +626 -273
  118. package/docs/reference/web.mdx +956 -533
  119. package/docs/security.mdx +6 -5
  120. package/docs/snippets/model-provider.jsx +763 -0
  121. package/docs/snippets/setup-prompt-card.jsx +22 -0
  122. package/docs/snippets/show-more.jsx +22 -0
  123. package/docs/snippets/video-card.jsx +34 -0
  124. package/docs/snippets/video.jsx +25 -0
  125. package/docs/subscriptions.mdx +10 -3
  126. package/docs/telemetry.mdx +39 -39
  127. package/package.json +3 -2
  128. package/schema/report-v1.schema.json +6 -0
  129. package/skills/e2e/references/agent.md +8 -3
  130. package/skills/e2e/references/debugging.md +1 -1
  131. package/skills/e2e/references/mcp.md +1 -1
  132. package/skills/e2e/references/setup.md +12 -3
  133. package/skills/e2e/references/writing-tests.md +1 -1
@@ -6,66 +6,46 @@ icon: "terminal"
6
6
  "twitter:image": "https://e2e.tester.army/docs/images/og/reference/cli.png?v=e0e03be674"
7
7
  ---
8
8
 
9
+ The `e2e` command scaffolds a project, runs, lists, and explores tests, and
10
+ manages the replay cache, subscription logins, and telemetry. Every command
11
+ takes `--help`.
12
+
9
13
  <CodeGroup>
10
14
  ```bash npm
11
- npx e2e init [directory]
12
- npx e2e login [provider]
13
- npx e2e logout [provider]
14
- npx e2e models [provider]
15
- npx e2e guide [topic]
16
- npx e2e mcp [options]
17
- npx e2e run [files...] [options]
18
- npx e2e explore [goal] [options]
19
- npx e2e list [files...] [options]
20
- npx e2e cache ls|clear|stats [--config <path>] [--cache-dir <dir>]
21
- npx e2e feedback -m <text> [options]
22
- npx e2e telemetry [status|enable|disable]
23
- npx e2e --help
24
- npx e2e --version
15
+ npx e2e init # scaffold a project
16
+ npx e2e run # run every test
17
+ npx e2e run tests/signup.e2e.ts --headed # one file, with a visible browser
25
18
  ```
26
19
 
27
20
  ```bash pnpm
28
- pnpm dlx e2e init [directory]
29
- pnpm exec e2e login [provider]
30
- pnpm exec e2e logout [provider]
31
- pnpm exec e2e models [provider]
32
- pnpm exec e2e guide [topic]
33
- pnpm exec e2e mcp [options]
34
- pnpm exec e2e run [files...] [options]
35
- pnpm exec e2e explore [goal] [options]
36
- pnpm exec e2e list [files...] [options]
37
- pnpm exec e2e cache ls|clear|stats [--config <path>] [--cache-dir <dir>]
38
- pnpm exec e2e feedback -m <text> [options]
39
- pnpm exec e2e telemetry [status|enable|disable]
40
- pnpm exec e2e --help
41
- pnpm exec e2e --version
21
+ pnpm dlx e2e init # scaffold a project
22
+ pnpm exec e2e run # run every test
23
+ pnpm exec e2e run tests/signup.e2e.ts --headed # one file, with a visible browser
42
24
  ```
43
25
 
44
26
  ```bash bun
45
- bunx e2e init [directory]
46
- bunx e2e login [provider]
47
- bunx e2e logout [provider]
48
- bunx e2e models [provider]
49
- bunx e2e guide [topic]
50
- bunx e2e mcp [options]
51
- bunx e2e run [files...] [options]
52
- bunx e2e explore [goal] [options]
53
- bunx e2e list [files...] [options]
54
- bunx e2e cache ls|clear|stats [--config <path>] [--cache-dir <dir>]
55
- bunx e2e feedback -m <text> [options]
56
- bunx e2e telemetry [status|enable|disable]
57
- bunx e2e --help
58
- bunx e2e --version
27
+ bunx e2e init # scaffold a project
28
+ bunx e2e run # run every test
29
+ bunx e2e run tests/signup.e2e.ts --headed # one file, with a visible browser
59
30
  ```
60
31
  </CodeGroup>
61
32
 
62
- The CLI needs Node.js 24.8 or newer, or 22.22.3 or newer on Node.js 22. Any package manager installs it (npm, pnpm, Yarn with or without Plug'n'Play, Bun); the CLI itself runs on Node.js, not on the Bun or Deno runtimes (`bunx e2e` runs it on Node.js). Under Yarn Plug'n'Play on Node.js 24.8 through 24.11.0, a `.cts` file fails to load (Node.js rejects the source Yarn's loader hands on); 24.11.1 and later load it.
33
+ <Note>
34
+ The CLI needs Node.js 24.8 or newer, or 22.22.3 or newer on Node.js 22. Any
35
+ package manager installs it: npm, pnpm, Yarn with or without Plug'n'Play, or
36
+ Bun. The CLI runs on Node.js, not on the Bun or Deno runtimes; `bunx e2e`
37
+ runs it on Node.js. Under Yarn Plug'n'Play on Node.js 24.8 through 24.11.0, a
38
+ `.cts` file fails to load: Node.js rejects the source Yarn's loader hands on.
39
+ 24.11.1 and later load it.
40
+ </Note>
41
+
42
+ ## Reference
63
43
 
64
- ## Help and version
44
+ ### Help and version
65
45
 
66
- | Flag | Behavior |
46
+ | Option | Description |
67
47
  | --- | --- |
68
- | `-h`, `--help` | Prints the help of the command it follows and exits 0. `e2e help run` is the same as `e2e run --help`. |
48
+ | `-h`, `--help` | Prints the help of the command it follows and exits 0. `e2e help run` equals `e2e run --help`. |
69
49
  | `-v`, `--version` | Prints the installed `e2e` version and exits 0. |
70
50
 
71
51
  ```console
@@ -73,72 +53,36 @@ $ npx e2e --version
73
53
  0.14.0
74
54
  ```
75
55
 
76
- Output is colored on a terminal or in CI, plain when piped or when `NO_COLOR`
77
- is set. `FORCE_COLOR` keeps the colors. `e2e` without a command prints the
78
- help on stderr and exits 2.
79
-
80
- On a terminal at least 55 columns wide, `e2e --help` opens with the e2e
81
- wordmark, the site's lettering drawn in block characters; piped output and
82
- narrower terminals get the text alone.
83
-
84
- ## e2e init
85
-
86
- Scaffolds a project and adds `e2e` to `devDependencies`. Existing
87
- config and test files are skipped. With a `directory` argument the project is
88
- scaffolded there.
89
-
90
- On a terminal, the wizard opens with the e2e wordmark written in over about
91
- a second, letter by letter, the way a pen writes it. With `--yes`, in CI, on
92
- a dumb terminal, and on a terminal shorter than 11 rows the wordmark prints at
93
- once, and piped output never shows it. Ctrl-C or SIGTERM while it is being
94
- written exits 130.
95
-
96
- For a new project, the wizard asks you to choose:
97
-
98
- - The engine. **Web** (Playwright, the default) adds `@e2e-dev/web`.
99
- **Mobile (iOS/Android)** adds `@e2e-dev/mobile`, defaults to iOS on
100
- macOS and Android elsewhere, and opens Settings. **None** adds nothing.
101
- - A model provider or subscription for agent steps. Choose Vercel AI Gateway,
102
- OpenRouter, an OpenAI-compatible endpoint, ChatGPT, GitHub Copilot,
103
- OpenCode Console, or SuperGrok. Choose None for tests that only use
104
- locators. The wizard adds the required packages and model config. Subscription users then sign in
105
- with `e2e login`. See [Models](/models) and [Subscriptions](/subscriptions).
106
- - Where to install the agent skill (see [e2e guide](#e2e-guide)):
107
- `.agents/skills/` for Codex, Cursor, Copilot, Gemini CLI, OpenCode, and
108
- Zed, and `.claude/skills/` for Claude Code. Both are preselected; with
109
- both, `.claude/skills/e2e` is a symlink to `.agents/skills/e2e`, the
110
- layout `npx skills add` produces.
111
- - Where to register the [MCP server](/reference/mcp): `.mcp.json` for
112
- Claude Code and `.cursor/mcp.json` for Cursor. Both are preselected.
113
-
114
- After confirming the file changes, choose whether to install the selected
115
- dependencies. The installer follows the project's `packageManager` field or
116
- lockfile, then the invoking package manager, then npm.
117
-
118
- | Argument or flag | Type | Default | Behavior |
119
- | --- | --- | --- | --- |
120
- | `[directory]` | path | the working directory | Where to scaffold; created when missing |
121
- | `-y`, `--yes` | boolean | `false` | Skips the prompts. Each answer not given as a flag takes its default: Web engine, Vercel AI Gateway, the skill and MCP config in both locations, no dependency installation |
122
- | `--engine <id>` | `web`, `mobile`, `none` | asked | The engine for a new config |
123
- | `--gateway <id>` | `vercel`, `openrouter`, `openai-compatible`, `chatgpt`, `copilot`, `opencode-console`, `grok`, `none` | asked | The model gateway for agent steps in a new config |
124
- | `--endpoint <url>` | URL | asked; `http://127.0.0.1:11434/v1` with `--yes` | Base URL for `--gateway openai-compatible`, which it implies: HTTPS, or HTTP on a loopback host |
125
- | `--skills <dirs>` | `.agents/skills`, `.claude/skills`, comma-separated or repeated, or `none` | asked | Where to install the agent skill |
126
- | `--mcp <files>` | `.mcp.json`, `.cursor/mcp.json`, comma-separated or repeated, or `none` | asked | Where to register the MCP server |
127
- | `--install`, `--no-install` | boolean | asked; `--no-install` with `--yes` | Whether to install dependencies |
128
-
129
- A flag answers its question and skips that prompt.
130
- Without a terminal, pass `--yes`.
131
- An unknown value exits 2 before any write and lists the valid ones.
132
- `--engine`, `--gateway`, and `--endpoint` apply only to a new config.
56
+ - Output is colored on a terminal or in CI, and plain when piped or when
57
+ `NO_COLOR` is set. `FORCE_COLOR` keeps the colors.
58
+ - `e2e` without a command prints the help on stderr and exits 2.
59
+ - On a terminal 55 or more columns wide, `e2e --help` opens with the e2e
60
+ wordmark: the site's lettering in block characters. Piped output and
61
+ narrower terminals get the text alone.
133
62
 
134
- | Creates | Content |
63
+ ### Commands
64
+
65
+ | Command | Description |
135
66
  | --- | --- |
136
- | `package.json` | Created as a private ESM package when missing; otherwise only missing dependencies and scripts are added. Sibling packages (`@e2e-dev/web`, `@e2e-dev/mobile`) get the caret range of the version released with the CLI, or that exact version when the CLI is a prerelease. A `test:e2e` script runs `e2e run` unless one exists |
137
- | `e2e.config.ts` | One target for the selected engine, with the app in its `app`. Playwright reads `APP_URL` (default `http://localhost:3000`) into `app.url` and shows `app.command` in a comment. Mobile sets `app: { bundleId: 'Settings' }` on iOS and `com.android.settings` on Android. With a gateway, `agents.default` is `{ model, system }` |
138
- | `tests/example.e2e.ts` | An app-open test that passes against any page, plus a commented agent test. Device setup gets a Settings check instead |
139
- | `.gitignore` | Appends `node_modules/`, `.e2e/results/`, `.e2e/cache/`, `.e2e/sessions/`, `.e2e/report.json`, `.e2e/ai-trace.json`, `.e2e/junit.xml`, `.e2e/summary.md`, `.e2e/logs/`, and `.e2e/videos/`. Committing the cache is opt-in; see [commit the replay cache](/cache#commit-the-replay-cache) |
140
- | `.agents/skills/e2e/`, `.claude/skills/e2e/` | `SKILL.md` and one `references/<topic>.md` per `e2e guide` topic. With both chosen, the Claude Code directory is a relative symlink (`../../.agents/skills/e2e`) to the copy |
141
- | `.mcp.json`, `.cursor/mcp.json` | Registers `e2e mcp` without changing other servers |
67
+ | [`e2e init`](#e2e-init) | Scaffolds a project |
68
+ | [`e2e login`](#e2e-login) | Signs in to a model subscription |
69
+ | [`e2e logout`](#e2e-logout) | Forgets a stored login |
70
+ | [`e2e models`](#e2e-models) | Lists a stored login's models |
71
+ | [`e2e guide`](#e2e-guide) | Prints the agent skill |
72
+ | [`e2e mcp`](#e2e-mcp) | Serves live sessions over MCP |
73
+ | [`e2e run`](#e2e-run) | Runs the tests |
74
+ | [`e2e explore`](#e2e-explore) | Explores the app toward a goal |
75
+ | [`e2e list`](#e2e-list) | Prints the tests a run selects |
76
+ | [`e2e cache`](#e2e-cache) | Inspects or clears the replay cache |
77
+ | [`e2e feedback`](#e2e-feedback) | Sends feedback to the e2e team |
78
+ | [`e2e telemetry`](#e2e-telemetry) | Shows or switches usage telemetry |
79
+
80
+ An option with no default listed is off unless given.
81
+
82
+ #### `e2e init`
83
+
84
+ Scaffolds a project and adds `e2e` to `devDependencies`. It skips existing
85
+ config and test files.
142
86
 
143
87
  <CodeGroup>
144
88
  ```bash npm
@@ -157,31 +101,97 @@ bunx e2e init --yes --gateway openrouter --skills .agents/skills --mcp .mcp.json
157
101
  ```
158
102
  </CodeGroup>
159
103
 
160
- Re-running `init` keeps an existing config and adds no packages for it. MCP
161
- registration keeps extra settings on the e2e server, such as environment
162
- variables and client timeouts, even when repairing its launch command. A
163
- current registration is left byte-for-byte unchanged. Once
164
- a skill directory exists, later runs refresh copies that differ from the
165
- installed version and never add a directory the project did not choose.
166
- `--skills` and `--mcp` name the exact locations instead; others are left as
167
- they are. A
168
- `.claude/skills/e2e/` copy from an earlier version becomes the symlink when
169
- it holds nothing but the shipped files; one with other files in it stays a
170
- copy and is refreshed. A skill directory that is a symlink elsewhere, or sits
171
- under one, is never written through: `--yes` skips it with a warning naming
172
- the link and its target, and an interactive run offers to replace a linked
173
- skill directory with a copy, or with the link where the link goes. A
174
- copy with a directory where a file goes, or a file where a directory goes, is
175
- skipped with a warning naming the entry; `init` removes nothing to repair it.
176
- An MCP config file reached through a symlink, or under a linked directory, is
177
- skipped with the same warning and left unchanged, so the server is not
178
- registered there; a plain config is merged as usual.
179
-
180
- Cancellation leaves files untouched and exits 0. An invalid `package.json`
181
- exits 2 before any write. Without a terminal on stdin and stdout, `init`
182
- without `--yes` exits 2 and names the flag.
183
-
184
- ## e2e login
104
+ | Option | Description |
105
+ | --- | --- |
106
+ | `[directory]` | Where to scaffold; created when missing. Default: the working directory. |
107
+ | `-y`, `--yes` | Skips the prompts. Each answer not given as a flag takes its default: Web engine, Vercel AI Gateway, skill and MCP config in both locations, no install. |
108
+ | `--engine <id>` | The engine for a new config: `web`, `mobile`, or `none`. Default: asked. |
109
+ | `--gateway <id>` | The model gateway for agent steps in a new config: `vercel`, `openrouter`, `openai-compatible`, `chatgpt`, `copilot`, `opencode-console`, `grok`, or `none`. Default: asked. |
110
+ | `--endpoint <url>` | Base URL for `--gateway openai-compatible`, and implies it. HTTPS, or HTTP on a loopback host. Default: asked; `http://127.0.0.1:11434/v1` with `--yes`. |
111
+ | `--skills <dirs>` | Where to install the agent skill: `.agents/skills`, `.claude/skills`, or `none`. Comma-separated or repeated. Default: asked. |
112
+ | `--mcp <files>` | Where to register the MCP server: `.mcp.json`, `.cursor/mcp.json`, or `none`. Comma-separated or repeated. Default: asked. |
113
+ | `--install`, `--no-install` | Whether to install dependencies. Default: asked; `--no-install` with `--yes`. |
114
+
115
+ - A flag answers its question and skips that prompt.
116
+ - `--engine`, `--gateway`, and `--endpoint` apply only to a new config.
117
+
118
+ | Exit code | When |
119
+ | --- | --- |
120
+ | 0 | The scaffold finished, or the wizard was cancelled. Cancelling leaves files untouched. |
121
+ | 2 | An unknown flag value. Exits before any write and lists the valid values. |
122
+ | 2 | An invalid `package.json`. Exits before any write. |
123
+ | 2 | No terminal on stdin and stdout, and no `--yes`. The message names the flag. |
124
+ | 2 | The dependency installation failed. |
125
+ | 130 | Ctrl-C or SIGTERM while the wordmark is written, or SIGINT during the dependency installation. |
126
+
127
+ ##### Prompts
128
+
129
+ On a terminal, the wizard opens with the e2e wordmark, written letter by
130
+ letter over about a second, the way a pen writes it. With `--yes`, in CI, on
131
+ a dumb terminal, or on a terminal under 11 rows, it prints at once. Piped
132
+ output never shows it.
133
+
134
+ For a new project, the wizard asks for:
135
+
136
+ - **The engine.** **Web** (Playwright, the default) adds `@e2e-dev/web`.
137
+ **Mobile (iOS/Android)** adds `@e2e-dev/mobile`, defaults to iOS on macOS
138
+ and Android elsewhere, and opens Settings. **None** adds nothing.
139
+ - **A model provider or subscription for agent steps:** Vercel AI Gateway,
140
+ OpenRouter, an OpenAI-compatible endpoint, ChatGPT, GitHub Copilot, OpenCode
141
+ Console, or SuperGrok. None is for tests that use only locators. The wizard
142
+ adds the required packages and model config. Subscription users then sign
143
+ in with [`e2e login`](#e2e-login). See [Models](/models) and
144
+ [Subscriptions](/subscriptions).
145
+ - **Where to install the agent skill** (see [e2e guide](#e2e-guide)).
146
+ `.agents/skills/` serves Codex, Cursor, Copilot, Gemini CLI, OpenCode, and
147
+ Zed; `.claude/skills/` serves Claude Code. Both are preselected.
148
+ - **Where to register the [MCP server](/reference/mcp).** `.mcp.json` for
149
+ Claude Code, `.cursor/mcp.json` for Cursor. Both are preselected.
150
+
151
+ After you confirm the file changes, the wizard asks whether to install the
152
+ selected dependencies. The installer follows the project's `packageManager`
153
+ field or lockfile, then the invoking package manager, then npm.
154
+
155
+ ##### Files
156
+
157
+ | Creates | Content |
158
+ | --- | --- |
159
+ | `package.json` | Created as a private ESM package when missing; otherwise gets only the missing dependencies and scripts. Sibling packages (`@e2e-dev/web`, `@e2e-dev/mobile`) get the caret range of the version released with the CLI, or that exact version when the CLI is a prerelease. Adds a `test:e2e` script that runs `e2e run`, unless one exists |
160
+ | `e2e.config.ts` | One target for the selected engine, with the app in its `app`. Playwright reads `APP_URL` (default `http://localhost:3000`) into `app.url` and shows `app.command` in a comment. Mobile sets `app: { bundleId: 'Settings' }` on iOS and `com.android.settings` on Android. With a gateway, `agents.default` is `{ model, system }` |
161
+ | `tests/example.e2e.ts` | An app-open test that passes against any page, plus a commented agent test. Device setup gets a Settings check instead |
162
+ | `.gitignore` | Appends `node_modules/`, `.e2e/results/`, `.e2e/cache/`, `.e2e/sessions/`, `.e2e/report.json`, `.e2e/ai-trace.json`, `.e2e/junit.xml`, `.e2e/summary.md`, `.e2e/logs/`, and `.e2e/videos/`. Committing the cache is opt-in; see [commit the replay cache](/cache#commit-the-replay-cache) |
163
+ | `.agents/skills/e2e/`, `.claude/skills/e2e/` | `SKILL.md` and one `references/<topic>.md` per `e2e guide` topic. With both chosen, the Claude Code directory is a relative symlink (`../../.agents/skills/e2e`) to the copy, the layout `npx skills add` produces |
164
+ | `.mcp.json`, `.cursor/mcp.json` | Registers `e2e mcp` without changing other servers |
165
+
166
+ ##### Re-running init
167
+
168
+ - An existing config is kept, and no packages are added for it.
169
+ - MCP registration keeps extra settings on the e2e server, such as
170
+ environment variables and client timeouts, even when it repairs the launch
171
+ command. A current registration stays byte-for-byte unchanged.
172
+ - Once a skill directory exists, later runs refresh copies that differ from
173
+ the installed version. They never add a directory the project did not
174
+ choose. `--skills` and `--mcp` name the exact locations instead and leave
175
+ the others as they are.
176
+ - A `.claude/skills/e2e/` copy from an earlier version becomes the symlink
177
+ when it holds only the shipped files. A copy with other files stays a copy
178
+ and is refreshed.
179
+ - `init` never writes through a skill directory that is a symlink elsewhere,
180
+ or sits under one. `--yes` skips it with a warning naming the link and its
181
+ target. An interactive run offers to replace it with a copy, or with the
182
+ link where the link goes.
183
+ - A copy with a directory where a file goes, or a file where a directory
184
+ goes, is skipped with a warning naming the entry. `init` removes nothing to
185
+ repair it.
186
+ - An MCP config file reached through a symlink, or under a linked directory,
187
+ is skipped with the same warning and left unchanged, so the server is not
188
+ registered there. A plain config is merged as usual.
189
+
190
+ #### `e2e login`
191
+
192
+ Signs in to a subscription and stores the credentials in
193
+ `$XDG_CONFIG_HOME/e2e/oauth.json`, by default `~/.config/e2e/oauth.json`. See
194
+ [Subscriptions](/subscriptions) for the model configuration.
185
195
 
186
196
  <CodeGroup>
187
197
  ```bash npm
@@ -215,20 +225,23 @@ bunx e2e login # choose a provider
215
225
  ```
216
226
  </CodeGroup>
217
227
 
218
- Signs in to a subscription and stores credentials in
219
- `$XDG_CONFIG_HOME/e2e/oauth.json`, by default `~/.config/e2e/oauth.json`. A failed sign-in exits 1, a cancelled one 130; an
220
- unknown provider exits 2. See [Subscriptions](/subscriptions) for model
221
- configuration.
228
+ | Option | Description |
229
+ | --- | --- |
230
+ | `[provider]` | The subscription: `openai`, `github-copilot`, `opencode-console`, or `spacexai`. Default: a picker. |
231
+ | `--device` | ChatGPT: shows a code to enter on another device instead of opening a browser. |
232
+ | `--from-gh` | GitHub Copilot: reuses the signed-in GitHub CLI's token. |
233
+ | `--client-id <id>` | GitHub Copilot: the client id of your GitHub OAuth App, with the device flow enabled. |
234
+ | `--enterprise-url <host>` | GitHub Copilot: the GitHub Enterprise host. |
235
+
236
+ | Exit code | When |
237
+ | --- | --- |
238
+ | 1 | The sign-in failed. |
239
+ | 2 | An unknown provider. |
240
+ | 130 | The sign-in was cancelled. |
222
241
 
223
- | Argument or flag | Type | Default | Behavior |
224
- | --- | --- | --- | --- |
225
- | `[provider]` | `openai`, `github-copilot`, `opencode-console`, or `spacexai` | a picker | The subscription to sign in to |
226
- | `--device` | boolean | `false` | ChatGPT: show a code to enter on another device instead of opening a browser |
227
- | `--from-gh` | boolean | `false` | GitHub Copilot: reuse the token of the signed-in GitHub CLI |
228
- | `--client-id <id>` | string | none | GitHub Copilot: the client id of your GitHub OAuth App with the device flow enabled |
229
- | `--enterprise-url <host>` | string | none | GitHub Copilot: the GitHub Enterprise host |
242
+ #### `e2e logout`
230
243
 
231
- ## e2e logout
244
+ Forgets the stored login of one provider.
232
245
 
233
246
  <CodeGroup>
234
247
  ```bash npm
@@ -247,7 +260,14 @@ bunx e2e logout openai # forget one
247
260
  ```
248
261
  </CodeGroup>
249
262
 
250
- ## e2e models
263
+ | Option | Description |
264
+ | --- | --- |
265
+ | `[provider]` | The login to forget: `openai`, `github-copilot`, `opencode-console`, or `spacexai`. Default: a picker over the stored logins. |
266
+
267
+ #### `e2e models`
268
+
269
+ Asks the vendor which models the stored login serves and prints one line per
270
+ model. For OpenCode Console it asks with `OPENCODE_API_KEY` when set.
251
271
 
252
272
  <CodeGroup>
253
273
  ```bash npm
@@ -266,19 +286,32 @@ bunx e2e models openai # one provider
266
286
  ```
267
287
  </CodeGroup>
268
288
 
269
- Asks the vendor which models the stored login serves, or for OpenCode Console
270
- `OPENCODE_API_KEY` when set, and prints one line per
271
- model: the id `chatgpt()`, `copilot()`, `opencodeConsole()`, or `grok()`
272
- takes, the vendor's name for it, and its notes (reasoning levels, vision,
273
- preview, hidden, for Copilot `not enabled` or `no chat or responses` when
274
- `copilot()` cannot call it, and for OpenCode Console the plan, Zen or Go,
275
- that serves it). Without a provider it lists every stored login, plus
276
- OpenCode Console when only the key is set. Without a credential, or when the
277
- named provider has none, exits 1 with the sign-in command; when one of
278
- several fails to list, the others still print and the command exits 1. An unknown provider exits 2. See
279
- [Pick a model](/subscriptions#pick-a-model).
289
+ | Option | Description |
290
+ | --- | --- |
291
+ | `[provider]` | `openai`, `github-copilot`, `opencode-console`, or `spacexai`. Default: every stored login, plus OpenCode Console when only `OPENCODE_API_KEY` is set. |
292
+
293
+ Each line holds:
280
294
 
281
- ## e2e guide
295
+ - the id `chatgpt()`, `copilot()`, `opencodeConsole()`, or `grok()` takes;
296
+ - the vendor's name for the model;
297
+ - its notes: reasoning levels, vision, preview, hidden; for Copilot
298
+ `not enabled` or `no chat or responses` when `copilot()` cannot call it;
299
+ for OpenCode Console the plan that serves it, Zen or Go.
300
+
301
+ Only a call finds out whether a Copilot account can use a listed model.
302
+
303
+ | Exit code | When |
304
+ | --- | --- |
305
+ | 1 | No credential, or none for the named provider. The message names the sign-in command. |
306
+ | 1 | One of several logins failed to list. The others still print. |
307
+ | 2 | An unknown provider. |
308
+
309
+ See [Pick a model](/subscriptions#pick-a-model).
310
+
311
+ #### `e2e guide`
312
+
313
+ Prints the skill that `init` installs; `npx skills add tester-army/e2e`
314
+ installs the same skill. See [Coding agents](/coding-agents).
282
315
 
283
316
  <CodeGroup>
284
317
  ```bash npm
@@ -297,13 +330,13 @@ bunx e2e guide setup # one topic
297
330
  ```
298
331
  </CodeGroup>
299
332
 
300
- Prints the skill that `init` installs. `guide` alone prints the overview;
301
- `guide <topic>` prints one of `setup`, `writing-tests`, `agent`, `running`,
302
- `explore`, `debugging`, `mcp`, or `bug-bash`. An unknown topic exits 2 and names the
303
- valid ones. The same skill installs with `npx skills add tester-army/e2e`.
304
- See [Coding agents](/coding-agents).
333
+ | Option | Description |
334
+ | --- | --- |
335
+ | `[topic]` | One of `setup`, `writing-tests`, `agent`, `running`, `explore`, `debugging`, `mcp`, or `bug-bash`. Default: the overview. An unknown topic exits 2 and names the valid ones. |
336
+
337
+ #### `e2e mcp`
305
338
 
306
- ## e2e mcp
339
+ An MCP server on stdio that gives a coding agent a live session on the app.
307
340
 
308
341
  <CodeGroup>
309
342
  ```bash npm
@@ -319,142 +352,20 @@ bunx e2e mcp [--config <path>] [--target <name>] [--headed] [--max-sessions <n>]
319
352
  ```
320
353
  </CodeGroup>
321
354
 
322
- An MCP server on stdio that gives a coding agent a live session on the app.
323
- Flags, tools, and session rules are on the [MCP server](/reference/mcp) page.
324
-
325
- ## e2e run
326
-
327
- ### Positional arguments
328
-
329
- ```bash
330
- e2e run [files...]
331
- ```
332
-
333
- Test files, directories, or globs relative to the project root:
334
-
335
- - a file path (`tests/signup.e2e.ts`) selects that file;
336
- - a directory (`tests/agent`, `.`) selects every discovered test file beneath
337
- it;
338
- - a bare name selects every discovered file whose path ends with it at a
339
- segment boundary. `signup.e2e.ts` and `agent/signup.e2e.ts` both select
340
- `tests/agent/signup.e2e.ts`. Without a `/` or `.` it matches the file name
341
- with extensions dropped, so `signup` selects it too;
342
- - any of these with `:line` appended (`tests/signup.e2e.ts:12`, `signup:12`)
343
- selects only the test declared at that line of each file it names: the line
344
- the `test(` call opens on, as an editor or a report's `source` shows it. A
345
- serial group runs whole when a line names one member. Several lines of one
346
- file add up; naming the file without a line selects it whole. When no test
347
- is declared at a named line, `NO_TESTS` names the positional and the lines
348
- the file declares tests at. A positional that exists on disk as written is
349
- a path, even when its name ends in `:12`;
350
- - a glob (`'tests/**/*.smoke.e2e.ts'`, quoted) uses the grammar of the config
351
- `tests` globs: `*`, `?`, and a whole `**` segment. Braces, character
352
- classes, and extglobs are `INVALID_GLOB` with a hint rather than a pattern
353
- that matches nothing: there is no expansion, so name one glob per
354
- alternative. `./`, `..`, a trailing `/`, and an absolute path inside the
355
- project resolve as they do for any positional path, and an existing file or
356
- directory is selected by its path whatever its name is spelled with.
357
-
358
- Positionals narrow the config globs and the tag, platform, and capability
359
- filters; they never bypass them. Names and globs are case-sensitive on every
360
- OS. A path outside the project root is `COLLECTION_ERROR`.
361
-
362
- The runner imports every discovered test file, even when you select only a
363
- few. This lets a selected test find its session setup in another file.
364
- Other ordinary tests remain unselected. A collection error in a selected
365
- file, or in any file of a run with no positionals, fails the run. A file the
366
- positionals left out that fails to import or register its tests is skipped
367
- with a notice, so a half-written file elsewhere in the suite does not stop
368
- `npx e2e run tests/login.e2e.ts`; if a selected test's session then has no
369
- setup test, the error names that file as where the setup may be.
370
-
371
- A positional starting with `-` that names no file is a usage error, exit 2.
372
- This is what `pnpm test:e2e -- --headed` produces: the `--` reaches e2e and
373
- `--headed` would otherwise be taken for a file. The message names the direct
374
- form, `pnpm exec e2e run --headed`.
375
- For npm, the direct form is `npm exec -- e2e run --headed`; npm needs the
376
- separator before the binary to pass its flags through.
377
-
378
- When nothing is left to run, `NO_TESTS` names the cause: no file matched the
379
- globs (with look-alike files such as `tests/login.test.ts`), a positional
380
- matched nothing (with the nearest path), a matched file registered no tests,
381
- or every test was filtered out.
382
-
383
- ### Flags
384
-
385
- | Flag | Type | Default | Behavior |
386
- | --- | --- | --- | --- |
387
- | `--config <path>` | string | discovered | Explicit config path. A missing file is `CONFIG_NOT_FOUND`. |
388
- | `--target <ids>` | comma-separated or repeated | every configured target | Selects targets by name. Only the app commands of selected targets start. An unknown ID is `UNKNOWN_TARGET`. |
389
- | `--tag <tags>` | comma-separated or repeated | none | Only tests carrying the tags: any of them, or every one with `--tag-mode all`. When the filter leaves nothing to run, `NO_TESTS` counts the tests it filtered, marks each tag no test declares, and names the nearest declared one. |
390
- | `--tag-mode <mode>` | `any` or `all` | `any` | How several tags combine: any of them, or every one. |
391
- | `--exclude-tag <tags>` | comma-separated or repeated | none | Leaves out every test carrying any of the tags, whatever `--tag` or a positional selected. A serial group still runs whole when any member is selected; tag the serial `describe` to exclude the flow. |
392
- | `--grep <pattern>` | regular expression, repeatable | none | Only tests whose title matches one of the patterns. The text matched is the describe titles and the test title joined by one space (`checkout pays`); the file and the tags are not part of it. A bare pattern is case-sensitive; `'/pattern/i'` adds flags. A pattern that does not compile is a usage error, exit 2. |
393
- | `--grep-invert <pattern>` | regular expression, repeatable | none | Leaves out every test whose title matches one of the patterns. Same text and pattern forms as `--grep`; both together keep the tests `--grep` matches minus the ones `--grep-invert` matches. |
394
- | `--last-failed` | boolean | `false` | Only the tests the previous run did not pass, read from the `report.json` this run overwrites, `<output>/report.json` (`--output`, else the config's [`output`](/reference/config#output)): failed, timed out, interrupted, or skipped because a setup, a serial predecessor, a hook, or the worker failed or `--max-failures` or `--max-duration` stopped the run first, and every test it selected in the describe or file, on that target, of a `beforeAll` or `afterAll` that failed (for a setup, the tests that consume its session). A rerun that runs the hook again reports it again if it still fails, and a rerun that runs any test of its scope without it failing resolves it. A test another filter (`--grep`, `--tag`, `--target`, `--agent`, a positional) leaves out of the rerun stays owed: the rerun's report carries its last result, and the hook failure whose scope it is in, under `run.carried`, the next `--last-failed` selects it again, and the [`@e2e-dev/github`](/github) page stays red until it passes (⏹️ for a test the run before interrupted). The rerun's own exit code and summary count only what it ran. It keeps the artifacts the previous report names and writes its own under each test's `rerun-<n>/`, so no attempt overwrites an earlier one's files. A test is named per target and agent. No report is `NO_LAST_RUN`; a previous run with nothing failed selects nothing, `NO_TESTS` unless `--pass-with-no-tests`. |
395
- | `--shard <index/total>` | `2/3` | none | Runs one contiguous slice of the selected tests, cut once every other filter applied. Slices differ in size by at most one test, a serial group counts as one and is never split, and each shard brings only the setup tests its own tests need. Every shard runs the same command with a different index; the order is the report order, so the same suite shards the same way on every machine. A shard that gets no test is `NO_TESTS` unless `--pass-with-no-tests`. See [Sharding](/ci#sharding). |
396
- | `--headed` | boolean | `false` | Visible UI when the engine supports it. |
397
- | `--agent <names>` | comma-separated or repeated | `default` | Runs unpinned tests once per selected agent. For a test with `{ agent }`, selects the shared names when there is overlap; otherwise keeps the test's selection. An unknown name is `INVALID_CONFIG`. |
398
- | `--retries <n>` | integer 0-10 | config | Replaces `config.retries`. |
399
- | `--workers <n>` | integer 1-1024 | config | Replaces `config.workers`. |
400
- | `--repeat-each <n>` | integer, at least 1 | `1` | Runs every selected test this many times. Each run is a result of its own, with `repeat` 0 through `n - 1` and its own `id`; the list reporter titles the later runs `(repeat #1)`, `(repeat #2)`, and their artifacts go under a `repeat-<n>` directory. Setup tests run once. With the replay cache on, the first run records and the later ones replay, so pair it with `--no-cache` to exercise the model each time. The `list` summary adds a `Repeats` row (`1 of 2 tests passed all 5 runs`) and one line per test that did not pass every run, naming each run that failed; the `markdown` page carries the same tally. |
401
- | `--max-failures <n>` | integer, at least 1 | none | Stops the run once this many tests have failed or timed out. Nothing more is dispatched, the tests running on other workers end as `interrupted`, and the tests not started, the rest of the file whose test reached the limit included, are skipped with cause `failure-limit` and the reason `run stopped after 3 failures (--max-failures 3)`. The exit code is the failures' own, 1, not 130. |
402
- | `--max-duration <ms>` | integer 1-2147483647 | config | Overrides [`maxDuration`](/reference/config#maxduration) for one run. Exits 1, not 130. |
403
- | `--reporter <ids>` | comma-separated list | config | Replaces the built-in IDs in `config.reporters`: `list`, `json`, `junit`, `markdown`. `json` with `list` is `INVALID_CONFIG`. [Reporter objects](/reference/reporters) in the config keep running. |
404
- | `--output <dir>` | string | config, else `.e2e` | The results directory for this run, over the config's [`output`](/reference/config#output): the report, the reporter files, the AI trace, `results/` (cleared once the run's tests start, down to what the previous report names on a `--last-failed` rerun), and `sessions/`. Resolved from the project root; one the run cannot own is `INVALID_CONFIG`. The removed `--artifacts <dir>` fails with the `--output` it maps to: `--artifacts out/artifacts` is `--output out`, and a directory whose parent is the project root or outside it maps to `--output .e2e`. |
405
- | `--cache-dir <dir>` | string | `E2E_CACHE_DIR`, else `cache.dir`, else `.e2e/cache` | The replay cache directory for this run, over [`E2E_CACHE_DIR`](/reference/environment#replay-cache) and [`cache.dir`](/reference/config#cache). Resolved from the project root, `~` or a leading `~/` as the home directory; the cache mode stays as configured. With a custom `cache.store` it is `INVALID_CONFIG`. |
406
- | `--no-cache` | boolean | `false` | Replay cache off for this run, whatever `config.cache` says. |
407
- | `--strict-cache` | boolean | `false` | Turns on `cache.strict` for this run: a step whose recording exists but no longer replays fails with `REPLAY_STALE` instead of handing off to the agent, retries replay too, and the run never writes the cache. See [Fail on a stale recording](/cache#fail-on-a-stale-recording). |
408
- | `-u`, `--update-snapshots` | boolean | `false` | Writes each stored screenshot [`toHaveScreenshot`](/reference/expect#screenshot-matchers) finds missing or different, and passes the matcher. Without it a missing one is written and fails, and in CI it is only attached to the results. |
409
- | `-w`, `--watch` | boolean | `false` | Runs, then runs again whenever a test file or the config changes, or a key asks for it; see [Watch mode](#watch-mode). Refused in CI (`CI` set), and with `--reporter json`. |
410
- | `--pass-with-no-tests` | boolean | `false` | Exit 0 instead of `NO_TESTS` when nothing is selected. |
411
- | `--debug` | boolean | `false` | Phase timings and an agent step table on stderr after the run. |
412
- | `--ai-trace` | boolean | `false` | Records every model call to `<output>/ai-trace.json`, an AI SDK devtools database. Open it with [unbox-ai](https://github.com/tester-army/unbox-ai). |
413
- | `--trace [mode]` | `off`, `on`, `retain-on-failure`, `on-first-retry`, or `on-all-retries` | config | Which tests keep a trace page (`<output>/results/<test>/trace.md`), over the config's and every target's `trace`; a test's own `trace` still wins. A bare `--trace` is `on`, a page for every test; `--trace off` writes none for a test without its own `trace`. The value is optional and greedy, so `--trace tests/a.e2e.ts` reads the file as a mode and fails: write `--trace=<mode>`, or put test files before `--trace`. See [trace](/reference/config#trace). |
414
- | `--video [mode]` | a recording mode, as `--trace` | config | Which attempts record a video (WebM on a browser engine, MP4 on a device engine), over the config's and every target's `video`; a test's own `video` still wins. A bare `--video` is `on`. It applies to the targets whose engine can record; the run names the others in one notice. Greedy like `--trace`: write `--video=<mode>`, or put test files before `--video`. See [video](/reference/config#video). |
415
-
416
- Out-of-range `--retries` and `--workers` fail with `INVALID_CONFIG` before
417
- anything starts, the same as the config value would. `--target`, `--tag`,
418
- `--exclude-tag`, and `--agent` take each name once; an empty value (`--tag ''`,
419
- or `--tag "$TAGS"` with the variable unset) is a usage error, exit 2, never a
420
- wider selection.
421
-
422
- Filters only narrow, and each applies after the one before: positionals, then
423
- `.only`, `--tag`, `--exclude-tag`, `--grep`, `--grep-invert`, `--last-failed`,
424
- then the platform and capability rules, then `--shard`. When they leave
425
- nothing to run, `NO_TESTS` counts the tests each filter removed, names the tags
426
- and patterns, and marks a tag no test declares.
427
-
428
- `--debug` appends two tables to stderr. Phase timings are summed across the
429
- runner and every worker. The agent step table lists every agent step in
430
- execution order with its per-phase split, model calls, tokens, and cost when
431
- the provider reported one.
355
+ | Option | Description |
356
+ | --- | --- |
357
+ | `--config <path>` | The config every session loads unless `open_session` names one. Default: the nearest `e2e.config.ts` or `e2e.config.mts`. |
358
+ | `--target <name>` | The target every session opens on. Default: the only target, or the one `open_session` names. |
359
+ | `--headed` | Shows the UI in sessions whose `open_session` does not set `headed`, when the engine supports it. |
360
+ | `--max-sessions <n>` | Sessions open at once, each with its own browser or device, 1 through 16. Default: 4. |
432
361
 
433
- ```console
434
- $ npx e2e run --debug
435
- ...
436
- [e2e debug] phase timings (wall 14.2s)
437
- phase count total avg max
438
- scheduler 1 11.8s 11.8s 11.8s
439
- test.body 6 10.1s 1.7s 3.9s
440
- session.launch 6 1.1s 183ms 410ms
441
- engine.prepare 1 612ms 612ms 612ms
442
- collect 1 420ms 420ms 420ms
443
- config.load 1 118ms 118ms 118ms
444
- realm.import 1 96ms 96ms 96ms
445
- session.close 6 88ms 15ms 31ms
446
- engine.init 1 74ms 74ms 74ms
362
+ The [MCP server](/reference/mcp) page covers these options in full, plus the
363
+ tools and the session rules.
447
364
 
448
- [e2e debug] agent steps (execution order, model anthropic/claude-sonnet-4.5, total $0.0312)
449
- step total model observe action calls tokens in/out cached cost
450
- act "add a todo named groceries" 3.9s 2.8s 410ms 620ms 3 9412/388 61% (5740) $0.0198
451
- assert "the list shows groceries" 1.2s 1.1s 88ms 0ms 1 3120/41 0% (0) $0.0114
452
- ```
365
+ #### `e2e run`
453
366
 
454
- `cached` is the share of input tokens the provider served from its prompt
455
- cache, `-` when it reports no split. The report records the same per step
456
- (`model.cacheReadTokens`, `model.cacheWriteTokens`) and per run
457
- (`usage.modelCachedTokens`).
367
+ Runs the tests the config discovers and writes `report.json` under the
368
+ [output directory](/reference/config#output).
458
369
 
459
370
  <CodeGroup>
460
371
  ```bash npm
@@ -509,40 +420,239 @@ bunx e2e run --reporter list,junit
509
420
  ```
510
421
  </CodeGroup>
511
422
 
512
- ### Output
423
+ <a id="positional-arguments" />
424
+
425
+ ##### Positional arguments
426
+
427
+ ```bash
428
+ e2e run [files...]
429
+ ```
430
+
431
+ Test files, directories, or globs relative to the project root:
432
+
433
+ | Positional | Selects |
434
+ | --- | --- |
435
+ | A file path (`tests/signup.e2e.ts`) | That file. |
436
+ | A directory (`tests/agent`, `.`) | Every discovered test file beneath it. |
437
+ | A bare name (`signup.e2e.ts`, `agent/signup.e2e.ts`, `signup`) | Every discovered file whose path ends with it at a segment boundary; the first two both select `tests/agent/signup.e2e.ts`. Without a `/` or `.`, it matches the file name without extensions, so `signup` selects it too. |
438
+ | Any of these with `:line` (`tests/signup.e2e.ts:12`, `signup:12`) | Only the test declared at that line of each file it names: the line its `test(` call opens on, as an editor or a report's `source` shows it. |
439
+ | A quoted glob (`'tests/**/*.smoke.e2e.ts'`) | Every discovered file it matches, in the grammar of the config `tests` globs: `*`, `?`, and a whole `**` segment. |
440
+
441
+ - **Lines.** A line that names one member of a serial group runs the group
442
+ whole. Several lines of one file add up; the file without a line selects it
443
+ whole. When no test is declared at a named line, `NO_TESTS` names the
444
+ positional and the lines the file declares tests at. A positional that
445
+ exists on disk as written is a path, even when it ends in `:12`.
446
+ - **Globs.** Braces, character classes, and extglobs are `INVALID_GLOB` with a
447
+ hint, not a pattern that matches nothing. There is no expansion, so name one
448
+ glob per alternative. `./`, `..`, a trailing `/`, and an absolute path
449
+ inside the project resolve as for any positional path. An existing file or
450
+ directory is selected by its path, whatever characters its name holds.
451
+ - Positionals narrow the config globs and the tag, platform, and capability
452
+ filters; they never bypass them. Names and globs are case-sensitive on
453
+ every OS. A path outside the project root is `COLLECTION_ERROR`.
454
+ - The runner imports every discovered test file, even when you select a few,
455
+ so a selected test finds its session setup in another file. Other ordinary
456
+ tests stay unselected. A collection error fails the run when it is in a
457
+ selected file, or in any file of a run with no positionals. A file the
458
+ positionals left out that fails to import or register its tests is skipped
459
+ with a notice. So a half-written file elsewhere does not stop
460
+ `npx e2e run tests/login.e2e.ts`. If a selected test's session then has no
461
+ setup test, the error names that file as where the setup may be.
462
+ - A positional that starts with `-` and names no file is a usage error, exit
463
+ 2. `pnpm test:e2e -- --headed` produces it: the `--` reaches e2e, which
464
+ would otherwise take `--headed` for a file. The message names the direct
465
+ form, `pnpm exec e2e run --headed`. For npm it is
466
+ `npm exec -- e2e run --headed`; npm needs the separator before the binary
467
+ to pass its flags through.
468
+ - When nothing is left to run, `NO_TESTS` names the cause: no file matched
469
+ the globs (with look-alikes such as `tests/login.test.ts`), a positional
470
+ matched nothing (with the nearest path), a matched file registered no
471
+ tests, or every test was filtered out.
472
+
473
+ <a id="flags" />
474
+
475
+ ##### Flags
476
+
477
+ **Selection**
478
+
479
+ | Option | Description |
480
+ | --- | --- |
481
+ | `--config <path>` | Explicit config path. Default: the nearest `e2e.config.ts` or `e2e.config.mts` (see [Loading](/reference/config#loading)). A missing file is `CONFIG_NOT_FOUND`. |
482
+ | `--target <ids>` | Targets by name, comma-separated or repeated. Default: every configured target. Only selected targets' app commands start. An unknown ID is `UNKNOWN_TARGET`. |
483
+ | `--tag <tags>` | Only tests with the tags, comma-separated or repeated: any of them, or every one with `--tag-mode all`. |
484
+ | `--tag-mode <mode>` | How several tags combine: `any` or `all`. Default: `any`. |
485
+ | `--exclude-tag <tags>` | Leaves out tests with any of the tags, comma-separated or repeated, whatever `--tag` or a positional selected. A serial group still runs whole when any member is selected; tag the serial `describe` to exclude the flow. |
486
+ | `--grep <pattern>` | Only tests whose title matches one of the regular expressions; repeatable. The title is the describe titles and the test title joined by one space (`checkout pays`), without the file or the tags. A bare pattern is case-sensitive; `'/pattern/i'` adds flags. A pattern that does not compile is a usage error, exit 2. |
487
+ | `--grep-invert <pattern>` | Leaves out tests whose title matches one of the regular expressions; repeatable, with the text and forms of `--grep`. With both, the run keeps what `--grep` matches minus what `--grep-invert` matches. |
488
+ | `--last-failed` | Only the tests the previous run did not pass; the list after these tables says which. |
489
+ | `--shard <index/total>` | Runs one contiguous slice of the selected tests, such as `2/3`, cut after every other filter. Slices differ by at most one test. A serial group counts as one and is never split. Each shard brings only the setup tests its own tests need. Skipped tests are not cut: the shard that runs the nearest test before one reports it, else the first shard, so only one shard does. Every shard runs the same command with a different index. The order is the report order, so a suite shards the same way on every machine. A shard with no test is `NO_TESTS` unless `--pass-with-no-tests`. See [Sharding](/ci#sharding). |
490
+ | `--pass-with-no-tests` | Exits 0 instead of `NO_TESTS` when nothing is selected. |
491
+
492
+ **Execution**
493
+
494
+ | Option | Description |
495
+ | --- | --- |
496
+ | `--headed` | Visible UI when the engine supports it. |
497
+ | `--agent <names>` | Runs unpinned tests once per selected agent, comma-separated or repeated. Default: `default`. For a test with `{ agent }`, it selects the shared names when they overlap, else keeps the test's own. An unknown name is `INVALID_CONFIG`. |
498
+ | `--retries <n>` | Replaces `config.retries`; an integer 0-10. Default: the config. |
499
+ | `--workers <n>` | Replaces `config.workers`; an integer 1-1024. Default: the config. |
500
+ | `--max-failures <n>` | Stops the run once this many tests (at least 1) have failed or timed out. Nothing more is dispatched, and tests running on other workers end `interrupted`. Tests not started, including the rest of the file whose test reached the limit, are skipped with cause `failure-limit` and the reason `run stopped after 3 failures (--max-failures 3)`. Exits 1, the failures' own code, not 130. |
501
+ | `--max-duration <ms>` | Overrides [`maxDuration`](/reference/config#maxduration) for one run; an integer 1-2147483647. Default: the config. Exits 1, not 130. |
502
+ | `--repeat-each <n>` | Runs every selected test this many times, at least 1. Default: `1`. Each run is its own result, with `repeat` 0 through `n - 1` and its own `id`. The list reporter titles later runs `(repeat #1)`, `(repeat #2)`, and puts their artifacts under a `repeat-<n>` directory. Setup tests run once. With the replay cache on, the first run records and the rest replay; add `--no-cache` to call the model each time. The `list` summary adds a `Repeats` row (`1 of 2 tests passed all 5 runs`) and one line per test that did not pass every run, naming the failed runs. The `markdown` page has the same tally. |
503
+ | `--no-cache` | Turns the replay cache off for this run, whatever `config.cache` says. |
504
+ | `--strict-cache` | Turns on `cache.strict` for this run. A step whose recording exists but no longer replays fails with `REPLAY_STALE` instead of handing off to the agent. Retries replay too, and the run never writes the cache. See [Fail on a stale recording](/cache#fail-on-a-stale-recording). |
505
+ | `-u`, `--update-snapshots` | Writes each stored screenshot [`toHaveScreenshot`](/reference/expect#screenshot-matchers) finds missing or different, and passes the matcher. Without it, a missing one is written and fails; in CI it is only attached to the results. |
506
+ | `-w`, `--watch` | Runs, then reruns when a test file or the config changes, or on a key. See [Watch mode](#watch-mode). |
507
+
508
+ **Output**
509
+
510
+ | Option | Description |
511
+ | --- | --- |
512
+ | `--reporter <ids>` | Replaces the built-in IDs in `config.reporters`, comma-separated: `list`, `json`, `junit`, `markdown`. Default: the config. `json` with `list` is `INVALID_CONFIG`. [Reporter objects](/reference/reporters) in the config keep running. |
513
+ | `--output <dir>` | The results directory for this run, over the config's [`output`](/reference/config#output): the report, the reporter files, the AI trace, `results/`, and `sessions/`. Default: the config, else `.e2e`. Resolved from the project root; a directory the run cannot own is `INVALID_CONFIG`. `results/` is cleared once the tests start; a `--last-failed` rerun keeps what the previous report names. |
514
+ | `--cache-dir <dir>` | The replay cache directory for this run, over [`E2E_CACHE_DIR`](/reference/environment#replay-cache) and [`cache.dir`](/reference/config#cache). Default: `E2E_CACHE_DIR`, else `cache.dir`, else `.e2e/cache`. Resolved from the project root; `~` or a leading `~/` is the home directory. The cache mode stays as configured. With a custom `cache.store` it is `INVALID_CONFIG`. |
515
+ | `--debug` | Phase timings and an agent step table on stderr after the run; see [Debug output](#debug-output). |
516
+ | `--ai-trace` | Records every model call to `<output>/ai-trace.json`; see [Output](#output). |
517
+ | `--trace [mode]` | Which tests keep a trace page (`<output>/results/<test>/trace.md`): `off`, `on`, `retain-on-failure`, `on-first-retry`, or `on-all-retries`. Default: the config. `--trace off` writes none for a test without its own `trace`. See [trace](/reference/config#trace). |
518
+ | `--video [mode]` | Which attempts record a video (WebM on a browser engine, MP4 on a device engine), with the modes of `--trace`. Default: the config. Only targets whose engine can record get one; the run names the others in one notice. See [video](/reference/config#video). |
519
+
520
+ `--trace` and `--video` apply over the config's and every target's value; a
521
+ test's own `trace` or `video` still wins. A bare flag is `on`: bare `--trace`
522
+ keeps a page for every test. The value is optional and greedy:
523
+ `--trace tests/a.e2e.ts` reads the file as a mode and fails. Write
524
+ `--trace=<mode>`, or put test files before the flag.
525
+
526
+ The removed `--artifacts <dir>` fails with the `--output` it maps to:
527
+ `--artifacts out/artifacts` is `--output out`, and a directory whose parent is
528
+ the project root or outside it maps to `--output .e2e`.
529
+
530
+ `--last-failed` selects the tests the previous run did not pass. It reads them
531
+ from the `report.json` this run overwrites, `<output>/report.json` (`--output`,
532
+ else the config's [`output`](/reference/config#output)):
533
+
534
+ - Tests that failed, timed out, or were interrupted.
535
+ - Tests skipped because a setup, a serial predecessor, a hook, or the worker
536
+ failed, or because `--max-failures` or `--max-duration` stopped the run
537
+ first.
538
+ - Every test a failed `beforeAll` or `afterAll` selected in its describe or
539
+ file, on that target; for a setup, the tests that consume its session. A
540
+ rerun that runs the hook again reports it again if it still fails. A rerun
541
+ that runs any test of its scope without it failing resolves it.
542
+ - A test another filter (`--grep`, `--tag`, `--target`, `--agent`, a
543
+ positional) leaves out of the rerun stays owed. The rerun's report carries
544
+ its last result, and the hook failure whose scope holds it, under
545
+ `run.carried`. The next `--last-failed` selects it again, and the
546
+ [`@e2e-dev/github`](/github) page stays red until it passes (⏹️ for a test
547
+ the run before interrupted). The rerun's own exit code and summary count
548
+ only what it ran.
549
+ - The rerun keeps the artifacts the previous report names and writes its own
550
+ under each test's `rerun-<n>/`, so no attempt overwrites an earlier one's
551
+ files.
552
+ - A test is named per target and agent.
553
+ - No report is `NO_LAST_RUN`. When nothing failed in the previous run, it
554
+ selects nothing: `NO_TESTS` unless `--pass-with-no-tests`.
555
+
556
+ Out-of-range `--retries` and `--workers` fail with `INVALID_CONFIG` before
557
+ anything starts, as the config value would. `--target`, `--tag`,
558
+ `--exclude-tag`, and `--agent` take each name once. An empty value (`--tag ''`,
559
+ or `--tag "$TAGS"` with the variable unset) is a usage error, exit 2, never a
560
+ wider selection.
561
+
562
+ Filters only narrow, and each applies after the one before: positionals, then
563
+ `.only`, `--tag`, `--exclude-tag`, `--grep`, `--grep-invert`, `--last-failed`,
564
+ then the platform and capability rules, then `--shard`. When they leave
565
+ nothing to run, `NO_TESTS` counts the tests each filter removed and names the
566
+ tags and patterns. It marks each tag no test declares and names the nearest
567
+ declared one.
568
+
569
+ <a id="debug-output" />
570
+
571
+ ##### Debug output
572
+
573
+ `--debug` appends two tables to stderr. Phase timings are summed across the
574
+ runner and every worker. The agent step table lists every agent step in
575
+ execution order: its per-phase split, model calls, tokens, and cost when the
576
+ provider reported one.
577
+
578
+ ```console
579
+ $ npx e2e run --debug
580
+ ...
581
+ [e2e debug] phase timings (wall 14.2s)
582
+ phase count total avg max
583
+ scheduler 1 11.8s 11.8s 11.8s
584
+ test.body 6 10.1s 1.7s 3.9s
585
+ session.launch 6 1.1s 183ms 410ms
586
+ engine.prepare 1 612ms 612ms 612ms
587
+ collect 1 420ms 420ms 420ms
588
+ config.load 1 118ms 118ms 118ms
589
+ realm.import 1 96ms 96ms 96ms
590
+ session.close 6 88ms 15ms 31ms
591
+ engine.init 1 74ms 74ms 74ms
592
+
593
+ [e2e debug] agent steps (execution order, model anthropic/claude-sonnet-4.5, total $0.0312)
594
+ step total model observe action calls tokens in/out cached cost
595
+ act "add a todo named groceries" 3.9s 2.8s 410ms 620ms 3 9412/388 61% (5740) $0.0198
596
+ assert "the list shows groceries" 1.2s 1.1s 88ms 0ms 1 3120/41 0% (0) $0.0114
597
+ ```
598
+
599
+ `cached` is the share of input tokens the provider served from its prompt
600
+ cache, `-` when it reports no split. The report records the same per step
601
+ (`model.cacheReadTokens`, `model.cacheWriteTokens`) and per run
602
+ (`usage.modelCachedTokens`).
603
+
604
+ <a id="output" />
605
+
606
+ ##### Output
513
607
 
514
608
  | Reporter | Writes |
515
609
  | --- | --- |
516
- | `list` | Terminal progress, failures with code frames, and a run summary. On a TTY, live progress includes the current agent step, model turns, tool calls, and reasoning excerpts. Untrusted fields are capped at 8 KiB and stripped of control characters. |
517
- | `json` | The report document to stdout and nothing else there, so `e2e run --reporter json \| jq` works. Same document as `<output>/report.json`, described by `schema/report-v1.schema.json` in the package. The shape changes only with the `schemaVersion` field (`report-1` today). Cannot be combined with `list`. |
610
+ | `list` | Terminal progress, failures with code frames, and a run summary. On a TTY, live progress shows the current agent step, model turns, tool calls, and reasoning excerpts. Untrusted fields are capped at 8 KiB and stripped of control characters. |
611
+ | `json` | The report document to stdout, and nothing else there, so `e2e run --reporter json \| jq` works. It is the `<output>/report.json` document, described by `schema/report-v1.schema.json` in the package. The shape changes only with the `schemaVersion` field (`report-1` today). |
518
612
  | `junit` | `junit.xml` beside the report. One `<testsuite>` per file, one `<testcase>` per test-target pair named `title > path [target]`. A test-class error is a `<failure>`, any other an `<error>`, a skip a `<skipped>` with its reason. An interrupted test reached no verdict, so it is a `<skipped>` whose message starts `interrupted:`. Run-level errors form a `run` suite. |
519
- | `markdown` | `summary.md` beside the report with counts, cost, errors, and failure details, each naming its trace page when one was kept. Exploration runs show findings. [`@e2e-dev/github`](/github) uses this summary for pull request comments. |
520
-
521
- Every run that gets to its tests writes `report.json`, for scripts, under the [output directory](/reference/config#output)
522
- (`.e2e` by default) whatever the reporters are, the trace pages its
523
- [`trace`](/reference/config#trace) mode keeps in each test's directory under `results/`, and the reporters' files beside it. A run that stops before (a config or collection error,
524
- `NO_TESTS`, `NO_LAST_RUN`, `UNSUPPORTED_ARTIFACT`, an app command that
525
- fails to start, an interrupt) writes none of them and leaves the previous
526
- run's files in place; its errors are on the terminal and in the exit code. A `junit.xml` or `summary.md` that could not be written is a
527
- line on stderr, never a run error.
528
-
529
- `run.summary` in the report counts the results. `discovered` is every
530
- test-target pair the runner collected and `selected` the ones this run
531
- chose; the selection's left-out pairs are in `results` as `selected: false`
532
- but in no other count. `passed`, `failed` (timed out included),
533
- `interrupted`, `flaky`, and `skipped` count the selected results and add up
534
- to `selected`. `interrupted` is a test the run stopped before its verdict
535
- (a signal, `--max-failures`, `--max-duration`, or a run-level error such as
536
- `MODEL_UNAVAILABLE`), never a failure. `executed` is the selected
537
- results that ran at least one attempt.
538
-
539
- With `--ai-trace`, the run also writes `<output>/ai-trace.json`: one entry per
540
- model round trip with the exact prompt, tool definitions, response, usage,
541
- and provider metadata. Observations in it are already redacted. Inline image
542
- and file bytes are replaced with byte counts. Open it with
543
- `npx unbox-ai .e2e/ai-trace.json` for the default output; see [Debugging a run](/debugging#agent-steps).
544
-
545
- ### Watch mode
613
+ | `markdown` | `summary.md` beside the report: counts, cost, errors, and failure details, each naming its trace page when one was kept. Exploration runs show findings. [`@e2e-dev/github`](/github) uses this summary for pull request comments. |
614
+
615
+ Every run that gets to its tests writes, under the
616
+ [output directory](/reference/config#output) (`.e2e` by default):
617
+
618
+ - `report.json`, for scripts, whatever the reporters are;
619
+ - the trace pages its [`trace`](/reference/config#trace) mode keeps, in each
620
+ test's directory under `results/`;
621
+ - the reporters' files beside the report.
622
+
623
+ A run that stops before its tests writes none of them and leaves the previous
624
+ run's files in place. That covers a config or collection error, `NO_TESTS`,
625
+ `NO_LAST_RUN`, `UNSUPPORTED_ARTIFACT`, an app command that fails to start, and
626
+ an interrupt. Its errors are on the terminal and in the exit code. A
627
+ `junit.xml` or `summary.md` that could not be written is a line on stderr,
628
+ never a run error.
629
+
630
+ `run.summary` in the report counts the results:
631
+
632
+ - `discovered` is every test-target pair the runner collected, and
633
+ `selected` the ones this run chose. The left-out pairs are in `results` as
634
+ `selected: false`, but in no other count.
635
+ - `passed`, `failed` (timed out included), `interrupted`, `flaky`, and
636
+ `skipped` count the selected results and add up to `selected`.
637
+ - `interrupted` is a test the run stopped before its verdict (a signal,
638
+ `--max-failures`, `--max-duration`, or a run-level error such as
639
+ `MODEL_UNAVAILABLE`), never a failure.
640
+ - `executed` is the selected results that ran at least one attempt.
641
+
642
+ With `--ai-trace`, the run also writes `<output>/ai-trace.json`, an AI SDK
643
+ devtools database. It holds one entry per model round trip: the exact prompt,
644
+ tool definitions, response, usage, and provider metadata. Observations in it
645
+ are already redacted, and inline image and file bytes become byte counts.
646
+ Open it with [unbox-ai](https://github.com/tester-army/unbox-ai):
647
+ `npx unbox-ai runs .e2e/ai-trace.json` lists its runs. See
648
+ [Debugging a run](/debugging#agent-steps).
649
+
650
+ <a id="watch-mode" />
651
+
652
+ ##### Watch mode
653
+
654
+ `--watch` runs the selection, prints the summary, and waits instead of
655
+ exiting.
546
656
 
547
657
  <CodeGroup>
548
658
  ```bash npm
@@ -561,18 +671,19 @@ bunx e2e run --watch tests/checkout.e2e.ts --headed
561
671
  ```
562
672
  </CodeGroup>
563
673
 
564
- `--watch` runs the selection, prints the summary, and waits instead of
565
- exiting. A saved test file (one the config's [`tests`](/reference/config#tests)
566
- globs match) runs again on its own, whole, in place of the positionals. A
567
- saved source file next to the tests (a helper, a fixture: `.ts`, `.js`,
568
- `.json` under the directories the globs name, or beside the config) and a
569
- saved config run the selection again. Both keep the command line's other
570
- filters and the `p` and `t` filters. Changes that land during a run wait for
571
- it and run next, together (during the first run, only the config and its
572
- directory are watched: the test directories come from the config). On a
573
- terminal the screen is cleared before each
574
- rerun, and a banner names what triggered it. Each rerun reads the project's
575
- files afresh, the helpers a test imports included.
674
+ - A saved test file (one the config's [`tests`](/reference/config#top-level)
675
+ globs match) runs again alone and whole, in place of the positionals.
676
+ - A saved source file near the tests (a helper or a fixture: `.ts`, `.js`, or
677
+ `.json` under the directories the globs name, or beside the config) or a
678
+ saved config reruns the selection.
679
+ - Both keep the command line's other filters and the `p` and `t` filters.
680
+ - Changes during a run wait for it and run next, together. During the first
681
+ run only the config and its directory are watched, since the test
682
+ directories come from the config.
683
+ - On a terminal the screen clears before each rerun, and a banner names the
684
+ trigger.
685
+ - Each rerun reads the project's files afresh, including the helpers a test
686
+ imports.
576
687
 
577
688
  On a terminal, keys drive it as in vitest:
578
689
 
@@ -580,124 +691,62 @@ On a terminal, keys drive it as in vitest:
580
691
  | --- | --- |
581
692
  | `a`, `enter` | every test the command line selected, with the `p` and `t` filters cleared |
582
693
  | `r` | the selection again, under the `p` and `t` filters |
583
- | `f` | the tests the last run did not pass, as `--last-failed` reads them from its report, wherever they are: the positionals and the `p` filter do not narrow it |
694
+ | `f` | the tests the last run did not pass, read from its report as `--last-failed` does, wherever they are; the positionals and the `p` filter do not narrow it |
584
695
  | `u` | the current selection with `--update-snapshots` |
585
696
  | `p` | prompts for a positional (a file, a bare name, a glob) the next runs select with; empty clears it |
586
697
  | `t` | prompts for a title pattern, as `--grep`; empty clears it |
587
698
  | `h` | shows the keys |
588
699
  | `q` | quits |
589
700
 
590
- During a run, any of these keys (and space or `c`) cancels it, as an
591
- interrupt would, a second one forces it, and the session waits again.
592
- Ctrl-C or Ctrl-D
593
- interrupts the run and ends the session, from inside a `p` or `t` prompt
594
- too; escape does the same outside a prompt and only closes the prompt
595
- inside one. Ctrl-Z suspends the process as usual.
596
-
597
- Every rerun is a complete run: once the config has loaded it writes
598
- `report.json` and the reporter files, and starts and stops the app command.
599
- To keep a dev server up across
600
- reruns, start it yourself and set
601
- [`reuseExisting`](/reference/config#command) on the command. `--last-failed`
602
- applies to the first run only; a rerun that selects nothing passes empty.
603
- The exit code on quit is that of the last run that finished.
701
+ - During a run, any of these keys (and space or `c`) cancels it, as an
702
+ interrupt would. A second one forces it, and the session waits again.
703
+ - Ctrl-C or Ctrl-D interrupts the run and ends the session, from inside a `p`
704
+ or `t` prompt too. Escape does the same outside a prompt, and inside one
705
+ only closes the prompt. Ctrl-Z suspends the process as usual.
706
+ - Every rerun is a complete run: once the config loads, it writes
707
+ `report.json` and the reporter files, and starts and stops the app command.
708
+ To keep a dev server up across reruns, start it yourself and set
709
+ [`reuseExisting`](/reference/config#command) on the command.
710
+ - `--last-failed` applies to the first run only; a rerun that selects nothing
711
+ passes empty.
712
+ - On quit, the exit code is that of the last run that finished.
713
+ - Without a terminal on stdin, only file changes rerun.
604
714
 
605
- Without a terminal on stdin, only file changes rerun. Not for CI: a process
606
- that never exits on its own hangs the job, so `--watch` is refused when `CI`
607
- is set. It is refused with `--reporter json` too, and ends with exit code 2
608
- after the first run when the config's `reporters` include `json`.
715
+ <Note>
716
+ `--watch` is not for CI: a process that never exits on its own hangs the job,
717
+ so it is refused when `CI` is set. It is also refused with `--reporter json`, and
718
+ exits 2 after the first run when the config's `reporters` include `json`.
719
+ </Note>
609
720
 
610
- ### Signals
721
+ ##### Signals
611
722
 
612
723
  Signals escalate.
613
724
 
614
- 1. The first `SIGINT` or `SIGTERM` interrupts. The running test ends with
615
- status `interrupted`, its `afterEach` hooks and session close run within
616
- the cleanup budget, and every worker disposes its engine. The tests not
617
- started are skipped with cause `infrastructure-unavailable` and the reason
618
- `run interrupted before execution`, so the report lists every planned
619
- test and `--last-failed` runs the interrupted and the unstarted ones
620
- again. The runner writes the report and exits 130. An interrupt before
621
- any test starts (during collection, an engine's `prepare`, or app
622
- startup) writes no report and leaves the previous one in place.
725
+ 1. The first `SIGINT` or `SIGTERM` interrupts. The running test ends
726
+ `interrupted`; its `afterEach` hooks and session close run within the
727
+ cleanup budget, and every worker disposes its engine. Tests not started
728
+ are skipped with cause `infrastructure-unavailable` and the reason
729
+ `run interrupted before execution`. So the report lists every planned
730
+ test, and `--last-failed` reruns the interrupted and the unstarted ones.
731
+ The runner writes the report and exits 130. An interrupt before any test
732
+ starts (during collection, an engine's `prepare`, or app startup) writes
733
+ no report and leaves the previous one in place.
623
734
  2. A second signal forces. Workers dispose their engines at once and are
624
735
  killed when the cleanup budget is spent. The report is still written,
625
- and so are `junit.xml` and `summary.md` when the `junit` and
626
- `markdown` reporters are on; a custom reporter's
627
- `onRunFinished` is abandoned.
628
- 3. A third signal exits on the spot, after killing the process groups of every
629
- app command the run started. A process attached through
736
+ and so are `junit.xml` and `summary.md` when the `junit` and `markdown`
737
+ reporters are on; a custom reporter's `onRunFinished` is abandoned.
738
+ 3. A third signal exits on the spot, after killing the process groups of
739
+ every app command the run started. A process attached through
630
740
  `reuseExisting` is left alone.
631
741
 
632
742
  A worker whose runner disappears disposes its engine and exits within the
633
743
  cleanup budget.
634
744
 
635
- ## e2e explore
636
-
637
- ```bash
638
- e2e explore [goal]
639
- ```
640
-
641
- A run with no test file. The agent gets a goal, plans one exploration step
642
- at a time, reports defects through its `report_finding` tool, and ends with an
643
- assessment. The default goal is `Explore the app and find bugs`. See
644
- [Exploring without a test](/explore).
645
-
646
- | Flag | Type | Default | Behavior |
647
- | --- | --- | --- | --- |
648
- | `--config <path>` | string | discovered | Explicit config path, with the same errors as `run`. |
649
- | `--target <id>` | string | the first configured target | The one target to explore. Several targets and no flag explores the first and says so on stderr. An unknown name is `UNKNOWN_TARGET`. |
650
- | `--agent <name>` | string | `default` | The configured agent the exploration runs as. Its model, `system`, tools, and context are reused; a hand-rolled `StepExecutor` is replaced by the built-in agent, with a notice. An entry with no model, a custom executor's included, is `MODEL_UNAVAILABLE` before anything starts. An unknown name is `INVALID_CONFIG`. |
651
- | `--session <name>` | string | none | Starts signed in. The setup test in the config's test files that declares this session runs first, and the exploration restores what it saved, as a test with `{ session }` does. No other test runs. A name no setup test declares is `COLLECTION_ERROR`, naming the declared ones, before any app process starts. See [Start signed in](/explore#start-signed-in). |
652
- | `--max-steps <n>` | integer 1-12 | `8` | Exploration steps at most. Out of range exits 2. |
653
- | `--timeout <ms>` | integer 180000-900000 | `600000` | The run's wall clock. The last minute is kept for the closing assessment. Out of range exits 2. |
654
- | `--headed` | boolean | `false` | Requests visible UI when the engine supports it. |
655
- | `--reporter <ids>` | comma-separated list | resolved config value | As for `run`. The exploration summary rows print under the `list` summary; `json` carries the record in the report; `markdown` writes the findings to `<output>/summary.md`. |
656
- | `--output <dir>` | string | config, else `.e2e` | As for `run`; a finding's evidence screenshot is one of the attempt's artifacts under `<output>/results`. |
657
- | `--debug` | boolean | `false` | As for `run`: phase timings and the agent step table. |
658
- | `--ai-trace` | boolean | `false` | As for `run`. |
659
- | `--trace [mode]` | a recording mode | config | As for `run`: a trace page of the exploration. A bare `--trace` is `on`; `retain-on-failure` keeps it only when the exploration did not pass. The exploration is one attempt, so a retry mode keeps none, and the plan-time notice says to pass `--trace on`. Put the goal before `--trace`, or write `--trace=<mode>`. |
660
- | `--video [mode]` | a recording mode | config | As for `run`: one video of the whole exploration (WebM on a browser engine, MP4 on a device engine). A bare `--video` is `on`; `retain-on-failure` keeps it only when the exploration did not pass. An engine that cannot record prints a notice. The exploration is one attempt, so a retry mode records nothing, and the plan-time notice says to pass `--video on`. Put the goal before `--video`, or write `--video=<mode>`. |
661
-
662
- The run has one result, under the virtual file `explore`, titled by the goal.
663
- The exploration opens the app first when the target declares `app.url`.
664
- With `--session`, the report also holds the setup test's result and lists the
665
- project's other tests as skipped with cause `filtered`, as `run` lists the
666
- tests of a file no positional named.
667
- The exploration runs with the replay cache off and zero retries whatever the
668
- config says. A `--session` setup test runs as it would for `run`, with the
669
- config's agents, cache, and retries.
670
- `<output>/report.json` gains `run.explore`: the goal, the budgets, why the run
671
- ended (`finished`, `step-limit`, `time`, `stuck`, or `aborted`), the
672
- assessment, the steps, and the findings.
673
-
674
- Each step has its own budget: 40 actions and 40 model calls, unless the agent
675
- sets `maxSteps` or `maxModelCalls` (1 to 100). A step that reaches its action
676
- or time budget is time-boxed, and its findings still count. A failed step
677
- does not end the run; three failed or blocked steps in a row with no new
678
- finding do (`stuck`).
679
-
680
- A finding of kind `issue` is a functional defect; a `warning` is cosmetic.
681
- Severity runs from `trivial` (1) to `critical` (5), and the report lists
682
- issues before warnings, the most severe first. Each finding has its location,
683
- the expected and observed behavior, the steps to reproduce, and a screenshot
684
- when capture is allowed and succeeds. After a secret fill, findings carry no
685
- screenshots; see
686
- [screenshots after a secret fill](/reference/security#screenshots-after-a-secret-fill).
745
+ #### `e2e explore`
687
746
 
688
- Artifacts go to a directory named from the goal's first words and a digest of
689
- the goal, target, and agent, such as
690
- `<output>/results/explore-check-the-cart-totals-1a2b3c4d5e6f7a8b/`, so a rerun
691
- of the same goal on the same target and agent writes to the same directory.
692
-
693
- | Outcome | Run status | Exit code |
694
- | --- | --- | --- |
695
- | Steps ran, no `issue` finding, and not every step was blocked | passed | 0 |
696
- | At least one `issue` finding | failed | 1 |
697
- | No step ran and nothing was found | blocked | 1 |
698
- | Every step was blocked, with no `issue` finding | blocked | 1, 2, or 3, from the first blocker |
699
- | Config, engine, or model provider error | error | 2 or 3 |
700
- | Ctrl-C | | 130 |
747
+ A run with no test file. The agent gets a goal, plans one exploration step at
748
+ a time, reports defects through its `report_finding` tool, and ends with an
749
+ assessment. See [Exploring without a test](/explore).
701
750
 
702
751
  <CodeGroup>
703
752
  ```bash npm
@@ -722,28 +771,82 @@ bunx e2e explore 'Hunt for broken forms and dead links' --agent thorough --video
722
771
  ```
723
772
  </CodeGroup>
724
773
 
725
- ## e2e list
774
+ | Option | Description |
775
+ | --- | --- |
776
+ | `[goal]` | What to explore, in a sentence. Default: `Explore the app and find bugs`. |
777
+ | `--config <path>` | Explicit config path, with the errors of `run`. Default: discovered. |
778
+ | `--target <id>` | The one target to explore. Default: the first configured target; with several targets and no flag, it explores the first and says so on stderr. An unknown name is `UNKNOWN_TARGET`. |
779
+ | `--agent <name>` | The configured agent to explore as. Default: `default`. Its model, `system`, tools, and context are reused. The built-in agent replaces a hand-rolled `StepExecutor`, with a notice. An entry with no model, a custom executor's included, is `MODEL_UNAVAILABLE` before anything starts. An unknown name is `INVALID_CONFIG`. |
780
+ | `--session <name>` | Starts signed in. The setup test in the config's test files that declares this session runs first. The exploration restores what it saved, as a test with `{ session }` does. No other test runs. A name no setup test declares is `COLLECTION_ERROR`, naming the declared ones, before any app process starts. See [Start signed in](/explore#start-signed-in). |
781
+ | `--max-steps <n>` | Exploration steps at most, an integer 1-12. Default: `8`. Out of range exits 2. |
782
+ | `--timeout <ms>` | The run's wall clock, an integer 180000-900000. Default: `600000`. The last minute is kept for the closing assessment. Out of range exits 2. |
783
+ | `--headed` | Requests visible UI when the engine supports it. |
784
+ | `--reporter <ids>` | As for `run`. Default: the resolved config value. `list` prints the exploration rows under its summary, `json` carries the record in the report, and `markdown` writes the findings to `<output>/summary.md`. |
785
+ | `--output <dir>` | As for `run`. Default: the config, else `.e2e`. A finding's evidence screenshot is an attempt artifact under `<output>/results`. |
786
+ | `--debug` | As for `run`: phase timings and the agent step table. |
787
+ | `--ai-trace` | As for `run`. |
788
+ | `--trace [mode]` | As for `run`: a trace page of the exploration. Default: the config. |
789
+ | `--video [mode]` | As for `run`: one video of the whole exploration (WebM on a browser engine, MP4 on a device engine). Default: the config. An engine that cannot record prints a notice. |
790
+
791
+ For `--trace` and `--video`, a bare flag is `on`, and `retain-on-failure`
792
+ keeps the page or video only when the exploration did not pass. The
793
+ exploration is one attempt, so a retry mode records nothing, and a plan-time
794
+ notice says to pass `--trace on` (or `--video on`). Put the goal before the
795
+ flag, or write `--trace=<mode>` or `--video=<mode>`.
796
+
797
+ ##### Report
798
+
799
+ - The run has one result, under the virtual file `explore`, titled by the
800
+ goal.
801
+ - The exploration opens the app first when the target declares `app.url`.
802
+ - With `--session`, the report also holds the setup test's result. It lists
803
+ the project's other tests as skipped with cause `filtered`, as `run` lists
804
+ the tests of a file no positional named.
805
+ - The exploration runs with the replay cache off and zero retries, whatever
806
+ the config says. A `--session` setup test runs as it would for `run`, with
807
+ the config's agents, cache, and retries.
808
+ - `<output>/report.json` gains `run.explore`: the goal, the budgets, why the
809
+ run ended (`finished`, `step-limit`, `time`, `stuck`, or `aborted`), the
810
+ assessment, the steps, and the findings.
811
+ - Artifacts go to a directory named from the goal's first words and a digest
812
+ of the goal, target, and agent, such as
813
+ `<output>/results/explore-check-the-cart-totals-1a2b3c4d5e6f7a8b/`. A rerun
814
+ of the same goal on the same target and agent writes to the same directory.
815
+
816
+ ##### Step budgets
726
817
 
727
- ```bash
728
- e2e list [files...]
729
- ```
818
+ Each step has its own budget: 40 actions and 40 model calls, unless the agent
819
+ sets `maxSteps` or `maxModelCalls` (1 to 100). A step that reaches its action
820
+ or time budget is time-boxed, and its findings still count. A failed step
821
+ does not end the run. Three failed or blocked steps in a row with no new
822
+ finding do (`stuck`).
730
823
 
731
- A dry run. `list` selects tests exactly as `run` does, prints one line per
732
- test-target pair, and exits. No app process, engine, or worker starts, and
733
- nothing is written. The same selection is [`list` from `e2e/runner`](/reference/runner),
734
- in process. That function also returns pairs a filter removed.
824
+ ##### Findings
735
825
 
736
- Positionals and the selection flags (`--config`, `--target`, `--tag`,
737
- `--tag-mode`, `--exclude-tag`, `--grep`, `--grep-invert`, `--last-failed`,
738
- `--shard`, `--pass-with-no-tests`) are those of `run`; `--last-failed` reads
739
- `report.json` under the config's [`output`](/reference/config#output). `--reporter` accepts `list` (the default) or `json`.
826
+ - A finding of kind `issue` is a functional defect; a `warning` is cosmetic.
827
+ - Severity runs from `trivial` (1) to `critical` (5). The report lists issues
828
+ before warnings, the most severe first.
829
+ - Each finding has its location, the expected and observed behavior, the
830
+ steps to reproduce, and a screenshot when capture is allowed and succeeds.
831
+ After a secret fill, findings carry no screenshots; see
832
+ [screenshots after a secret fill](/reference/security#screenshots-after-a-secret-fill).
740
833
 
741
- | Reporter | Writes |
742
- | --- | --- |
743
- | `list` | One line per pair, `file › title [target]`, in selection order, with the describe path joined by ` › ` (`tests/billing.e2e.ts › checkout › pays [web]`), then each tag the test declares as `#tag` (`tests/billing.e2e.ts › checkout › pays [web] #smoke #billing`). A skipped test keeps its line with `(skipped: <reason>)` appended. Setup tests are listed like any other. |
744
- | `json` | `{ "pairs": [...] }` to stdout. Each pair carries `id`, `file`, `title`, `titlePath`, `kind` (`test` or `setup`), `tags` (the tags the test declares, `[]` when none), `source` when the declaration is known, `session` when the test consumes one, `sessions`, `serialId` when the test is in a serial group, `target`, `agent`, `disposition` (`run` or `skip`), and `reason` when skipped. |
834
+ ##### Outcomes
745
835
 
746
- Pairs a filter removed are not listed.
836
+ | Outcome | Run status | Exit code |
837
+ | --- | --- | --- |
838
+ | Steps ran, no `issue` finding, and not every step was blocked | passed | 0 |
839
+ | At least one `issue` finding | failed | 1 |
840
+ | No step ran and nothing was found | blocked | 1 |
841
+ | Every step was blocked, with no `issue` finding | blocked | 1, 2, or 3, from the first blocker |
842
+ | Config, engine, or model provider error | error | 2 or 3 |
843
+ | Ctrl-C | | 130 |
844
+
845
+ #### `e2e list`
846
+
847
+ A dry run: `list` selects tests exactly as `run` does, prints one line per
848
+ test-target pair, and exits. It starts no app process, engine, or worker, and
849
+ writes nothing.
747
850
 
748
851
  <CodeGroup>
749
852
  ```bash npm
@@ -771,18 +874,25 @@ bunx e2e list --reporter json
771
874
  ```
772
875
  </CodeGroup>
773
876
 
774
- ## e2e cache
775
-
776
- Inspect or clear the replay cache. These commands load the discovered config
777
- or `--config <path>`, read the directory a run would (`--cache-dir`, else
778
- `E2E_CACHE_DIR`, else `cache.dir`), and exit 0 on an empty store. The
779
- default directory is `.e2e/cache/`.
877
+ | Option | Description |
878
+ | --- | --- |
879
+ | `[files...]` | As for [`run`](#positional-arguments). |
880
+ | `--config`, `--target`, `--tag`, `--tag-mode`, `--exclude-tag`, `--grep`, `--grep-invert`, `--last-failed`, `--shard`, `--pass-with-no-tests` | As for [`run`](#flags). `--last-failed` reads `report.json` under the config's [`output`](/reference/config#output). |
881
+ | `--reporter <id>` | `list` or `json`. Default: `list`. |
780
882
 
781
- | Command | Prints |
883
+ | Reporter | Writes |
782
884
  | --- | --- |
783
- | `e2e cache ls` | One row per entry: the test and target it was recorded for, the first 12 characters of its instruction digest, its age, and how many actions a replay would run (`(truncated)` when recording hit the action cap). Rows are grouped by test, then target, oldest first. |
784
- | `e2e cache stats` | The store directory, the number of readable entries, their total size, and the number of unreadable files when there are any. |
785
- | `e2e cache clear` | Deletes every cache file, then the directory when nothing else is left in it, and reports how many files went. |
885
+ | `list` | One line per pair in selection order: `file › title [target]`, with the describe path joined by ` › ` (`tests/billing.e2e.ts › checkout › pays [web]`), then each declared tag as `#tag` (`tests/billing.e2e.ts › checkout › pays [web] #smoke #billing`). A skipped test keeps its line, with `(skipped: <reason>)` appended. Setup tests are listed like any other. |
886
+ | `json` | `{ "pairs": [...] }` to stdout. Each pair carries `id`, `file`, `title`, `titlePath`, `kind` (`test` or `setup`), `tags` (the tags the test declares, `[]` when none), `source` when the declaration is known, `session` when the test consumes one, `sessions`, `serialId` when the test is in a serial group, `target`, `agent`, `disposition` (`run` or `skip`), and `reason` when skipped. |
887
+
888
+ Pairs a filter removed are not listed. [`list` from `e2e/runner`](/reference/runner)
889
+ runs the same selection in process and also returns the removed pairs.
890
+
891
+ #### `e2e cache`
892
+
893
+ Inspects or clears the replay cache. Each subcommand loads the discovered
894
+ config or `--config <path>`, reads the directory a run would, and exits 0 on
895
+ an empty store.
786
896
 
787
897
  <CodeGroup>
788
898
  ```bash npm
@@ -807,71 +917,41 @@ bunx e2e cache ls --config packages/web/e2e.config.ts
807
917
  ```
808
918
  </CodeGroup>
809
919
 
920
+ | Command | Prints |
921
+ | --- | --- |
922
+ | `e2e cache ls` | One row per entry: the test and target it was recorded for, the first 12 characters of its instruction digest, its age, and how many actions a replay would run (`(truncated)` when recording hit the action cap). Rows are grouped by test, then target, oldest first. |
923
+ | `e2e cache stats` | The store directory, the number of readable entries, their total size, and the number of unreadable files when there are any. |
924
+ | `e2e cache clear` | Deletes every cache file, then the directory when nothing else is left in it, and reports how many files went. |
925
+
926
+ | Option | Description |
927
+ | --- | --- |
928
+ | `--config <path>` | Explicit config path. Default: the nearest `e2e.config.ts` or `e2e.config.mts` (see [Loading](/reference/config#loading)). |
929
+ | `--cache-dir <dir>` | The store directory. Default: `E2E_CACHE_DIR`, else `cache.dir`, else `.e2e/cache/`. |
930
+
810
931
  ```text
811
932
  TEST TARGET INSTRUCTION AGE ACTIONS
812
933
  tests/billing.e2e.ts::upgrades web 4f1c0a9d3b62 2h 4
813
934
  tests/signup.e2e.ts::signs%20up web 9ab7715ce004 3d 7 (truncated)
814
935
  ```
815
936
 
816
- An entry recorded before entries carried provenance prints `-` in those
817
- columns and still replays. A file the store cannot read (truncated, over the
818
- 1 MiB ceiling, or from a newer schema) is counted separately on stderr, treated
819
- as a miss by a run, and removed by `clear`. Files the runner never wrote are
820
- left alone.
937
+ - An entry recorded before entries carried provenance prints `-` in those
938
+ columns and still replays.
939
+ - A file the store cannot read (truncated, over the 1 MiB ceiling, or from a
940
+ newer schema) is counted separately on stderr, treated as a miss by a run,
941
+ and removed by `clear`.
942
+ - Files the runner never wrote are left alone.
821
943
 
822
944
  <Note>
823
945
  These commands read the file store only. With a custom `cache.store` they
824
946
  exit 2 and say so.
825
947
  </Note>
826
948
 
827
- ## e2e feedback
828
-
829
- <CodeGroup>
830
- ```bash npm
831
- npx e2e feedback -m <text> [--type bug|docs|feature|other] [options]
832
- ```
833
-
834
- ```bash pnpm
835
- pnpm exec e2e feedback -m <text> [--type bug|docs|feature|other] [options]
836
- ```
837
-
838
- ```bash bun
839
- bunx e2e feedback -m <text> [--type bug|docs|feature|other] [options]
840
- ```
841
- </CodeGroup>
949
+ #### `e2e feedback`
842
950
 
843
951
  Sends one report about e2e itself to the e2e team: something broke, the docs
844
- or the skill misled, or a capability is missing. Coding agents may send one
845
- when they hit a problem with e2e, not with the app under test, after showing
846
- the user the `--dry-run` output and getting their agreement.
847
-
848
- | Flag | Meaning |
849
- | --- | --- |
850
- | `-m, --message <text>` | Required. One or two sentences on the problem, at most 2000 characters |
851
- | `--type <type>` | `bug`, `docs`, `feature`, or `other` (default) |
852
- | `--task <text>` | What you were trying to do |
853
- | `--expected <text>` | What you expected to happen |
854
- | `--actual <text>` | What happened instead, with the error message |
855
- | `--error-code <code>` | The e2e error code involved, such as `LOCATOR_AMBIGUOUS` |
856
- | `--repro <text>` | For a bug: steps that reproduce it every time, without app content |
857
- | `--approach <text>` | What you tried, workarounds included |
858
- | `--command <text>` | The e2e command or API involved, at most 200 characters |
859
- | `--agent <text>` | The coding agent and model sending it, at most 200 characters |
860
- | `--allow-outdated` | Send even when a newer e2e release exists |
861
- | `--dry-run` | Print the event that would be sent and send nothing |
862
-
863
- `--task`, `--expected`, `--actual`, `--approach`, and `--repro` take up to
864
- 1500 characters each. A blank or longer value exits 2.
865
-
866
- A bug without `--repro` isn't sent. The command prints a prefilled
867
- [GitHub issue](https://github.com/tester-army/e2e/issues/new?template=bug_report.yml)
868
- instead, where the e2e team can ask for a reproduction. Add a minimal one and
869
- submit it yourself.
870
-
871
- Before sending, the command asks the npm registry for the latest e2e
872
- release. When yours is older, it exits 2 and sends nothing, since the problem
873
- may already be fixed: upgrade, check it still happens, and send it again, or
874
- pass `--allow-outdated`.
952
+ or the skill misled, or a capability is missing. A coding agent may send one
953
+ when it hits a problem with e2e, not with the app under test. It first shows
954
+ the user the `--dry-run` output and gets their agreement.
875
955
 
876
956
  <CodeGroup>
877
957
  ```bash npm
@@ -896,36 +976,63 @@ bunx e2e feedback --type feature -m "Need a way to set the viewport per test" --
896
976
  ```
897
977
  </CodeGroup>
898
978
 
899
- The report goes to the same PostHog project as [telemetry](/telemetry#feedback),
900
- as one `e2e_feedback` event, together with the anonymous machine facts every
901
- telemetry event carries. Before sending, the value of every environment
902
- variable whose name holds `KEY`, `TOKEN`, `SECRET`, `PASS`, `_PWD`,
903
- `CREDENTIAL`, `AUTH`, `COOKIE`, or `PRIVATE` becomes `<secret:NAME>` when it
904
- is at least six characters long, the minimum for a config secret, since a
905
- shorter one would rewrite ordinary words. Bearer tokens,
906
- URL credentials, private keys, JWTs, and common API key shapes become
907
- `<redacted>`. Do not paste app data, page content, or test files.
908
-
909
- Feedback is sent only when you run the command, so a saved
910
- `e2e telemetry disable` does not stop it. `E2E_TELEMETRY_DISABLED=1` and
911
- `DO_NOT_TRACK=1` do: with either set the command exits 2 and sends nothing,
912
- so a CI job or a fleet that opted out sends nothing even when an agent runs
913
- it. `E2E_TELEMETRY_DEBUG=1` prints it like `--dry-run`. It prints a reference
914
- id and exits 0 once PostHog accepts the event, or exits 3 when PostHog did
915
- not confirm it; the report may still have arrived, so the message names its
916
- reference id.
917
-
918
- ## e2e telemetry
979
+ | Option | Description |
980
+ | --- | --- |
981
+ | `-m`, `--message <text>` | Required. One or two sentences on the problem. |
982
+ | `--type <type>` | `bug`, `docs`, `feature`, or `other`. Default: `other`. |
983
+ | `--task <text>` | What you were trying to do. |
984
+ | `--expected <text>` | What you expected to happen. |
985
+ | `--actual <text>` | What happened instead, with the error message. |
986
+ | `--error-code <code>` | The e2e error code involved, such as `LOCATOR_AMBIGUOUS`. |
987
+ | `--repro <text>` | For a bug: steps that reproduce it every time, without app content. |
988
+ | `--approach <text>` | What you tried, workarounds included. |
989
+ | `--command <text>` | The e2e command or API involved. |
990
+ | `--agent <text>` | The coding agent and model sending it. |
991
+ | `--allow-outdated` | Sends even when a newer e2e release exists. |
992
+ | `--dry-run` | Prints the event that would be sent and sends nothing. |
993
+
994
+ `--message` takes at most 2000 characters, `--command` and `--agent` 200, and
995
+ `--task`, `--expected`, `--actual`, `--repro`, and `--approach` 1500.
996
+
997
+ - **A bug needs `--repro`.** Without it, the command sends nothing and prints
998
+ a prefilled
999
+ [GitHub issue](https://github.com/tester-army/e2e/issues/new?template=bug_report.yml)
1000
+ instead, where the e2e team can ask for a reproduction. Add a minimal one
1001
+ and submit it yourself.
1002
+ - **Outdated releases are refused.** Before sending, the command asks the npm
1003
+ registry for the latest e2e release. When yours is older, it exits 2 and
1004
+ sends nothing, since the problem may be fixed already. Upgrade, check that
1005
+ it still happens, and send it again, or pass `--allow-outdated`.
1006
+ - **Where it goes.** One `e2e_feedback` event, to the same PostHog project as
1007
+ [telemetry](/telemetry#feedback), with the anonymous machine facts every
1008
+ telemetry event carries.
1009
+ - **Opting out.** Feedback is sent only when you run the command, so a saved
1010
+ `e2e telemetry disable` does not stop it. `E2E_TELEMETRY_DISABLED=1` and
1011
+ `DO_NOT_TRACK=1` do: with either set, the command exits 2 and sends
1012
+ nothing. So a CI job or a fleet that opted out sends nothing, even when an
1013
+ agent runs it. `E2E_TELEMETRY_DEBUG=1` prints the event like `--dry-run`.
1014
+
1015
+ | Exit code | When |
1016
+ | --- | --- |
1017
+ | 0 | PostHog accepted the event, and the command printed a reference id. Also `--dry-run`, `E2E_TELEMETRY_DEBUG=1`, and `--type bug` without `--repro`: none of them send, so they exit 0 even with an opt-out variable set. The outdated-release check runs first, so an outdated install exits 2 on these too, unless `--allow-outdated` or an opt-out variable is set. |
1018
+ | 2 | A blank or too long text value, an outdated release without `--allow-outdated`, or `E2E_TELEMETRY_DISABLED=1` or `DO_NOT_TRACK=1` set. |
1019
+ | 3 | PostHog did not confirm the event. The report may still have arrived, so the message names its reference id. |
1020
+
1021
+ <Warning>
1022
+ Before sending, the value of every environment variable whose name holds
1023
+ `KEY`, `TOKEN`, `SECRET`, `PASS`, `_PWD`, `CREDENTIAL`, `AUTH`, `COOKIE`, or
1024
+ `PRIVATE` becomes `<secret:NAME>` when it is six or more characters long. That
1025
+ is the minimum for a config secret; a shorter one would rewrite ordinary
1026
+ words. Bearer tokens, URL credentials, private keys, JWTs, and common API key
1027
+ shapes become `<redacted>`. Do not paste app data, page content, or test
1028
+ files.
1029
+ </Warning>
1030
+
1031
+ #### `e2e telemetry`
919
1032
 
920
1033
  Shows whether anonymous usage telemetry is on, and switches it.
921
1034
  [Telemetry](/telemetry) lists every property sent.
922
1035
 
923
- | Command | Effect |
924
- | --- | --- |
925
- | `e2e telemetry`, `e2e telemetry status` | Prints `Status: enabled`, or `Status: disabled` with the reason: the variable that is set, the saved choice, or a preferences directory that cannot be written. |
926
- | `e2e telemetry disable` | Saves the opt-out to `~/.config/e2e/telemetry.json` (`$XDG_CONFIG_HOME/e2e/` when set, `%APPDATA%\e2e\` on Windows) and prints the status. |
927
- | `e2e telemetry enable` | Saves the opt-in and prints the status. A variable set in the environment still wins, and the status says so. |
928
-
929
1036
  <CodeGroup>
930
1037
  ```bash npm
931
1038
  npx e2e telemetry
@@ -946,10 +1053,18 @@ E2E_TELEMETRY_DEBUG=1 bunx e2e run
946
1053
  ```
947
1054
  </CodeGroup>
948
1055
 
949
- `E2E_TELEMETRY_DISABLED=1` and `DO_NOT_TRACK=1` turn telemetry off without
950
- touching the file. `E2E_TELEMETRY_DEBUG=1` prints every event to stderr and
951
- sends nothing. The first command on a machine other than `e2e init` and
952
- `e2e telemetry` prints a notice once; in CI nothing is printed and no file is written.
1056
+ | Command | Effect |
1057
+ | --- | --- |
1058
+ | `e2e telemetry`, `e2e telemetry status` | Prints `Status: enabled`, or `Status: disabled` with the reason: the variable that is set, the saved choice, or a preferences directory that cannot be written. `status` is the default. |
1059
+ | `e2e telemetry disable` | Saves the opt-out to `~/.config/e2e/telemetry.json` (`$XDG_CONFIG_HOME/e2e/` when set, `%APPDATA%\e2e\` on Windows) and prints the status. |
1060
+ | `e2e telemetry enable` | Saves the opt-in and prints the status. A variable set in the environment still wins, and the status says so. |
1061
+
1062
+ - `E2E_TELEMETRY_DISABLED=1` and `DO_NOT_TRACK=1` turn telemetry off without
1063
+ touching the file.
1064
+ - `E2E_TELEMETRY_DEBUG=1` prints every event to stderr and sends nothing.
1065
+ - The first command on a machine, other than `e2e init` and
1066
+ `e2e telemetry`, prints a notice one time. In CI nothing is printed and no
1067
+ file is written.
953
1068
 
954
1069
  ## Exit codes
955
1070
 
@@ -962,16 +1077,17 @@ sends nothing. The first command on a machine other than `e2e init` and
962
1077
  | 4 | Internal runner invariant or unhandled runner error |
963
1078
  | 130 | Interrupted by a user or CI signal |
964
1079
 
965
- Precedence for a mixed run is `130 > 4 > 3 > 2 > 1 > 0`. The report keeps every
966
- individual result; precedence affects only the process exit code.
967
-
968
- Argument parsing failures exit 2 with a one-line error on stderr.
1080
+ - Precedence for a mixed run is `130 > 4 > 3 > 2 > 1 > 0`. The report keeps
1081
+ every individual result; precedence affects only the process exit code.
1082
+ - Argument parsing failures exit 2 with a one-line error on stderr.
969
1083
 
970
1084
  ## Diagnostics
971
1085
 
972
- A message that rejects a name suggests the nearest valid one. Keys from other
973
- runners' configs (`testDir`, `baseURL`, `webServer`, `projects`) say where
974
- that fact lives here. A failing `import` in the config or a test file ends
975
- with the install or add command for the project's package manager, or the
976
- subpaths the package exports. A model credential the provider rejects names
977
- the variable the provider reads.
1086
+ - A message that rejects a name suggests the nearest valid one.
1087
+ - Keys from other runners' configs (`testDir`, `baseURL`, `webServer`,
1088
+ `projects`) say where that fact lives here.
1089
+ - A failing `import` in the config or a test file ends with the install or
1090
+ add command for the project's package manager, or the subpaths the package
1091
+ exports.
1092
+ - A model credential the provider rejects names the variable the provider
1093
+ reads.