@e2edev/e2e 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/README.md +14 -8
  2. package/dist/agent/act-validation.d.ts +2 -0
  3. package/dist/agent/act-validation.d.ts.map +1 -1
  4. package/dist/agent/act-validation.js +1 -1
  5. package/dist/agent/act-validation.js.map +1 -1
  6. package/dist/agent/act.d.ts +30 -2
  7. package/dist/agent/act.d.ts.map +1 -1
  8. package/dist/agent/act.js +18 -4
  9. package/dist/agent/act.js.map +1 -1
  10. package/dist/agent/ai-sdk.d.ts +2 -2
  11. package/dist/agent/ai-sdk.d.ts.map +1 -1
  12. package/dist/agent/ai-sdk.js +3 -4
  13. package/dist/agent/ai-sdk.js.map +1 -1
  14. package/dist/agent/default-agent.d.ts +33 -9
  15. package/dist/agent/default-agent.d.ts.map +1 -1
  16. package/dist/agent/default-agent.js +46 -31
  17. package/dist/agent/default-agent.js.map +1 -1
  18. package/dist/agent/error.d.ts.map +1 -1
  19. package/dist/agent/error.js +1 -0
  20. package/dist/agent/error.js.map +1 -1
  21. package/dist/agent/executor.d.ts +13 -0
  22. package/dist/agent/executor.d.ts.map +1 -1
  23. package/dist/agent/executor.js.map +1 -1
  24. package/dist/agent/interactive-step.d.ts +38 -0
  25. package/dist/agent/interactive-step.d.ts.map +1 -0
  26. package/dist/agent/interactive-step.js +131 -0
  27. package/dist/agent/interactive-step.js.map +1 -0
  28. package/dist/agent/model/adapter.d.ts +3 -0
  29. package/dist/agent/model/adapter.d.ts.map +1 -1
  30. package/dist/agent/model/adapter.js.map +1 -1
  31. package/dist/agent/model/overflow.d.ts +20 -0
  32. package/dist/agent/model/overflow.d.ts.map +1 -0
  33. package/dist/agent/model/overflow.js +82 -0
  34. package/dist/agent/model/overflow.js.map +1 -0
  35. package/dist/agent/model/prompt-cache.d.ts +47 -0
  36. package/dist/agent/model/prompt-cache.d.ts.map +1 -0
  37. package/dist/agent/model/prompt-cache.js +78 -0
  38. package/dist/agent/model/prompt-cache.js.map +1 -0
  39. package/dist/agent/model/sdk.d.ts +41 -24
  40. package/dist/agent/model/sdk.d.ts.map +1 -1
  41. package/dist/agent/model/sdk.js +64 -91
  42. package/dist/agent/model/sdk.js.map +1 -1
  43. package/dist/agent/model/tool-choice.d.ts +21 -0
  44. package/dist/agent/model/tool-choice.d.ts.map +1 -0
  45. package/dist/agent/model/tool-choice.js +49 -0
  46. package/dist/agent/model/tool-choice.js.map +1 -0
  47. package/dist/agent/primitives.d.ts.map +1 -1
  48. package/dist/agent/primitives.js +2 -1
  49. package/dist/agent/primitives.js.map +1 -1
  50. package/dist/agent/public.d.ts +1 -1
  51. package/dist/agent/public.d.ts.map +1 -1
  52. package/dist/agent/public.js.map +1 -1
  53. package/dist/agent/screen-update.d.ts +8 -3
  54. package/dist/agent/screen-update.d.ts.map +1 -1
  55. package/dist/agent/screen-update.js +18 -5
  56. package/dist/agent/screen-update.js.map +1 -1
  57. package/dist/agent/tool-loop.d.ts +4 -4
  58. package/dist/agent/tool-loop.d.ts.map +1 -1
  59. package/dist/agent/tool-loop.js +241 -48
  60. package/dist/agent/tool-loop.js.map +1 -1
  61. package/dist/agent/tool-output.d.ts +31 -0
  62. package/dist/agent/tool-output.d.ts.map +1 -0
  63. package/dist/agent/tool-output.js +60 -0
  64. package/dist/agent/tool-output.js.map +1 -0
  65. package/dist/agent/tool.d.ts +9 -3
  66. package/dist/agent/tool.d.ts.map +1 -1
  67. package/dist/agent/tool.js +2 -2
  68. package/dist/agent/tool.js.map +1 -1
  69. package/dist/agent/usage.d.ts +7 -0
  70. package/dist/agent/usage.d.ts.map +1 -1
  71. package/dist/agent/usage.js +11 -0
  72. package/dist/agent/usage.js.map +1 -1
  73. package/dist/cli/docs-url.d.ts +1 -1
  74. package/dist/cli/docs-url.d.ts.map +1 -1
  75. package/dist/cli/docs-url.js +1 -1
  76. package/dist/cli/docs-url.js.map +1 -1
  77. package/dist/cli/index.d.ts.map +1 -1
  78. package/dist/cli/index.js +139 -35
  79. package/dist/cli/index.js.map +1 -1
  80. package/dist/cli/init/engine-versions.json +1 -1
  81. package/dist/cli/init/engines.d.ts +17 -16
  82. package/dist/cli/init/engines.d.ts.map +1 -1
  83. package/dist/cli/init/engines.js +26 -26
  84. package/dist/cli/init/engines.js.map +1 -1
  85. package/dist/cli/init/gateways.d.ts +22 -0
  86. package/dist/cli/init/gateways.d.ts.map +1 -0
  87. package/dist/cli/init/gateways.js +50 -0
  88. package/dist/cli/init/gateways.js.map +1 -0
  89. package/dist/cli/init/mcp-config.d.ts +31 -0
  90. package/dist/cli/init/mcp-config.d.ts.map +1 -0
  91. package/dist/cli/init/mcp-config.js +72 -0
  92. package/dist/cli/init/mcp-config.js.map +1 -0
  93. package/dist/cli/init/scaffold.d.ts +8 -2
  94. package/dist/cli/init/scaffold.d.ts.map +1 -1
  95. package/dist/cli/init/scaffold.js +22 -11
  96. package/dist/cli/init/scaffold.js.map +1 -1
  97. package/dist/cli/init.d.ts.map +1 -1
  98. package/dist/cli/init.js +109 -29
  99. package/dist/cli/init.js.map +1 -1
  100. package/dist/cli/mcp.d.ts +15 -0
  101. package/dist/cli/mcp.d.ts.map +1 -0
  102. package/dist/cli/mcp.js +63 -0
  103. package/dist/cli/mcp.js.map +1 -0
  104. package/dist/collect/collect.d.ts +7 -0
  105. package/dist/collect/collect.d.ts.map +1 -1
  106. package/dist/collect/collect.js +16 -0
  107. package/dist/collect/collect.js.map +1 -1
  108. package/dist/collect/registry.d.ts.map +1 -1
  109. package/dist/collect/registry.js +33 -6
  110. package/dist/collect/registry.js.map +1 -1
  111. package/dist/collect/select.d.ts +9 -2
  112. package/dist/collect/select.d.ts.map +1 -1
  113. package/dist/collect/select.js +57 -11
  114. package/dist/collect/select.js.map +1 -1
  115. package/dist/config/agent.d.ts +13 -32
  116. package/dist/config/agent.d.ts.map +1 -1
  117. package/dist/config/agent.js +32 -123
  118. package/dist/config/agent.js.map +1 -1
  119. package/dist/config/resolve.d.ts +8 -5
  120. package/dist/config/resolve.d.ts.map +1 -1
  121. package/dist/config/resolve.js +11 -9
  122. package/dist/config/resolve.js.map +1 -1
  123. package/dist/credentials.d.ts +6 -0
  124. package/dist/credentials.d.ts.map +1 -1
  125. package/dist/credentials.js +9 -0
  126. package/dist/credentials.js.map +1 -1
  127. package/dist/explore/body.d.ts +36 -0
  128. package/dist/explore/body.d.ts.map +1 -0
  129. package/dist/explore/body.js +168 -0
  130. package/dist/explore/body.js.map +1 -0
  131. package/dist/explore/executor.d.ts +25 -0
  132. package/dist/explore/executor.d.ts.map +1 -0
  133. package/dist/explore/executor.js +86 -0
  134. package/dist/explore/executor.js.map +1 -0
  135. package/dist/explore/index.d.ts +64 -0
  136. package/dist/explore/index.d.ts.map +1 -0
  137. package/dist/explore/index.js +185 -0
  138. package/dist/explore/index.js.map +1 -0
  139. package/dist/explore/plan.d.ts +74 -0
  140. package/dist/explore/plan.d.ts.map +1 -0
  141. package/dist/explore/plan.js +149 -0
  142. package/dist/explore/plan.js.map +1 -0
  143. package/dist/explore/reporter.d.ts +11 -0
  144. package/dist/explore/reporter.d.ts.map +1 -0
  145. package/dist/explore/reporter.js +54 -0
  146. package/dist/explore/reporter.js.map +1 -0
  147. package/dist/explore/state.d.ts +58 -0
  148. package/dist/explore/state.d.ts.map +1 -0
  149. package/dist/explore/state.js +147 -0
  150. package/dist/explore/state.js.map +1 -0
  151. package/dist/internal/ai-trace.d.ts +2 -0
  152. package/dist/internal/ai-trace.d.ts.map +1 -1
  153. package/dist/internal/ai-trace.js +3 -0
  154. package/dist/internal/ai-trace.js.map +1 -1
  155. package/dist/internal/ids.d.ts +2 -2
  156. package/dist/internal/ids.d.ts.map +1 -1
  157. package/dist/internal/ids.js +3 -3
  158. package/dist/internal/ids.js.map +1 -1
  159. package/dist/internal/package-manager.d.ts +4 -0
  160. package/dist/internal/package-manager.d.ts.map +1 -1
  161. package/dist/internal/package-manager.js +26 -0
  162. package/dist/internal/package-manager.js.map +1 -1
  163. package/dist/mcp/catalog.d.ts +50 -0
  164. package/dist/mcp/catalog.d.ts.map +1 -0
  165. package/dist/mcp/catalog.js +172 -0
  166. package/dist/mcp/catalog.js.map +1 -0
  167. package/dist/mcp/config.d.ts +11 -0
  168. package/dist/mcp/config.d.ts.map +1 -0
  169. package/dist/mcp/config.js +16 -0
  170. package/dist/mcp/config.js.map +1 -0
  171. package/dist/mcp/server.d.ts +29 -0
  172. package/dist/mcp/server.d.ts.map +1 -0
  173. package/dist/mcp/server.js +86 -0
  174. package/dist/mcp/server.js.map +1 -0
  175. package/dist/mcp/session.d.ts +68 -0
  176. package/dist/mcp/session.d.ts.map +1 -0
  177. package/dist/mcp/session.js +317 -0
  178. package/dist/mcp/session.js.map +1 -0
  179. package/dist/mcp/tools.d.ts +77 -0
  180. package/dist/mcp/tools.d.ts.map +1 -0
  181. package/dist/mcp/tools.js +179 -0
  182. package/dist/mcp/tools.js.map +1 -0
  183. package/dist/report/build.d.ts +61 -0
  184. package/dist/report/build.d.ts.map +1 -1
  185. package/dist/report/build.js +13 -2
  186. package/dist/report/build.js.map +1 -1
  187. package/dist/report/debug-steps.d.ts.map +1 -1
  188. package/dist/report/debug-steps.js +15 -0
  189. package/dist/report/debug-steps.js.map +1 -1
  190. package/dist/report/format.d.ts +30 -1
  191. package/dist/report/format.d.ts.map +1 -1
  192. package/dist/report/format.js +57 -3
  193. package/dist/report/format.js.map +1 -1
  194. package/dist/report/junit.d.ts.map +1 -1
  195. package/dist/report/junit.js +4 -1
  196. package/dist/report/junit.js.map +1 -1
  197. package/dist/report/list.d.ts +10 -1
  198. package/dist/report/list.d.ts.map +1 -1
  199. package/dist/report/list.js +35 -13
  200. package/dist/report/list.js.map +1 -1
  201. package/dist/run/events.d.ts +13 -4
  202. package/dist/run/events.d.ts.map +1 -1
  203. package/dist/run/events.js.map +1 -1
  204. package/dist/run/execute.d.ts +2 -2
  205. package/dist/run/execute.d.ts.map +1 -1
  206. package/dist/run/execute.js +19 -14
  207. package/dist/run/execute.js.map +1 -1
  208. package/dist/run/fixtures.d.ts +10 -4
  209. package/dist/run/fixtures.d.ts.map +1 -1
  210. package/dist/run/fixtures.js +23 -23
  211. package/dist/run/fixtures.js.map +1 -1
  212. package/dist/run/in-process.d.ts +8 -0
  213. package/dist/run/in-process.d.ts.map +1 -1
  214. package/dist/run/in-process.js +7 -6
  215. package/dist/run/in-process.js.map +1 -1
  216. package/dist/run/provision.d.ts +51 -0
  217. package/dist/run/provision.d.ts.map +1 -0
  218. package/dist/run/provision.js +94 -0
  219. package/dist/run/provision.js.map +1 -0
  220. package/dist/run/records.d.ts +4 -0
  221. package/dist/run/records.d.ts.map +1 -1
  222. package/dist/run/runner.d.ts +25 -3
  223. package/dist/run/runner.d.ts.map +1 -1
  224. package/dist/run/runner.js +38 -81
  225. package/dist/run/runner.js.map +1 -1
  226. package/dist/run/scheduler.d.ts +5 -2
  227. package/dist/run/scheduler.d.ts.map +1 -1
  228. package/dist/run/scheduler.js +13 -10
  229. package/dist/run/scheduler.js.map +1 -1
  230. package/dist/run/serial.d.ts +1 -1
  231. package/dist/run/serial.d.ts.map +1 -1
  232. package/dist/run/serial.js +7 -2
  233. package/dist/run/serial.js.map +1 -1
  234. package/dist/run/standalone.d.ts +63 -0
  235. package/dist/run/standalone.d.ts.map +1 -0
  236. package/dist/run/standalone.js +152 -0
  237. package/dist/run/standalone.js.map +1 -0
  238. package/dist/run/steps.d.ts +7 -0
  239. package/dist/run/steps.d.ts.map +1 -1
  240. package/dist/run/steps.js.map +1 -1
  241. package/dist/run/units.d.ts +6 -3
  242. package/dist/run/units.d.ts.map +1 -1
  243. package/dist/run/units.js +0 -0
  244. package/dist/run/units.js.map +1 -1
  245. package/dist/run/worker/entry.js +2 -2
  246. package/dist/run/worker/entry.js.map +1 -1
  247. package/dist/run/worker/protocol.d.ts +5 -0
  248. package/dist/run/worker/protocol.d.ts.map +1 -1
  249. package/dist/run/worker/protocol.js.map +1 -1
  250. package/dist/run/worker/session.d.ts +3 -4
  251. package/dist/run/worker/session.d.ts.map +1 -1
  252. package/dist/run/worker/session.js +5 -4
  253. package/dist/run/worker/session.js.map +1 -1
  254. package/dist/run/worker-models.d.ts +4 -4
  255. package/dist/run/worker-models.js +6 -13
  256. package/dist/run/worker-models.js.map +1 -1
  257. package/dist/telemetry/events.d.ts.map +1 -1
  258. package/dist/telemetry/events.js +17 -2
  259. package/dist/telemetry/events.js.map +1 -1
  260. package/dist/types.d.ts +20 -20
  261. package/dist/types.d.ts.map +1 -1
  262. package/package.json +9 -6
  263. package/skills/e2e/SKILL.md +16 -6
  264. package/skills/e2e/references/agent.md +21 -16
  265. package/skills/e2e/references/debugging.md +2 -1
  266. package/skills/e2e/references/explore.md +83 -0
  267. package/skills/e2e/references/mcp.md +88 -0
  268. package/skills/e2e/references/running.md +7 -5
  269. package/skills/e2e/references/setup.md +17 -13
@@ -18,6 +18,7 @@ target names an engine, `@e2edev/playwright` for browsers or
18
18
  import type { E2EConfig } from '@e2edev/e2e';
19
19
  import { createAgent } from '@e2edev/e2e/agent';
20
20
  import { playwright } from '@e2edev/playwright';
21
+ import { gateway } from 'ai';
21
22
 
22
23
  export default {
23
24
  targets: [
@@ -28,8 +29,13 @@ export default {
28
29
  }),
29
30
  },
30
31
  ],
31
- // Only needed for agent.* steps; the model comes from E2E_MODEL.
32
- agents: { default: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }) },
32
+ // Only needed for agent.* steps. The model is an AI SDK instance; gateway() from 'ai' reads AI_GATEWAY_API_KEY.
33
+ agents: {
34
+ default: createAgent({
35
+ model: gateway('openai/gpt-5.4-mini'),
36
+ system: 'You are a thorough QA agent. Verify every outcome on screen.',
37
+ }),
38
+ },
33
39
  } satisfies E2EConfig;
34
40
  ```
35
41
 
@@ -58,7 +64,9 @@ one. Without them, the installed CLI prints the same text:
58
64
  | `writing-tests` | [references/writing-tests.md](references/writing-tests.md) | Writing or fixing tests: fixtures, locators, actions, matchers, sign-in sessions, the `web` fixture |
59
65
  | `agent` | [references/agent.md](references/agent.md) | Adding `agent.*` steps, picking a model, cost and budgets, the trace cache |
60
66
  | `running` | [references/running.md](references/running.md) | CLI flags, reporters, `.e2e/report.json`, exit codes, CI |
67
+ | `explore` | [references/explore.md](references/explore.md) | Exploring an app toward a goal without a test file: `e2e explore`, its budgets, verdict, and `run.explore` |
61
68
  | `debugging` | [references/debugging.md](references/debugging.md) | A run failed: error codes and their fixes, `--headed`, `--debug`, `--ai-trace` |
69
+ | `mcp` | [references/mcp.md](references/mcp.md) | Driving the live app from a coding agent over MCP: `e2e mcp`, its tools, and the explore-then-write loop |
62
70
 
63
71
  ## Workflow
64
72
 
@@ -67,13 +75,15 @@ one. Without them, the installed CLI prints the same text:
67
75
  there: follow `setup`.
68
76
  2. Learn the screens you will drive before writing a test: routes, labels,
69
77
  roles, button text. Semantic locators need the accessible names the app
70
- renders, so read the templates or components, or open the page with
71
- `--headed`.
78
+ renders, so read the templates or components, open the page with
79
+ `--headed`, or drive the live app through the `e2e mcp` server when it is
80
+ registered (topic `mcp`): `open_session`, `observe`, and `locate` show the
81
+ exact names and check a locator before you write it.
72
82
  3. Write `tests/<feature>.e2e.ts`. Deterministic steps first. One `agent.act`
73
83
  per goal where the flow varies, and an `expect` on its outcome right after.
74
84
  4. Run one file: `npx --no-install e2e run tests/<feature>.e2e.ts`. Agent
75
- steps need `E2E_MODEL=provider/model-id` and `E2E_MODEL_API_KEY` in the
76
- environment; deterministic tests need neither.
85
+ steps need a model in the config and its provider's key in the environment
86
+ (`AI_GATEWAY_API_KEY` for `gateway()`); deterministic tests need neither.
77
87
  5. Read the failure: the reporter prints the error code, the message, and a
78
88
  code frame; `.e2e/report.json` has every step and artifact path. Fix the
79
89
  locator, the expectation, or the app. Never add a sleep.
@@ -6,18 +6,19 @@ transcript between calls. Deterministic tests never load a model.
6
6
 
7
7
  ## Configure a model
8
8
 
9
- Agents live under `agents` by name; `default` is the one tests use, `e2e run --agent <name>` re-points that default, a test or describe pins one with `{ agent: 'name' }`, and any `agent.*` call can name one with `{ agent: 'name' }` (innermost wins). There is no default model. Pick one of three shapes for an entry:
9
+ Agents live under `agents` by name; `default` is the one tests use, `e2e run --agent <name>` re-points that default, a test or describe pins one with `{ agent: 'name' }` or several with `{ agent: ['buyer', 'admin'] }` (the test runs once per agent, one result each, tagged `[admin]` in the terminal and `agent` in the report), and any `agent.*` call can name one with `{ agent: 'name' }` (innermost wins). `--agent buyer,admin` runs every unpinned test once per agent; on a pinned list it narrows to the names both give and never overrides a pin it does not name. A signed-in persona pairs the pin with a `session`, one per describe block, so a loop over describe blocks sweeps signed-in personas. There is no default model. Pick one of three shapes for an entry:
10
10
 
11
11
  ```ts
12
12
  // e2e.config.ts
13
13
  import { createAgent } from '@e2edev/e2e/agent';
14
+ import { gateway } from 'ai';
14
15
 
15
16
  export default {
16
- // 1. The built-in agent; the model comes from E2E_MODEL at run time.
17
- agents: { default: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }) },
17
+ // 1. The built-in agent with an AI SDK model: gateway() from 'ai' is the Vercel AI Gateway and reads AI_GATEWAY_API_KEY.
18
+ agents: { default: createAgent({ model: gateway('openai/gpt-5.4-mini'), system: 'You are a thorough QA agent. Verify every outcome on screen.' }) },
18
19
 
19
- // 2. An options block: a gateway model string plus project vocabulary.
20
- // agents: { default: { model: 'anthropic/claude-sonnet-4.5', context: 'A billing dashboard. Plans are Free, Team, and Pro.' } },
20
+ // 2. An options block: a model plus project vocabulary. openrouter() from '@openrouter/ai-sdk-provider' reads OPENROUTER_API_KEY.
21
+ // agents: { default: { model: openrouter('anthropic/claude-sonnet-4.5'), context: 'A billing dashboard. Plans are Free, Team, and Pro.' } },
21
22
 
22
23
  // 3. A live AI SDK model instance for a provider called directly.
23
24
  // agents: { default: createAgent({ model: openai('gpt-5.4-mini') }) },
@@ -25,21 +26,25 @@ export default {
25
26
  ```
26
27
 
27
28
  ```bash
28
- E2E_MODEL=anthropic/claude-sonnet-4.5 E2E_MODEL_API_KEY=... npx --no-install e2e run
29
+ AI_GATEWAY_API_KEY=... npx --no-install e2e run
29
30
  ```
30
31
 
31
- - A `provider/model-id` string is routed through the Vercel AI Gateway; one
32
- `E2E_MODEL_API_KEY` (or `AI_GATEWAY_API_KEY`) reaches every provider.
33
- - With no `agents` key at all, the built-in agent still runs as `default` and takes its
34
- model from `E2E_MODEL`. Use `createAgent` for a `system` prompt, tools, or
35
- a pinned model.
32
+ - The model is always an AI SDK instance the config constructs; the runner
33
+ implies no gateway and reads no model variable. `gateway()` from `ai` is the
34
+ Vercel AI Gateway, `openrouter()` from `@openrouter/ai-sdk-provider` is
35
+ OpenRouter, `createOpenAICompatible({ baseURL }).chatModel()` from
36
+ `@ai-sdk/openai-compatible` is any /v1 chat endpoint (Ollama, vLLM), and a
37
+ provider's own package (`openai()`) calls it directly. Each reads its own
38
+ key variable. A string in a model slot is `INVALID_CONFIG`.
39
+ - With no model anywhere, acquiring `agent` is `MODEL_UNAVAILABLE`. Use
40
+ `createAgent` for the model, a `system` prompt, and tools.
36
41
  - `ai@^7` must be installed for any `agent.*` step; the runner loads it
37
42
  lazily and fails without it.
38
43
  - `context` in the config and `agentContext` on a test or group add trusted
39
44
  project vocabulary to every prompt.
40
- - `visionModel` (or `E2E_VISION_MODEL`) serves the calls that send pixels.
45
+ - `visionModel` serves the calls that send pixels.
41
46
  - The model passed to `createAgent({ model })` is the one model for every
42
- `agent.*` call, `act` and the judgments alike, and outranks `E2E_MODEL`.
47
+ `agent.*` call, `act` and the judgments alike.
43
48
  An agent `model` naming a different model is `INVALID_CONFIG`.
44
49
  - Missing model or key: checked once per run when the first test acquires
45
50
  the `agent` fixture. One run-level `MODEL_UNAVAILABLE` (exit 2) stops the
@@ -217,12 +222,12 @@ arbitrary tool result JSON or other fields are preserved.
217
222
  can be the `agent`; the runner still owns observations, actions, budgets,
218
223
  and the report.
219
224
 
220
- Full reference: https://e2e-docs.vercel.app/agents
225
+ Full reference: https://e2e.mintlify.app/agents
221
226
 
222
227
  ## In CI
223
228
 
224
229
  Deterministic tests gate merges; agentic tests are opt-in. Keep them in a
225
230
  separate config (`e2e.agent.config.ts` with its own `tests` glob and a larger
226
- `timeout`), run them on a schedule or `workflow_dispatch`, and pass the model
227
- through `env: { E2E_MODEL, E2E_MODEL_API_KEY }` from CI variables and
231
+ `timeout`), run them on a schedule or `workflow_dispatch`, and pass the key
232
+ the config's model reads (`env: { AI_GATEWAY_API_KEY }` for `gateway()`) from
228
233
  secrets.
@@ -36,9 +36,10 @@ jq '.run.results[] | select(.status != "passed") | .attempts[-1]
36
36
  | `ASSERTION_FAILED` | The expectation is wrong, or the state settles later than 5 s; for `agent.assert`, the judgment was false (explanation in the report) | Compare with the actual text in the report or screenshot; `{ timeout }` on the matcher; rewrite the question |
37
37
  | `ACTION_FAILED` | Element not actionable (covered, disabled, detached) or an operation timed out | Wait on the right condition with `expect` first; close overlays; check `actionTimeout` |
38
38
  | `TEST_TIMEOUT` | The attempt exceeded `timeout` (120 s) | Split the test, or raise `timeout` for slow flows and agent steps |
39
- | `MODEL_UNAVAILABLE` | No model (`createAgent({ model })`, `agent.model`, or `E2E_MODEL`), no key, or an unknown provider. Reported once for the run under `run.errors`; the run stops | Export `E2E_MODEL=provider/model-id` and `E2E_MODEL_API_KEY`, or pass a model to `createAgent`; confirm the model id |
39
+ | `MODEL_UNAVAILABLE` | No model: neither `createAgent({ model })` nor `agent.model` holds an AI SDK instance. Reported once for the run under `run.errors`; the run stops | Construct one in the config, e.g. `gateway('openai/gpt-5.4-mini')` from `ai`, and export the key its provider reads (`AI_GATEWAY_API_KEY`) |
40
40
  | `MODEL_PROVIDER_FAILED` | Network, 5xx, rate limit, or no credits after the transport retries | Check the key and the quota; retry; exit code 3 |
41
41
  | `STEP_TIMEOUT`, `STEP_BUDGET_EXHAUSTED` | The goal was too big or ambiguous, or the provider slow | Split the goal, use on-screen wording, add `context`, raise `timeout` and `actionTimeout`, `--debug` to read the transcript |
42
+ | `CONTEXT_OVERFLOW` | The screen plus the step's history did not fit the model's context window, even after the loop shrank the history and retried once | Lower `agent.maxObservationBytes`, split the step, or pick a model with a larger window |
42
43
  | `POLICY_DENIED` | Navigation outside `allowedOrigins`; a `Secret` given to a sink that is not a password field; reading a secure field | Add the origin to `allowedOrigins`; fill secrets only into password inputs; assert the outcome instead of the value |
43
44
  | `UNSUPPORTED_CAPABILITY` | A fixture the engine does not contribute (`web` on a device), `schema` or `vision` on `act`, an action the surface lacks | Declare `requires: ['web']`; drop the option; use a supported action |
44
45
  | `SESSION_UNAVAILABLE`, `SESSION_CONTRACT` | `session: 'x'` with no setup saving `x`; a setup that did not save every declared name | Add or fix the `test.setup` |
@@ -0,0 +1,83 @@
1
+ # Exploring without a test
2
+
3
+ `e2e explore` runs the agent against the app with a goal instead of a test
4
+ file. Use it to see what the agent can do with an app before tests exist, to
5
+ hunt for regressions on a branch, or to find what is worth turning into a
6
+ test. It needs a config with a target and an agent that holds a model,
7
+ nothing else.
8
+
9
+ ```bash
10
+ npx --no-install e2e explore # goal: "Explore the app and find bugs"
11
+ npx --no-install e2e explore 'Explore checkout like a first-time buyer and report anything off'
12
+ npx --no-install e2e explore --target web --max-steps 4 --headed
13
+ npx --no-install e2e explore 'Hunt for broken forms' --video
14
+ ```
15
+
16
+ ## What a run does
17
+
18
+ 1. Opens the app on the target's URL.
19
+ 2. Plans one step: a structured model call reads the goal, the steps and
20
+ findings so far, and the current screen, and answers with a title and a
21
+ concrete charter for one flow, or decides the goal is covered.
22
+ 3. Runs the charter as an `agent.act()` step with the project's tools plus
23
+ `report_finding`. The agent reports each defect the moment it has evidence:
24
+ title, `issue` or `warning`, severity 1 to 5, expected, actual, reproduction
25
+ steps. The runner adds the path and a redacted screenshot.
26
+ 4. Repeats until the planner finishes, the step limit, the clock, or three
27
+ failed or blocked steps in a row that reported nothing; then asks for a closing assessment.
28
+
29
+ A failed step does not end the run: it is recorded, and only findings the
30
+ agent reported count toward the verdict. A step that hits its action or time
31
+ budget ended at its limit and counts as neither. Configured `credentials`
32
+ reach the explorer as step secrets: the planner knows the account names and
33
+ usernames, and the agent fills passwords with `type_secret` by name.
34
+
35
+ ## Flags
36
+
37
+ | Flag | Default | Effect |
38
+ | --- | --- | --- |
39
+ | `[goal]` | `Explore the app and find bugs` | One quoted sentence: the area and the posture. |
40
+ | `--target <id>` | first configured target | The one target to explore. |
41
+ | `--agent <name>` | `default` | Build the explorer from another configured agent (`agents.<name>`). |
42
+ | `--max-steps <n>` | 8 (1 to 12) | Exploration steps at most. |
43
+ | `--timeout <ms>` | 600000 (180000 to 900000) | Wall clock; the last minute is for the assessment. |
44
+ | `--headed`, `--reporter`, `--artifacts`, `--debug`, `--ai-trace`, `--video` | as `run` | Same meaning as for `e2e run`. |
45
+
46
+ Per-step action and model-call budgets default to 40 each; `agent.maxSteps`
47
+ and `agent.maxModelCalls` in the config override them. The trace cache is off
48
+ and retries are zero for the run.
49
+
50
+ ## Reading the result
51
+
52
+ Exit code `0`: steps ran and no `issue` was reported (warnings allowed). Exit
53
+ code `1`: at least one `issue`, or no step ran and nothing was found (the run
54
+ is `blocked`). `2` and `3` as for `run`.
55
+
56
+ The terminal prints each step live and, under the summary, `Explored`,
57
+ `Findings`, one row per finding sorted by severity, and `Assessment`.
58
+ `.e2e/report.json` has the record under `run.explore`:
59
+
60
+ ```json
61
+ {
62
+ "goal": "...",
63
+ "budgets": { "maxSteps": 8, "timeoutMs": 600000 },
64
+ "ended": "finished | step-limit | time | stuck | aborted",
65
+ "summary": "the closing assessment",
66
+ "steps": [{ "index": 1, "title": "...", "instruction": "...", "status": "passed | failed | blocked | exhausted", "summary": "...", "startedAt": "...", "durationMs": 0 }],
67
+ "findings": [{ "index": 0, "step": 1, "kind": "issue", "severity": 4, "title": "...", "expected": "...", "actual": "...", "reproduction": ["..."], "path": "/cart", "artifactId": "<attempt id>:artifact:2", "reportedAt": "..." }]
68
+ }
69
+ ```
70
+
71
+ `artifactId` names the evidence screenshot among the attempt's `artifacts` in
72
+ `run.results[0]`, where its path, size, and digest are.
73
+
74
+ Turn a finding into a test: its `reproduction` steps are the `agent.act()`
75
+ instructions or `screen.*` actions, and `expected` is the assertion.
76
+
77
+ ## When it does not fit
78
+
79
+ An agent built with `createAgent({ tools, system })` in the config lends its
80
+ tools and guidance to the explorer; a hand-rolled `StepExecutor` is replaced by
81
+ the built-in agent for the run, with a notice on stderr. Findings are the
82
+ model's claims plus evidence, not verified reproductions: read `actual` against
83
+ the screenshot before filing a bug.
@@ -0,0 +1,88 @@
1
+ # Driving the app over MCP
2
+
3
+ `e2e mcp` serves a project's live app to a coding agent over MCP (stdio).
4
+ It gives the agent the same hands the testing agent has on the app: look at
5
+ a screen before writing a test, and check a locator before committing to it.
6
+ Running tests and reading a failed run stay on the CLI (topics `running` and
7
+ `debugging`).
8
+
9
+ ## Setup
10
+
11
+ The server ships with `@e2edev/e2e`. `e2e init` offers to register it; by
12
+ hand:
13
+
14
+ ```bash
15
+ claude mcp add e2e -- npx --no-install e2e mcp # Claude Code
16
+ ```
17
+
18
+ Or declare it in the client's project config (`.mcp.json` for Claude Code,
19
+ `.cursor/mcp.json` for Cursor, `.vscode/mcp.json` for VS Code):
20
+
21
+ ```json
22
+ { "mcpServers": { "e2e": { "command": "npx", "args": ["--no-install", "e2e", "mcp"] } } }
23
+ ```
24
+
25
+ Flags: `--config <path>` names the default config file, `--target <name>`
26
+ fixes the target every session opens on, `--headless` hides the browser or
27
+ simulator (sessions are headed by default outside CI, so the developer can
28
+ watch).
29
+
30
+ ## Tools
31
+
32
+ The server has four tools, and the list never changes. Everything a session
33
+ can do is a catalog behind `call`.
34
+
35
+ | Tool | Does |
36
+ | --- | --- |
37
+ | `open_session` | Loads the config (`config` names another file; default the nearest `e2e.config.ts`), starts the declared app command if any, boots the engine, opens the app URL, and returns the session id, the catalog, and the first observation. `target` is required when the config declares several. One session at a time. |
38
+ | `tools` | The catalog: one line per tool with its argument names (`?` marks optional), the first sentence of its description, and `[read-only]` where it changes nothing. `tools {tool}` shows one tool's full description and the JSON Schema of its arguments. |
39
+ | `call` | Runs one catalog tool: `call {tool: "tap", args: {target: "n42"}}`. Arguments are checked against the tool's schema first; a wrong one fails with `INVALID_ARGUMENT` naming the field. |
40
+ | `close_session` | Ends the attempt, disposes the engine, stops the app processes the session started. |
41
+
42
+ The catalog, per session:
43
+
44
+ | Catalog tool | Does |
45
+ | --- | --- |
46
+ | `observe` | A fresh observation: one node per line as `#id role "name" ...`, plus the current path. |
47
+ | `tap`, `type`, `press`, `select`, `scroll`, `navigate` | The grammar verbs, exactly as the testing agent gets them. Each reports what changed on screen; `observe` shows the whole screen. A verb the engine cannot honor is not listed and fails with `UNSUPPORTED_CAPABILITY`. |
48
+ | `type_secret` | Fills a configured credential's password into a secure field by credential name; the plaintext never reaches the agent. Listed when the config declares `credentials`. |
49
+ | `locate` | Tries a semantic locator (`role` + `name`, `text`, `label`, `placeholder`, `testId`, `exact`) and returns how many nodes match, which, and the `screen.*` call to write. |
50
+ | `screenshot` | The masked pixels as an image, withheld once a secret was filled in the session. |
51
+ | Project tools | Every `defineTool` passed to `createAgent({ tools })` that applies to the target's platform, under its own name; an engine pack such as `agentDeviceTools` adds `open_app`, `swipe`, `type_text`, `alert`. |
52
+
53
+ Resources: `e2e://guide` and `e2e://guide/<topic>` hold this skill.
54
+
55
+ ## Workflow
56
+
57
+ 1. `open_session`, then `call {tool: "observe"}` and act until the screen you
58
+ want to test is in front of you. Node ids are valid only for the newest
59
+ observation; an action reports what changed, so observe again before
60
+ using new ids. The opening text lists every tool with its arguments;
61
+ `tools {tool}` when you need the full contract.
62
+ 2. `call {tool: "locate", args: {...}}` for each locator you intend to write.
63
+ One match: use the printed `screen.getByRole(...)` call. Zero or several:
64
+ adjust before writing the test, the same failure would hit the test as
65
+ `LOCATOR_NOT_FOUND` or `LOCATOR_AMBIGUOUS`.
66
+ 3. Write `tests/<feature>.e2e.ts` (topic `writing-tests`). Deterministic steps
67
+ where you saw exact names; `agent.act` where the flow varies.
68
+ 4. Run it from the shell: `npx --no-install e2e run tests/<feature>.e2e.ts`,
69
+ read the failure (topic `debugging`), fix, repeat.
70
+ 5. `close_session` when you are done exploring; an idle session closes on its
71
+ own after 30 minutes and never outlives 4 hours. To look at another
72
+ project or config, `open_session {config: "path/to/e2e.config.ts"}`; no
73
+ restart needed.
74
+
75
+ ## Rules
76
+
77
+ - Sessions enforce the same policy as tests: navigation stays inside the
78
+ engine's allowed origins, secrets fill only through `type_secret`, and
79
+ pixels are withheld once a secret is on screen.
80
+ - Nothing a session does is recorded as a test or into the trace cache. A
81
+ session is for looking and trying; the test is what you write afterwards.
82
+ - A run from the shell and a live session can share the app only if the
83
+ engine's `command` uses `reuseExisting`; otherwise close the session before
84
+ running.
85
+ - `TARGET_REQUIRED`: pass `target` to `open_session` or start with `--target`.
86
+ `NO_SESSION`: call `open_session` first. `SESSION_OPEN`: one is already
87
+ open; use it or `close_session`. `UNKNOWN_TOOL`: the name is not in this
88
+ session's catalog; the message lists what is.
@@ -4,10 +4,12 @@
4
4
 
5
5
  ```bash
6
6
  npx --no-install e2e run [files...] [options] # run tests
7
+ npx --no-install e2e explore [goal] [options] # explore the app toward a goal, no test file (see the explore topic)
7
8
  npx --no-install e2e list [files...] [options] # print what run would select, without running
8
9
  npx --no-install e2e init [--yes] # scaffold a project, refresh the agent skill
9
- npx --no-install e2e guide [topic] # print this skill: setup, writing-tests, agent, running, debugging
10
+ npx --no-install e2e guide [topic] # print this skill: setup, writing-tests, agent, running, explore, debugging, mcp
10
11
  npx --no-install e2e cache ls|clear|stats # read or empty the trace cache
12
+ npx --no-install e2e mcp [--target <name>] # serve the project to a coding agent over MCP (topic mcp)
11
13
  npx --no-install e2e telemetry [disable|enable] # anonymous usage telemetry: status, or the switch
12
14
  ```
13
15
 
@@ -20,7 +22,7 @@ npx --no-install e2e telemetry [disable|enable] # anonymous usage telemetry: sta
20
22
  | `--target <ids>` | Comma-separated target names. Only selected targets start app commands and services; unknown names fail before startup. |
21
23
  | `--tag <tag>` | Repeatable tag filter; `--tag-mode all` requires every tag. |
22
24
  | `--headed` | Visible browser or simulator when the engine supports it. |
23
- | `--agent <name>` | Run with another configured agent (`agents.<name>`); default is `agents.default`. |
25
+ | `--agent <names>` | Run unpinned tests as other configured agents (`agents.<name>`), comma-separated or repeated; several names run each such test once per agent. Default is `agents.default`. |
24
26
  | `--workers <n>`, `--retries <n>` | Override the resolved values. |
25
27
  | `--reporter <ids>` | `list`, `json`, `junit`, comma-separated. `json` cannot combine with `list`. |
26
28
  | `--artifacts <dir>` | Artifact root, default `.e2e/artifacts`. |
@@ -70,7 +72,7 @@ recorded flow is stale — `--no-cache` only skips the cache for one run.
70
72
 
71
73
  - `list` (default): one line per file and target, a `Failed Tests` section
72
74
  with each error, its code, the failing line and a code frame, then a
73
- summary (`Test Files`, `Tests`, `AI`, `Start at`, `Duration`, `Report`); setup steps (a first-run browser download, each service and app command) print above the tests and stay out of `Duration` or are split out of it as `(startup …)`.
75
+ summary (`Test Files`, `Tests`, `AI`, `Cache` when the trace cache was on: agent steps `replayed` whole, `handed off` to the model part-way, or `missed`, `Start at`, `Duration`, `Report`); setup steps (a first-run browser download, each service and app command) print above the tests and stay out of `Duration` or are split out of it as `(startup …)`.
74
76
  - `.e2e/report.json` is written on every run whatever the reporters:
75
77
  `run.status`, `run.exitCode`, `run.errors[]` (run-level failures such as
76
78
  `APP_UNREACHABLE`), and `run.results[]`, one per test and target, with
@@ -155,5 +157,5 @@ jobs:
155
157
  - Start the app through the engine's `command`; the runner tears it down on
156
158
  every exit path.
157
159
  - Agentic suites: a separate config, run on `schedule` or
158
- `workflow_dispatch`, `E2E_MODEL` as a CI variable and `E2E_MODEL_API_KEY`
159
- as a secret, never a required check.
160
+ `workflow_dispatch`, the key the config's model reads (`AI_GATEWAY_API_KEY`
161
+ for `gateway()` from `ai`) as a secret, never a required check.
@@ -21,14 +21,15 @@ pnpm dlx @e2edev/e2e@beta init # pnpm
21
21
  When `@e2edev/e2e` is already installed, run `npx --no-install e2e init`
22
22
  instead, so the installed version scaffolds.
23
23
 
24
- The wizard asks for the engine (Playwright by default; None and agent-device
25
- are the alternatives), whether to enable AI testing (adds `ai@^7.0.0` and
26
- `createAgent` to the config), which agent directories receive this skill
24
+ The wizard asks for the engine (Web with Playwright by default; Mobile with
25
+ agent-device and None are the alternatives), which model gateway agent steps
26
+ use (adds AI SDK v7 and the provider package), which agent directories receive this skill
27
27
  (`.agents/skills/` and `.claude/skills/`), a confirmation of the files it
28
28
  will write, and whether to install. `--yes` skips every prompt (use it from
29
- scripts and from a shell without a TTY): Playwright, AI on, no installation,
30
- skill in both directories. The closing line prints the run command,
31
- `APP_URL=http://localhost:3000 npx --no-install e2e run` for Playwright, and
29
+ scripts and from a shell without a TTY): Playwright, the Vercel AI Gateway,
30
+ no installation, skill in both directories. The closing line prints the run
31
+ command through the project's package manager,
32
+ `APP_URL=http://localhost:3000 npm run test:e2e` for Playwright under npm, and
32
33
  suggests a `tsconfig.json` when the project has none.
33
34
 
34
35
  Init writes `package.json` (a private ESM package when missing; otherwise
@@ -56,6 +57,7 @@ unknown keys are `INVALID_CONFIG`.
56
57
  import type { E2EConfig } from '@e2edev/e2e';
57
58
  import { createAgent } from '@e2edev/e2e/agent';
58
59
  import { playwright } from '@e2edev/playwright';
60
+ import { gateway } from 'ai';
59
61
 
60
62
  export default {
61
63
  tests: 'tests/**/*.e2e.ts',
@@ -67,8 +69,13 @@ export default {
67
69
  }),
68
70
  },
69
71
  ],
70
- // Only for agent.* steps. The model comes from E2E_MODEL unless set here.
71
- agents: { default: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }) },
72
+ // Only for agent.* steps. The model is an AI SDK instance; gateway() from 'ai' reads AI_GATEWAY_API_KEY.
73
+ agents: {
74
+ default: createAgent({
75
+ model: gateway('openai/gpt-5.4-mini'),
76
+ system: 'You are a thorough QA agent. Verify every outcome on screen.',
77
+ }),
78
+ },
72
79
  credentials: {
73
80
  admin: { username: 'admin@example.test', password: process.env.ADMIN_PASSWORD ?? '' },
74
81
  },
@@ -86,7 +93,7 @@ export default {
86
93
  | `workers` | half the cores, `1` in CI | Test files run in parallel across workers, at most the `workers` the engine declares per target (a device target: one per device). |
87
94
  | `reporters` | `['list']` | `list`, `json`, `junit`, and reporter objects (`{ name, onEvent?, onRunFinished? }`) that receive the finished run. `json` excludes `list`; `--reporter` keeps the objects. |
88
95
  | `cache` | `'read-write'`, `'read-only'` in CI | The trace cache for `agent.act`; `'off'` disables it. |
89
- | `agents` | `{ default: built-in }` | Agents by name. `default` is what tests run with; `e2e run --agent <name>` runs with another. Each entry is `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. Omitted, the built-in agent runs with `E2E_MODEL`. |
96
+ | `agents` | `{ default: built-in }` | Agents by name. `default` is what tests run with; `e2e run --agent <name>` runs with another. Each entry is `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. `model` is an AI SDK instance; without one anywhere, acquiring `agent` is `MODEL_UNAVAILABLE`. |
90
97
  | `credentials` | `{}` | Named `{ username, password, allowedOrigins? }` entries; `password` may be a function returning the value. |
91
98
  | `screen.testIdAttribute` | `'data-testid'` | Attribute read by `getByTestId`. |
92
99
  | `artifacts` | `['screenshot', 'trace']` | Kinds to keep (`screenshot`, `trace`, and the opt-in `video`), or `{ kinds, store, video }`; `video: { retain: 'on-failure' }` keeps only the recordings of attempts that did not pass. |
@@ -184,10 +191,7 @@ or read the address from the environment:
184
191
 
185
192
  | Variable | Effect |
186
193
  | --- | --- |
187
- | `E2E_MODEL` | `provider/model-id` for `agent.*` steps when the config sets no model. Routed through the Vercel AI Gateway. |
188
- | `E2E_MODEL_API_KEY` | Credential for that model. `AI_GATEWAY_API_KEY` is the fallback. |
189
- | `E2E_MODEL_ENDPOINT` | Any OpenAI-compatible endpoint URL for that model; the gateway is the default. HTTPS unless loopback. The agent's `model.endpoint` wins. |
190
- | `E2E_VISION_MODEL` | Model for calls with `vision`; falls back to the main model. |
194
+ | `AI_GATEWAY_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, ... | Read by the provider package the config constructs the model with (`gateway()` from `ai`, `openrouter()`, `openai()`), not by the runner. The runner reads no model variable; the model is always an AI SDK instance in the config. |
191
195
  | `E2E_USER_<NAME>_USERNAME`, `E2E_USER_<NAME>_PASSWORD` | Override `credentials.<name>`. `<NAME>` is the credential name uppercased, other characters as `_`. |
192
196
  | `CI` | Turns on CI defaults: `retries: 1`, `workers: 1`, `test.only` rejected, cache read-only, `reuseExisting` ignored. |
193
197
  | `E2E_TELEMETRY_DISABLED`, `DO_NOT_TRACK` | Turn anonymous usage telemetry off; `e2e telemetry disable` saves the same choice. `E2E_TELEMETRY_DEBUG=1` prints each event instead of sending it. |