e2e 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/README.md +10 -0
  2. package/dist/.build.json +1 -1
  3. package/dist/agent/act.js +7 -4
  4. package/dist/agent/action-dispatcher.js +4 -5
  5. package/dist/agent/actions.d.ts +32 -23
  6. package/dist/agent/actions.js +38 -24
  7. package/dist/agent/ai-sdk.js +5 -4
  8. package/dist/agent/error.js +3 -3
  9. package/dist/agent/invocation.js +3 -4
  10. package/dist/agent/model/sdk.js +2 -0
  11. package/dist/agent/model/tool-choice.d.ts +9 -0
  12. package/dist/agent/model/tool-choice.js +14 -0
  13. package/dist/agent/observation-feed.d.ts +5 -6
  14. package/dist/agent/observation-feed.js +8 -11
  15. package/dist/agent/observation.d.ts +40 -11
  16. package/dist/agent/observation.js +109 -59
  17. package/dist/agent/phases.d.ts +7 -1
  18. package/dist/agent/phases.js +12 -3
  19. package/dist/agent/point-tap.d.ts +8 -8
  20. package/dist/agent/point-tap.js +1 -1
  21. package/dist/agent/prompts.js +3 -0
  22. package/dist/agent/replay.d.ts +19 -8
  23. package/dist/agent/replay.js +29 -29
  24. package/dist/agent/scroll-target.d.ts +2 -3
  25. package/dist/agent/step-cache.d.ts +43 -15
  26. package/dist/agent/step-cache.js +156 -67
  27. package/dist/agent/tool-loop.js +94 -5
  28. package/dist/cache/anchors.d.ts +72 -23
  29. package/dist/cache/anchors.js +233 -60
  30. package/dist/cache/context.d.ts +25 -14
  31. package/dist/cache/context.js +50 -8
  32. package/dist/cache/decide.d.ts +2 -2
  33. package/dist/cache/decide.js +4 -6
  34. package/dist/cache/identity.d.ts +18 -4
  35. package/dist/cache/identity.js +9 -4
  36. package/dist/cache/recorder.d.ts +15 -1
  37. package/dist/cache/recorder.js +36 -23
  38. package/dist/cache/rekeyed.d.ts +46 -0
  39. package/dist/cache/rekeyed.js +89 -0
  40. package/dist/cache/relocate.d.ts +14 -14
  41. package/dist/cache/relocate.js +33 -25
  42. package/dist/cache/route.d.ts +32 -33
  43. package/dist/cache/route.js +94 -50
  44. package/dist/cache/store.d.ts +6 -0
  45. package/dist/cache/store.js +21 -1
  46. package/dist/cache/trace.d.ts +53 -14
  47. package/dist/cache/trace.js +76 -30
  48. package/dist/cli/bin.js +2 -2
  49. package/dist/cli/index.js +43 -32
  50. package/dist/cli/init/engines.d.ts +1 -3
  51. package/dist/cli/init/engines.js +2 -2
  52. package/dist/cli/init/gateways.d.ts +1 -1
  53. package/dist/cli/init/gateways.js +11 -1
  54. package/dist/cli/init/mcp-config.d.ts +2 -1
  55. package/dist/cli/init/mcp-config.js +8 -3
  56. package/dist/cli/init/sibling-versions.json +1 -1
  57. package/dist/cli/init/versions.d.ts +0 -8
  58. package/dist/cli/init/versions.js +1 -10
  59. package/dist/cli/mcp.d.ts +3 -2
  60. package/dist/cli/mcp.js +52 -16
  61. package/dist/cli/run-output.d.ts +17 -0
  62. package/dist/cli/run-output.js +45 -0
  63. package/dist/collect/collect.js +8 -4
  64. package/dist/collect/registry.js +27 -16
  65. package/dist/config/compiled-files.d.ts +36 -0
  66. package/dist/config/compiled-files.js +50 -0
  67. package/dist/config/diagnose.js +21 -19
  68. package/dist/config/esm-hooks.d.ts +39 -44
  69. package/dist/config/esm-hooks.js +238 -63
  70. package/dist/config/load.d.ts +1 -7
  71. package/dist/config/load.js +23 -66
  72. package/dist/config/register.d.ts +8 -0
  73. package/dist/config/register.js +9 -0
  74. package/dist/config/resolve.d.ts +1 -0
  75. package/dist/config/resolve.js +33 -1
  76. package/dist/config/specifiers.d.ts +19 -0
  77. package/dist/config/specifiers.js +33 -0
  78. package/dist/config/tsconfig.d.ts +56 -0
  79. package/dist/config/tsconfig.js +115 -0
  80. package/dist/config/typescript.d.ts +19 -0
  81. package/dist/config/typescript.js +253 -0
  82. package/dist/engine/contract.d.ts +1 -0
  83. package/dist/engine/index.d.ts +4 -2
  84. package/dist/engine/session.js +2 -2
  85. package/dist/expect/async.js +4 -2
  86. package/dist/expect/poll-scope.d.ts +52 -0
  87. package/dist/expect/poll-scope.js +113 -0
  88. package/dist/expect/poll.js +33 -6
  89. package/dist/expect/values.js +11 -4
  90. package/dist/internal/abandoned.d.ts +11 -0
  91. package/dist/internal/abandoned.js +23 -0
  92. package/dist/internal/brands.d.ts +2 -0
  93. package/dist/internal/brands.js +2 -0
  94. package/dist/internal/client-identity.d.ts +9 -0
  95. package/dist/internal/client-identity.js +14 -0
  96. package/dist/internal/errors.d.ts +10 -3
  97. package/dist/internal/errors.js +23 -8
  98. package/dist/internal/node-version.d.ts +16 -7
  99. package/dist/internal/node-version.js +50 -15
  100. package/dist/internal/package-manager.js +1 -1
  101. package/dist/internal/realm-slot.d.ts +5 -4
  102. package/dist/internal/redact.d.ts +9 -4
  103. package/dist/internal/redact.js +120 -23
  104. package/dist/internal/time.d.ts +26 -4
  105. package/dist/internal/time.js +58 -14
  106. package/dist/internal/urls.d.ts +9 -5
  107. package/dist/internal/urls.js +19 -10
  108. package/dist/locator/engine.d.ts +12 -2
  109. package/dist/locator/engine.js +55 -14
  110. package/dist/locator/screen.js +59 -9
  111. package/dist/mcp/catalog.d.ts +3 -2
  112. package/dist/mcp/catalog.js +6 -6
  113. package/dist/mcp/output.d.ts +59 -0
  114. package/dist/mcp/output.js +129 -0
  115. package/dist/mcp/server.d.ts +4 -0
  116. package/dist/mcp/server.js +9 -12
  117. package/dist/mcp/session.d.ts +8 -1
  118. package/dist/mcp/session.js +56 -15
  119. package/dist/mcp/tools.js +7 -5
  120. package/dist/oauth/chatgpt.js +3 -23
  121. package/dist/oauth/cli.js +1 -0
  122. package/dist/oauth/copilot.d.ts +5 -4
  123. package/dist/oauth/copilot.js +80 -7
  124. package/dist/oauth/grok.js +2 -1
  125. package/dist/oauth/models.js +2 -1
  126. package/dist/oauth/opencode-console.d.ts +10 -0
  127. package/dist/oauth/opencode-console.js +133 -0
  128. package/dist/oauth/providers/github-copilot.d.ts +12 -0
  129. package/dist/oauth/providers/github-copilot.js +77 -23
  130. package/dist/oauth/providers/opencode-console.d.ts +31 -0
  131. package/dist/oauth/providers/opencode-console.js +148 -0
  132. package/dist/oauth/providers.d.ts +2 -6
  133. package/dist/oauth/providers.js +3 -7
  134. package/dist/oauth/responses.d.ts +9 -0
  135. package/dist/oauth/responses.js +23 -0
  136. package/dist/report/build.d.ts +26 -0
  137. package/dist/report/build.js +16 -8
  138. package/dist/report/failure-text.d.ts +6 -1
  139. package/dist/report/failure-text.js +14 -4
  140. package/dist/report/format.d.ts +10 -5
  141. package/dist/report/format.js +12 -5
  142. package/dist/report/junit.js +11 -12
  143. package/dist/report/list.d.ts +12 -0
  144. package/dist/report/list.js +61 -24
  145. package/dist/report/markdown.d.ts +1 -1
  146. package/dist/report/markdown.js +37 -14
  147. package/dist/report/repeats.d.ts +6 -1
  148. package/dist/report/repeats.js +12 -3
  149. package/dist/run/artifacts.d.ts +22 -4
  150. package/dist/run/artifacts.js +71 -3
  151. package/dist/run/execute.d.ts +34 -5
  152. package/dist/run/execute.js +93 -44
  153. package/dist/run/extended-fixtures.js +6 -2
  154. package/dist/run/failure-evidence.js +10 -4
  155. package/dist/run/fixtures.js +6 -8
  156. package/dist/run/in-process.d.ts +2 -0
  157. package/dist/run/in-process.js +52 -10
  158. package/dist/run/last-run.d.ts +42 -8
  159. package/dist/run/last-run.js +121 -14
  160. package/dist/run/process-output.d.ts +73 -0
  161. package/dist/run/process-output.js +130 -0
  162. package/dist/run/provision.d.ts +2 -0
  163. package/dist/run/provision.js +1 -0
  164. package/dist/run/realm.d.ts +4 -1
  165. package/dist/run/realm.js +20 -6
  166. package/dist/run/records.d.ts +20 -4
  167. package/dist/run/records.js +26 -0
  168. package/dist/run/retry.d.ts +10 -1
  169. package/dist/run/retry.js +30 -12
  170. package/dist/run/runner.d.ts +9 -1
  171. package/dist/run/runner.js +101 -39
  172. package/dist/run/scheduler.js +255 -19
  173. package/dist/run/secrecy.d.ts +22 -19
  174. package/dist/run/secrecy.js +22 -18
  175. package/dist/run/serial.d.ts +75 -2
  176. package/dist/run/serial.js +243 -59
  177. package/dist/run/standalone.js +1 -1
  178. package/dist/run/steps.d.ts +0 -6
  179. package/dist/run/steps.js +3 -20
  180. package/dist/run/trace-redaction.d.ts +5 -6
  181. package/dist/run/trace-redaction.js +5 -6
  182. package/dist/run/worker/entry.js +40 -18
  183. package/dist/run/worker/handle.js +10 -5
  184. package/dist/run/worker/protocol.d.ts +85 -3
  185. package/dist/run/worker/session.d.ts +2 -0
  186. package/dist/run/worker/session.js +29 -0
  187. package/dist/telemetry/environment.d.ts +2 -1
  188. package/dist/telemetry/environment.js +12 -2
  189. package/dist/telemetry/events.js +1 -0
  190. package/dist/telemetry/project.d.ts +5 -6
  191. package/dist/telemetry/project.js +6 -11
  192. package/dist/telemetry/store.d.ts +5 -5
  193. package/dist/telemetry/store.js +5 -7
  194. package/dist/telemetry/telemetry.d.ts +2 -3
  195. package/dist/telemetry/telemetry.js +2 -4
  196. package/dist/types.d.ts +15 -6
  197. package/docs/agent-steps.mdx +1 -0
  198. package/docs/agents.mdx +7 -3
  199. package/docs/api-testing.mdx +2 -1
  200. package/docs/authentication.mdx +6 -3
  201. package/docs/browser.mdx +49 -12
  202. package/docs/bug-bash.mdx +2 -1
  203. package/docs/cache.mdx +66 -19
  204. package/docs/ci.mdx +14 -9
  205. package/docs/coding-agents.mdx +2 -1
  206. package/docs/core-concepts.mdx +208 -0
  207. package/docs/debugging.mdx +25 -12
  208. package/docs/decision-models.mdx +142 -0
  209. package/docs/example-projects.mdx +32 -0
  210. package/docs/executors.mdx +7 -2
  211. package/docs/explore.mdx +1 -0
  212. package/docs/github.mdx +14 -5
  213. package/docs/index.mdx +9 -5
  214. package/docs/integrations/eas.mdx +1 -0
  215. package/docs/integrations/index.mdx +1 -0
  216. package/docs/integrations/kernel.mdx +3 -2
  217. package/docs/migrate/cypress.mdx +26 -3
  218. package/docs/migrate/detox.mdx +24 -1
  219. package/docs/migrate/maestro.mdx +28 -4
  220. package/docs/migrate/playwright.mdx +44 -18
  221. package/docs/migrate/selenium.mdx +27 -3
  222. package/docs/mobile-ci.mdx +6 -5
  223. package/docs/mobile.mdx +96 -8
  224. package/docs/models.mdx +60 -9
  225. package/docs/quickstart.mdx +28 -16
  226. package/docs/reference/agent.mdx +13 -3
  227. package/docs/reference/app.mdx +2 -1
  228. package/docs/reference/cli.mdx +41 -18
  229. package/docs/reference/config.mdx +67 -13
  230. package/docs/reference/engine.mdx +25 -13
  231. package/docs/reference/environment.mdx +10 -2
  232. package/docs/reference/errors.mdx +13 -10
  233. package/docs/reference/expect.mdx +36 -13
  234. package/docs/reference/mcp.mdx +21 -8
  235. package/docs/reference/mobile.mdx +24 -13
  236. package/docs/reference/reporters.mdx +9 -2
  237. package/docs/reference/screen.mdx +49 -16
  238. package/docs/reference/test.mdx +32 -5
  239. package/docs/reference/web.mdx +133 -23
  240. package/docs/security.mdx +58 -42
  241. package/docs/subscriptions.mdx +92 -12
  242. package/docs/telemetry.mdx +9 -6
  243. package/docs/tools.mdx +1 -0
  244. package/docs/{starting-your-app.mdx → web.mdx} +157 -11
  245. package/docs/writing-an-engine.mdx +2 -1
  246. package/package.json +25 -8
  247. package/schema/report-v1.schema.json +14 -1
  248. package/skills/e2e/SKILL.md +5 -3
  249. package/skills/e2e/references/agent.md +33 -11
  250. package/skills/e2e/references/debugging.md +4 -4
  251. package/skills/e2e/references/mcp.md +8 -3
  252. package/skills/e2e/references/running.md +15 -9
  253. package/skills/e2e/references/setup.md +45 -16
  254. package/skills/e2e/references/writing-tests.md +35 -18
  255. package/docs/viewports.mdx +0 -146
  256. package/docs/writing-tests.mdx +0 -256
package/docs/browser.mdx CHANGED
@@ -1,6 +1,7 @@
1
1
  ---
2
2
  title: Working with the browser
3
3
  description: Mock an API, handle a dialog, download a file, reach into an iframe, and pick a browser.
4
+ icon: "window-maximize"
4
5
  ---
5
6
 
6
7
  `@e2e-dev/web` runs tests in Chromium, Firefox, and WebKit. It is built on
@@ -39,7 +40,9 @@ await app.open('/checkout');
39
40
  await expect(screen.getByText('$42.00')).toBeVisible();
40
41
  ```
41
42
 
42
- Each handler must call exactly one of `fulfill`, `continue`, or `abort`. An
43
+ Each handler must call exactly one of `fulfill`, `continue`, `fallback`, or
44
+ `abort`. The newest route runs first; `continue` sends the request straight to
45
+ the network, while `fallback` passes it to the route registered before. An
43
46
  `expect` that fails inside the handler fails the test, at the next step or at
44
47
  the end of the test when nothing follows. To inspect a real response, start
45
48
  waiting before the action that sends it:
@@ -114,16 +117,35 @@ const flag = await browser.evaluate((name: string) => window.localStorage.getIte
114
117
  Use `evaluate` for state that `screen` cannot read. Prefer locators and
115
118
  assertions for visible UI so tests depend less on the DOM structure.
116
119
 
120
+ ## Run a script before the page
121
+
122
+ An init script runs in every document before the page's own scripts: mock a
123
+ browser API, seed `Math.random`, or set a flag the app reads at boot. For every
124
+ test:
125
+
126
+ ```ts
127
+ web({ initScripts: [{ path: 'tests/init.js' }] })
128
+ ```
129
+
130
+ For one test, before the page opens:
131
+
132
+ ```ts
133
+ await browser.addInitScript((value) => { Math.random = () => value; }, 0.5);
134
+ await app.open('/raffle');
135
+ ```
136
+
137
+ See [init scripts](/reference/web#init-scripts).
138
+
117
139
  ## Cookies
118
140
 
119
141
  ```ts
120
142
  await browser.setCookies([{ url: 'http://localhost:3000', name: 'tz', value: 'UTC' }]);
121
143
  ```
122
144
 
123
- A cookie URL that does not parse, or uses `file:`, `data:`, or
124
- `javascript:`, is `POLICY_DENIED`; a domain cookie is checked as the origin
145
+ A cookie URL that does not parse, or uses a scheme other than `http:` or
146
+ `https:` (`about:blank` included), is `POLICY_DENIED`; a domain cookie is checked as the origin
125
147
  it would be sent to. If any entry is refused, no cookies are set. To resize
126
- the page, see [Testing viewports](/viewports).
148
+ the page, see [Viewports](/web#viewports).
127
149
 
128
150
  ## Pick a browser
129
151
 
@@ -144,10 +166,25 @@ export default {
144
166
  } satisfies E2EConfig;
145
167
  ```
146
168
 
147
- `npx playwright install webkit --with-deps` provisions the extra browser in
148
- CI. `connect` attaches to a remote Chromium over CDP instead of launching
169
+ In CI, provision the extra browser with the [install command](/reference/web#installing-browsers):
170
+
171
+ <CodeGroup>
172
+ ```bash npm
173
+ npx @e2e-dev/web install webkit --with-deps
174
+ ```
175
+
176
+ ```bash pnpm
177
+ pnpm exec e2e-web install webkit --with-deps
178
+ ```
179
+
180
+ ```bash bun
181
+ bunx @e2e-dev/web install webkit --with-deps
182
+ ```
183
+ </CodeGroup>
184
+
185
+ `connect` attaches to a remote Chromium over CDP instead of launching
149
186
  one; a [browser provider](#hosted-browsers) does the same and also leases
150
- and releases the browser. [Testing viewports](/viewports) covers sizes.
187
+ and releases the browser. [Viewports](/web#viewports) covers sizes.
151
188
 
152
189
  ## Hosted browsers
153
190
 
@@ -161,7 +198,7 @@ when the run no longer uses it. The engine knows no vendor: for
161
198
  provider, and for any other service a provider is a few dozen lines in your
162
199
  project. Add `viewport: null` when
163
200
  the service shows the window in a live view: the page then
164
- [fills the window](/viewports#fill-the-window) instead of a fixed 1280 by
201
+ [fills the window](/web#fill-the-window) instead of a fixed 1280 by
165
202
  720 area of it.
166
203
 
167
204
  ```ts
@@ -237,7 +274,7 @@ The attempt runs in the browser's persistent default context, on the
237
274
  browser's own first tab, so the remote must contain only that context, and
238
275
  the limits of
239
276
  [`connect.reconnectEndpoint`](/reference/web#recovering-a-cdp-transport)
240
- apply: no `headers`, `basicAuth`, or `userAgent` (each is `INVALID_CONFIG`), no
277
+ apply: no `headers`, `basicAuth`, `userAgent`, `locale`, or `timezoneId` (each is `INVALID_CONFIG`), no
241
278
  `app.clearState()` (`UNSUPPORTED_CAPABILITY` naming the scope), and no session
242
279
  state capture or restore, so a run whose tests consume a
243
280
  [session](/authentication) on the target fails with `COLLECTION_ERROR`
@@ -249,7 +286,7 @@ same browser.
249
286
  A provider against a typical session API has three requests: create a
250
287
  session, poll it until it exposes a DevTools endpoint, stop it.
251
288
 
252
- ```ts title="browser-provider.ts"
289
+ ```ts title="browser-provider.ts" expandable
253
290
  import type { BrowserProvider } from '@e2e-dev/web';
254
291
 
255
292
  /**
@@ -434,7 +471,7 @@ root declared in HTML (`<template shadowrootmode="closed">`) is out of reach.
434
471
  <Card title="Web engine reference" icon="masks-theater" href="/reference/web">
435
472
  Every `browser` method, its timeout, and what it throws.
436
473
  </Card>
437
- <Card title="Starting your app" icon="play" href="/starting-your-app">
438
- Let the runner start the dev server and reach protected previews.
474
+ <Card title="Web" icon="play" href="/web">
475
+ Let the runner start the dev server, reach protected previews, and test viewports.
439
476
  </Card>
440
477
  </CardGroup>
package/docs/bug-bash.mdx CHANGED
@@ -1,6 +1,7 @@
1
1
  ---
2
2
  title: Bug bashes
3
3
  description: Run many explorations at once, then prove each bug with a failing test before anyone reads about it.
4
+ icon: "bug"
4
5
  ---
5
6
 
6
7
  A bug bash runs several `e2e explore` sessions at once, each on one area
@@ -50,7 +51,7 @@ itself: `npx e2e guide bug-bash`.
50
51
  `reuseExisting: true` on the target's `command`, or declare the URL with
51
52
  port `0` so each run starts its own copy. `reuseExisting` does not apply
52
53
  when `CI` is set; use port `0` there. See
53
- [Starting your app](/starting-your-app).
54
+ [Starting your app](/web#starting-your-app).
54
55
  - When the target's `app.command` brings up a stack of its own, such as a
55
56
  database from `docker compose` that it removes on exit, start the stack once
56
57
  and explore with a config that points at the running app and declares no
package/docs/cache.mdx CHANGED
@@ -1,6 +1,7 @@
1
1
  ---
2
- title: Caching agent steps
2
+ title: Caching
3
3
  description: Replay verified agent actions without model calls, and let the agent take over when the app changes.
4
+ icon: "box-archive"
4
5
  ---
5
6
 
6
7
  The replay cache records the actions in an `agent.act()` step after a later
@@ -45,27 +46,60 @@ bunx e2e run --no-cache
45
46
 
46
47
  ## How a replay works
47
48
 
48
- 1. Check the starting screen: the same route, whatever the record id, the
49
- query, or the fragment. Retries always run live.
49
+ 1. Check the starting screen: the same route, whatever the record id or the
50
+ fragment. Retries always run live.
50
51
  2. Find each recorded control and repeat its action. The runner matches
51
52
  controls by their role, name, test id, and surrounding context. It waits
52
53
  for missing controls and stops if a match is ambiguous. A control with no
53
- name is found by its place among the unnamed controls of its kind.
54
- 3. Check the final path and controls that appeared during the original step.
55
- Text that changes with the data, a count, a date, an id, is not used.
54
+ name is found by its place among the unnamed controls of its kind inside
55
+ the same named row, list item, or group; with no such container to tell
56
+ it from its twins, the agent takes the step.
57
+ 3. Check the final route and what the original step changed: the controls
58
+ that appeared, with their checked, selected, expanded, or pressed state,
59
+ and the ones that went away. At least one of those changes must happen
60
+ during the replay, counted from the first screen the replay saw on the
61
+ route it ended on, so an outcome already on screen before the actions
62
+ that should produce it proves nothing; a step whose last action moved to
63
+ another route has that move as its evidence. An alert the replay raised
64
+ that the recording never saw stops it too. Text that reads differently on
65
+ every run, a date, a time, a duration, an id, is not used, except in an
66
+ alert, where it is read as a placeholder.
56
67
  4. Finish without a model call, or let the agent continue with a record of
57
68
  what ran and why replay stopped.
58
69
 
70
+ A route is the origin, the path, and the query. Ids, tokens, timestamps,
71
+ and cache busters in either the path or the query are read as an id
72
+ placeholder, so `/orders/42?t=1727780000` and `/orders/7?t=1727780999` are
73
+ one route. Every other query value is part of it: `?mode=safe` and
74
+ `?mode=unsafe`, or `/companies` and `/companies?tab=notes`, are different
75
+ screens, and so is a tracking parameter that was not there when the step
76
+ was recorded. Two routes that differ in one word, `/products/summer-sneaker`
77
+ and `/products/winter-boot`, are different screens too, even when they look
78
+ alike. A location on the app's own origin is compared by its path, so a
79
+ recording follows the app to a preview deployment that shares its
80
+ `app.identity`; a page on any other origin is another route.
81
+
82
+ A count that names what it counts, `3 records imported` or `Showing 1 to 9
83
+ of 9 results`, is part of the effect when the step made it appear on the
84
+ screen it acted on, and must read the same on replay. A count that was
85
+ already on screen and only moved, `41 items` to `42 items`, or one on a
86
+ screen the step navigated to, is not used, since leftover data from earlier
87
+ runs moves it again. A bare number such as a badge is never used.
88
+
59
89
  Typed values replay as recorded, even when an earlier run left the same
60
90
  value on screen. Secret fills resolve the current value by name and run
61
91
  through authorization again.
62
92
 
63
93
  ## What must match
64
94
 
65
- A cache entry belongs to a specific test, target, instruction, and set of
66
- params. Renaming the test or target, changing the instruction or an ordinary
67
- param, or upgrading the engine's major or minor version causes a miss.
68
- Changing the model does not.
95
+ A cache entry belongs to a specific test, target, instruction, set of
96
+ params, and agent. Renaming the test or target, changing the instruction or
97
+ an ordinary param, running with another configured agent, changing the
98
+ agent's `context` or the test's `agentContext`, or upgrading the engine's
99
+ major or minor version causes a miss. A test that runs once per agent
100
+ (`agent: ['buyer', 'admin']`) records once per agent. Changing the model
101
+ does not. The agent context enters the key as a hash of its redacted text,
102
+ so a secret in it contributes only its name.
69
103
 
70
104
  A timestamp or a fresh email address would cause a miss on every run. Wrap
71
105
  values like these in `unique()` so the runner substitutes the current value
@@ -96,7 +130,9 @@ identity. On devices, replay uses the
96
130
 
97
131
  In `read-write` mode, the runner saves a recording at the end of the
98
132
  attempt when a later verification step passed. A step that took no action
99
- records nothing.
133
+ records nothing, and neither does one that changed nothing the replay could
134
+ check: no control appeared, went away, or changed state, and the route
135
+ stayed the same.
100
136
 
101
137
  | Counts as verification | Does not count |
102
138
  | --- | --- |
@@ -145,14 +181,14 @@ such as an `afterEach` assertion.
145
181
  | `retry` | Retries never replay. | Nothing. |
146
182
  | `invalid-entry` | The file is unreadable, over 1 MiB, or from another schema version. | `npx e2e cache clear`. |
147
183
  | `truncated` | The recording hit a limit, see [below](#limits-and-gaps). | Split the step, or pass the secret as a `Secret`. |
148
- | `wrong-context` | The app was on another screen than when the step was recorded: a different route, not a different record, query, or fragment. | Open the same screen before the step. |
149
- | `target-not-found` | A recorded control did not appear within 15 seconds. | The UI changed. The next passing run re-records. |
184
+ | `wrong-context` | The app was on another screen than when the step was recorded: a different origin, path, or query, not a different record id or fragment. | Open the same screen before the step. |
185
+ | `target-not-found` | A recorded control did not appear within 15 seconds, or it has no name and no named container to tell it from its twins. | The UI changed: the next passing run re-records. For an unnamed control, give it a label or put it in a named row or group. |
150
186
  | `target-ambiguous` | The number of matching controls changed. | Give repeated controls distinct names or use unique test data. |
151
187
  | `gap` | An action cannot be replayed, see [below](#limits-and-gaps). | The agent continues this part live. Put stable values in `params`. |
152
188
  | `action-failed` | The app rejected a replayed action. | Check the app. |
153
189
  | `action-uncertain` | A replayed action failed with an unknown commit state. The agent is told to check before acting again. | Check the app. |
154
190
  | `viewport-changed` | A bare-point tap or hover was recorded on a viewport of another size. | Keep the viewport fixed. |
155
- | `end-mismatch` | Every action ran, but the recorded path or a control that should have appeared did not come back. | Leftover data from an earlier run, a banner that comes and goes, or a regression. |
191
+ | `end-mismatch` | Every action ran, but the recorded route, a control that should have appeared or gone away, or a recorded state did not come back; the outcome was already on screen before the replay; or the replay raised an alert the recording never saw. | Leftover data from an earlier run, a banner that comes and goes, or a regression. |
156
192
 
157
193
  A stop before any action ran is a miss. After replay starts, it is a
158
194
  hand-off. Unique test data and cleanup in `afterEach` help avoid stale
@@ -176,8 +212,9 @@ the instruction or `params`, or ones the agent composed itself, replay.
176
212
 
177
213
  Each entry is a JSON file under `.e2e/cache/`, named by a hash of its key.
178
214
  It contains actions, target descriptions, typed text, a step summary, and
179
- checks for the final state. Those checks include the path and up to eight
180
- controls that appeared during the step.
215
+ checks for the final state. Those checks include the route, up to eight
216
+ controls that appeared during the step, alerts and status messages first,
217
+ and up to eight that went away.
181
218
 
182
219
  Entries contain no prompts, conversations, screenshots, observed trees, or
183
220
  secret values. A secret fill stores only the secret's name. Other typed
@@ -220,12 +257,16 @@ Local runs leave a committed directory clean unless something changed. A
220
257
  replayed step never rewrites its entry, and a live re-recording with the same
221
258
  actions, paths, and anchors is not written: the summary, the measured end
222
259
  wait, and the rule that flagged a typed run-time value differ between two
223
- live runs of the same flow and do not count. A file changes when:
260
+ live runs of the same flow and do not count. Upgrading `e2e` to a version
261
+ that changes how entries are matched re-keys every entry, so the first run
262
+ after it records them again. A file changes when:
224
263
 
225
264
  - a step records for the first time,
226
265
  - the agent finishes a step after a hand-off, re-recording what it saw,
227
266
  - a step fails after replaying, or ends in an `end-mismatch` the agent had
228
267
  to repair, which deletes the entry,
268
+ - a step with an entry passes without leaving anything to record, which
269
+ deletes the entry,
229
270
  - no verification step confirmed the entry by the end of the attempt, which
230
271
  deletes it.
231
272
 
@@ -257,8 +298,14 @@ CI=1 bunx e2e run --strict-cache
257
298
  The reasons that fail are the ones that mean the app or the entry changed:
258
299
  `invalid-entry`, `wrong-context`, `target-not-found`, `target-ambiguous`,
259
300
  `viewport-changed`, `action-failed`, `action-uncertain`, and `end-mismatch`.
260
- A step with no recording (`no-entry`), a retry, a `gap`, and a `truncated`
261
- recording still run live. `REPLAY_STALE` exits 2 and is not retried, since a
301
+ A `no-entry` miss fails too when the cache directory holds a recording of the
302
+ same step under another key, from before an `e2e` or engine upgrade or a
303
+ change to the project id, the app's identity, or the agent's context. The
304
+ message names that file; delete it once nothing replays it. A custom
305
+ `cache.store` is not checked this way. Entries recorded before this check
306
+ existed are covered once a `read-write` run replays or re-records them. A
307
+ step never recorded (a new or edited instruction or params), a retry, a
308
+ `gap`, and a `truncated` recording still run live. `REPLAY_STALE` exits 2 and is not retried, since a
262
309
  retry never replays and would pass live. The entry stays where it is, even
263
310
  in `read-write` mode, so every strict run fails on it until it is re-recorded:
264
311
  run the test in `read-write` mode without the flag, and with `cache.strict`
package/docs/ci.mdx CHANGED
@@ -1,13 +1,14 @@
1
1
  ---
2
2
  title: Continuous integration
3
3
  description: Run tests on pull requests, upload reports, and configure model credentials in CI.
4
+ icon: "rotate"
4
5
  ---
5
6
 
6
7
  Run deterministic and agent tests in the same CI job. This workflow assumes
7
- your config includes a [command](/starting-your-app) to start the app and
8
+ your config includes a [command](/web#starting-your-app) to start the app and
8
9
  uses AI Gateway for model calls:
9
10
 
10
- ```yaml title=".github/workflows/e2e.yml"
11
+ ```yaml title=".github/workflows/e2e.yml" expandable
11
12
  name: e2e
12
13
 
13
14
  on:
@@ -35,7 +36,7 @@ jobs:
35
36
  - run: pnpm install --frozen-lockfile
36
37
 
37
38
  - name: Install browsers
38
- run: npx playwright install chromium --with-deps
39
+ run: pnpm exec e2e-web install chromium --with-deps
39
40
 
40
41
  - name: Run e2e
41
42
  run: npx e2e run --reporter list,junit
@@ -66,7 +67,9 @@ jobs:
66
67
  ```
67
68
 
68
69
  The workflow pins actions to commit SHAs and installs dependencies from the
69
- lockfile. `npx e2e` runs the installed version.
70
+ lockfile. `npx e2e` runs the installed version. The
71
+ [install command](/reference/web#installing-browsers) downloads the browsers
72
+ for the Playwright version the engine pins, with their system libraries.
70
73
 
71
74
  Artifacts upload on every run that is not cancelled, so a test that failed
72
75
  and then passed on retry still has its evidence.
@@ -139,10 +142,12 @@ implemented yet.
139
142
 
140
143
  `--last-failed` runs only the tests the previous run did not pass, read from
141
144
  `report.json` in the [`output`](/reference/config#output) directory (`.e2e`
142
- by default). The rerun empties `artifacts/` there when it starts, so
143
- upload the first run's artifacts before rerunning in the same job if you want
144
- both. The [CLI reference](/reference/cli#flags) lists what counts as not
145
- passed. Locally it is the loop after a red run:
145
+ by default). The rerun keeps the artifacts the first run's report names and
146
+ writes its own under `artifacts/rerun-1/`, so one upload after both steps
147
+ holds the evidence of each. A test another filter keeps out of the rerun
148
+ stays owed: the next `--last-failed` runs it. The
149
+ [CLI reference](/reference/cli#flags) lists what counts as not passed.
150
+ Locally it is the loop after a red run:
146
151
 
147
152
  <CodeGroup>
148
153
  ```bash npm
@@ -193,7 +198,7 @@ flaky, so you can identify flows that need investigation.
193
198
  `--max-failures <n>` stops the run once that many tests have failed, so a
194
199
  broken deploy does not spend a model call on every remaining agent step. The
195
200
  tests not started are skipped with cause `failure-limit`, the ones running
196
- end as `interrupted`, and the job exits 1.
201
+ end as `interrupted`, counted apart from the failures, and the job exits 1.
197
202
 
198
203
  ## Exit codes
199
204
 
@@ -1,6 +1,7 @@
1
1
  ---
2
2
  title: Coding agents
3
3
  description: Give a coding agent the skill, a live app session, and reports it can use to write and debug tests.
4
+ icon: "robot"
4
5
  ---
5
6
 
6
7
  `e2e init` installs a skill for writing and running tests, plus MCP
@@ -168,7 +169,7 @@ model call in the flow.
168
169
  Do not edit recordings by hand. A failed read-write run removes implicated
169
170
  entries so a later verified run can replace them. A failure where no model
170
171
  answered implicates nothing and removes none. See
171
- [Caching agent steps](/cache).
172
+ [Caching](/cache).
172
173
 
173
174
  <CardGroup cols={2}>
174
175
  <Card title="CLI reference" icon="terminal" href="/reference/cli">
@@ -0,0 +1,208 @@
1
+ ---
2
+ title: Core concepts
3
+ description: Goals, assertions, and locators, and how to structure and run tests with them.
4
+ icon: "lightbulb"
5
+ ---
6
+
7
+ A test has three types of step:
8
+
9
+ | Step | API | Model call |
10
+ | --- | --- | --- |
11
+ | [Goal](#goals) | `agent.act` | Yes, unless the [cache](/cache) replays it |
12
+ | [Assertion](#assertions) | `agent.assert`, `agent.waitFor`, `agent.extract` | Yes |
13
+ | [Locator](#locators) | `screen` and `expect` | No |
14
+
15
+ The model column is for the built-in agent. A
16
+ [custom executor](/executors) can run `act` and `assert` without a model.
17
+
18
+ You can use all three in one test:
19
+
20
+ ```ts title="tests/todos.e2e.ts"
21
+ import { test, expect } from 'e2e';
22
+
23
+ test('todos survive a filter round-trip', async ({ app, agent, screen }) => {
24
+ await app.open('/todos');
25
+
26
+ await agent.act('add todos "Buy milk" and "Walk the dog", complete the first');
27
+ await agent.assert('only one todo remains open');
28
+
29
+ await screen.getByRole('tab', 'Done').tap();
30
+ await expect(screen.getByTestId('todo')).toHaveCount(1);
31
+ });
32
+ ```
33
+
34
+ Await each agent call, screen call, and `expect.poll`. If a test returns
35
+ while a step still runs, the test fails with `STEP_NOT_AWAITED`.
36
+
37
+ ## Goals
38
+
39
+ `agent.act` takes one goal. The agent selects and does the actions. It
40
+ returns when the goal is complete. It throws when the goal fails.
41
+
42
+ ```ts
43
+ await agent.act('complete checkout with the test card');
44
+ await agent.act('invite {email} as an editor', { params: { email: 'ada@example.test' } });
45
+ ```
46
+
47
+ - Write one goal for each call. The agent controls the flow in a goal. You
48
+ control the order of the goals.
49
+ - Use the words that the screen shows.
50
+ - Give test data in `params`. For a password, give a `Secret`. The model
51
+ does not see a secret. See [Signing in](/authentication).
52
+ - For a value that changes on each run, use `unique()`. Then the
53
+ [replay cache](/cache#what-must-match) can replay the step:
54
+
55
+ ```ts
56
+ import { unique } from 'e2e';
57
+
58
+ const email = `ada+${Date.now()}@example.test`;
59
+ await agent.act('sign up with email {email}', { params: { email: unique(email) } });
60
+ ```
61
+
62
+ If a goal fails, read the report. Then make the instruction more specific.
63
+ Put words that many goals use in the agent's `context`; see
64
+ [Agents and personas](/agents). For verdicts and exit codes, see
65
+ [How agent steps work](/agent-steps#how-a-step-ends).
66
+
67
+ `act` gets screenshots when it needs them. On some engines, it can tap and
68
+ type at points in the image.
69
+
70
+ ## Assertions
71
+
72
+ An assertion examines the screen. The judging model sees the statement, the
73
+ current screen, and the agent's context. It does not see the earlier steps
74
+ or the summaries of `act`.
75
+
76
+ - `agent.assert` examines the screen now. A false statement fails with
77
+ `ASSERTION_FAILED`. Not sufficient evidence fails with
78
+ `ASSERTION_INCONCLUSIVE`.
79
+ - `agent.waitFor` waits until a condition is true. It calls the model only
80
+ when the screen changes.
81
+ - `agent.extract` reads data from the screen into a
82
+ [Standard Schema](https://standardschema.dev) type, for example Zod.
83
+
84
+ ```ts
85
+ import { expect } from 'e2e';
86
+ import { z } from 'zod';
87
+
88
+ await agent.assert('the dashboard shows a trial badge');
89
+ await agent.waitFor('a download link appears', { timeout: 120_000 });
90
+
91
+ const data = await agent.extract('every todo title', {
92
+ schema: z.object({ titles: z.array(z.string()) }),
93
+ });
94
+ expect(data.titles).toContain('Buy milk');
95
+ ```
96
+
97
+ For text that the model wrote, examine the meaning, not the full sentence.
98
+ When the exact text is important, use a [locator](#locators).
99
+
100
+ ### Judging pixels
101
+
102
+ Judgments read a redacted text snapshot of the screen. For a visual check,
103
+ such as a chart or a layout, add `vision: true`. To judge only the
104
+ screenshot, use `vision: 'only'`.
105
+
106
+ ```ts
107
+ await agent.assert('the chart trends upward', { vision: true });
108
+ ```
109
+
110
+ Images add input tokens. After a secret fill, the attempt sends no more
111
+ screenshots. See the [agent reference](/reference/agent#vision).
112
+
113
+ ## Locators
114
+
115
+ A locator finds a known control. Locator steps do not call a model. Use them
116
+ when you know the exact action or the exact value.
117
+
118
+ ```ts
119
+ // Role and name first, label second, test ID last.
120
+ await screen.getByRole('button', 'Sign in').tap();
121
+ await screen.getByLabel('Email').fill('ada@example.test');
122
+
123
+ // Scope into a row.
124
+ const row = screen.getByRole('listitem').filter({ hasText: 'Design review' });
125
+ await row.getByRole('button', 'Archive').tap();
126
+
127
+ // Gestures and coordinates.
128
+ await screen.swipe({ direction: 'up' });
129
+ await screen.getByRole('image', 'Seat map').tap({ position: { x: 120, y: 48 } });
130
+
131
+ // Assertions retry until they pass or time out.
132
+ await expect(screen.getByRole('status')).toHaveText('2 remaining');
133
+ ```
134
+
135
+ - An action, a read, or a single-node assertion fails when its query finds
136
+ more than one node. Use `.filter()`, `.first()`, or `.nth()` to find one
137
+ node.
138
+ - A direct read, such as `textContent()`, does not retry.
139
+ - Put an `expect` after an `agent.act` goal. It examines the exact result.
140
+ The [replay cache](/cache) records a goal only after a later check passes.
141
+
142
+ See the [screen reference](/reference/screen) and the
143
+ [expect reference](/reference/expect#locator-matchers). For routes, cookies,
144
+ dialogs, and downloads, see [Working with the browser](/browser).
145
+
146
+ ## Structure
147
+
148
+ ```ts
149
+ import { beforeEach, describe, test } from 'e2e';
150
+
151
+ describe('billing', { tags: ['billing'] }, () => {
152
+ beforeEach(async ({ app }) => { await app.open('/billing'); });
153
+
154
+ test('upgrades', { retries: 2, timeout: 60_000 }, async ({ agent }) => {
155
+ await agent.act('upgrade to Pro');
156
+ });
157
+ });
158
+
159
+ test('needs two organizations', async ({ app }) => {
160
+ test.skip((await countOrganizations()) < 2, 'nothing to switch to');
161
+ await app.open('/organizations');
162
+ });
163
+ ```
164
+
165
+ - Each test runs in its own attempt. Prepare its data in fixtures or hooks.
166
+ - `test.skip(condition, reason)` stops the test when the condition is true.
167
+ The report shows the test as `skipped`.
168
+ - For a flow across many tests, set `{ serial: true }` on the group. The
169
+ tests run in sequence on one worker, and retry together.
170
+ - For setup and cleanup, use `test.extend`. See
171
+ [Your own fixtures](/reference/test#your-own-fixtures).
172
+ - Import `test` from `@e2e-dev/web` or `@e2e-dev/mobile` to have the
173
+ `browser` or `device` fixture typed.
174
+
175
+ The [test reference](/reference/test) shows all options.
176
+
177
+ ## Run one test
178
+
179
+ <CodeGroup>
180
+ ```bash npm
181
+ npx e2e run tests/signup.e2e.ts:12 # the test at line 12
182
+ npx e2e run --grep checkout # tests with a matching title
183
+ npx e2e run --target web # one target
184
+ ```
185
+
186
+ ```bash pnpm
187
+ pnpm exec e2e run tests/signup.e2e.ts:12 # the test at line 12
188
+ pnpm exec e2e run --grep checkout # tests with a matching title
189
+ pnpm exec e2e run --target web # one target
190
+ ```
191
+
192
+ ```bash bun
193
+ bunx e2e run tests/signup.e2e.ts:12 # the test at line 12
194
+ bunx e2e run --grep checkout # tests with a matching title
195
+ bunx e2e run --target web # one target
196
+ ```
197
+ </CodeGroup>
198
+
199
+ The [CLI reference](/reference/cli#e2e-run) shows all flags.
200
+
201
+ <CardGroup cols={2}>
202
+ <Card title="Caching" icon="box-archive" href="/cache">
203
+ Replay passed goals with no model calls.
204
+ </Card>
205
+ <Card title="How agent steps work" icon="list-check" href="/agent-steps">
206
+ Verdicts, exit codes, and what the model sees.
207
+ </Card>
208
+ </CardGroup>