e2e 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/.build.json +1 -1
- package/dist/agent/act.js +7 -4
- package/dist/agent/action-dispatcher.js +4 -5
- package/dist/agent/actions.d.ts +32 -23
- package/dist/agent/actions.js +38 -24
- package/dist/agent/ai-sdk.js +5 -4
- package/dist/agent/error.js +3 -3
- package/dist/agent/invocation.js +3 -4
- package/dist/agent/model/sdk.js +2 -0
- package/dist/agent/model/tool-choice.d.ts +9 -0
- package/dist/agent/model/tool-choice.js +14 -0
- package/dist/agent/observation-feed.d.ts +5 -6
- package/dist/agent/observation-feed.js +8 -11
- package/dist/agent/observation.d.ts +40 -11
- package/dist/agent/observation.js +109 -59
- package/dist/agent/phases.d.ts +7 -1
- package/dist/agent/phases.js +12 -3
- package/dist/agent/point-tap.d.ts +8 -8
- package/dist/agent/point-tap.js +1 -1
- package/dist/agent/prompts.js +3 -0
- package/dist/agent/replay.d.ts +19 -8
- package/dist/agent/replay.js +29 -29
- package/dist/agent/scroll-target.d.ts +2 -3
- package/dist/agent/step-cache.d.ts +43 -15
- package/dist/agent/step-cache.js +156 -67
- package/dist/agent/tool-loop.js +94 -5
- package/dist/cache/anchors.d.ts +72 -23
- package/dist/cache/anchors.js +233 -60
- package/dist/cache/context.d.ts +25 -14
- package/dist/cache/context.js +50 -8
- package/dist/cache/decide.d.ts +2 -2
- package/dist/cache/decide.js +4 -6
- package/dist/cache/identity.d.ts +18 -4
- package/dist/cache/identity.js +9 -4
- package/dist/cache/recorder.d.ts +15 -1
- package/dist/cache/recorder.js +36 -23
- package/dist/cache/rekeyed.d.ts +46 -0
- package/dist/cache/rekeyed.js +89 -0
- package/dist/cache/relocate.d.ts +14 -14
- package/dist/cache/relocate.js +33 -25
- package/dist/cache/route.d.ts +32 -33
- package/dist/cache/route.js +94 -50
- package/dist/cache/store.d.ts +6 -0
- package/dist/cache/store.js +21 -1
- package/dist/cache/trace.d.ts +53 -14
- package/dist/cache/trace.js +76 -30
- package/dist/cli/bin.js +2 -2
- package/dist/cli/index.js +43 -32
- package/dist/cli/init/engines.d.ts +1 -3
- package/dist/cli/init/engines.js +2 -2
- package/dist/cli/init/gateways.d.ts +1 -1
- package/dist/cli/init/gateways.js +11 -1
- package/dist/cli/init/mcp-config.d.ts +2 -1
- package/dist/cli/init/mcp-config.js +8 -3
- package/dist/cli/init/sibling-versions.json +1 -1
- package/dist/cli/init/versions.d.ts +0 -8
- package/dist/cli/init/versions.js +1 -10
- package/dist/cli/mcp.d.ts +3 -2
- package/dist/cli/mcp.js +52 -16
- package/dist/cli/run-output.d.ts +17 -0
- package/dist/cli/run-output.js +45 -0
- package/dist/collect/collect.js +8 -4
- package/dist/collect/registry.js +27 -16
- package/dist/config/compiled-files.d.ts +36 -0
- package/dist/config/compiled-files.js +50 -0
- package/dist/config/diagnose.js +21 -19
- package/dist/config/esm-hooks.d.ts +39 -44
- package/dist/config/esm-hooks.js +238 -63
- package/dist/config/load.d.ts +1 -7
- package/dist/config/load.js +23 -66
- package/dist/config/register.d.ts +8 -0
- package/dist/config/register.js +9 -0
- package/dist/config/resolve.d.ts +1 -0
- package/dist/config/resolve.js +33 -1
- package/dist/config/specifiers.d.ts +19 -0
- package/dist/config/specifiers.js +33 -0
- package/dist/config/tsconfig.d.ts +56 -0
- package/dist/config/tsconfig.js +115 -0
- package/dist/config/typescript.d.ts +19 -0
- package/dist/config/typescript.js +253 -0
- package/dist/engine/contract.d.ts +1 -0
- package/dist/engine/index.d.ts +4 -2
- package/dist/engine/session.js +2 -2
- package/dist/expect/async.js +4 -2
- package/dist/expect/poll-scope.d.ts +52 -0
- package/dist/expect/poll-scope.js +113 -0
- package/dist/expect/poll.js +33 -6
- package/dist/expect/values.js +11 -4
- package/dist/internal/abandoned.d.ts +11 -0
- package/dist/internal/abandoned.js +23 -0
- package/dist/internal/brands.d.ts +2 -0
- package/dist/internal/brands.js +2 -0
- package/dist/internal/client-identity.d.ts +9 -0
- package/dist/internal/client-identity.js +14 -0
- package/dist/internal/errors.d.ts +10 -3
- package/dist/internal/errors.js +23 -8
- package/dist/internal/node-version.d.ts +16 -7
- package/dist/internal/node-version.js +50 -15
- package/dist/internal/package-manager.js +1 -1
- package/dist/internal/realm-slot.d.ts +5 -4
- package/dist/internal/redact.d.ts +9 -4
- package/dist/internal/redact.js +120 -23
- package/dist/internal/time.d.ts +26 -4
- package/dist/internal/time.js +58 -14
- package/dist/internal/urls.d.ts +9 -5
- package/dist/internal/urls.js +19 -10
- package/dist/locator/engine.d.ts +12 -2
- package/dist/locator/engine.js +55 -14
- package/dist/locator/screen.js +59 -9
- package/dist/mcp/catalog.d.ts +3 -2
- package/dist/mcp/catalog.js +6 -6
- package/dist/mcp/output.d.ts +59 -0
- package/dist/mcp/output.js +129 -0
- package/dist/mcp/server.d.ts +4 -0
- package/dist/mcp/server.js +9 -12
- package/dist/mcp/session.d.ts +8 -1
- package/dist/mcp/session.js +56 -15
- package/dist/mcp/tools.js +7 -5
- package/dist/oauth/chatgpt.js +3 -23
- package/dist/oauth/cli.js +1 -0
- package/dist/oauth/copilot.d.ts +5 -4
- package/dist/oauth/copilot.js +80 -7
- package/dist/oauth/grok.js +2 -1
- package/dist/oauth/models.js +2 -1
- package/dist/oauth/opencode-console.d.ts +10 -0
- package/dist/oauth/opencode-console.js +133 -0
- package/dist/oauth/providers/github-copilot.d.ts +12 -0
- package/dist/oauth/providers/github-copilot.js +77 -23
- package/dist/oauth/providers/opencode-console.d.ts +31 -0
- package/dist/oauth/providers/opencode-console.js +148 -0
- package/dist/oauth/providers.d.ts +2 -6
- package/dist/oauth/providers.js +3 -7
- package/dist/oauth/responses.d.ts +9 -0
- package/dist/oauth/responses.js +23 -0
- package/dist/report/build.d.ts +26 -0
- package/dist/report/build.js +16 -8
- package/dist/report/failure-text.d.ts +6 -1
- package/dist/report/failure-text.js +14 -4
- package/dist/report/format.d.ts +10 -5
- package/dist/report/format.js +12 -5
- package/dist/report/junit.js +11 -12
- package/dist/report/list.d.ts +12 -0
- package/dist/report/list.js +61 -24
- package/dist/report/markdown.d.ts +1 -1
- package/dist/report/markdown.js +37 -14
- package/dist/report/repeats.d.ts +6 -1
- package/dist/report/repeats.js +12 -3
- package/dist/run/artifacts.d.ts +22 -4
- package/dist/run/artifacts.js +71 -3
- package/dist/run/execute.d.ts +34 -5
- package/dist/run/execute.js +93 -44
- package/dist/run/extended-fixtures.js +6 -2
- package/dist/run/failure-evidence.js +10 -4
- package/dist/run/fixtures.js +6 -8
- package/dist/run/in-process.d.ts +2 -0
- package/dist/run/in-process.js +52 -10
- package/dist/run/last-run.d.ts +42 -8
- package/dist/run/last-run.js +121 -14
- package/dist/run/process-output.d.ts +73 -0
- package/dist/run/process-output.js +130 -0
- package/dist/run/provision.d.ts +2 -0
- package/dist/run/provision.js +1 -0
- package/dist/run/realm.d.ts +4 -1
- package/dist/run/realm.js +20 -6
- package/dist/run/records.d.ts +20 -4
- package/dist/run/records.js +26 -0
- package/dist/run/retry.d.ts +10 -1
- package/dist/run/retry.js +30 -12
- package/dist/run/runner.d.ts +9 -1
- package/dist/run/runner.js +101 -39
- package/dist/run/scheduler.js +255 -19
- package/dist/run/secrecy.d.ts +22 -19
- package/dist/run/secrecy.js +22 -18
- package/dist/run/serial.d.ts +75 -2
- package/dist/run/serial.js +243 -59
- package/dist/run/standalone.js +1 -1
- package/dist/run/steps.d.ts +0 -6
- package/dist/run/steps.js +3 -20
- package/dist/run/trace-redaction.d.ts +5 -6
- package/dist/run/trace-redaction.js +5 -6
- package/dist/run/worker/entry.js +40 -18
- package/dist/run/worker/handle.js +10 -5
- package/dist/run/worker/protocol.d.ts +85 -3
- package/dist/run/worker/session.d.ts +2 -0
- package/dist/run/worker/session.js +29 -0
- package/dist/telemetry/environment.d.ts +2 -1
- package/dist/telemetry/environment.js +12 -2
- package/dist/telemetry/events.js +1 -0
- package/dist/telemetry/project.d.ts +5 -6
- package/dist/telemetry/project.js +6 -11
- package/dist/telemetry/store.d.ts +5 -5
- package/dist/telemetry/store.js +5 -7
- package/dist/telemetry/telemetry.d.ts +2 -3
- package/dist/telemetry/telemetry.js +2 -4
- package/dist/types.d.ts +15 -6
- package/docs/agent-steps.mdx +1 -0
- package/docs/agents.mdx +7 -3
- package/docs/api-testing.mdx +2 -1
- package/docs/authentication.mdx +6 -3
- package/docs/browser.mdx +49 -12
- package/docs/bug-bash.mdx +2 -1
- package/docs/cache.mdx +66 -19
- package/docs/ci.mdx +14 -9
- package/docs/coding-agents.mdx +2 -1
- package/docs/core-concepts.mdx +208 -0
- package/docs/debugging.mdx +25 -12
- package/docs/decision-models.mdx +142 -0
- package/docs/example-projects.mdx +32 -0
- package/docs/executors.mdx +7 -2
- package/docs/explore.mdx +1 -0
- package/docs/github.mdx +14 -5
- package/docs/index.mdx +9 -5
- package/docs/integrations/eas.mdx +1 -0
- package/docs/integrations/index.mdx +1 -0
- package/docs/integrations/kernel.mdx +3 -2
- package/docs/migrate/cypress.mdx +26 -3
- package/docs/migrate/detox.mdx +24 -1
- package/docs/migrate/maestro.mdx +28 -4
- package/docs/migrate/playwright.mdx +44 -18
- package/docs/migrate/selenium.mdx +27 -3
- package/docs/mobile-ci.mdx +6 -5
- package/docs/mobile.mdx +96 -8
- package/docs/models.mdx +60 -9
- package/docs/quickstart.mdx +28 -16
- package/docs/reference/agent.mdx +13 -3
- package/docs/reference/app.mdx +2 -1
- package/docs/reference/cli.mdx +41 -18
- package/docs/reference/config.mdx +67 -13
- package/docs/reference/engine.mdx +25 -13
- package/docs/reference/environment.mdx +10 -2
- package/docs/reference/errors.mdx +13 -10
- package/docs/reference/expect.mdx +36 -13
- package/docs/reference/mcp.mdx +21 -8
- package/docs/reference/mobile.mdx +24 -13
- package/docs/reference/reporters.mdx +9 -2
- package/docs/reference/screen.mdx +49 -16
- package/docs/reference/test.mdx +32 -5
- package/docs/reference/web.mdx +133 -23
- package/docs/security.mdx +58 -42
- package/docs/subscriptions.mdx +92 -12
- package/docs/telemetry.mdx +9 -6
- package/docs/tools.mdx +1 -0
- package/docs/{starting-your-app.mdx → web.mdx} +157 -11
- package/docs/writing-an-engine.mdx +2 -1
- package/package.json +25 -8
- package/schema/report-v1.schema.json +14 -1
- package/skills/e2e/SKILL.md +5 -3
- package/skills/e2e/references/agent.md +33 -11
- package/skills/e2e/references/debugging.md +4 -4
- package/skills/e2e/references/mcp.md +8 -3
- package/skills/e2e/references/running.md +15 -9
- package/skills/e2e/references/setup.md +45 -16
- package/skills/e2e/references/writing-tests.md +35 -18
- package/docs/viewports.mdx +0 -146
- package/docs/writing-tests.mdx +0 -256
|
@@ -11,7 +11,7 @@ npx e2e guide [topic] # print this skill; topics: setup, writing-te
|
|
|
11
11
|
# running, explore, debugging, mcp, bug-bash
|
|
12
12
|
npx e2e cache ls|clear|stats # inspect or empty the replay cache
|
|
13
13
|
npx e2e login|logout|models [provider] # e2e/oauth subscription logins: openai,
|
|
14
|
-
# github-copilot, spacexai
|
|
14
|
+
# github-copilot, opencode-console, spacexai
|
|
15
15
|
npx e2e mcp [--target <name>] # MCP server for a coding agent (topic mcp)
|
|
16
16
|
npx e2e feedback -m <text> [opts] # report a problem with e2e itself
|
|
17
17
|
npx e2e telemetry [disable|enable] # anonymous usage telemetry: status or switch
|
|
@@ -27,7 +27,7 @@ npx e2e telemetry [disable|enable] # anonymous usage telemetry: status or switch
|
|
|
27
27
|
| `--tag <tags>` | Any of the tags, comma-separated or repeated; all of them with `--tag-mode all`. An empty `--target`, `--tag`, or `--agent` value is a usage error, exit 2. |
|
|
28
28
|
| `--exclude-tag <tags>` | Drop tests carrying any of these tags, however selected. |
|
|
29
29
|
| `--grep <pattern>`, `--grep-invert <pattern>` | Keep, or drop, tests whose title (describe titles and test title joined by spaces, `checkout pays`; not file or tags) matches a regular expression. Bare pattern or `'/pattern/i'`; repeat for alternatives. |
|
|
30
|
-
| `--last-failed` | The tests the previous run (`<output>/report.json`) did not pass, plus every test in a failed `beforeAll` or `afterAll` scope. No report is `NO_LAST_RUN`, exit 2. |
|
|
30
|
+
| `--last-failed` | The tests the previous run (`<output>/report.json`) did not pass or never ran (`--max-failures` included), plus every test in a failed `beforeAll` or `afterAll` scope. A test another filter leaves out stays owed (`run.carried` in the report) until a rerun runs it. No report is `NO_LAST_RUN`, exit 2. |
|
|
31
31
|
| `--shard <index/total>` | One contiguous slice of the selected tests (`--shard 2/3`), cut after every other filter; serial groups stay whole, each shard brings its own setup tests. |
|
|
32
32
|
| `--headed` | Visible browser or simulator when the engine supports it. |
|
|
33
33
|
| `--agent <names>` | Run unpinned tests as these `agents.<name>` entries (default `agents.default`), comma-separated or repeated; several names run each such test once per agent. |
|
|
@@ -67,6 +67,7 @@ npx e2e list tests/signup.e2e.ts --tag smoke --reporter json
|
|
|
67
67
|
With a `package.json` script `"test:e2e": "e2e run"`, pnpm forwards `--`
|
|
68
68
|
literally: `pnpm test:e2e -- --headed` reaches e2e as `run -- --headed` and
|
|
69
69
|
exits 2. Write `pnpm test:e2e --headed` or `pnpm exec e2e run --headed`.
|
|
70
|
+
With npm, use `npm exec -- e2e run --headed` to pass flags to e2e.
|
|
70
71
|
|
|
71
72
|
## The replay cache
|
|
72
73
|
|
|
@@ -87,7 +88,8 @@ its own tools.
|
|
|
87
88
|
`summary.md`, `failures/`, `ai-trace.json`, `sessions/`, and `artifacts/`
|
|
88
89
|
(screenshots, Playwright traces, videos, `--debug` transcripts, downloads).
|
|
89
90
|
`artifacts/` is cleared once a run's tests start; a run stopping before
|
|
90
|
-
leaves the last run's files
|
|
91
|
+
leaves the last run's files, and a `--last-failed` rerun keeps the files
|
|
92
|
+
the report it reruns names and writes its own under `artifacts/rerun-<n>/`. The report records every artifact path, a
|
|
91
93
|
hosted service's video by URL.
|
|
92
94
|
|
|
93
95
|
- `list` (default): setup steps, one line per file and target, a `Failed
|
|
@@ -102,11 +104,14 @@ hosted service's video by URL.
|
|
|
102
104
|
and `run.results[]`, one per test and target: `titlePath`, `file`,
|
|
103
105
|
`source`, `tags` (`[]` when none), `agent`, `repeat` (0 unless
|
|
104
106
|
`--repeat-each`), `selected`, `status`, `attempts[]` of `steps[]`,
|
|
105
|
-
`artifacts[]`, `error`.
|
|
107
|
+
`artifacts[]`, `error`. `run.summary` counts the selected results:
|
|
108
|
+
`passed`, `failed` (timed out included), `interrupted` (stopped by the
|
|
109
|
+
run: a signal, `--max-failures`, a run-level error; not a failure), `flaky`, and `skipped` add up
|
|
110
|
+
to `selected`; `discovered - selected` were left out.
|
|
106
111
|
- `junit`: `junit.xml` for CI summaries; `--reporter list,junit` keeps the
|
|
107
112
|
terminal output.
|
|
108
113
|
- `markdown` (`--reporter list,markdown`): `summary.md` plus one page per
|
|
109
|
-
failed or flaky test under `failures
|
|
114
|
+
failed or flaky test under `failures/` (an interrupted test gets none). The summary holds counts and
|
|
110
115
|
spend, a block per failed test (error, facts, failing step, whether every
|
|
111
116
|
attempt failed alike, last model turns, screen location and closest
|
|
112
117
|
nodes, the line to look at, evidence paths), the flaky tests folded
|
|
@@ -142,8 +147,9 @@ interrupted workers keeps the failures' or the error's code. Exit 2 is
|
|
|
142
147
|
deterministic, never retry it; only exit 3 is worth a job-level retry.
|
|
143
148
|
|
|
144
149
|
The first SIGINT or SIGTERM interrupts and writes the report if any test
|
|
145
|
-
had started; the second forces engine teardown
|
|
146
|
-
|
|
150
|
+
had started; the second forces engine teardown and still writes the
|
|
151
|
+
report, plus `junit.xml` and `summary.md` when those reporters are on; the
|
|
152
|
+
third kills the app process groups and exits 130 at once.
|
|
147
153
|
|
|
148
154
|
## Continuous integration
|
|
149
155
|
|
|
@@ -171,10 +177,10 @@ jobs:
|
|
|
171
177
|
- uses: pnpm/action-setup@9fd676a19091d4595eefd76e4bd31c97133911f1 # v4.2.0
|
|
172
178
|
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
173
179
|
with:
|
|
174
|
-
node-version: 26 #
|
|
180
|
+
node-version: 26 # Node >= 24.8, or >= 22.22.3 on 22
|
|
175
181
|
cache: pnpm
|
|
176
182
|
- run: pnpm install --frozen-lockfile
|
|
177
|
-
- run:
|
|
183
|
+
- run: pnpm exec e2e-web install chromium --with-deps
|
|
178
184
|
- run: npx e2e run --reporter list,junit
|
|
179
185
|
env:
|
|
180
186
|
AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
|
|
@@ -2,17 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
## Requirements
|
|
4
4
|
|
|
5
|
-
- Node.js 22.
|
|
5
|
+
- Node.js 24.8 or newer, or 22.22.3 or newer on Node.js 22.
|
|
6
6
|
- ES modules: `.ts` config, tests, helpers, and workspace packages exporting
|
|
7
7
|
`.ts` source load as ESM regardless of the nearest `package.json` `type`
|
|
8
8
|
(CommonJS packages need no change); never `require` or `module.exports`.
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
browsers download on first boot; in CI run
|
|
14
|
-
--with-deps
|
|
15
|
-
|
|
9
|
+
Imports follow TypeScript: `./x.js` or `./x` loads `x.ts`, and the nearest
|
|
10
|
+
`tsconfig.json` `paths` and `baseUrl` apply. Decorators need
|
|
11
|
+
`experimentalDecorators`; CommonJS TypeScript goes in `.cts`.
|
|
12
|
+
- Browser tests: `@e2e-dev/web`, pinning `playwright-core` exactly; do not
|
|
13
|
+
add `playwright` for it. Missing browsers download on first boot; in CI run
|
|
14
|
+
`npx @e2e-dev/web install chromium --with-deps` (pnpm: `pnpm exec e2e-web
|
|
15
|
+
install chromium --with-deps`). Mobile tests: `@e2e-dev/mobile`, pinning
|
|
16
|
+
`agent-device` exactly; each pin moves with its engine release.
|
|
16
17
|
|
|
17
18
|
## Scaffold
|
|
18
19
|
|
|
@@ -41,7 +42,7 @@ MCP entries.
|
|
|
41
42
|
Without the wizard (`ai`, Vercel AI SDK v7, only for `agent.*` steps):
|
|
42
43
|
|
|
43
44
|
```bash
|
|
44
|
-
npm install --save-dev e2e @e2e-dev/web
|
|
45
|
+
npm install --save-dev e2e @e2e-dev/web ai@^7
|
|
45
46
|
```
|
|
46
47
|
|
|
47
48
|
## Subscriptions and API keys
|
|
@@ -53,6 +54,7 @@ key, or a local endpoint. Authenticate:
|
|
|
53
54
|
| --- | --- |
|
|
54
55
|
| ChatGPT Plus or Pro | `npx e2e login openai` |
|
|
55
56
|
| GitHub Copilot | `npx e2e login github-copilot` (GitHub CLI signed in, or your own `--client-id`) |
|
|
57
|
+
| OpenCode Console (OpenCode Zen and OpenCode Go) | `npx e2e login opencode-console` (approve the device code, pick the workspace) |
|
|
56
58
|
| SuperGrok or X Premium+ | `npx e2e login spacexai` |
|
|
57
59
|
| Vercel AI Gateway | Set `AI_GATEWAY_API_KEY`, or sign in to the Vercel CLI and `npx vercel link`; without the key `gateway()` uses a Vercel OIDC token |
|
|
58
60
|
| OpenRouter | Set `OPENROUTER_API_KEY` |
|
|
@@ -63,6 +65,22 @@ set `model: chatgpt('gpt-6-luna')` from `e2e/oauth/chatgpt`, run `npx e2e
|
|
|
63
65
|
login openai`. `npx e2e models` lists the ids each login serves. Use API keys
|
|
64
66
|
in CI.
|
|
65
67
|
|
|
68
|
+
Switching to Copilot: install `ai`, `@ai-sdk/openai-compatible`, and
|
|
69
|
+
`@ai-sdk/openai`, set `model: copilot('<id>')` from `e2e/oauth/copilot`, run
|
|
70
|
+
`npx e2e login github-copilot`. `copilot()` calls a model over chat completions, or
|
|
71
|
+
over Copilot's Responses API when the plan serves that model only there, choosing
|
|
72
|
+
per model from the plan's listing. `npx e2e models github-copilot` marks the models
|
|
73
|
+
it cannot call at all: those served only over an API `copilot()` does not speak, and
|
|
74
|
+
those the plan has not enabled.
|
|
75
|
+
|
|
76
|
+
Switching to OpenCode Console: install `ai`, `@ai-sdk/openai-compatible`,
|
|
77
|
+
`@ai-sdk/openai`, `@ai-sdk/anthropic`, and `@ai-sdk/google`, set
|
|
78
|
+
`model: opencodeConsole('<id>')` from `e2e/oauth/opencode-console`, run
|
|
79
|
+
`npx e2e login opencode-console`. A bare id is an OpenCode Zen model; a `go/` id
|
|
80
|
+
(`go/deepseek-v4.1-flash`) is an OpenCode Go model and needs the workspace's Go
|
|
81
|
+
subscription. `npx e2e models opencode-console` lists the ids, tagged Zen or Go. In CI,
|
|
82
|
+
set a Console service account key as `OPENCODE_API_KEY`.
|
|
83
|
+
|
|
66
84
|
## The config
|
|
67
85
|
|
|
68
86
|
`e2e.config.ts` sits at the project root and default-exports an object literal
|
|
@@ -168,6 +186,8 @@ start a script that brings them up and serves the app.
|
|
|
168
186
|
| `headers` | Sent to the app's site only (Vercel's `x-vercel-protection-bypass`, ngrok's `ngrok-skip-browser-warning`), `agent.act` included; disables the browser HTTP cache and service workers. |
|
|
169
187
|
| `basicAuth` | `{ username, password }` for a `401` challenge; `password` may be `secrets.get('name')`, resolved per attempt and redacted like any secret, the base64 `Authorization` credential too. |
|
|
170
188
|
| `userAgent` | The `User-Agent` every attempt sends and `navigator.userAgent` reports. |
|
|
189
|
+
| `locale`, `timezoneId` | The language (`'de-DE'`: `navigator.language`, `Intl`, `Accept-Language`) and IANA time zone (`'Europe/Berlin'`) every attempt runs in. |
|
|
190
|
+
| `initScripts` | Scripts every document runs before the page's own: source, `{ path }`, or a function with no closures. |
|
|
171
191
|
| `testIdAttribute` | What `getByTestId` reads; default `data-testid`. |
|
|
172
192
|
| `screencast` | `{ size?, quality? }` for the engine's own video: frame size (default the viewport's), JPEG quality 0 to 100. |
|
|
173
193
|
|
|
@@ -176,7 +196,7 @@ start a script that brings them up and serves the app.
|
|
|
176
196
|
the handle (a reference, not the value) in `app.command.env`, a template
|
|
177
197
|
literal, or `context`; read those from `process.env`.
|
|
178
198
|
- `reconnectEndpoint` or an attempt-scoped provider rides one persistent
|
|
179
|
-
context without `headers`, `basicAuth`, `userAgent`, `app.clearState()`, or
|
|
199
|
+
context without `headers`, `basicAuth`, `userAgent`, `locale`, `timezoneId`, `app.clearState()`, or
|
|
180
200
|
session state. Recovery never repeats a dispatched operation; exhausting
|
|
181
201
|
the budget is `OPERATION_TIMEOUT`. Without `reconnectEndpoint` a dropped
|
|
182
202
|
connection is reacquired at the next attempt.
|
|
@@ -246,16 +266,20 @@ For an app started elsewhere, point `app.url` at it, literally or via
|
|
|
246
266
|
| --- | --- |
|
|
247
267
|
| `AI_GATEWAY_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, ... | Read by provider packages, not the runner. |
|
|
248
268
|
| `E2E_USER_<NAME>_USERNAME`, `E2E_USER_<NAME>_PASSWORD` | Override `credentials.<name>`; `<NAME>` is the name uppercased, other characters `_`. |
|
|
249
|
-
| `E2E_SECRET_<NAME>` | Overrides `secrets.<name>`, same rule. |
|
|
269
|
+
| `E2E_SECRET_<NAME>` | Overrides `secrets.<name>`, same rule. Two entries of one namespace mapping to one variable are `INVALID_CONFIG`. |
|
|
250
270
|
| `CI` | CI defaults; list in topic `running`. |
|
|
251
271
|
| `E2E_TELEMETRY_DISABLED`, `DO_NOT_TRACK` | Disable anonymous telemetry, as does `e2e telemetry disable`; `E2E_TELEMETRY_DEBUG=1` prints events instead of sending. |
|
|
252
272
|
|
|
253
273
|
## Mobile targets
|
|
254
274
|
|
|
255
|
-
`@e2e-dev/mobile` drives iOS simulators
|
|
256
|
-
[agent-device](https://github.com/callstack/agent-device);
|
|
257
|
-
simulator runtime or the Android SDK with an emulator
|
|
258
|
-
|
|
275
|
+
`@e2e-dev/mobile` drives iOS simulators, Android emulators, and connected
|
|
276
|
+
phones through [agent-device](https://github.com/callstack/agent-device);
|
|
277
|
+
needs Xcode with a simulator runtime or the Android SDK with an emulator (a
|
|
278
|
+
phone needs Xcode with Developer Mode and runner signing, or adb with USB
|
|
279
|
+
debugging authorized); run `npx agent-device doctor` once. Name a phone in `device` by the name
|
|
280
|
+
`npx agent-device devices` lists, not its UDID or serial. On an iPhone, device
|
|
281
|
+
settings (permissions, `clearState`, network, location, appearance,
|
|
282
|
+
biometrics, keychain) and the clipboard are simulator-only.
|
|
259
283
|
|
|
260
284
|
```ts
|
|
261
285
|
import type { E2EConfig } from 'e2e';
|
|
@@ -292,7 +316,7 @@ export default {
|
|
|
292
316
|
`app.command` (`npx expo start --port 8081`) with `readyUrl:
|
|
293
317
|
'http://localhost:8081/status'` starts it.
|
|
294
318
|
- One worker per device. No `device`: every booted simulator or emulator of
|
|
295
|
-
the platform is the pool, up to `workers` (none booted: agent-device boots
|
|
319
|
+
the platform, and every connected phone, is the pool, up to `workers` (none booted: agent-device boots
|
|
296
320
|
one); one `device`: one worker whatever `workers` says; a list (`device:
|
|
297
321
|
['iPhone 17', 'iPhone 17 Pro']`): an explicit pool. Devices boot in
|
|
298
322
|
`prepare`, before the run's clock. Two sessions on one device fight over
|
|
@@ -333,3 +357,8 @@ export default {
|
|
|
333
357
|
- `.gitignore` lists the `.e2e/` outputs (init adds them); drop the
|
|
334
358
|
`.e2e/cache/` line to commit replays.
|
|
335
359
|
- CI runs the whole suite, agent steps included, on PRs; see `running`.
|
|
360
|
+
|
|
361
|
+
### Existing MCP registration
|
|
362
|
+
|
|
363
|
+
Re-running `init` preserves the e2e MCP server's environment and client settings
|
|
364
|
+
while repairing its launch command. A current registration is left unchanged.
|
|
@@ -168,7 +168,9 @@ substring; a `RegExp` matches as written.
|
|
|
168
168
|
scoping under another locator. Any other `filter` key (`hasNot`,
|
|
169
169
|
`hasNotText`) or an empty `filter({})` is `INVALID_LOCATOR`.
|
|
170
170
|
- `visible: true` drops nodes the page hides (a closed drawer) before the
|
|
171
|
-
exactly-one rule.
|
|
171
|
+
exactly-one rule. Visibility is what renders, as `toBeVisible()` reads it:
|
|
172
|
+
an `aria-hidden` spinner that paints is visible, so `toBeHidden()` waits
|
|
173
|
+
for it to go.
|
|
172
174
|
- On the web, queries reach open shadow roots and closed roots attached with
|
|
173
175
|
`attachShadow`, not declarative closed roots. `browser.locator(css)`,
|
|
174
176
|
`frameLocator`, and `filter({ hasText })` stop at a closed root; query the
|
|
@@ -201,7 +203,7 @@ field and types one character per `delay`, plain string only (a `Secret` is
|
|
|
201
203
|
Coordinates are CSS pixels, for what the tree does not list: `tap({ position:
|
|
202
204
|
{ x, y } })` offsets from the node's top-left corner (with `modifiers` it is
|
|
203
205
|
`INVALID_ARGUMENT`), `screen.tapAt({ x, y })` taps a viewport point,
|
|
204
|
-
`screen.swipe({ from, to })` swipes along a path,
|
|
206
|
+
`screen.swipe({ from, to, duration? })` swipes along a path,
|
|
205
207
|
`screen.swipe({ direction, momentum? })` swipes the viewport. Prefer a
|
|
206
208
|
locator; a point moves with the layout.
|
|
207
209
|
|
|
@@ -215,7 +217,7 @@ text, both `[]` at zero. Text is the rendered text, whitespace collapsed: on
|
|
|
215
217
|
the web what `innerText` reads (`text-transform` applies, `display: none`
|
|
216
218
|
drops out, `<br>` is a space); `toHaveText` reads the same. `isChecked()` is
|
|
217
219
|
`false`, not an error, on a node with no checked state, so query checkable
|
|
218
|
-
controls by role. `waitFor({ state?: 'visible' | 'hidden', timeout? })` waits
|
|
220
|
+
controls by role. `waitFor({ state?: 'attached' | 'detached' | 'visible' | 'hidden', timeout? })` waits
|
|
219
221
|
within `actionTimeout`, else `LOCATOR_NOT_FOUND`. For a value that has to
|
|
220
222
|
settle use `expect`, not a read. Reading a password field's value or
|
|
221
223
|
attributes is `POLICY_DENIED`, as is `toHaveAttribute` on one, negated too.
|
|
@@ -224,13 +226,20 @@ attributes is `POLICY_DENIED`, as is `toHaveAttribute` on one, negated too.
|
|
|
224
226
|
|
|
225
227
|
`expect(locator)` polls up to `config.assertionTimeout` (5 s) or
|
|
226
228
|
`{ timeout }`; `.not` inverts and passes once the negation has held 1 s
|
|
227
|
-
continuously,
|
|
229
|
+
continuously, or for the whole `timeout` when that is shorter, so it never
|
|
230
|
+
returns sooner. `expect(value,
|
|
228
231
|
message?)` is synchronous.
|
|
229
232
|
`expect.poll(read, { timeout?, interval?, message? })` re-reads until a value
|
|
230
233
|
matcher passes (`assertionTimeout` and 100 ms by default, stopping with the
|
|
231
|
-
attempt); a throwing read keeps polling, and it is not a report step.
|
|
234
|
+
attempt); a throwing read keeps polling, and it is not a report step. A
|
|
235
|
+
poll that the body or a hook returns without awaiting is cancelled and fails
|
|
236
|
+
that phase with `STEP_NOT_AWAITED`.
|
|
232
237
|
`expect.soft(x)` keeps a failure instead of throwing; the attempt fails
|
|
233
|
-
after the body with every soft failure listed.
|
|
238
|
+
after the body with every soft failure listed. If the body calls `test.skip`,
|
|
239
|
+
the test stays skipped and the failures remain in `secondaryErrors`. The CLI
|
|
240
|
+
shows `Skipped After Failure`, also when a retry skips after an earlier
|
|
241
|
+
failure. Set `failOnSkippedFailure: true` in the config to fail the run with
|
|
242
|
+
exit code 1 in either case. The default is `false`; clean skips stay green.
|
|
234
243
|
`expect.any(Class)`, `expect.anything()`, `expect.objectContaining(obj)`,
|
|
235
244
|
`expect.arrayContaining(arr)`, `expect.stringContaining(s)`, and
|
|
236
245
|
`expect.stringMatching(s | RegExp)` stand in for values inside `toEqual`,
|
|
@@ -265,8 +274,8 @@ list or tests a RegExp. A failed matcher is `ASSERTION_FAILED`, exit code 1.
|
|
|
265
274
|
|
|
266
275
|
Playwright's `{ checked: false }`, `{ enabled: false }`, `{ visible: false }`,
|
|
267
276
|
and `{ attached: false }` flip their matchers, and `{ ignoreCase: true }`
|
|
268
|
-
works on `toHaveText`, `toContainText`, `toHaveAccessibleName`,
|
|
269
|
-
`toHaveAttribute(name, value)`. Any other option, `indeterminate` or
|
|
277
|
+
works on `toHaveText`, `toContainText`, `toHaveAccessibleName`,
|
|
278
|
+
`toHaveAttribute(name, value)`, and `expect(browser).toHaveURL`. Any other option, `indeterminate` or
|
|
270
279
|
`useInnerText` included, is `INVALID_ARGUMENT`, in JavaScript too.
|
|
271
280
|
|
|
272
281
|
## Sign-in sessions
|
|
@@ -323,7 +332,9 @@ credentials: {
|
|
|
323
332
|
A function is read at fill time and redacted only from that fill on.
|
|
324
333
|
- `E2E_USER_<NAME>_USERNAME` and `E2E_USER_<NAME>_PASSWORD` override either
|
|
325
334
|
field per run, even over a function; `<NAME>` is the credential name
|
|
326
|
-
uppercased, every character outside `[A-Z0-9]` as `_`.
|
|
335
|
+
uppercased, every character outside `[A-Z0-9]` as `_`. Two credentials (or
|
|
336
|
+
two secrets) mapping to one variable, `svc-a` and `svc_a`, are
|
|
337
|
+
`INVALID_CONFIG`.
|
|
327
338
|
- `credentials.user('admin').password` is a `Secret` with no plaintext
|
|
328
339
|
accessor, named `admin.password` to the agent and in reports; `secrets.get()`
|
|
329
340
|
never returns it (separate namespaces). Only `fill()` and `agent.act` params accept it, stringifying it
|
|
@@ -360,14 +371,20 @@ the app opens itself (`target="_blank"`, `window.open`) is not followed:
|
|
|
360
371
|
- `evaluate(fn | source, arg?)`: runs a function or source string in the
|
|
361
372
|
page, JSON in and out, no closures; a throw in the page is
|
|
362
373
|
`EVALUATE_FAILED`.
|
|
363
|
-
- `
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
`{ url
|
|
370
|
-
|
|
374
|
+
- `addInitScript(source | { path } | fn)`, `addInitScript(fn, arg)`: runs
|
|
375
|
+
before the page's own scripts; call it before `app.open`. `arg` is JSON.
|
|
376
|
+
- `route(pattern, handler)`, `unroute(pattern)`: intercept requests, newest
|
|
377
|
+
route first; `route.request` has `url`, `method`, `headers`, `postData`.
|
|
378
|
+
The handler calls exactly one of
|
|
379
|
+
`fulfill({ status?, headers?, contentType?, json | body | path })`,
|
|
380
|
+
`continue({ url?, method?, headers?, postData? })` (straight to the
|
|
381
|
+
network), `fallback()` (the route registered before it), or `abort()`;
|
|
382
|
+
none or two fails the next step with `ACTION_FAILED`, an unsupported
|
|
383
|
+
option with `INVALID_ARGUMENT`.
|
|
384
|
+
- `waitForResponse(pattern, { timeout? })`: resolves once the headers
|
|
385
|
+
arrive, with `{ url, status, headers, json(), text() }`; `text()` and
|
|
386
|
+
`json()` wait for the body (up to the action timeout) and reject with
|
|
387
|
+
`ACTION_FAILED` when it could not be read.
|
|
371
388
|
- `cookies()`, `setCookies([...])`: a target is an http(s) URL or a domain.
|
|
372
389
|
- `setViewport({ width, height })`: resize.
|
|
373
390
|
- `onDialog('accept' | 'dismiss' | handler)`: awaited; resolves to an async
|
|
@@ -396,5 +413,5 @@ A `route`, `unroute`, or `waitForResponse` pattern is a glob string or
|
|
|
396
413
|
plus value matchers, in a `test.extend` fixture that reads `browser.cookies()`
|
|
397
414
|
when the API needs the session.
|
|
398
415
|
- No sleeps or polling loops; a matcher with a longer `timeout` instead.
|
|
399
|
-
- `await` every step call
|
|
416
|
+
- `await` every step call and `expect.poll`, else `STEP_NOT_AWAITED` at the line of the call.
|
|
400
417
|
- Assert the fact a model produced with `toContain`, not its exact sentence.
|
package/docs/viewports.mdx
DELETED
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
title: Testing viewports
|
|
3
|
-
description: Run a suite at several sizes, resize inside a test, and know what the agent sees at each.
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
Set a viewport on the target to run a suite at that size. Use
|
|
7
|
-
`browser.setViewport()` to resize during a test. The agent uses the current size
|
|
8
|
-
and cannot resize it.
|
|
9
|
-
|
|
10
|
-
## One size per target
|
|
11
|
-
|
|
12
|
-
`viewport` sets the initial size of each attempt, in pixels. The default is
|
|
13
|
-
1280 by 720. Add one target for each size you want to test:
|
|
14
|
-
|
|
15
|
-
```ts title="e2e.config.ts"
|
|
16
|
-
import type { E2EConfig } from 'e2e';
|
|
17
|
-
import { web } from '@e2e-dev/web';
|
|
18
|
-
|
|
19
|
-
const app = { url: 'http://localhost:3000' };
|
|
20
|
-
|
|
21
|
-
export default {
|
|
22
|
-
targets: [
|
|
23
|
-
{ name: 'desktop', engine: web(), app },
|
|
24
|
-
{ name: 'tablet', engine: web({ viewport: { width: 820, height: 1180 } }), app },
|
|
25
|
-
{ name: 'phone', engine: web({ browser: 'webkit', viewport: { width: 390, height: 844 } }), app },
|
|
26
|
-
],
|
|
27
|
-
} satisfies E2EConfig;
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
Each test runs once per target, with a separate result labeled `desktop`,
|
|
31
|
-
`tablet`, or `phone`. Run one size with `npx e2e run --target phone`, or list
|
|
32
|
-
the selected test-target pairs with `npx e2e list`.
|
|
33
|
-
|
|
34
|
-
To have the runner start one dev server for every size, give every target
|
|
35
|
-
the same `app.command`: targets declaring one command share one process; see
|
|
36
|
-
[several targets](/starting-your-app#several-targets). Every retry starts at
|
|
37
|
-
the target's configured size.
|
|
38
|
-
|
|
39
|
-
## Fill the window
|
|
40
|
-
|
|
41
|
-
`viewport: null` sets no size, so the page fills the browser window. The
|
|
42
|
-
agent, screenshots, and videos use that size. Use it in a headed run, or
|
|
43
|
-
with a hosted browser whose live view would show a fixed 1280 by 720 page
|
|
44
|
-
in one corner:
|
|
45
|
-
|
|
46
|
-
```ts title="e2e.config.ts"
|
|
47
|
-
{ name: 'window', engine: web({ viewport: null }), app }
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
The page takes the window's size, minus the toolbar in a headed browser. A
|
|
51
|
-
local headed launch opens a small window; a hosted browser uses the window
|
|
52
|
-
size you requested from the service. `browser.setViewport` still works.
|
|
53
|
-
|
|
54
|
-
## Resize inside a test
|
|
55
|
-
|
|
56
|
-
`browser.setViewport` resizes the page for the rest of the attempt, through
|
|
57
|
-
`app.restart()` and `app.clearState()`:
|
|
58
|
-
|
|
59
|
-
```ts
|
|
60
|
-
import { test } from '@e2e-dev/web';
|
|
61
|
-
import { expect } from 'e2e';
|
|
62
|
-
|
|
63
|
-
test('the menu collapses on a narrow screen', async ({ app, screen, browser }) => {
|
|
64
|
-
await app.open('/');
|
|
65
|
-
await expect(screen.getByRole('navigation')).toBeVisible();
|
|
66
|
-
|
|
67
|
-
await browser.setViewport({ width: 390, height: 844 });
|
|
68
|
-
await expect(screen.getByRole('button', 'Menu')).toBeVisible();
|
|
69
|
-
await expect(screen.getByRole('navigation')).toBeHidden();
|
|
70
|
-
});
|
|
71
|
-
```
|
|
72
|
-
|
|
73
|
-
Use this when one test needs to check a layout before and after resizing.
|
|
74
|
-
For a whole suite, configure separate targets instead.
|
|
75
|
-
|
|
76
|
-
Call it before `app.open` to load the first page at that size, so the app
|
|
77
|
-
never renders its desktop layout first:
|
|
78
|
-
|
|
79
|
-
```ts
|
|
80
|
-
await browser.setViewport({ width: 390, height: 844 });
|
|
81
|
-
await app.open('/');
|
|
82
|
-
```
|
|
83
|
-
|
|
84
|
-
The resize appears as a step in the report. To check the exact width, read
|
|
85
|
-
it from the page:
|
|
86
|
-
|
|
87
|
-
```ts
|
|
88
|
-
const width = await browser.evaluate(() => window.innerWidth);
|
|
89
|
-
expect(width).toBe(390);
|
|
90
|
-
```
|
|
91
|
-
|
|
92
|
-
## What the agent sees
|
|
93
|
-
|
|
94
|
-
The agent receives the current viewport size with each observation.
|
|
95
|
-
Screenshots cover that viewport. Off-screen elements can still appear in
|
|
96
|
-
the tree, and tapping them scrolls them into view. The agent can scroll to
|
|
97
|
-
reveal lazy-loaded content.
|
|
98
|
-
|
|
99
|
-
Resize before calling `agent.act`:
|
|
100
|
-
|
|
101
|
-
```ts
|
|
102
|
-
test('the mobile menu reaches pricing', async ({ app, agent, browser }) => {
|
|
103
|
-
await browser.setViewport({ width: 390, height: 844 });
|
|
104
|
-
await app.open('/');
|
|
105
|
-
await agent.act('open the navigation menu and go to Pricing');
|
|
106
|
-
await expect(browser).toHaveURL('/pricing');
|
|
107
|
-
});
|
|
108
|
-
```
|
|
109
|
-
|
|
110
|
-
## Scoping a test to one size
|
|
111
|
-
|
|
112
|
-
`platforms` and `requires` cannot select a viewport; all Playwright targets
|
|
113
|
-
use platform `web`. To skip a test at some sizes, check the width in the
|
|
114
|
-
test:
|
|
115
|
-
|
|
116
|
-
```ts
|
|
117
|
-
test('the sidebar stays open', async ({ app, screen, browser }) => {
|
|
118
|
-
await app.open('/');
|
|
119
|
-
test.skip((await browser.evaluate(() => window.innerWidth)) < 768, 'desktop layout only');
|
|
120
|
-
await expect(screen.getByRole('complementary')).toBeVisible();
|
|
121
|
-
});
|
|
122
|
-
```
|
|
123
|
-
|
|
124
|
-
To keep a whole group of size-specific tests apart, give them their own
|
|
125
|
-
config and run it with `--config`.
|
|
126
|
-
|
|
127
|
-
## Limits
|
|
128
|
-
|
|
129
|
-
A narrow viewport tests responsive layout. It still uses a desktop browser
|
|
130
|
-
with a scale factor of 1, desktop pointer behavior, and its usual user agent.
|
|
131
|
-
It does not emulate touch, color scheme, or reduced-motion preferences.
|
|
132
|
-
Use the [device engine](/mobile) for tests on a simulator or emulator.
|
|
133
|
-
|
|
134
|
-
A video keeps the size the page had when it started recording; set
|
|
135
|
-
`web({ screencast: { size } })` for one fixed size. A browser provider that
|
|
136
|
-
records, such as [Kernel](/integrations/kernel#recordings), records the whole
|
|
137
|
-
screen instead.
|
|
138
|
-
|
|
139
|
-
<CardGroup cols={2}>
|
|
140
|
-
<Card title="Working with the browser" icon="globe" href="/browser">
|
|
141
|
-
Routes, cookies, dialogs, downloads, and the rest of the `browser` fixture.
|
|
142
|
-
</Card>
|
|
143
|
-
<Card title="Web engine reference" icon="masks-theater" href="/reference/web">
|
|
144
|
-
Every engine option and `browser` method, with timeouts and errors.
|
|
145
|
-
</Card>
|
|
146
|
-
</CardGroup>
|