@e2edev/e2e 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -8
- package/dist/agent/act-validation.d.ts +2 -0
- package/dist/agent/act-validation.d.ts.map +1 -1
- package/dist/agent/act-validation.js +1 -1
- package/dist/agent/act-validation.js.map +1 -1
- package/dist/agent/act.d.ts +30 -2
- package/dist/agent/act.d.ts.map +1 -1
- package/dist/agent/act.js +18 -4
- package/dist/agent/act.js.map +1 -1
- package/dist/agent/ai-sdk.d.ts +2 -2
- package/dist/agent/ai-sdk.d.ts.map +1 -1
- package/dist/agent/ai-sdk.js +3 -4
- package/dist/agent/ai-sdk.js.map +1 -1
- package/dist/agent/default-agent.d.ts +33 -9
- package/dist/agent/default-agent.d.ts.map +1 -1
- package/dist/agent/default-agent.js +46 -31
- package/dist/agent/default-agent.js.map +1 -1
- package/dist/agent/error.d.ts.map +1 -1
- package/dist/agent/error.js +1 -0
- package/dist/agent/error.js.map +1 -1
- package/dist/agent/executor.d.ts +13 -0
- package/dist/agent/executor.d.ts.map +1 -1
- package/dist/agent/executor.js.map +1 -1
- package/dist/agent/interactive-step.d.ts +38 -0
- package/dist/agent/interactive-step.d.ts.map +1 -0
- package/dist/agent/interactive-step.js +131 -0
- package/dist/agent/interactive-step.js.map +1 -0
- package/dist/agent/model/adapter.d.ts +3 -0
- package/dist/agent/model/adapter.d.ts.map +1 -1
- package/dist/agent/model/adapter.js.map +1 -1
- package/dist/agent/model/overflow.d.ts +20 -0
- package/dist/agent/model/overflow.d.ts.map +1 -0
- package/dist/agent/model/overflow.js +82 -0
- package/dist/agent/model/overflow.js.map +1 -0
- package/dist/agent/model/prompt-cache.d.ts +47 -0
- package/dist/agent/model/prompt-cache.d.ts.map +1 -0
- package/dist/agent/model/prompt-cache.js +78 -0
- package/dist/agent/model/prompt-cache.js.map +1 -0
- package/dist/agent/model/sdk.d.ts +41 -24
- package/dist/agent/model/sdk.d.ts.map +1 -1
- package/dist/agent/model/sdk.js +64 -91
- package/dist/agent/model/sdk.js.map +1 -1
- package/dist/agent/model/tool-choice.d.ts +21 -0
- package/dist/agent/model/tool-choice.d.ts.map +1 -0
- package/dist/agent/model/tool-choice.js +49 -0
- package/dist/agent/model/tool-choice.js.map +1 -0
- package/dist/agent/primitives.d.ts.map +1 -1
- package/dist/agent/primitives.js +2 -1
- package/dist/agent/primitives.js.map +1 -1
- package/dist/agent/public.d.ts +1 -1
- package/dist/agent/public.d.ts.map +1 -1
- package/dist/agent/public.js.map +1 -1
- package/dist/agent/screen-update.d.ts +8 -3
- package/dist/agent/screen-update.d.ts.map +1 -1
- package/dist/agent/screen-update.js +18 -5
- package/dist/agent/screen-update.js.map +1 -1
- package/dist/agent/tool-loop.d.ts +4 -4
- package/dist/agent/tool-loop.d.ts.map +1 -1
- package/dist/agent/tool-loop.js +241 -48
- package/dist/agent/tool-loop.js.map +1 -1
- package/dist/agent/tool-output.d.ts +31 -0
- package/dist/agent/tool-output.d.ts.map +1 -0
- package/dist/agent/tool-output.js +60 -0
- package/dist/agent/tool-output.js.map +1 -0
- package/dist/agent/tool.d.ts +9 -3
- package/dist/agent/tool.d.ts.map +1 -1
- package/dist/agent/tool.js +2 -2
- package/dist/agent/tool.js.map +1 -1
- package/dist/agent/usage.d.ts +7 -0
- package/dist/agent/usage.d.ts.map +1 -1
- package/dist/agent/usage.js +11 -0
- package/dist/agent/usage.js.map +1 -1
- package/dist/cli/docs-url.d.ts +1 -1
- package/dist/cli/docs-url.d.ts.map +1 -1
- package/dist/cli/docs-url.js +1 -1
- package/dist/cli/docs-url.js.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +139 -35
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init/engine-versions.json +1 -1
- package/dist/cli/init/engines.d.ts +17 -16
- package/dist/cli/init/engines.d.ts.map +1 -1
- package/dist/cli/init/engines.js +26 -26
- package/dist/cli/init/engines.js.map +1 -1
- package/dist/cli/init/gateways.d.ts +22 -0
- package/dist/cli/init/gateways.d.ts.map +1 -0
- package/dist/cli/init/gateways.js +50 -0
- package/dist/cli/init/gateways.js.map +1 -0
- package/dist/cli/init/mcp-config.d.ts +31 -0
- package/dist/cli/init/mcp-config.d.ts.map +1 -0
- package/dist/cli/init/mcp-config.js +72 -0
- package/dist/cli/init/mcp-config.js.map +1 -0
- package/dist/cli/init/scaffold.d.ts +8 -2
- package/dist/cli/init/scaffold.d.ts.map +1 -1
- package/dist/cli/init/scaffold.js +22 -11
- package/dist/cli/init/scaffold.js.map +1 -1
- package/dist/cli/init.d.ts.map +1 -1
- package/dist/cli/init.js +109 -29
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/mcp.d.ts +15 -0
- package/dist/cli/mcp.d.ts.map +1 -0
- package/dist/cli/mcp.js +63 -0
- package/dist/cli/mcp.js.map +1 -0
- package/dist/collect/collect.d.ts +7 -0
- package/dist/collect/collect.d.ts.map +1 -1
- package/dist/collect/collect.js +16 -0
- package/dist/collect/collect.js.map +1 -1
- package/dist/collect/registry.d.ts.map +1 -1
- package/dist/collect/registry.js +33 -6
- package/dist/collect/registry.js.map +1 -1
- package/dist/collect/select.d.ts +9 -2
- package/dist/collect/select.d.ts.map +1 -1
- package/dist/collect/select.js +57 -11
- package/dist/collect/select.js.map +1 -1
- package/dist/config/agent.d.ts +13 -32
- package/dist/config/agent.d.ts.map +1 -1
- package/dist/config/agent.js +32 -123
- package/dist/config/agent.js.map +1 -1
- package/dist/config/resolve.d.ts +8 -5
- package/dist/config/resolve.d.ts.map +1 -1
- package/dist/config/resolve.js +11 -9
- package/dist/config/resolve.js.map +1 -1
- package/dist/credentials.d.ts +6 -0
- package/dist/credentials.d.ts.map +1 -1
- package/dist/credentials.js +9 -0
- package/dist/credentials.js.map +1 -1
- package/dist/explore/body.d.ts +36 -0
- package/dist/explore/body.d.ts.map +1 -0
- package/dist/explore/body.js +168 -0
- package/dist/explore/body.js.map +1 -0
- package/dist/explore/executor.d.ts +25 -0
- package/dist/explore/executor.d.ts.map +1 -0
- package/dist/explore/executor.js +86 -0
- package/dist/explore/executor.js.map +1 -0
- package/dist/explore/index.d.ts +64 -0
- package/dist/explore/index.d.ts.map +1 -0
- package/dist/explore/index.js +185 -0
- package/dist/explore/index.js.map +1 -0
- package/dist/explore/plan.d.ts +74 -0
- package/dist/explore/plan.d.ts.map +1 -0
- package/dist/explore/plan.js +149 -0
- package/dist/explore/plan.js.map +1 -0
- package/dist/explore/reporter.d.ts +11 -0
- package/dist/explore/reporter.d.ts.map +1 -0
- package/dist/explore/reporter.js +54 -0
- package/dist/explore/reporter.js.map +1 -0
- package/dist/explore/state.d.ts +58 -0
- package/dist/explore/state.d.ts.map +1 -0
- package/dist/explore/state.js +147 -0
- package/dist/explore/state.js.map +1 -0
- package/dist/internal/ai-trace.d.ts +2 -0
- package/dist/internal/ai-trace.d.ts.map +1 -1
- package/dist/internal/ai-trace.js +3 -0
- package/dist/internal/ai-trace.js.map +1 -1
- package/dist/internal/ids.d.ts +2 -2
- package/dist/internal/ids.d.ts.map +1 -1
- package/dist/internal/ids.js +3 -3
- package/dist/internal/ids.js.map +1 -1
- package/dist/internal/package-manager.d.ts +4 -0
- package/dist/internal/package-manager.d.ts.map +1 -1
- package/dist/internal/package-manager.js +26 -0
- package/dist/internal/package-manager.js.map +1 -1
- package/dist/mcp/catalog.d.ts +50 -0
- package/dist/mcp/catalog.d.ts.map +1 -0
- package/dist/mcp/catalog.js +172 -0
- package/dist/mcp/catalog.js.map +1 -0
- package/dist/mcp/config.d.ts +11 -0
- package/dist/mcp/config.d.ts.map +1 -0
- package/dist/mcp/config.js +16 -0
- package/dist/mcp/config.js.map +1 -0
- package/dist/mcp/server.d.ts +29 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +86 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/session.d.ts +68 -0
- package/dist/mcp/session.d.ts.map +1 -0
- package/dist/mcp/session.js +317 -0
- package/dist/mcp/session.js.map +1 -0
- package/dist/mcp/tools.d.ts +77 -0
- package/dist/mcp/tools.d.ts.map +1 -0
- package/dist/mcp/tools.js +179 -0
- package/dist/mcp/tools.js.map +1 -0
- package/dist/report/build.d.ts +61 -0
- package/dist/report/build.d.ts.map +1 -1
- package/dist/report/build.js +13 -2
- package/dist/report/build.js.map +1 -1
- package/dist/report/debug-steps.d.ts.map +1 -1
- package/dist/report/debug-steps.js +15 -0
- package/dist/report/debug-steps.js.map +1 -1
- package/dist/report/format.d.ts +30 -1
- package/dist/report/format.d.ts.map +1 -1
- package/dist/report/format.js +57 -3
- package/dist/report/format.js.map +1 -1
- package/dist/report/junit.d.ts.map +1 -1
- package/dist/report/junit.js +4 -1
- package/dist/report/junit.js.map +1 -1
- package/dist/report/list.d.ts +10 -1
- package/dist/report/list.d.ts.map +1 -1
- package/dist/report/list.js +35 -13
- package/dist/report/list.js.map +1 -1
- package/dist/run/events.d.ts +13 -4
- package/dist/run/events.d.ts.map +1 -1
- package/dist/run/events.js.map +1 -1
- package/dist/run/execute.d.ts +2 -2
- package/dist/run/execute.d.ts.map +1 -1
- package/dist/run/execute.js +19 -14
- package/dist/run/execute.js.map +1 -1
- package/dist/run/fixtures.d.ts +10 -4
- package/dist/run/fixtures.d.ts.map +1 -1
- package/dist/run/fixtures.js +23 -23
- package/dist/run/fixtures.js.map +1 -1
- package/dist/run/in-process.d.ts +8 -0
- package/dist/run/in-process.d.ts.map +1 -1
- package/dist/run/in-process.js +7 -6
- package/dist/run/in-process.js.map +1 -1
- package/dist/run/provision.d.ts +51 -0
- package/dist/run/provision.d.ts.map +1 -0
- package/dist/run/provision.js +94 -0
- package/dist/run/provision.js.map +1 -0
- package/dist/run/records.d.ts +4 -0
- package/dist/run/records.d.ts.map +1 -1
- package/dist/run/runner.d.ts +25 -3
- package/dist/run/runner.d.ts.map +1 -1
- package/dist/run/runner.js +38 -81
- package/dist/run/runner.js.map +1 -1
- package/dist/run/scheduler.d.ts +5 -2
- package/dist/run/scheduler.d.ts.map +1 -1
- package/dist/run/scheduler.js +13 -10
- package/dist/run/scheduler.js.map +1 -1
- package/dist/run/serial.d.ts +1 -1
- package/dist/run/serial.d.ts.map +1 -1
- package/dist/run/serial.js +7 -2
- package/dist/run/serial.js.map +1 -1
- package/dist/run/standalone.d.ts +63 -0
- package/dist/run/standalone.d.ts.map +1 -0
- package/dist/run/standalone.js +152 -0
- package/dist/run/standalone.js.map +1 -0
- package/dist/run/steps.d.ts +7 -0
- package/dist/run/steps.d.ts.map +1 -1
- package/dist/run/steps.js.map +1 -1
- package/dist/run/units.d.ts +6 -3
- package/dist/run/units.d.ts.map +1 -1
- package/dist/run/units.js +0 -0
- package/dist/run/units.js.map +1 -1
- package/dist/run/worker/entry.js +2 -2
- package/dist/run/worker/entry.js.map +1 -1
- package/dist/run/worker/protocol.d.ts +5 -0
- package/dist/run/worker/protocol.d.ts.map +1 -1
- package/dist/run/worker/protocol.js.map +1 -1
- package/dist/run/worker/session.d.ts +3 -4
- package/dist/run/worker/session.d.ts.map +1 -1
- package/dist/run/worker/session.js +5 -4
- package/dist/run/worker/session.js.map +1 -1
- package/dist/run/worker-models.d.ts +4 -4
- package/dist/run/worker-models.js +6 -13
- package/dist/run/worker-models.js.map +1 -1
- package/dist/telemetry/events.d.ts.map +1 -1
- package/dist/telemetry/events.js +17 -2
- package/dist/telemetry/events.js.map +1 -1
- package/dist/types.d.ts +20 -20
- package/dist/types.d.ts.map +1 -1
- package/package.json +9 -6
- package/skills/e2e/SKILL.md +16 -6
- package/skills/e2e/references/agent.md +21 -16
- package/skills/e2e/references/debugging.md +2 -1
- package/skills/e2e/references/explore.md +83 -0
- package/skills/e2e/references/mcp.md +88 -0
- package/skills/e2e/references/running.md +7 -5
- package/skills/e2e/references/setup.md +17 -13
package/skills/e2e/SKILL.md
CHANGED
|
@@ -18,6 +18,7 @@ target names an engine, `@e2edev/playwright` for browsers or
|
|
|
18
18
|
import type { E2EConfig } from '@e2edev/e2e';
|
|
19
19
|
import { createAgent } from '@e2edev/e2e/agent';
|
|
20
20
|
import { playwright } from '@e2edev/playwright';
|
|
21
|
+
import { gateway } from 'ai';
|
|
21
22
|
|
|
22
23
|
export default {
|
|
23
24
|
targets: [
|
|
@@ -28,8 +29,13 @@ export default {
|
|
|
28
29
|
}),
|
|
29
30
|
},
|
|
30
31
|
],
|
|
31
|
-
// Only needed for agent.* steps
|
|
32
|
-
agents: {
|
|
32
|
+
// Only needed for agent.* steps. The model is an AI SDK instance; gateway() from 'ai' reads AI_GATEWAY_API_KEY.
|
|
33
|
+
agents: {
|
|
34
|
+
default: createAgent({
|
|
35
|
+
model: gateway('openai/gpt-5.4-mini'),
|
|
36
|
+
system: 'You are a thorough QA agent. Verify every outcome on screen.',
|
|
37
|
+
}),
|
|
38
|
+
},
|
|
33
39
|
} satisfies E2EConfig;
|
|
34
40
|
```
|
|
35
41
|
|
|
@@ -58,7 +64,9 @@ one. Without them, the installed CLI prints the same text:
|
|
|
58
64
|
| `writing-tests` | [references/writing-tests.md](references/writing-tests.md) | Writing or fixing tests: fixtures, locators, actions, matchers, sign-in sessions, the `web` fixture |
|
|
59
65
|
| `agent` | [references/agent.md](references/agent.md) | Adding `agent.*` steps, picking a model, cost and budgets, the trace cache |
|
|
60
66
|
| `running` | [references/running.md](references/running.md) | CLI flags, reporters, `.e2e/report.json`, exit codes, CI |
|
|
67
|
+
| `explore` | [references/explore.md](references/explore.md) | Exploring an app toward a goal without a test file: `e2e explore`, its budgets, verdict, and `run.explore` |
|
|
61
68
|
| `debugging` | [references/debugging.md](references/debugging.md) | A run failed: error codes and their fixes, `--headed`, `--debug`, `--ai-trace` |
|
|
69
|
+
| `mcp` | [references/mcp.md](references/mcp.md) | Driving the live app from a coding agent over MCP: `e2e mcp`, its tools, and the explore-then-write loop |
|
|
62
70
|
|
|
63
71
|
## Workflow
|
|
64
72
|
|
|
@@ -67,13 +75,15 @@ one. Without them, the installed CLI prints the same text:
|
|
|
67
75
|
there: follow `setup`.
|
|
68
76
|
2. Learn the screens you will drive before writing a test: routes, labels,
|
|
69
77
|
roles, button text. Semantic locators need the accessible names the app
|
|
70
|
-
renders, so read the templates or components,
|
|
71
|
-
`--headed
|
|
78
|
+
renders, so read the templates or components, open the page with
|
|
79
|
+
`--headed`, or drive the live app through the `e2e mcp` server when it is
|
|
80
|
+
registered (topic `mcp`): `open_session`, `observe`, and `locate` show the
|
|
81
|
+
exact names and check a locator before you write it.
|
|
72
82
|
3. Write `tests/<feature>.e2e.ts`. Deterministic steps first. One `agent.act`
|
|
73
83
|
per goal where the flow varies, and an `expect` on its outcome right after.
|
|
74
84
|
4. Run one file: `npx --no-install e2e run tests/<feature>.e2e.ts`. Agent
|
|
75
|
-
steps need
|
|
76
|
-
|
|
85
|
+
steps need a model in the config and its provider's key in the environment
|
|
86
|
+
(`AI_GATEWAY_API_KEY` for `gateway()`); deterministic tests need neither.
|
|
77
87
|
5. Read the failure: the reporter prints the error code, the message, and a
|
|
78
88
|
code frame; `.e2e/report.json` has every step and artifact path. Fix the
|
|
79
89
|
locator, the expectation, or the app. Never add a sleep.
|
|
@@ -6,18 +6,19 @@ transcript between calls. Deterministic tests never load a model.
|
|
|
6
6
|
|
|
7
7
|
## Configure a model
|
|
8
8
|
|
|
9
|
-
Agents live under `agents` by name; `default` is the one tests use, `e2e run --agent <name>` re-points that default, a test or describe pins one with `{ agent: 'name' }
|
|
9
|
+
Agents live under `agents` by name; `default` is the one tests use, `e2e run --agent <name>` re-points that default, a test or describe pins one with `{ agent: 'name' }` or several with `{ agent: ['buyer', 'admin'] }` (the test runs once per agent, one result each, tagged `[admin]` in the terminal and `agent` in the report), and any `agent.*` call can name one with `{ agent: 'name' }` (innermost wins). `--agent buyer,admin` runs every unpinned test once per agent; on a pinned list it narrows to the names both give and never overrides a pin it does not name. A signed-in persona pairs the pin with a `session`, one per describe block, so a loop over describe blocks sweeps signed-in personas. There is no default model. Pick one of three shapes for an entry:
|
|
10
10
|
|
|
11
11
|
```ts
|
|
12
12
|
// e2e.config.ts
|
|
13
13
|
import { createAgent } from '@e2edev/e2e/agent';
|
|
14
|
+
import { gateway } from 'ai';
|
|
14
15
|
|
|
15
16
|
export default {
|
|
16
|
-
// 1. The built-in agent
|
|
17
|
-
agents: { default: createAgent({ system: 'You are a thorough QA agent. Verify every outcome on screen.' }) },
|
|
17
|
+
// 1. The built-in agent with an AI SDK model: gateway() from 'ai' is the Vercel AI Gateway and reads AI_GATEWAY_API_KEY.
|
|
18
|
+
agents: { default: createAgent({ model: gateway('openai/gpt-5.4-mini'), system: 'You are a thorough QA agent. Verify every outcome on screen.' }) },
|
|
18
19
|
|
|
19
|
-
// 2. An options block: a
|
|
20
|
-
// agents: { default: { model: 'anthropic/claude-sonnet-4.5', context: 'A billing dashboard. Plans are Free, Team, and Pro.' } },
|
|
20
|
+
// 2. An options block: a model plus project vocabulary. openrouter() from '@openrouter/ai-sdk-provider' reads OPENROUTER_API_KEY.
|
|
21
|
+
// agents: { default: { model: openrouter('anthropic/claude-sonnet-4.5'), context: 'A billing dashboard. Plans are Free, Team, and Pro.' } },
|
|
21
22
|
|
|
22
23
|
// 3. A live AI SDK model instance for a provider called directly.
|
|
23
24
|
// agents: { default: createAgent({ model: openai('gpt-5.4-mini') }) },
|
|
@@ -25,21 +26,25 @@ export default {
|
|
|
25
26
|
```
|
|
26
27
|
|
|
27
28
|
```bash
|
|
28
|
-
|
|
29
|
+
AI_GATEWAY_API_KEY=... npx --no-install e2e run
|
|
29
30
|
```
|
|
30
31
|
|
|
31
|
-
-
|
|
32
|
-
`
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
- The model is always an AI SDK instance the config constructs; the runner
|
|
33
|
+
implies no gateway and reads no model variable. `gateway()` from `ai` is the
|
|
34
|
+
Vercel AI Gateway, `openrouter()` from `@openrouter/ai-sdk-provider` is
|
|
35
|
+
OpenRouter, `createOpenAICompatible({ baseURL }).chatModel()` from
|
|
36
|
+
`@ai-sdk/openai-compatible` is any /v1 chat endpoint (Ollama, vLLM), and a
|
|
37
|
+
provider's own package (`openai()`) calls it directly. Each reads its own
|
|
38
|
+
key variable. A string in a model slot is `INVALID_CONFIG`.
|
|
39
|
+
- With no model anywhere, acquiring `agent` is `MODEL_UNAVAILABLE`. Use
|
|
40
|
+
`createAgent` for the model, a `system` prompt, and tools.
|
|
36
41
|
- `ai@^7` must be installed for any `agent.*` step; the runner loads it
|
|
37
42
|
lazily and fails without it.
|
|
38
43
|
- `context` in the config and `agentContext` on a test or group add trusted
|
|
39
44
|
project vocabulary to every prompt.
|
|
40
|
-
- `visionModel`
|
|
45
|
+
- `visionModel` serves the calls that send pixels.
|
|
41
46
|
- The model passed to `createAgent({ model })` is the one model for every
|
|
42
|
-
`agent.*` call, `act` and the judgments alike
|
|
47
|
+
`agent.*` call, `act` and the judgments alike.
|
|
43
48
|
An agent `model` naming a different model is `INVALID_CONFIG`.
|
|
44
49
|
- Missing model or key: checked once per run when the first test acquires
|
|
45
50
|
the `agent` fixture. One run-level `MODEL_UNAVAILABLE` (exit 2) stops the
|
|
@@ -217,12 +222,12 @@ arbitrary tool result JSON or other fields are preserved.
|
|
|
217
222
|
can be the `agent`; the runner still owns observations, actions, budgets,
|
|
218
223
|
and the report.
|
|
219
224
|
|
|
220
|
-
Full reference: https://e2e
|
|
225
|
+
Full reference: https://e2e.mintlify.app/agents
|
|
221
226
|
|
|
222
227
|
## In CI
|
|
223
228
|
|
|
224
229
|
Deterministic tests gate merges; agentic tests are opt-in. Keep them in a
|
|
225
230
|
separate config (`e2e.agent.config.ts` with its own `tests` glob and a larger
|
|
226
|
-
`timeout`), run them on a schedule or `workflow_dispatch`, and pass the
|
|
227
|
-
|
|
231
|
+
`timeout`), run them on a schedule or `workflow_dispatch`, and pass the key
|
|
232
|
+
the config's model reads (`env: { AI_GATEWAY_API_KEY }` for `gateway()`) from
|
|
228
233
|
secrets.
|
|
@@ -36,9 +36,10 @@ jq '.run.results[] | select(.status != "passed") | .attempts[-1]
|
|
|
36
36
|
| `ASSERTION_FAILED` | The expectation is wrong, or the state settles later than 5 s; for `agent.assert`, the judgment was false (explanation in the report) | Compare with the actual text in the report or screenshot; `{ timeout }` on the matcher; rewrite the question |
|
|
37
37
|
| `ACTION_FAILED` | Element not actionable (covered, disabled, detached) or an operation timed out | Wait on the right condition with `expect` first; close overlays; check `actionTimeout` |
|
|
38
38
|
| `TEST_TIMEOUT` | The attempt exceeded `timeout` (120 s) | Split the test, or raise `timeout` for slow flows and agent steps |
|
|
39
|
-
| `MODEL_UNAVAILABLE` | No model
|
|
39
|
+
| `MODEL_UNAVAILABLE` | No model: neither `createAgent({ model })` nor `agent.model` holds an AI SDK instance. Reported once for the run under `run.errors`; the run stops | Construct one in the config, e.g. `gateway('openai/gpt-5.4-mini')` from `ai`, and export the key its provider reads (`AI_GATEWAY_API_KEY`) |
|
|
40
40
|
| `MODEL_PROVIDER_FAILED` | Network, 5xx, rate limit, or no credits after the transport retries | Check the key and the quota; retry; exit code 3 |
|
|
41
41
|
| `STEP_TIMEOUT`, `STEP_BUDGET_EXHAUSTED` | The goal was too big or ambiguous, or the provider slow | Split the goal, use on-screen wording, add `context`, raise `timeout` and `actionTimeout`, `--debug` to read the transcript |
|
|
42
|
+
| `CONTEXT_OVERFLOW` | The screen plus the step's history did not fit the model's context window, even after the loop shrank the history and retried once | Lower `agent.maxObservationBytes`, split the step, or pick a model with a larger window |
|
|
42
43
|
| `POLICY_DENIED` | Navigation outside `allowedOrigins`; a `Secret` given to a sink that is not a password field; reading a secure field | Add the origin to `allowedOrigins`; fill secrets only into password inputs; assert the outcome instead of the value |
|
|
43
44
|
| `UNSUPPORTED_CAPABILITY` | A fixture the engine does not contribute (`web` on a device), `schema` or `vision` on `act`, an action the surface lacks | Declare `requires: ['web']`; drop the option; use a supported action |
|
|
44
45
|
| `SESSION_UNAVAILABLE`, `SESSION_CONTRACT` | `session: 'x'` with no setup saving `x`; a setup that did not save every declared name | Add or fix the `test.setup` |
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# Exploring without a test
|
|
2
|
+
|
|
3
|
+
`e2e explore` runs the agent against the app with a goal instead of a test
|
|
4
|
+
file. Use it to see what the agent can do with an app before tests exist, to
|
|
5
|
+
hunt for regressions on a branch, or to find what is worth turning into a
|
|
6
|
+
test. It needs a config with a target and an agent that holds a model,
|
|
7
|
+
nothing else.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npx --no-install e2e explore # goal: "Explore the app and find bugs"
|
|
11
|
+
npx --no-install e2e explore 'Explore checkout like a first-time buyer and report anything off'
|
|
12
|
+
npx --no-install e2e explore --target web --max-steps 4 --headed
|
|
13
|
+
npx --no-install e2e explore 'Hunt for broken forms' --video
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## What a run does
|
|
17
|
+
|
|
18
|
+
1. Opens the app on the target's URL.
|
|
19
|
+
2. Plans one step: a structured model call reads the goal, the steps and
|
|
20
|
+
findings so far, and the current screen, and answers with a title and a
|
|
21
|
+
concrete charter for one flow, or decides the goal is covered.
|
|
22
|
+
3. Runs the charter as an `agent.act()` step with the project's tools plus
|
|
23
|
+
`report_finding`. The agent reports each defect the moment it has evidence:
|
|
24
|
+
title, `issue` or `warning`, severity 1 to 5, expected, actual, reproduction
|
|
25
|
+
steps. The runner adds the path and a redacted screenshot.
|
|
26
|
+
4. Repeats until the planner finishes, the step limit, the clock, or three
|
|
27
|
+
failed or blocked steps in a row that reported nothing; then asks for a closing assessment.
|
|
28
|
+
|
|
29
|
+
A failed step does not end the run: it is recorded, and only findings the
|
|
30
|
+
agent reported count toward the verdict. A step that hits its action or time
|
|
31
|
+
budget ended at its limit and counts as neither. Configured `credentials`
|
|
32
|
+
reach the explorer as step secrets: the planner knows the account names and
|
|
33
|
+
usernames, and the agent fills passwords with `type_secret` by name.
|
|
34
|
+
|
|
35
|
+
## Flags
|
|
36
|
+
|
|
37
|
+
| Flag | Default | Effect |
|
|
38
|
+
| --- | --- | --- |
|
|
39
|
+
| `[goal]` | `Explore the app and find bugs` | One quoted sentence: the area and the posture. |
|
|
40
|
+
| `--target <id>` | first configured target | The one target to explore. |
|
|
41
|
+
| `--agent <name>` | `default` | Build the explorer from another configured agent (`agents.<name>`). |
|
|
42
|
+
| `--max-steps <n>` | 8 (1 to 12) | Exploration steps at most. |
|
|
43
|
+
| `--timeout <ms>` | 600000 (180000 to 900000) | Wall clock; the last minute is for the assessment. |
|
|
44
|
+
| `--headed`, `--reporter`, `--artifacts`, `--debug`, `--ai-trace`, `--video` | as `run` | Same meaning as for `e2e run`. |
|
|
45
|
+
|
|
46
|
+
Per-step action and model-call budgets default to 40 each; `agent.maxSteps`
|
|
47
|
+
and `agent.maxModelCalls` in the config override them. The trace cache is off
|
|
48
|
+
and retries are zero for the run.
|
|
49
|
+
|
|
50
|
+
## Reading the result
|
|
51
|
+
|
|
52
|
+
Exit code `0`: steps ran and no `issue` was reported (warnings allowed). Exit
|
|
53
|
+
code `1`: at least one `issue`, or no step ran and nothing was found (the run
|
|
54
|
+
is `blocked`). `2` and `3` as for `run`.
|
|
55
|
+
|
|
56
|
+
The terminal prints each step live and, under the summary, `Explored`,
|
|
57
|
+
`Findings`, one row per finding sorted by severity, and `Assessment`.
|
|
58
|
+
`.e2e/report.json` has the record under `run.explore`:
|
|
59
|
+
|
|
60
|
+
```json
|
|
61
|
+
{
|
|
62
|
+
"goal": "...",
|
|
63
|
+
"budgets": { "maxSteps": 8, "timeoutMs": 600000 },
|
|
64
|
+
"ended": "finished | step-limit | time | stuck | aborted",
|
|
65
|
+
"summary": "the closing assessment",
|
|
66
|
+
"steps": [{ "index": 1, "title": "...", "instruction": "...", "status": "passed | failed | blocked | exhausted", "summary": "...", "startedAt": "...", "durationMs": 0 }],
|
|
67
|
+
"findings": [{ "index": 0, "step": 1, "kind": "issue", "severity": 4, "title": "...", "expected": "...", "actual": "...", "reproduction": ["..."], "path": "/cart", "artifactId": "<attempt id>:artifact:2", "reportedAt": "..." }]
|
|
68
|
+
}
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
`artifactId` names the evidence screenshot among the attempt's `artifacts` in
|
|
72
|
+
`run.results[0]`, where its path, size, and digest are.
|
|
73
|
+
|
|
74
|
+
Turn a finding into a test: its `reproduction` steps are the `agent.act()`
|
|
75
|
+
instructions or `screen.*` actions, and `expected` is the assertion.
|
|
76
|
+
|
|
77
|
+
## When it does not fit
|
|
78
|
+
|
|
79
|
+
An agent built with `createAgent({ tools, system })` in the config lends its
|
|
80
|
+
tools and guidance to the explorer; a hand-rolled `StepExecutor` is replaced by
|
|
81
|
+
the built-in agent for the run, with a notice on stderr. Findings are the
|
|
82
|
+
model's claims plus evidence, not verified reproductions: read `actual` against
|
|
83
|
+
the screenshot before filing a bug.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# Driving the app over MCP
|
|
2
|
+
|
|
3
|
+
`e2e mcp` serves a project's live app to a coding agent over MCP (stdio).
|
|
4
|
+
It gives the agent the same hands the testing agent has on the app: look at
|
|
5
|
+
a screen before writing a test, and check a locator before committing to it.
|
|
6
|
+
Running tests and reading a failed run stay on the CLI (topics `running` and
|
|
7
|
+
`debugging`).
|
|
8
|
+
|
|
9
|
+
## Setup
|
|
10
|
+
|
|
11
|
+
The server ships with `@e2edev/e2e`. `e2e init` offers to register it; by
|
|
12
|
+
hand:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
claude mcp add e2e -- npx --no-install e2e mcp # Claude Code
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Or declare it in the client's project config (`.mcp.json` for Claude Code,
|
|
19
|
+
`.cursor/mcp.json` for Cursor, `.vscode/mcp.json` for VS Code):
|
|
20
|
+
|
|
21
|
+
```json
|
|
22
|
+
{ "mcpServers": { "e2e": { "command": "npx", "args": ["--no-install", "e2e", "mcp"] } } }
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Flags: `--config <path>` names the default config file, `--target <name>`
|
|
26
|
+
fixes the target every session opens on, `--headless` hides the browser or
|
|
27
|
+
simulator (sessions are headed by default outside CI, so the developer can
|
|
28
|
+
watch).
|
|
29
|
+
|
|
30
|
+
## Tools
|
|
31
|
+
|
|
32
|
+
The server has four tools, and the list never changes. Everything a session
|
|
33
|
+
can do is a catalog behind `call`.
|
|
34
|
+
|
|
35
|
+
| Tool | Does |
|
|
36
|
+
| --- | --- |
|
|
37
|
+
| `open_session` | Loads the config (`config` names another file; default the nearest `e2e.config.ts`), starts the declared app command if any, boots the engine, opens the app URL, and returns the session id, the catalog, and the first observation. `target` is required when the config declares several. One session at a time. |
|
|
38
|
+
| `tools` | The catalog: one line per tool with its argument names (`?` marks optional), the first sentence of its description, and `[read-only]` where it changes nothing. `tools {tool}` shows one tool's full description and the JSON Schema of its arguments. |
|
|
39
|
+
| `call` | Runs one catalog tool: `call {tool: "tap", args: {target: "n42"}}`. Arguments are checked against the tool's schema first; a wrong one fails with `INVALID_ARGUMENT` naming the field. |
|
|
40
|
+
| `close_session` | Ends the attempt, disposes the engine, stops the app processes the session started. |
|
|
41
|
+
|
|
42
|
+
The catalog, per session:
|
|
43
|
+
|
|
44
|
+
| Catalog tool | Does |
|
|
45
|
+
| --- | --- |
|
|
46
|
+
| `observe` | A fresh observation: one node per line as `#id role "name" ...`, plus the current path. |
|
|
47
|
+
| `tap`, `type`, `press`, `select`, `scroll`, `navigate` | The grammar verbs, exactly as the testing agent gets them. Each reports what changed on screen; `observe` shows the whole screen. A verb the engine cannot honor is not listed and fails with `UNSUPPORTED_CAPABILITY`. |
|
|
48
|
+
| `type_secret` | Fills a configured credential's password into a secure field by credential name; the plaintext never reaches the agent. Listed when the config declares `credentials`. |
|
|
49
|
+
| `locate` | Tries a semantic locator (`role` + `name`, `text`, `label`, `placeholder`, `testId`, `exact`) and returns how many nodes match, which, and the `screen.*` call to write. |
|
|
50
|
+
| `screenshot` | The masked pixels as an image, withheld once a secret was filled in the session. |
|
|
51
|
+
| Project tools | Every `defineTool` passed to `createAgent({ tools })` that applies to the target's platform, under its own name; an engine pack such as `agentDeviceTools` adds `open_app`, `swipe`, `type_text`, `alert`. |
|
|
52
|
+
|
|
53
|
+
Resources: `e2e://guide` and `e2e://guide/<topic>` hold this skill.
|
|
54
|
+
|
|
55
|
+
## Workflow
|
|
56
|
+
|
|
57
|
+
1. `open_session`, then `call {tool: "observe"}` and act until the screen you
|
|
58
|
+
want to test is in front of you. Node ids are valid only for the newest
|
|
59
|
+
observation; an action reports what changed, so observe again before
|
|
60
|
+
using new ids. The opening text lists every tool with its arguments;
|
|
61
|
+
`tools {tool}` when you need the full contract.
|
|
62
|
+
2. `call {tool: "locate", args: {...}}` for each locator you intend to write.
|
|
63
|
+
One match: use the printed `screen.getByRole(...)` call. Zero or several:
|
|
64
|
+
adjust before writing the test, the same failure would hit the test as
|
|
65
|
+
`LOCATOR_NOT_FOUND` or `LOCATOR_AMBIGUOUS`.
|
|
66
|
+
3. Write `tests/<feature>.e2e.ts` (topic `writing-tests`). Deterministic steps
|
|
67
|
+
where you saw exact names; `agent.act` where the flow varies.
|
|
68
|
+
4. Run it from the shell: `npx --no-install e2e run tests/<feature>.e2e.ts`,
|
|
69
|
+
read the failure (topic `debugging`), fix, repeat.
|
|
70
|
+
5. `close_session` when you are done exploring; an idle session closes on its
|
|
71
|
+
own after 30 minutes and never outlives 4 hours. To look at another
|
|
72
|
+
project or config, `open_session {config: "path/to/e2e.config.ts"}`; no
|
|
73
|
+
restart needed.
|
|
74
|
+
|
|
75
|
+
## Rules
|
|
76
|
+
|
|
77
|
+
- Sessions enforce the same policy as tests: navigation stays inside the
|
|
78
|
+
engine's allowed origins, secrets fill only through `type_secret`, and
|
|
79
|
+
pixels are withheld once a secret is on screen.
|
|
80
|
+
- Nothing a session does is recorded as a test or into the trace cache. A
|
|
81
|
+
session is for looking and trying; the test is what you write afterwards.
|
|
82
|
+
- A run from the shell and a live session can share the app only if the
|
|
83
|
+
engine's `command` uses `reuseExisting`; otherwise close the session before
|
|
84
|
+
running.
|
|
85
|
+
- `TARGET_REQUIRED`: pass `target` to `open_session` or start with `--target`.
|
|
86
|
+
`NO_SESSION`: call `open_session` first. `SESSION_OPEN`: one is already
|
|
87
|
+
open; use it or `close_session`. `UNKNOWN_TOOL`: the name is not in this
|
|
88
|
+
session's catalog; the message lists what is.
|
|
@@ -4,10 +4,12 @@
|
|
|
4
4
|
|
|
5
5
|
```bash
|
|
6
6
|
npx --no-install e2e run [files...] [options] # run tests
|
|
7
|
+
npx --no-install e2e explore [goal] [options] # explore the app toward a goal, no test file (see the explore topic)
|
|
7
8
|
npx --no-install e2e list [files...] [options] # print what run would select, without running
|
|
8
9
|
npx --no-install e2e init [--yes] # scaffold a project, refresh the agent skill
|
|
9
|
-
npx --no-install e2e guide [topic] # print this skill: setup, writing-tests, agent, running, debugging
|
|
10
|
+
npx --no-install e2e guide [topic] # print this skill: setup, writing-tests, agent, running, explore, debugging, mcp
|
|
10
11
|
npx --no-install e2e cache ls|clear|stats # read or empty the trace cache
|
|
12
|
+
npx --no-install e2e mcp [--target <name>] # serve the project to a coding agent over MCP (topic mcp)
|
|
11
13
|
npx --no-install e2e telemetry [disable|enable] # anonymous usage telemetry: status, or the switch
|
|
12
14
|
```
|
|
13
15
|
|
|
@@ -20,7 +22,7 @@ npx --no-install e2e telemetry [disable|enable] # anonymous usage telemetry: sta
|
|
|
20
22
|
| `--target <ids>` | Comma-separated target names. Only selected targets start app commands and services; unknown names fail before startup. |
|
|
21
23
|
| `--tag <tag>` | Repeatable tag filter; `--tag-mode all` requires every tag. |
|
|
22
24
|
| `--headed` | Visible browser or simulator when the engine supports it. |
|
|
23
|
-
| `--agent <
|
|
25
|
+
| `--agent <names>` | Run unpinned tests as other configured agents (`agents.<name>`), comma-separated or repeated; several names run each such test once per agent. Default is `agents.default`. |
|
|
24
26
|
| `--workers <n>`, `--retries <n>` | Override the resolved values. |
|
|
25
27
|
| `--reporter <ids>` | `list`, `json`, `junit`, comma-separated. `json` cannot combine with `list`. |
|
|
26
28
|
| `--artifacts <dir>` | Artifact root, default `.e2e/artifacts`. |
|
|
@@ -70,7 +72,7 @@ recorded flow is stale — `--no-cache` only skips the cache for one run.
|
|
|
70
72
|
|
|
71
73
|
- `list` (default): one line per file and target, a `Failed Tests` section
|
|
72
74
|
with each error, its code, the failing line and a code frame, then a
|
|
73
|
-
summary (`Test Files`, `Tests`, `AI`, `Start at`, `Duration`, `Report`); setup steps (a first-run browser download, each service and app command) print above the tests and stay out of `Duration` or are split out of it as `(startup …)`.
|
|
75
|
+
summary (`Test Files`, `Tests`, `AI`, `Cache` when the trace cache was on: agent steps `replayed` whole, `handed off` to the model part-way, or `missed`, `Start at`, `Duration`, `Report`); setup steps (a first-run browser download, each service and app command) print above the tests and stay out of `Duration` or are split out of it as `(startup …)`.
|
|
74
76
|
- `.e2e/report.json` is written on every run whatever the reporters:
|
|
75
77
|
`run.status`, `run.exitCode`, `run.errors[]` (run-level failures such as
|
|
76
78
|
`APP_UNREACHABLE`), and `run.results[]`, one per test and target, with
|
|
@@ -155,5 +157,5 @@ jobs:
|
|
|
155
157
|
- Start the app through the engine's `command`; the runner tears it down on
|
|
156
158
|
every exit path.
|
|
157
159
|
- Agentic suites: a separate config, run on `schedule` or
|
|
158
|
-
`workflow_dispatch`,
|
|
159
|
-
as a secret, never a required check.
|
|
160
|
+
`workflow_dispatch`, the key the config's model reads (`AI_GATEWAY_API_KEY`
|
|
161
|
+
for `gateway()` from `ai`) as a secret, never a required check.
|
|
@@ -21,14 +21,15 @@ pnpm dlx @e2edev/e2e@beta init # pnpm
|
|
|
21
21
|
When `@e2edev/e2e` is already installed, run `npx --no-install e2e init`
|
|
22
22
|
instead, so the installed version scaffolds.
|
|
23
23
|
|
|
24
|
-
The wizard asks for the engine (Playwright by default;
|
|
25
|
-
are the alternatives),
|
|
26
|
-
|
|
24
|
+
The wizard asks for the engine (Web with Playwright by default; Mobile with
|
|
25
|
+
agent-device and None are the alternatives), which model gateway agent steps
|
|
26
|
+
use (adds AI SDK v7 and the provider package), which agent directories receive this skill
|
|
27
27
|
(`.agents/skills/` and `.claude/skills/`), a confirmation of the files it
|
|
28
28
|
will write, and whether to install. `--yes` skips every prompt (use it from
|
|
29
|
-
scripts and from a shell without a TTY): Playwright,
|
|
30
|
-
skill in both directories. The closing line prints the run
|
|
31
|
-
|
|
29
|
+
scripts and from a shell without a TTY): Playwright, the Vercel AI Gateway,
|
|
30
|
+
no installation, skill in both directories. The closing line prints the run
|
|
31
|
+
command through the project's package manager,
|
|
32
|
+
`APP_URL=http://localhost:3000 npm run test:e2e` for Playwright under npm, and
|
|
32
33
|
suggests a `tsconfig.json` when the project has none.
|
|
33
34
|
|
|
34
35
|
Init writes `package.json` (a private ESM package when missing; otherwise
|
|
@@ -56,6 +57,7 @@ unknown keys are `INVALID_CONFIG`.
|
|
|
56
57
|
import type { E2EConfig } from '@e2edev/e2e';
|
|
57
58
|
import { createAgent } from '@e2edev/e2e/agent';
|
|
58
59
|
import { playwright } from '@e2edev/playwright';
|
|
60
|
+
import { gateway } from 'ai';
|
|
59
61
|
|
|
60
62
|
export default {
|
|
61
63
|
tests: 'tests/**/*.e2e.ts',
|
|
@@ -67,8 +69,13 @@ export default {
|
|
|
67
69
|
}),
|
|
68
70
|
},
|
|
69
71
|
],
|
|
70
|
-
// Only for agent.* steps. The model
|
|
71
|
-
agents: {
|
|
72
|
+
// Only for agent.* steps. The model is an AI SDK instance; gateway() from 'ai' reads AI_GATEWAY_API_KEY.
|
|
73
|
+
agents: {
|
|
74
|
+
default: createAgent({
|
|
75
|
+
model: gateway('openai/gpt-5.4-mini'),
|
|
76
|
+
system: 'You are a thorough QA agent. Verify every outcome on screen.',
|
|
77
|
+
}),
|
|
78
|
+
},
|
|
72
79
|
credentials: {
|
|
73
80
|
admin: { username: 'admin@example.test', password: process.env.ADMIN_PASSWORD ?? '' },
|
|
74
81
|
},
|
|
@@ -86,7 +93,7 @@ export default {
|
|
|
86
93
|
| `workers` | half the cores, `1` in CI | Test files run in parallel across workers, at most the `workers` the engine declares per target (a device target: one per device). |
|
|
87
94
|
| `reporters` | `['list']` | `list`, `json`, `junit`, and reporter objects (`{ name, onEvent?, onRunFinished? }`) that receive the finished run. `json` excludes `list`; `--reporter` keeps the objects. |
|
|
88
95
|
| `cache` | `'read-write'`, `'read-only'` in CI | The trace cache for `agent.act`; `'off'` disables it. |
|
|
89
|
-
| `agents` | `{ default: built-in }` | Agents by name. `default` is what tests run with; `e2e run --agent <name>` runs with another. Each entry is `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`.
|
|
96
|
+
| `agents` | `{ default: built-in }` | Agents by name. `default` is what tests run with; `e2e run --agent <name>` runs with another. Each entry is `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. `model` is an AI SDK instance; without one anywhere, acquiring `agent` is `MODEL_UNAVAILABLE`. |
|
|
90
97
|
| `credentials` | `{}` | Named `{ username, password, allowedOrigins? }` entries; `password` may be a function returning the value. |
|
|
91
98
|
| `screen.testIdAttribute` | `'data-testid'` | Attribute read by `getByTestId`. |
|
|
92
99
|
| `artifacts` | `['screenshot', 'trace']` | Kinds to keep (`screenshot`, `trace`, and the opt-in `video`), or `{ kinds, store, video }`; `video: { retain: 'on-failure' }` keeps only the recordings of attempts that did not pass. |
|
|
@@ -184,10 +191,7 @@ or read the address from the environment:
|
|
|
184
191
|
|
|
185
192
|
| Variable | Effect |
|
|
186
193
|
| --- | --- |
|
|
187
|
-
| `
|
|
188
|
-
| `E2E_MODEL_API_KEY` | Credential for that model. `AI_GATEWAY_API_KEY` is the fallback. |
|
|
189
|
-
| `E2E_MODEL_ENDPOINT` | Any OpenAI-compatible endpoint URL for that model; the gateway is the default. HTTPS unless loopback. The agent's `model.endpoint` wins. |
|
|
190
|
-
| `E2E_VISION_MODEL` | Model for calls with `vision`; falls back to the main model. |
|
|
194
|
+
| `AI_GATEWAY_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, ... | Read by the provider package the config constructs the model with (`gateway()` from `ai`, `openrouter()`, `openai()`), not by the runner. The runner reads no model variable; the model is always an AI SDK instance in the config. |
|
|
191
195
|
| `E2E_USER_<NAME>_USERNAME`, `E2E_USER_<NAME>_PASSWORD` | Override `credentials.<name>`. `<NAME>` is the credential name uppercased, other characters as `_`. |
|
|
192
196
|
| `CI` | Turns on CI defaults: `retries: 1`, `workers: 1`, `test.only` rejected, cache read-only, `reuseExisting` ignored. |
|
|
193
197
|
| `E2E_TELEMETRY_DISABLED`, `DO_NOT_TRACK` | Turn anonymous usage telemetry off; `e2e telemetry disable` saves the same choice. `E2E_TELEMETRY_DEBUG=1` prints each event instead of sending it. |
|