e2e 0.21.1 → 0.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +2 -0
  2. package/dist/.build.json +1 -1
  3. package/dist/agent/act-validation.js +1 -0
  4. package/dist/agent/act.js +7 -3
  5. package/dist/agent/action-dispatcher.js +30 -8
  6. package/dist/agent/actions.d.ts +2 -0
  7. package/dist/agent/actions.js +5 -2
  8. package/dist/agent/error.js +1 -0
  9. package/dist/agent/executor.d.ts +14 -6
  10. package/dist/agent/index.js +5 -1
  11. package/dist/agent/invocation.d.ts +3 -0
  12. package/dist/agent/model/sdk.d.ts +5 -0
  13. package/dist/agent/model/sdk.js +18 -1
  14. package/dist/agent/observation.js +1 -0
  15. package/dist/agent/primitives.js +14 -3
  16. package/dist/agent/replay.js +9 -3
  17. package/dist/agent/step-cache.d.ts +3 -2
  18. package/dist/agent/step-cache.js +7 -4
  19. package/dist/agent/tool-loop.js +4 -1
  20. package/dist/cache/context.js +1 -1
  21. package/dist/cache/recorder.d.ts +2 -1
  22. package/dist/cache/recorder.js +11 -2
  23. package/dist/cache/trace-fields.d.ts +10 -0
  24. package/dist/cache/trace-fields.js +17 -0
  25. package/dist/cache/trace.d.ts +9 -1
  26. package/dist/cache/trace.js +26 -7
  27. package/dist/cli/feedback.js +2 -1
  28. package/dist/cli/init/sibling-versions.json +1 -1
  29. package/dist/collect/select.js +12 -2
  30. package/dist/config/agent.d.ts +1 -1
  31. package/dist/config/diagnose.js +9 -3
  32. package/dist/config/esm-hooks.js +3 -2
  33. package/dist/config/resolve.js +3 -2
  34. package/dist/config/typescript.js +52 -2
  35. package/dist/engine/contract.d.ts +3 -2
  36. package/dist/engine/manifest.js +2 -2
  37. package/dist/internal/errors.js +5 -2
  38. package/dist/internal/source.js +2 -0
  39. package/dist/internal/unserved-model.d.ts +5 -0
  40. package/dist/internal/unserved-model.js +22 -0
  41. package/dist/locator/screen.js +1 -2
  42. package/dist/oauth/copilot.js +23 -5
  43. package/dist/oauth/device-code.js +35 -19
  44. package/dist/oauth/errors.d.ts +3 -0
  45. package/dist/oauth/errors.js +26 -11
  46. package/dist/oauth/providers/github-copilot.d.ts +4 -0
  47. package/dist/oauth/providers/github-copilot.js +64 -16
  48. package/dist/oauth/providers/openai.d.ts +2 -2
  49. package/dist/oauth/providers/openai.js +47 -19
  50. package/dist/oauth/providers/opencode-console.js +36 -15
  51. package/dist/oauth/providers/xai.js +18 -9
  52. package/dist/oauth/token-endpoint.d.ts +15 -6
  53. package/dist/oauth/token-endpoint.js +18 -6
  54. package/dist/report/code-frame.js +9 -3
  55. package/dist/report/failure-text.d.ts +10 -2
  56. package/dist/report/failure-text.js +24 -8
  57. package/dist/report/list-model.d.ts +13 -0
  58. package/dist/report/list-steps.d.ts +6 -1
  59. package/dist/report/list-steps.js +29 -0
  60. package/dist/report/list.d.ts +1 -0
  61. package/dist/report/list.js +16 -7
  62. package/dist/report/live-window.js +2 -1
  63. package/dist/report/markdown.js +1 -1
  64. package/dist/run/fixtures.js +2 -0
  65. package/dist/run/host-trace.d.ts +3 -2
  66. package/dist/run/host-trace.js +2 -1
  67. package/dist/run/last-run.d.ts +9 -0
  68. package/dist/run/last-run.js +28 -0
  69. package/dist/run/open-session.d.ts +3 -1
  70. package/dist/run/open-session.js +1 -1
  71. package/dist/run/runner.js +6 -2
  72. package/dist/run/sessions.js +3 -3
  73. package/dist/run/standalone-attempt.d.ts +2 -1
  74. package/dist/run/standalone-attempt.js +3 -1
  75. package/dist/run/steps.d.ts +3 -1
  76. package/dist/run/units.d.ts +8 -0
  77. package/dist/run/units.js +14 -0
  78. package/dist/run/worker-models.d.ts +2 -0
  79. package/dist/run/worker-models.js +7 -0
  80. package/dist/types.d.ts +2 -2
  81. package/docs/authentication.mdx +26 -23
  82. package/docs/browser.mdx +7 -4
  83. package/docs/bug-bash.mdx +2 -2
  84. package/docs/cache.mdx +6 -1
  85. package/docs/ci/bitrise.mdx +1 -1
  86. package/docs/ci/codemagic.mdx +2 -2
  87. package/docs/ci/eas-workflows.mdx +1 -1
  88. package/docs/coding-agents.mdx +7 -35
  89. package/docs/debugging.mdx +24 -36
  90. package/docs/executors.mdx +1 -1
  91. package/docs/github.mdx +2 -1
  92. package/docs/integrations/eas.mdx +1 -1
  93. package/docs/integrations/index.mdx +6 -0
  94. package/docs/integrations/lightpanda.mdx +192 -0
  95. package/docs/integrations/testmu.mdx +152 -0
  96. package/docs/mcp.mdx +123 -0
  97. package/docs/migrate/detox.mdx +1 -1
  98. package/docs/migrate/maestro.mdx +101 -105
  99. package/docs/migrate/playwright.mdx +65 -70
  100. package/docs/mobile.mdx +18 -3
  101. package/docs/models.mdx +4 -2
  102. package/docs/quickstart.mdx +7 -0
  103. package/docs/reference/agent.mdx +369 -277
  104. package/docs/reference/app.mdx +141 -32
  105. package/docs/reference/cli.mdx +674 -558
  106. package/docs/reference/config.mdx +1406 -752
  107. package/docs/reference/engine.mdx +1062 -256
  108. package/docs/reference/environment.mdx +156 -84
  109. package/docs/reference/errors.mdx +106 -72
  110. package/docs/reference/expect.mdx +752 -372
  111. package/docs/reference/mcp.mdx +366 -141
  112. package/docs/reference/mobile.mdx +844 -403
  113. package/docs/reference/reporters.mdx +312 -106
  114. package/docs/reference/runner.mdx +373 -174
  115. package/docs/reference/screen.mdx +931 -398
  116. package/docs/reference/security.mdx +106 -40
  117. package/docs/reference/test.mdx +626 -273
  118. package/docs/reference/web.mdx +956 -533
  119. package/docs/security.mdx +6 -5
  120. package/docs/snippets/model-provider.jsx +763 -0
  121. package/docs/snippets/setup-prompt-card.jsx +22 -0
  122. package/docs/snippets/show-more.jsx +22 -0
  123. package/docs/snippets/video-card.jsx +34 -0
  124. package/docs/snippets/video.jsx +25 -0
  125. package/docs/subscriptions.mdx +10 -3
  126. package/docs/telemetry.mdx +39 -39
  127. package/package.json +3 -2
  128. package/schema/report-v1.schema.json +6 -0
  129. package/skills/e2e/references/agent.md +8 -3
  130. package/skills/e2e/references/debugging.md +1 -1
  131. package/skills/e2e/references/mcp.md +1 -1
  132. package/skills/e2e/references/setup.md +12 -3
  133. package/skills/e2e/references/writing-tests.md +1 -1
@@ -0,0 +1,22 @@
1
+ /** Shows a faded prompt preview with an accessible expand and collapse button. */
2
+ export const SetupPrompt = ({ children }) => {
3
+ const [expanded, setExpanded] = useState(false);
4
+
5
+ return (
6
+ <div className="setup-prompt" data-expanded={expanded}>
7
+ <div className="setup-prompt-header">
8
+ <span>Agent prompt</span>
9
+ </div>
10
+ <div id="setup-prompt-content">{children}</div>
11
+ <button
12
+ type="button"
13
+ className="setup-prompt-toggle"
14
+ aria-expanded={expanded}
15
+ aria-controls="setup-prompt-content"
16
+ onClick={() => setExpanded(!expanded)}
17
+ >
18
+ {expanded ? 'Show less' : 'Show more'}
19
+ </button>
20
+ </div>
21
+ );
22
+ };
@@ -0,0 +1,22 @@
1
+ /** Shows the start of its content in a hairline box, faded out over a "Show more" button that expands it. */
2
+ export const ShowMore = ({ id, children }) => {
3
+ const [expanded, setExpanded] = useState(false);
4
+
5
+ return (
6
+ <div className="show-more" data-expanded={expanded}>
7
+ {/* Tabbing into the folded part opens it, so focus never lands on clipped text. */}
8
+ <div id={id} className="show-more-content" onFocus={() => setExpanded(true)}>
9
+ {children}
10
+ </div>
11
+ <button
12
+ type="button"
13
+ className="show-more-toggle"
14
+ aria-expanded={expanded}
15
+ aria-controls={id}
16
+ onClick={() => setExpanded(!expanded)}
17
+ >
18
+ {expanded ? 'Show less' : 'Show more'}
19
+ </button>
20
+ </div>
21
+ );
22
+ };
@@ -0,0 +1,34 @@
1
+ /** Links to a YouTube video in a new tab: thumbnail on the left, title and subtitle on the right. */
2
+ export const VideoCard = ({ id, title, subtitle }) => (
3
+ <a
4
+ className="video-card"
5
+ href={`https://www.youtube.com/watch?v=${id}`}
6
+ target="_blank"
7
+ rel="noopener noreferrer"
8
+ >
9
+ <span className="video-card-thumbnail">
10
+ <img src={`https://i.ytimg.com/vi/${id}/hqdefault.jpg`} alt="" loading="lazy" noZoom />
11
+ <svg className="video-card-play" viewBox="0 0 48 48" aria-hidden="true">
12
+ <circle cx="24" cy="24" r="24" fill="rgb(0 0 0 / 0.6)" />
13
+ <path d="M19 15.5v17l14-8.5z" fill="#fff" />
14
+ </svg>
15
+ </span>
16
+ <span className="video-card-text">
17
+ <span className="video-card-title">{title}</span>
18
+ <span className="video-card-subtitle">{subtitle}</span>
19
+ <span className="video-card-new-tab"> (opens in a new tab)</span>
20
+ </span>
21
+ <svg
22
+ className="video-card-arrow"
23
+ viewBox="0 0 24 24"
24
+ fill="none"
25
+ stroke="currentColor"
26
+ strokeWidth="2"
27
+ strokeLinecap="round"
28
+ strokeLinejoin="round"
29
+ aria-hidden="true"
30
+ >
31
+ <path d="M7 17 17 7M7 7h10v10" />
32
+ </svg>
33
+ </a>
34
+ );
@@ -0,0 +1,25 @@
1
+ /**
2
+ * A muted, looping clip in a hairline frame with its caption underneath. It
3
+ * plays on its own unless the reader prefers reduced motion; the controls
4
+ * are always there.
5
+ *
6
+ * The page writes the `<video>` element itself, followed by the caption:
7
+ * Mintlify resolves the `src` of a `<video>` in MDX to the deployed asset,
8
+ * but never a prop passed to a custom component, so `src="/images/x.mp4"`
9
+ * as a prop 404s wherever the docs are served under a subpath (`/docs`).
10
+ */
11
+ export const Video = ({ children }) => {
12
+ const figure = useRef(null);
13
+
14
+ useEffect(() => {
15
+ if (window.matchMedia('(prefers-reduced-motion: no-preference)').matches) {
16
+ figure.current?.querySelector('video')?.play().catch(() => {});
17
+ }
18
+ }, []);
19
+
20
+ return (
21
+ <div ref={figure} className="video-figure">
22
+ {children}
23
+ </div>
24
+ );
25
+ };
@@ -71,6 +71,8 @@ import { copilot } from 'e2e/oauth/copilot';
71
71
 
72
72
  const agent = { model: copilot('claude-sonnet-5') };
73
73
  ```
74
+
75
+ Which models your plan serves is up to GitHub; see [Pick a model](#pick-a-model).
74
76
  </Tab>
75
77
  <Tab title="OpenCode Console">
76
78
  <CodeGroup>
@@ -191,9 +193,14 @@ The id is not always the name the vendor's app shows.
191
193
  With `OPENCODE_API_KEY` set, `e2e models opencode-console` lists the ids that
192
194
  key serves, the same key `opencodeConsole()` uses in place of the login.
193
195
 
194
- - **Copilot** marks the models `copilot()` can't call. For `not enabled`,
195
- turn the model on in your Copilot settings on GitHub, or ask the admin who
196
- manages your seat. For `no chat or responses`, use a different model.
196
+ - **Copilot** marks `not enabled` (turn the model on in your Copilot
197
+ settings on GitHub, or ask the admin who manages your seat) and `no chat
198
+ or responses` (use a different model). It can't see what your plan
199
+ serves: on Copilot Free, GitHub serves models through
200
+ [auto model selection only](https://docs.github.com/en/copilot/get-started/plans),
201
+ so a model called by id is usually refused. The run then stops with
202
+ `MODEL_UNAVAILABLE` naming it; pick another id or an
203
+ [API provider](/models).
197
204
  - **OpenCode Console** lists `go/` models only if you have Go. It leaves out
198
205
  models your workspace turned off, and Zen's free models. Key
199
206
  `providerOptions` by `opencode` for chat completions models, else by
@@ -34,10 +34,10 @@ export DO_NOT_TRACK=1 # the cross-tool convention
34
34
  ```
35
35
  </CodeGroup>
36
36
 
37
- `e2e telemetry` prints the status and, when off, the reason.
38
- `e2e telemetry enable` reverses a saved `disable`. The environment variables
39
- win over the saved setting. The preferences file is under
40
- `$XDG_CONFIG_HOME/e2e/` when that is set, and `%APPDATA%\e2e\` on Windows.
37
+ The environment variables win over the saved setting. `e2e telemetry enable`
38
+ reverses a saved `disable`. `e2e telemetry` prints the status and, when off,
39
+ the reason. The preferences file is under `$XDG_CONFIG_HOME/e2e/` when that is
40
+ set, and `%APPDATA%\e2e\` on Windows.
41
41
 
42
42
  A `disable` also stops commands already running in another terminal. Nothing
43
43
  is sent when the preferences directory isn't writable, or from a source
@@ -67,13 +67,19 @@ sends nothing.
67
67
  Each command sends one `e2e_cli_session` event. `run` and `explore` also
68
68
  send `e2e_run_completed`, `init` sends `e2e_init_completed`, and `mcp` sends
69
69
  one `e2e_mcp_session` per MCP session. `--help` and `--version` send
70
- nothing, also on a command. `run()` from `e2e/runner` sends one
71
- `e2e_run_completed` per run that returns a report, and no session event. Its
72
- first call can print the notice below on stderr.
70
+ nothing, on any command. `run()` from `e2e/runner` sends no session event,
71
+ only one `e2e_run_completed` per run that returns a report.
73
72
 
74
- The events go in one request when the command ends, with a two-second
75
- deadline. `e2e mcp` sends each MCP session event in its own request. Events
76
- hold only the properties below.
73
+ The events go in one request when the command or `run()` ends, with a
74
+ two-second deadline. `e2e mcp` sends each session event in its own request,
75
+ and `e2e run --watch` sends each run's event when that run ends.
76
+
77
+ Events hold only the properties below. An error code that isn't an upper-case
78
+ token is sent as `OTHER`. An engine, platform, model, fleet, or MCP client
79
+ name, or an MCP client version, that isn't a plain token of letters, digits,
80
+ dots, dashes, and underscores is sent as `other`. An engine version that isn't
81
+ semver is sent as `unversioned`. In action and tool call counts, a project's
82
+ own tools count together as `tool`, and any other name as `other`.
77
83
 
78
84
  <AccordionGroup>
79
85
  <Accordion title="Every event">
@@ -87,7 +93,7 @@ hold only the properties below.
87
93
  | `os`, `os_release`, `arch`, `cpu_count`, `memory_gb`, `docker`, `wsl`, `tty` | The machine class |
88
94
  | `sandbox` | The microVM the kernel announces itself as (`firecracker`), else `null` |
89
95
  | `ci`, `ci_name` | Whether `CI` is set, and the vendor when one announces itself (`github-actions`, `gitlab`, `circleci`, ...) |
90
- | `fleet` | The value of `E2E_TELEMETRY_FLEET` when it is a plain token, `other` when it isn't, `null` when unset |
96
+ | `fleet` | The value of `E2E_TELEMETRY_FLEET`, `null` when unset |
91
97
  | `coding_agent` | The coding agent whose shell ran the command, when one announces itself (`claude-code`, `cursor`, `codex`, ...) |
92
98
  | `e2e_version`, `package_manager` | The e2e version, and the package manager that started the CLI |
93
99
  | `runtime`, `runtime_version`, `node_version` | `node`, `bun`, or `deno`, and its version. `node_version` is Node's own version, or the Node compatibility version under Bun or Deno |
@@ -101,7 +107,7 @@ hold only the properties below.
101
107
  | `command` | `init`, `run`, `list`, `cache ls`, `guide`, `telemetry`, ... |
102
108
  | `flags` | The names of the flags given, such as `--headed` or `--tag`. Never their values |
103
109
  | `exit_code`, `duration_ms` | How the command ended and how long it took |
104
- | `error_code` | The runner code of the failure that stopped the command before it could run (`CONFIG_NOT_FOUND`, `INVALID_CONFIG`, ...), `CLI_USAGE` for a flag or argument the CLI rejected, else `null`. A code that isn't an upper-case token is sent as `OTHER` |
110
+ | `error_code` | The runner code of the failure that stopped the command before it could run (`CONFIG_NOT_FOUND`, `INVALID_CONFIG`, ...), `CLI_USAGE` for a flag or argument the CLI rejected, else `null` |
105
111
 
106
112
  </Accordion>
107
113
  <Accordion title="e2e_init_completed, once per e2e init">
@@ -123,16 +129,16 @@ hold only the properties below.
123
129
  | `flags` | The names of the flags given to the run command, never their values |
124
130
  | `tests_discovered`, `tests_selected`, `tests_executed`, `tests_passed`, `tests_failed`, `tests_interrupted`, `tests_flaky`, `tests_skipped` | The summary counts |
125
131
  | `attempts_total`, `tests_retried` | How many attempts the run made across tests and serial groups, and how many of these needed more than one |
126
- | `targets`, `platforms`, `engines` | How many targets, on which platforms, and with which engines and versions: `web`, `ios`, `web@0.6.1`, or your own engine's name and version. A name that isn't a plain token of letters, digits, dots, dashes, and underscores is sent as `other` |
132
+ | `targets`, `platforms`, `engines` | How many targets, on which platforms, and with which engines and versions: `web`, `ios`, `web@0.6.1`, or your own engine's name and version |
127
133
  | `steps_total`, `steps_agent`, `steps_locator`, `steps_assertion`, `steps_screen`, `steps_app`, `steps_session`, `steps_resource` | Step counts by kind |
128
134
  | `steps_agent_act`, `steps_agent_assert`, `steps_agent_extract`, `steps_agent_waitfor` | The agent steps by entry point |
129
- | `tests_with_agent_steps`, `tests_with_act` | How many of the executed tests ran any agent step (the rest are deterministic end to end), and how many of those ran `agent.act` |
135
+ | `tests_with_agent_steps`, `tests_with_act` | How many executed tests ran any agent step, and how many ran `agent.act` |
130
136
  | `agent_steps_replayed`, `agent_steps_partial`, `agent_steps_missed`, `agent_steps_vision` | How the replay cache served the agent steps, and how many used pixel evidence |
131
- | `agent_actions` | How often the agent took each action, by the runner's own names: `{"tap": 12, "typeText": 3, "scroll": 2}`. A project's own tools count together as `tool`, anything else as `other`. No other name is sent |
132
- | `model_gateway`, `model_provider`, `model_id`, `model_calls`, `model_tokens`, `model_cached_tokens`, `estimated_cost_usd` | For the first model-backed step: the AI SDK provider that served it (`gateway` for the Vercel AI Gateway, `openrouter.chat`, `openai.chat`, ...), the vendor (the first segment of a `vendor/model` id, else the provider), and the public model id (a fine-tuned or routed id is sent as `other`). Then the run's totals |
137
+ | `agent_actions` | How often the agent took each action, by the runner's own names: `{"tap": 12, "typeText": 3, "scroll": 2}` |
138
+ | `model_gateway`, `model_provider`, `model_id`, `model_calls`, `model_tokens`, `model_cached_tokens`, `estimated_cost_usd` | For the first model-backed step: the AI SDK provider that served it (`gateway` for the Vercel AI Gateway, `openrouter.chat`, `openai.chat`, ...), the vendor (the first segment of a `vendor/model` id, else the provider), and the public model id. Then the run's totals |
133
139
  | `cost_source` | `provider` when the provider priced the calls, `none` when nobody priced them, `null` for a run without model calls |
134
140
  | `artifact_bytes` | The size of the artifacts the run wrote |
135
- | `errors`, `error_codes` | How many run-level errors, and the distinct error codes seen anywhere in the run (`LOCATOR_NOT_FOUND`, `APP_UNREACHABLE`, ...), at most 20. A code that isn't an upper-case token is sent as `OTHER` |
141
+ | `errors`, `error_codes` | How many run-level errors, and the distinct error codes seen anywhere in the run (`LOCATOR_NOT_FOUND`, `APP_UNREACHABLE`, ...), at most 20 |
136
142
  | `primary_error_code` | The first error the run recorded, which decided its status: a run-level error, else the failed attempt's verdict, else a step's. `null` for a clean run |
137
143
  | `error_kinds` | For `ENGINE_FAILURE`, `MODEL_PROVIDER_FAILED`, and `ERROR`: the code and a kind read off the message against a closed list (`auth`, `rate-limit`, `overloaded`, `timeout`, `network`, `server`, `bad-request`, `device`, `daemon`, `browser`, `permission`, `not-found`, `memory`, `other`), as `MODEL_PROVIDER_FAILED:rate-limit`. Only the kind's name leaves the machine, never the message |
138
144
 
@@ -168,13 +174,13 @@ client kills the server isn't sent.
168
174
 
169
175
  | Property | Value |
170
176
  | --- | --- |
171
- | `client`, `client_version` | The name and version the MCP client gives itself (`claude-code`, `cursor-vscode`, ...). A value that isn't a plain token is sent as `other`. `null` before the client introduced itself |
177
+ | `client`, `client_version` | The name and version the MCP client gives itself (`claude-code`, `cursor-vscode`, ...). `null` before the client introduced itself |
172
178
  | `outcome`, `error_code` | `closed` for a session that opened, `open-failed` with the runner code of the failure (`APP_UNREACHABLE`, `ENGINE_FAILURE`, ...) |
173
179
  | `ended_by` | `agent` for `close_session`, `idle`, `step` for the session's time limit, `server` when the server shut down. `null` for a failed open |
174
180
  | `platform`, `engine`, `headed` | The target's platform and engine with its version, as for a run, and whether the UI was shown |
175
181
  | `duration_ms`, `sessions_open` | How long the session lived, and how many other sessions were open when it opened |
176
- | `calls_total`, `calls_failed`, `calls` | Tool calls, failed calls, and calls by the session's own tool names: `{"observe": 4, "tap": 9, "locate": 2, "screenshot": 1}`. A project's own tools count together as `tool`, anything else as `other`. No other name is sent |
177
- | `error_codes` | How often each runner code failed a call: `{"LOCATOR_NOT_FOUND": 2}`, at most 20 codes. A code that isn't an upper-case token is sent as `OTHER` |
182
+ | `calls_total`, `calls_failed`, `calls` | Tool calls, failed calls, and calls by the session's own tool names: `{"observe": 4, "tap": 9, "locate": 2, "screenshot": 1}` |
183
+ | `error_codes` | How often each runner code failed a call: `{"LOCATOR_NOT_FOUND": 2}`, at most 20 codes |
178
184
 
179
185
  </Accordion>
180
186
  </AccordionGroup>
@@ -217,25 +223,19 @@ other connection a run can make.
217
223
 
218
224
  ## Feedback
219
225
 
220
- [`e2e feedback`](/reference/cli#e2e-feedback) isn't telemetry. When you or
221
- your coding agent run it, it sends the words in its flags as one
222
- `e2e_feedback` event to the same PostHog project, with the machine facts
223
- above.
224
-
225
- - A saved `e2e telemetry disable` doesn't stop it.
226
- `E2E_TELEMETRY_DISABLED=1` and `DO_NOT_TRACK=1` do.
227
- - `E2E_TELEMETRY_DEBUG=1` and `--dry-run` print the event instead of sending
228
- it.
229
- - When telemetry is on, the event has the same anonymous id as your
230
- telemetry events.
231
- - The CLI redacts well-known token shapes and the values of secret-named
232
- environment variables, if 6 characters or longer, before it sends.
233
- - A bug without `--repro` isn't sent. You get a prefilled GitHub issue to
234
- finish and submit yourself.
226
+ [`e2e feedback`](/reference/cli#e2e-feedback) isn't telemetry. It sends the
227
+ words in its flags as one `e2e_feedback` event to the same PostHog project,
228
+ with the machine facts above. When telemetry is on, it has the same anonymous
229
+ id as your telemetry events.
230
+
231
+ A saved `e2e telemetry disable` doesn't stop it. `E2E_TELEMETRY_DISABLED=1`
232
+ and `DO_NOT_TRACK=1` do. The [CLI reference](/reference/cli#e2e-feedback)
233
+ covers redaction, `--dry-run`, and bugs without `--repro`.
235
234
 
236
235
  ## The notice
237
236
 
238
- The first command prints a notice on stderr that tells how to opt out, and
239
- leaves `--reporter json` output intact. `e2e init` and `e2e telemetry` don't
240
- print it, so it waits for the next command. The notice shows again after a
241
- large change to the data collected.
237
+ The first command, or the first `run()` from `e2e/runner`, prints a notice
238
+ on stderr that tells how to opt out. It leaves `--reporter json` output
239
+ intact. `e2e init` and `e2e telemetry` don't print it, so it waits for the
240
+ next command. The notice shows again after a large change to the data
241
+ collected.
package/package.json CHANGED
@@ -1,7 +1,8 @@
1
1
  {
2
2
  "name": "e2e",
3
- "version": "0.21.1",
3
+ "version": "0.21.2",
4
4
  "description": "SDK, runner, and CLI for agentic end-to-end testing",
5
+ "mcpName": "io.github.tester-army/e2e",
5
6
  "keywords": [
6
7
  "e2e",
7
8
  "end-to-end",
@@ -132,7 +133,7 @@
132
133
  "@ai-sdk/openai": "4.0.90",
133
134
  "@ai-sdk/openai-compatible": "3.0.53",
134
135
  "@ai-sdk/xai": "5.0.4",
135
- "@e2e-dev/web": "0.15.1",
136
+ "@e2e-dev/web": "0.15.2",
136
137
  "@modelcontextprotocol/client": "2.2.0",
137
138
  "@openrouter/ai-sdk-provider": "3.1.0",
138
139
  "@types/node": "26.6.2",
@@ -233,6 +233,12 @@
233
233
  "required": ["mode", "replayedActions", "totalActions"],
234
234
  "properties": {
235
235
  "mode": { "enum": ["self-finalized", "agent-concluded", "missed"] },
236
+ "recordedBy": {
237
+ "type": "object",
238
+ "additionalProperties": false,
239
+ "required": ["provider", "model"],
240
+ "properties": { "provider": { "type": "string", "maxLength": 300, "pattern": "\\S" }, "model": { "type": "string", "maxLength": 300, "pattern": "\\S" } }
241
+ },
236
242
  "reason": {
237
243
  "enum": [
238
244
  "retry",
@@ -59,9 +59,11 @@ gates, and limits.
59
59
  for a custom brain; none inherits `model` or `context` from `default`.
60
60
  - `model` drives `agent.act`. Judgments use `judge` when set, else `model`.
61
61
  - A missing model for the built-in agent raises one run-level
62
- `MODEL_UNAVAILABLE` when the first test acquires `agent`, exit 2; auth
63
- failures surface on the first model call as `MODEL_PROVIDER_FAILED`, or as
64
- `CONFIG_LOAD_FAILED` for a provider that reads its key when the config loads.
62
+ `MODEL_UNAVAILABLE` when the first test acquires `agent`, exit 2, and so
63
+ does a model GitHub Copilot does not serve to the account, at the first
64
+ call; auth failures surface on the first model call as
65
+ `MODEL_PROVIDER_FAILED`, or as `CONFIG_LOAD_FAILED` for a provider that
66
+ reads its key when the config loads.
65
67
  - `context` is what the app calls things, sent to every model call, judges
66
68
  included; `system` is how the acting agent works, read only by the act
67
69
  loop. Both live on the agents entry; `agentContext` on a test or group
@@ -279,6 +281,9 @@ selectors: the runner validates and authorizes every tool call first.
279
281
 
280
282
  ## The replay cache
281
283
 
284
+ Replay reports expose the original model as `step.cache.recordedBy` when
285
+ available; older recordings omit it. Use `--no-cache` to evaluate a new model.
286
+
282
287
  A passing `agent.act` saves its actions once a later check verifies the
283
288
  outcome; the next run replays them without model calls, and the live agent
284
289
  continues from the current screen when the app or final state no longer
@@ -92,7 +92,7 @@ jq '.run.results[] | select(.selected and .status != "passed") | .attempts[-1]
92
92
  | `ACTION_FAILED` | Element not actionable (covered, disabled, detached), or an operation timed out | `expect` the condition first; close overlays; check `actionTimeout` |
93
93
  | `TEST_TIMEOUT` | The attempt exceeded `timeout` (120 s) | Split the test, or raise `timeout` |
94
94
  | `STEP_NOT_AWAITED` | The body returned while a step or `expect.poll` still ran, or an `afterEach` hook or fixture teardown returned while an `expect.poll` still ran: a call without `await` | `await` the call the code frame names (every `app`, `agent`, `screen`, `expect`, `expect.poll` call) |
95
- | `MODEL_UNAVAILABLE` | `agents.<name>.model` holds no AI SDK instance: reported once under `run.errors` at the first `agent` fixture; the run stops. A model-less custom executor fails only `waitFor` and `extract` this way | Construct one, e.g. `gateway('openai/gpt-6-luna-fast')` from `ai`, and export its key (`AI_GATEWAY_API_KEY`) |
95
+ | `MODEL_UNAVAILABLE` | `agents.<name>.model` holds no AI SDK instance: reported once under `run.errors` at the first `agent` fixture; the run stops. A model-less custom executor fails only `waitFor` and `extract` this way. Also GitHub Copilot refusing the configured model to the account (`model_not_supported`, a plan or settings limit the listing cannot show): reported once at the first call, the run stops | Construct one, e.g. `gateway('openai/gpt-6-luna-fast')` from `ai`, and export its key (`AI_GATEWAY_API_KEY`); for Copilot, pick an id the plan serves |
96
96
  | `MODEL_PROVIDER_FAILED` | Network, 5xx, rate limit, no credits, or no response in 120 s, after the transport retries | Check the credential (the key; for keyless `gateway()` the Vercel CLI login and `.vercel/project.json`) and quota; retry (exit 3) |
97
97
  | `REPLAY_STALE` | `--strict-cache` or `cache.strict`, and a committed recording no longer replays; `step.cache.reason` says why | Rerun read-write with the knob the message names off; commit the changed entry under the directory it names (a custom `cache.store` is written by that run) |
98
98
  | `AUTOMATION_UNSUPPORTED` | The step needs an interaction the engine's toolset lacks (drag on a device, say); blocked, exit 1 | Do that step with `screen` actions, or run the test on a target that supports it |
@@ -71,7 +71,7 @@ The catalog, per session:
71
71
  | Catalog tool | Does |
72
72
  | --- | --- |
73
73
  | `observe` | A fresh observation: one node per line as `#id role "name" ...`, plus the current path. On the web engine a link's `href` is its path on the app URL's origin, else origin and path (`mailto:` and `tel:` keep their scheme, `blob:` its inner origin and path); `?…` and `#…` mark a dropped query or fragment, `data:…` and `javascript:…` an inline payload, a trailing `…` a target cut at 256 characters. |
74
- | `tap`, `double_tap`, `long_press`, `right_click`, `hover`, `type`, `press`, `select`, `check`, `scroll`, `scroll_to`, `drag`, `upload`, `navigate`, `back`, `dismiss_keyboard` | The grammar verbs, as the testing agent gets them: `check` takes `checked`, `drag` a `to` node, `upload` project-relative `files`; `dismiss_keyboard` comes with device engines. Each reports what changed on screen; `observe` shows the whole screen. A verb the engine cannot honor is not listed and fails with `UNSUPPORTED_CAPABILITY`. |
74
+ | `tap`, `double_tap`, `long_press`, `right_click`, `hover`, `type`, `press`, `select`, `check`, `scroll`, `scroll_to`, `drag`, `upload`, `navigate`, `back`, `dismiss_keyboard` | The grammar verbs, as the testing agent gets them: `check` takes `checked`, `drag` a `to` node, `upload` project-relative `files`; `dismiss_keyboard` comes with device engines. On an engine that declares `tapModifiers`, `tap`, `double_tap`, and `right_click` take an optional `modifiers` list (`Shift`, `Control`, `Alt`, `Meta`, `ControlOrMeta`) to hold keys for the click. Each reports what changed on screen; `observe` shows the whole screen. A verb the engine cannot honor is not listed and fails with `UNSUPPORTED_CAPABILITY`. |
75
75
  | `type_secret` | Fills a configured secret by name: a credential's password into a password field, a `secrets` entry into any editable input; the plaintext never reaches the agent. Listed when the config declares `credentials` or `secrets` and the engine can fill secrets. |
76
76
  | `locate` | Tries a semantic locator with exactly one of `role`, `text`, `label`, `placeholder`, `testId` (`name` only with `role`; `exact` optional; anything else is `INVALID_ARGUMENT`) and returns how many nodes match, which, and the `screen.*` call to write. |
77
77
  | `screenshot` | The masked pixels as an image. |
@@ -90,7 +90,10 @@ Switching to Copilot: install `ai`, `@ai-sdk/openai-compatible`, and
90
90
  over Copilot's Responses API when the plan serves that model only there, choosing
91
91
  per model from the plan's listing. `npx e2e models github-copilot` marks the models
92
92
  it cannot call at all: those served only over an API `copilot()` does not speak, and
93
- those the plan has not enabled.
93
+ those the plan has not enabled. Whether the plan serves a model shows nowhere: on
94
+ Copilot Free, GitHub serves models through auto model selection only, so a model
95
+ called by id is usually refused, and the run stops with `MODEL_UNAVAILABLE` naming
96
+ it. Pick another id or an API provider.
94
97
 
95
98
  Switching to OpenCode Console: install `ai`, `@ai-sdk/openai-compatible`,
96
99
  `@ai-sdk/openai`, `@ai-sdk/anthropic`, and `@ai-sdk/google`, set
@@ -201,9 +204,9 @@ start a script that brings them up and serves the app.
201
204
 
202
205
  | Option | Meaning |
203
206
  | --- | --- |
204
- | `browser` | `'chromium'` (default), `'firefox'`, `'webkit'`, or a `BrowserProvider` leasing hosted browsers over CDP (`kernel()` from `@e2e-dev/kernel`, `smol()` from `@e2e-dev/smol`, or your own), which implies chromium and excludes `connect`. Scope `'worker'` (default): one browser per worker slot from `prepare` to `finish`; `'attempt'`: one per attempt, with `reconnectEndpoint`'s limits. |
207
+ | `browser` | `'chromium'` (default), `'firefox'`, `'webkit'`, or a `BrowserProvider` leasing hosted browsers over CDP (`kernel()` from `@e2e-dev/kernel`, `smol()` from `@e2e-dev/smol`, `lightpanda()` from `@e2e-dev/lightpanda`, or your own), which implies chromium and excludes `connect`. Scope `'worker'` (default): one browser per worker slot from `prepare` to `finish`; `'attempt'`: one per attempt, with `reconnectEndpoint`'s limits (`lightpanda()` is worker scope only). |
205
208
  | `viewport` | `{ width, height }`, default 1280x720; `null` follows the browser window (hosted live view, headed run). On a headed hosted browser (Kernel) use `null` and size the service's screen; a fixed size gives a smaller, unmaximized window. |
206
- | `connect` | `{ cdpEndpoint }` attaches to a remote Chromium over CDP; both it and `reconnectEndpoint` are resolvers `(signal) => url`, not strings. With `reconnectEndpoint` it rides one persistent default context and reconnects only to the original browser and page. |
209
+ | `connect` | `{ cdpEndpoint }` attaches to a remote Chromium over CDP; `cdpEndpoint` is required, and both it and `reconnectEndpoint` are resolvers `(signal) => url`, not strings, else `INVALID_CONFIG` at load. With `reconnectEndpoint` it rides one persistent default context, needs a fresh browser from `cdpEndpoint` every attempt, and reconnects only to the original browser and page. |
207
210
  | `headers` | Sent to the app's site only (Vercel's `x-vercel-protection-bypass`, ngrok's `ngrok-skip-browser-warning`), `agent.act` included; disables the browser HTTP cache and service workers. |
208
211
  | `basicAuth` | `{ username, password }` for a `401` challenge; `password` may be `secrets.get('name')`, resolved per attempt and redacted like any secret, the base64 `Authorization` credential too. |
209
212
  | `userAgent` | The `User-Agent` every attempt sends and `navigator.userAgent` reports. |
@@ -361,6 +364,12 @@ export default {
361
364
  iOS keyboard actions such as `locator.press('Enter')` return
362
365
  `UNSUPPORTED_CAPABILITY`; tap an app button when available.
363
366
  See `examples/with-limrun` for the same form and restart tests on both platforms.
367
+ - `testmu({ device, osVersion, app })` from `@e2e-dev/testmu` leases TestMu
368
+ AI (formerly LambdaTest) emulators, simulators, or real devices
369
+ (`deviceType: 'real'`); `device` and `osVersion` match its catalog
370
+ exactly, TestMu AI installs `app` (omit `app.appPath`), and it reads
371
+ `LT_USERNAME` and `LT_ACCESS_KEY`. An attempt's video links TestMu AI's
372
+ recording of the whole session.
364
373
  - Only a control that appeared or moved with the previous action waits out
365
374
  `transition` (default 500 ms); agent actions settle `settle` ms (default
366
375
  150) before the next observation, `settle: false` skips it.
@@ -144,7 +144,7 @@ assertion. Every query also exists on a locator, scoped to its subtree.
144
144
  | `getByPlaceholder(text, { exact?, visible? })` | Inputs by placeholder. |
145
145
  | `getByText(text, { exact?, visible? })` | Visible text. |
146
146
  | `getByDisplayValue(value, { exact?, visible? })` | Inputs by current value; on the web it cannot scope child queries or be a `filter({ has })` target. |
147
- | `getByTestId(id, { visible? })` | `data-testid` on the web (or `web({ testIdAttribute })`), accessibility identifier or resource id on a device (React Native `testID`, SwiftUI `.accessibilityIdentifier`, Compose `testTag` with `testTagsAsResourceId` on Android, Flutter `Semantics(identifier:)`); a string matches the whole id, a RegExp tests it. Last resort. |
147
+ | `getByTestId(id, { visible? })` | `data-testid` on the web (or `web({ testIdAttribute })`), accessibility identifier or resource id on a device (React Native `testID`, SwiftUI `.accessibilityIdentifier`, Compose `testTag` with `testTagsAsResourceId` on Android, Flutter `Semantics(identifier:)`, .NET MAUI `AutomationId`); a string matches the whole id, a RegExp tests it; on Android the `<package>:id/` prefix may be left off. Last resort. |
148
148
 
149
149
  Roles: `button`, `link`, `textbox`, `searchbox`, `combobox`, `listbox`,
150
150
  `option`, `checkbox`, `radio`, `radiogroup`, `switch`, `slider`, `spinbutton`,