@e2edev/e2e 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. package/README.md +4 -4
  2. package/dist/agent/act.d.ts +13 -7
  3. package/dist/agent/act.d.ts.map +1 -1
  4. package/dist/agent/act.js +122 -730
  5. package/dist/agent/act.js.map +1 -1
  6. package/dist/agent/action-dispatcher.d.ts +98 -0
  7. package/dist/agent/action-dispatcher.d.ts.map +1 -0
  8. package/dist/agent/action-dispatcher.js +319 -0
  9. package/dist/agent/action-dispatcher.js.map +1 -0
  10. package/dist/agent/actions.d.ts +16 -1
  11. package/dist/agent/actions.d.ts.map +1 -1
  12. package/dist/agent/actions.js +5 -1
  13. package/dist/agent/actions.js.map +1 -1
  14. package/dist/agent/default-agent.d.ts +1 -1
  15. package/dist/agent/default-agent.d.ts.map +1 -1
  16. package/dist/agent/default-agent.js +30 -9
  17. package/dist/agent/default-agent.js.map +1 -1
  18. package/dist/agent/executor.d.ts +30 -2
  19. package/dist/agent/executor.d.ts.map +1 -1
  20. package/dist/agent/executor.js.map +1 -1
  21. package/dist/agent/index.js +1 -1
  22. package/dist/agent/index.js.map +1 -1
  23. package/dist/agent/invocation.d.ts +4 -9
  24. package/dist/agent/invocation.d.ts.map +1 -1
  25. package/dist/agent/invocation.js +1 -6
  26. package/dist/agent/invocation.js.map +1 -1
  27. package/dist/agent/model/sdk.d.ts +1 -1
  28. package/dist/agent/model/sdk.js +1 -1
  29. package/dist/agent/observation-feed.d.ts +123 -0
  30. package/dist/agent/observation-feed.d.ts.map +1 -0
  31. package/dist/agent/observation-feed.js +281 -0
  32. package/dist/agent/observation-feed.js.map +1 -0
  33. package/dist/agent/observation.d.ts +14 -1
  34. package/dist/agent/observation.d.ts.map +1 -1
  35. package/dist/agent/observation.js +58 -2
  36. package/dist/agent/observation.js.map +1 -1
  37. package/dist/agent/operation-queue.d.ts +21 -0
  38. package/dist/agent/operation-queue.d.ts.map +1 -0
  39. package/dist/agent/operation-queue.js +27 -0
  40. package/dist/agent/operation-queue.js.map +1 -0
  41. package/dist/agent/pixels.d.ts +21 -0
  42. package/dist/agent/pixels.d.ts.map +1 -0
  43. package/dist/agent/pixels.js +85 -0
  44. package/dist/agent/pixels.js.map +1 -0
  45. package/dist/agent/point-tap.d.ts +52 -0
  46. package/dist/agent/point-tap.d.ts.map +1 -0
  47. package/dist/agent/point-tap.js +88 -0
  48. package/dist/agent/point-tap.js.map +1 -0
  49. package/dist/agent/primitives.d.ts +7 -1
  50. package/dist/agent/primitives.d.ts.map +1 -1
  51. package/dist/agent/primitives.js +96 -25
  52. package/dist/agent/primitives.js.map +1 -1
  53. package/dist/agent/replay.d.ts +8 -3
  54. package/dist/agent/replay.d.ts.map +1 -1
  55. package/dist/agent/replay.js +44 -15
  56. package/dist/agent/replay.js.map +1 -1
  57. package/dist/agent/screen-update.d.ts +50 -18
  58. package/dist/agent/screen-update.d.ts.map +1 -1
  59. package/dist/agent/screen-update.js +76 -94
  60. package/dist/agent/screen-update.js.map +1 -1
  61. package/dist/agent/step-accounting.d.ts +95 -0
  62. package/dist/agent/step-accounting.d.ts.map +1 -0
  63. package/dist/agent/step-accounting.js +201 -0
  64. package/dist/agent/step-accounting.js.map +1 -0
  65. package/dist/agent/step-cache.d.ts.map +1 -1
  66. package/dist/agent/step-cache.js +2 -2
  67. package/dist/agent/step-cache.js.map +1 -1
  68. package/dist/agent/tool-loop.d.ts.map +1 -1
  69. package/dist/agent/tool-loop.js +9 -2
  70. package/dist/agent/tool-loop.js.map +1 -1
  71. package/dist/agent/transcript-compaction.d.ts +38 -0
  72. package/dist/agent/transcript-compaction.d.ts.map +1 -0
  73. package/dist/agent/transcript-compaction.js +165 -0
  74. package/dist/agent/transcript-compaction.js.map +1 -0
  75. package/dist/cache/recorder.d.ts.map +1 -1
  76. package/dist/cache/recorder.js +20 -0
  77. package/dist/cache/recorder.js.map +1 -1
  78. package/dist/cache/trace.d.ts +28 -1
  79. package/dist/cache/trace.d.ts.map +1 -1
  80. package/dist/cache/trace.js +44 -0
  81. package/dist/cache/trace.js.map +1 -1
  82. package/dist/cli/index.d.ts.map +1 -1
  83. package/dist/cli/index.js +51 -25
  84. package/dist/cli/index.js.map +1 -1
  85. package/dist/cli/init/engine-versions.json +1 -1
  86. package/dist/cli/init/engines.d.ts +10 -1
  87. package/dist/cli/init/engines.d.ts.map +1 -1
  88. package/dist/cli/init/engines.js +10 -1
  89. package/dist/cli/init/engines.js.map +1 -1
  90. package/dist/cli/init/gateways.js +3 -3
  91. package/dist/cli/init/mcp-config.js +1 -1
  92. package/dist/cli/init/mcp-config.js.map +1 -1
  93. package/dist/cli/init/scaffold.js +1 -1
  94. package/dist/cli/init.js +2 -2
  95. package/dist/cli/init.js.map +1 -1
  96. package/dist/collect/collect.d.ts +8 -3
  97. package/dist/collect/collect.d.ts.map +1 -1
  98. package/dist/collect/collect.js +29 -5
  99. package/dist/collect/collect.js.map +1 -1
  100. package/dist/collect/registry.d.ts +20 -5
  101. package/dist/collect/registry.d.ts.map +1 -1
  102. package/dist/collect/registry.js +139 -54
  103. package/dist/collect/registry.js.map +1 -1
  104. package/dist/collect/select.js +3 -1
  105. package/dist/collect/select.js.map +1 -1
  106. package/dist/config/agent.d.ts +1 -7
  107. package/dist/config/agent.d.ts.map +1 -1
  108. package/dist/config/agent.js +5 -8
  109. package/dist/config/agent.js.map +1 -1
  110. package/dist/config/app.d.ts +24 -2
  111. package/dist/config/app.d.ts.map +1 -1
  112. package/dist/config/app.js +82 -15
  113. package/dist/config/app.js.map +1 -1
  114. package/dist/config/resolve.d.ts +15 -1
  115. package/dist/config/resolve.d.ts.map +1 -1
  116. package/dist/config/resolve.js +23 -5
  117. package/dist/config/resolve.js.map +1 -1
  118. package/dist/engine/contract.d.ts +9 -2
  119. package/dist/engine/contract.d.ts.map +1 -1
  120. package/dist/engine/contract.js.map +1 -1
  121. package/dist/engine/index.d.ts +18 -4
  122. package/dist/engine/index.d.ts.map +1 -1
  123. package/dist/engine/index.js +8 -0
  124. package/dist/engine/index.js.map +1 -1
  125. package/dist/engine/session.d.ts.map +1 -1
  126. package/dist/engine/session.js +3 -0
  127. package/dist/engine/session.js.map +1 -1
  128. package/dist/engine/surface.d.ts +6 -2
  129. package/dist/engine/surface.d.ts.map +1 -1
  130. package/dist/engine/surface.js.map +1 -1
  131. package/dist/expect/attempt.d.ts +15 -0
  132. package/dist/expect/attempt.d.ts.map +1 -0
  133. package/dist/expect/attempt.js +33 -0
  134. package/dist/expect/attempt.js.map +1 -0
  135. package/dist/expect/index.d.ts +7 -5
  136. package/dist/expect/index.d.ts.map +1 -1
  137. package/dist/expect/index.js +8 -1
  138. package/dist/expect/index.js.map +1 -1
  139. package/dist/expect/poll.d.ts +5 -0
  140. package/dist/expect/poll.d.ts.map +1 -0
  141. package/dist/expect/poll.js +118 -0
  142. package/dist/expect/poll.js.map +1 -0
  143. package/dist/expect/values.d.ts.map +1 -1
  144. package/dist/expect/values.js +6 -1
  145. package/dist/expect/values.js.map +1 -1
  146. package/dist/explore/body.d.ts.map +1 -1
  147. package/dist/explore/body.js +2 -0
  148. package/dist/explore/body.js.map +1 -1
  149. package/dist/explore/executor.js +16 -11
  150. package/dist/explore/executor.js.map +1 -1
  151. package/dist/explore/index.d.ts +3 -2
  152. package/dist/explore/index.d.ts.map +1 -1
  153. package/dist/explore/index.js +5 -5
  154. package/dist/explore/index.js.map +1 -1
  155. package/dist/explore/plan.d.ts.map +1 -1
  156. package/dist/explore/plan.js +16 -5
  157. package/dist/explore/plan.js.map +1 -1
  158. package/dist/explore/progress.d.ts +43 -0
  159. package/dist/explore/progress.d.ts.map +1 -0
  160. package/dist/explore/progress.js +9 -0
  161. package/dist/explore/progress.js.map +1 -0
  162. package/dist/explore/state.d.ts +13 -12
  163. package/dist/explore/state.d.ts.map +1 -1
  164. package/dist/explore/state.js +28 -11
  165. package/dist/explore/state.js.map +1 -1
  166. package/dist/internal/brands.d.ts +2 -0
  167. package/dist/internal/brands.d.ts.map +1 -1
  168. package/dist/internal/brands.js +2 -0
  169. package/dist/internal/brands.js.map +1 -1
  170. package/dist/internal/geometry.d.ts +13 -0
  171. package/dist/internal/geometry.d.ts.map +1 -0
  172. package/dist/internal/geometry.js +16 -0
  173. package/dist/internal/geometry.js.map +1 -0
  174. package/dist/internal/urls.d.ts +6 -0
  175. package/dist/internal/urls.d.ts.map +1 -1
  176. package/dist/internal/urls.js +28 -0
  177. package/dist/internal/urls.js.map +1 -1
  178. package/dist/mcp/catalog.d.ts +5 -4
  179. package/dist/mcp/catalog.d.ts.map +1 -1
  180. package/dist/mcp/catalog.js +5 -36
  181. package/dist/mcp/catalog.js.map +1 -1
  182. package/dist/mcp/server.js +1 -1
  183. package/dist/mcp/server.js.map +1 -1
  184. package/dist/mcp/session.d.ts.map +1 -1
  185. package/dist/mcp/session.js +4 -2
  186. package/dist/mcp/session.js.map +1 -1
  187. package/dist/report/list-explore.d.ts +109 -0
  188. package/dist/report/list-explore.d.ts.map +1 -0
  189. package/dist/report/list-explore.js +388 -0
  190. package/dist/report/list-explore.js.map +1 -0
  191. package/dist/report/list.d.ts +21 -0
  192. package/dist/report/list.d.ts.map +1 -1
  193. package/dist/report/list.js +79 -3
  194. package/dist/report/list.js.map +1 -1
  195. package/dist/report/running-tree.d.ts +7 -7
  196. package/dist/report/running-tree.d.ts.map +1 -1
  197. package/dist/report/running-tree.js +12 -13
  198. package/dist/report/running-tree.js.map +1 -1
  199. package/dist/run/app-ports.d.ts +17 -0
  200. package/dist/run/app-ports.d.ts.map +1 -0
  201. package/dist/run/app-ports.js +74 -0
  202. package/dist/run/app-ports.js.map +1 -0
  203. package/dist/run/events.d.ts +10 -5
  204. package/dist/run/events.d.ts.map +1 -1
  205. package/dist/run/events.js.map +1 -1
  206. package/dist/run/execute.d.ts.map +1 -1
  207. package/dist/run/execute.js +23 -6
  208. package/dist/run/execute.js.map +1 -1
  209. package/dist/run/extended-fixtures.d.ts +33 -0
  210. package/dist/run/extended-fixtures.d.ts.map +1 -0
  211. package/dist/run/extended-fixtures.js +91 -0
  212. package/dist/run/extended-fixtures.js.map +1 -0
  213. package/dist/run/fixtures.d.ts.map +1 -1
  214. package/dist/run/fixtures.js +17 -4
  215. package/dist/run/fixtures.js.map +1 -1
  216. package/dist/run/managed-process.d.ts +18 -0
  217. package/dist/run/managed-process.d.ts.map +1 -1
  218. package/dist/run/managed-process.js +101 -6
  219. package/dist/run/managed-process.js.map +1 -1
  220. package/dist/run/realm.d.ts +5 -4
  221. package/dist/run/realm.d.ts.map +1 -1
  222. package/dist/run/realm.js +8 -7
  223. package/dist/run/realm.js.map +1 -1
  224. package/dist/run/runner.d.ts +9 -3
  225. package/dist/run/runner.d.ts.map +1 -1
  226. package/dist/run/runner.js +10 -5
  227. package/dist/run/runner.js.map +1 -1
  228. package/dist/run/worker/entry.js +2 -1
  229. package/dist/run/worker/entry.js.map +1 -1
  230. package/dist/run/worker/protocol.d.ts +7 -1
  231. package/dist/run/worker/protocol.d.ts.map +1 -1
  232. package/dist/run/worker/protocol.js.map +1 -1
  233. package/dist/types.d.ts +69 -13
  234. package/dist/types.d.ts.map +1 -1
  235. package/package.json +6 -3
  236. package/skills/e2e/SKILL.md +4 -5
  237. package/skills/e2e/references/agent.md +25 -10
  238. package/skills/e2e/references/debugging.md +5 -5
  239. package/skills/e2e/references/explore.md +13 -7
  240. package/skills/e2e/references/mcp.md +3 -3
  241. package/skills/e2e/references/running.md +21 -17
  242. package/skills/e2e/references/setup.md +25 -12
  243. package/skills/e2e/references/writing-tests.md +12 -5
  244. package/dist/agent/model/router.d.ts +0 -36
  245. package/dist/agent/model/router.d.ts.map +0 -1
  246. package/dist/agent/model/router.js +0 -45
  247. package/dist/agent/model/router.js.map +0 -1
  248. package/dist/explore/reporter.d.ts +0 -11
  249. package/dist/explore/reporter.d.ts.map +0 -1
  250. package/dist/explore/reporter.js +0 -54
  251. package/dist/explore/reporter.js.map +0 -1
@@ -7,9 +7,13 @@
7
7
  whatever the nearest `package.json` `type` says, so a CommonJS package (a
8
8
  Next.js app, for instance) needs no change. Write them with `import`, never
9
9
  `require` or `module.exports`.
10
- - For browser tests, `@e2edev/playwright`. Missing browsers download when the
11
- engine first boots. In CI install them up front:
12
- `npx playwright install chromium --with-deps`.
10
+ - For browser tests, `@e2edev/playwright` plus `playwright` (`>=1.63.0 <2`),
11
+ a peer dependency the engine does not install itself: an app that already
12
+ depends on Playwright keeps its version and its browser cache. A version
13
+ outside the range may be rejected by the package manager as an unmet peer
14
+ (npm's `ERESOLVE`); upgrade `playwright` within the range.
15
+ Missing browsers download when the engine first boots. In CI install them
16
+ up front: `npx playwright install chromium --with-deps`.
13
17
 
14
18
  ## Scaffold
15
19
 
@@ -18,8 +22,8 @@ npx @e2edev/e2e@beta init # npm
18
22
  pnpm dlx @e2edev/e2e@beta init # pnpm
19
23
  ```
20
24
 
21
- When `@e2edev/e2e` is already installed, run `npx --no-install e2e init`
22
- instead, so the installed version scaffolds.
25
+ When `@e2edev/e2e` is already installed, run `npx e2e init` instead, so the
26
+ installed version scaffolds.
23
27
 
24
28
  The wizard asks for the engine (Web with Playwright by default; Mobile with
25
29
  agent-device and None are the alternatives), which model gateway agent steps
@@ -42,7 +46,7 @@ project.
42
46
  Without the wizard:
43
47
 
44
48
  ```bash
45
- npm install --save-dev @e2edev/e2e@beta @e2edev/playwright@beta ai@^7
49
+ npm install --save-dev @e2edev/e2e@beta @e2edev/playwright@beta playwright ai@^7
46
50
  ```
47
51
 
48
52
  `ai` (the Vercel AI SDK, v7) is only needed for `agent.*` steps.
@@ -72,7 +76,7 @@ export default {
72
76
  // Only for agent.* steps. The model is an AI SDK instance; gateway() from 'ai' reads AI_GATEWAY_API_KEY.
73
77
  agents: {
74
78
  default: createAgent({
75
- model: gateway('openai/gpt-5.4-mini'),
79
+ model: gateway('openai/gpt-5.6-luna'),
76
80
  system: 'You are a thorough QA agent. Verify every outcome on screen.',
77
81
  }),
78
82
  },
@@ -93,7 +97,7 @@ export default {
93
97
  | `workers` | half the cores, `1` in CI | Test files run in parallel across workers, at most the `workers` the engine declares per target (a device target: one per device). |
94
98
  | `reporters` | `['list']` | `list`, `json`, `junit`, and reporter objects (`{ name, onEvent?, onRunFinished? }`) that receive the finished run. `json` excludes `list`; `--reporter` keeps the objects. |
95
99
  | `cache` | `'read-write'`, `'read-only'` in CI | The trace cache for `agent.act`; `'off'` disables it. |
96
- | `agents` | `{ default: built-in }` | Agents by name. `default` is what tests run with; `e2e run --agent <name>` runs with another. Each entry is `createAgent(...)`, an options block `{ model, context, visionModel, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. `model` is an AI SDK instance; without one anywhere, acquiring `agent` is `MODEL_UNAVAILABLE`. |
100
+ | `agents` | `{ default: built-in }` | Agents by name. `default` is what tests run with; `e2e run --agent <name>` runs with another. Each entry is `createAgent(...)`, an options block `{ model, context, maxSteps, maxModelCalls, vision, providerOptions }`, or a custom `StepExecutor`. `model` is an AI SDK instance; without one anywhere, acquiring `agent` is `MODEL_UNAVAILABLE`. |
97
101
  | `credentials` | `{}` | Named `{ username, password, allowedOrigins? }` entries; `password` may be a function returning the value. |
98
102
  | `screen.testIdAttribute` | `'data-testid'` | Attribute read by `getByTestId`. |
99
103
  | `artifacts` | `['screenshot', 'trace']` | Kinds to keep (`screenshot`, `trace`, and the opt-in `video`), or `{ kinds, store, video }`; `video: { retain: 'on-failure' }` keeps only the recordings of attempts that did not pass. |
@@ -106,9 +110,9 @@ process, and identity. `playwright()` accepts:
106
110
 
107
111
  | Option | Meaning |
108
112
  | --- | --- |
109
- | `url` | Base URL for `app.open()` and relative navigation. A missing scheme becomes `https://`, or `http://` for a loopback host. Required once a test navigates. |
110
- | `command` | The process that serves `url`. See below. |
111
- | `readyUrl` | Readiness probe when it differs from `url`. |
113
+ | `url` | Base URL for `app.open()` and relative navigation. A missing scheme becomes `https://`, or `http://` for a loopback host. Required once a test navigates. Port `0` on `127.0.0.1` or `[::1]` asks the run for a free port. |
114
+ | `command` | The process that serves `url`. `{port}` in `args` and `env` expands to the port of `url`. See below. |
115
+ | `readyUrl` | Readiness probe when it differs from `url`. `{port}` expands here too. |
112
116
  | `services` | Dependency processes started before `command`, in order. |
113
117
  | `allowedOrigins` | Origins tests and the agent may navigate to. Default: the origin of `url`. |
114
118
  | `environment` | `'test'`, `'staging'`, `'production'`. Inferred from the host; a label for the report and the cache key. |
@@ -181,6 +185,15 @@ How it behaves:
181
185
  then runs their `teardown` commands in reverse.
182
186
  - `executable` is resolved on `PATH` and never shell-interpreted. Point it at
183
187
  the server itself rather than at a wrapper script.
188
+ - `url: 'http://127.0.0.1:0'` (or `[::1]:0`, never `localhost:0`) asks the
189
+ run for a free port, so two checkouts can run at once. The command must take
190
+ it through `{port}` in `args` or `env`; the token also expands in `readyUrl`
191
+ and the services:
192
+ `command: { executable: 'pnpm', args: ['dev', '--port', '{port}'], env: { PORT: '{port}' } }`.
193
+ Tests read the allocated URL from `app.baseUrl`; the cache identity keeps
194
+ the declared `:0`. Services keep their own ports. A port another process
195
+ grabs between allocation and spawn fails the start with `APP_UNREACHABLE`;
196
+ rerun.
184
197
 
185
198
  To test an app started elsewhere, point `url` at it and start it yourself,
186
199
  or read the address from the environment:
@@ -244,7 +257,7 @@ export default {
244
257
 
245
258
  ## Done when
246
259
 
247
- - `npx --no-install e2e run tests/example.e2e.ts` passes against the app.
260
+ - `npx e2e run tests/example.e2e.ts` passes against the app.
248
261
  - `package.json` has a script such as `"test:e2e": "e2e run"`.
249
262
  - `.gitignore` lists the `.e2e/` outputs (init adds them). Committing
250
263
  `.e2e/cache/` is opt-in: remove that line to share `agent.act` replays.
@@ -53,6 +53,8 @@ test.afterAll(async () => {});
53
53
  test.skip('later', async () => {});
54
54
  test.only('focus', async () => {}); // local only: CI fails with ONLY_IN_CI
55
55
  test.setup('sign in', { sessions: ['admin'] }, async ({ app, screen, session }) => {}); // see Sign-in sessions
56
+ const wsTest = test.extend<{ ws: Ws }>({ ws: async ({ web }, use) => { await use(await seed()); await drop(); } });
57
+ wsTest('uses the workspace', async ({ ws }) => {}); // code after use() is teardown, runs after failures too
56
58
  ```
57
59
 
58
60
  | Option | Default | Notes |
@@ -94,7 +96,7 @@ Fixtures are lazy; destructure them in the callback.
94
96
  | `back()` | One history step back. |
95
97
  | `restart()` | Recreates the context and keeps persisted state, including a restored session. |
96
98
  | `clearState()` | Clears cookies and storage, then relaunches. Not inside a serial group. |
97
- | `screenshot(label?)` | Saves a redacted screenshot as an artifact and returns its path. |
99
+ | `screenshot(label?)` | Saves a redacted screenshot as an artifact and returns its path. Fails with `POLICY_DENIED` after a secret fill. |
98
100
 
99
101
  ## Locators
100
102
 
@@ -166,7 +168,10 @@ read. Reading a password field's value is `POLICY_DENIED`.
166
168
 
167
169
  `expect(locator)` polls for up to `config.assertionTimeout` (5 s) or
168
170
  `{ timeout }`; `.not` inverts. `expect(web)` gives the browser matchers.
169
- `expect(value)` is synchronous.
171
+ `expect(value)` is synchronous. `expect.poll(read, { timeout?, interval?,
172
+ message? })` re-reads a value until a value matcher passes
173
+ (`assertionTimeout` and 100 ms by default, stopping with the attempt); a
174
+ throwing read keeps polling, and it is not a report step.
170
175
 
171
176
  ```ts
172
177
  await expect(screen.getByRole('status')).toHaveText('Saved');
@@ -175,6 +180,7 @@ await expect(screen.getByTestId('todo')).toHaveCount(3);
175
180
  await expect(web).toHaveURL('/dashboard'); // relative to the base URL, or a RegExp
176
181
  await expect(web).toHaveTitle(/Dashboard/);
177
182
  expect(await screen.getByTestId('total').textContent()).toContain('$');
183
+ await expect.poll(() => db.orders.count(), { timeout: 15_000 }).toBe(1); // re-reads until it holds
178
184
  ```
179
185
 
180
186
  | Locator matchers | Web matchers | Value matchers |
@@ -232,9 +238,10 @@ credentials: {
232
238
  `E2E_USER_ADMIN_USERNAME` and `E2E_USER_ADMIN_PASSWORD` override either
233
239
  field per run. `credentials.user('admin').password` is a `Secret` with no
234
240
  plaintext accessor; only `fill()` and `agent.act` params accept it. Once a
235
- secret is filled, screenshots stop being attached for the rest of that
236
- attempt, so sign in inside a setup test and keep the evidence in the tests
237
- that matter.
241
+ secret is filled, model pixels and assertion screenshots are withheld for
242
+ the rest of that session, including later tests sharing a serial session.
243
+ `app.screenshot()` fails with `POLICY_DENIED` before capture. Sign in inside
244
+ a setup test and keep the evidence in the tests that matter.
238
245
 
239
246
  ## The web fixture (browser only)
240
247
 
@@ -1,36 +0,0 @@
1
- /** Which model an invocation talks to. */
2
- import type { ResolvedAgentConfig } from '../../config/agent.ts';
3
- import type { ModelAdapter } from './adapter.ts';
4
- /**
5
- * Picks the adapter for one invocation.
6
- *
7
- * `agent.visionModel` exists because visual grounding is a much higher bar than
8
- * accepting an image, and pinning it separately keeps every text-only call on the
9
- * cheaper model. The vision adapter is built on first use rather than at fixture
10
- * acquisition: a project that pins a vision model must not fail a run that never
11
- * asks for pixels, for example because that model's credential is absent from
12
- * this environment.
13
- */
14
- export declare class ModelRouter {
15
- private readonly buildBase;
16
- private readonly buildVision;
17
- private baseAdapter;
18
- private visionAdapter;
19
- constructor(buildBase: () => ModelAdapter, buildVision: (() => ModelAdapter) | undefined);
20
- /**
21
- * The adapter for a call. An invocation that is sending pixels keeps the vision
22
- * model for the rest of its lifetime, including rounds whose pixels were
23
- * withheld, so a polling method never switches models between rounds.
24
- */
25
- select(vision: boolean): ModelAdapter;
26
- }
27
- /**
28
- * Builds the router for one attempt from resolved agent config.
29
- *
30
- * The base adapter is built on first use, like the vision adapter: a run whose
31
- * `agent.act()` steps go to a custom executor may legitimately have no model at
32
- * all, and must not fail on a MODEL_UNAVAILABLE it would never hit. Judgment
33
- * and locate methods still fail with MODEL_UNAVAILABLE on their first call.
34
- */
35
- export declare function createModelRouter(agent: ResolvedAgentConfig, build: (model: ResolvedAgentConfig['model']) => ModelAdapter): ModelRouter;
36
- //# sourceMappingURL=router.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../../src/agent/model/router.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAE1C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD;;;;;;;;;GASG;AACH,qBAAa,WAAW;IAKpB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAL9B,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,aAAa,CAA2B;IAEhD,YACmB,SAAS,EAAE,MAAM,YAAY,EAC7B,WAAW,EAAE,CAAC,MAAM,YAAY,CAAC,GAAG,SAAS,EAC5D;IAEJ;;;;OAIG;IACH,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,YAAY,CAKpC;CACF;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,mBAAmB,EAC1B,KAAK,EAAE,CAAC,KAAK,EAAE,mBAAmB,CAAC,OAAO,CAAC,KAAK,YAAY,GAC3D,WAAW,CAMb"}
@@ -1,45 +0,0 @@
1
- /** Which model an invocation talks to. */
2
- /**
3
- * Picks the adapter for one invocation.
4
- *
5
- * `agent.visionModel` exists because visual grounding is a much higher bar than
6
- * accepting an image, and pinning it separately keeps every text-only call on the
7
- * cheaper model. The vision adapter is built on first use rather than at fixture
8
- * acquisition: a project that pins a vision model must not fail a run that never
9
- * asks for pixels, for example because that model's credential is absent from
10
- * this environment.
11
- */
12
- export class ModelRouter {
13
- buildBase;
14
- buildVision;
15
- baseAdapter;
16
- visionAdapter;
17
- constructor(buildBase, buildVision) {
18
- this.buildBase = buildBase;
19
- this.buildVision = buildVision;
20
- }
21
- /**
22
- * The adapter for a call. An invocation that is sending pixels keeps the vision
23
- * model for the rest of its lifetime, including rounds whose pixels were
24
- * withheld, so a polling method never switches models between rounds.
25
- */
26
- select(vision) {
27
- if (!vision || this.buildVision === undefined) {
28
- return (this.baseAdapter ??= this.buildBase());
29
- }
30
- return (this.visionAdapter ??= this.buildVision());
31
- }
32
- }
33
- /**
34
- * Builds the router for one attempt from resolved agent config.
35
- *
36
- * The base adapter is built on first use, like the vision adapter: a run whose
37
- * `agent.act()` steps go to a custom executor may legitimately have no model at
38
- * all, and must not fail on a MODEL_UNAVAILABLE it would never hit. Judgment
39
- * and locate methods still fail with MODEL_UNAVAILABLE on their first call.
40
- */
41
- export function createModelRouter(agent, build) {
42
- const visionModel = agent.visionModel;
43
- return new ModelRouter(() => build(agent.model), visionModel === undefined ? undefined : () => build(visionModel));
44
- }
45
- //# sourceMappingURL=router.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"router.js","sourceRoot":"","sources":["../../../src/agent/model/router.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAK1C;;;;;;;;;GASG;AACH,MAAM,OAAO,WAAW;IAKH,SAAS;IACT,WAAW;IALtB,WAAW,CAA2B;IACtC,aAAa,CAA2B;IAEhD,YACmB,SAA6B,EAC7B,WAA6C;yBAD7C,SAAS;2BACT,WAAW;IAC3B,CAAC;IAEJ;;;;OAIG;IACH,MAAM,CAAC,MAAe;QACpB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IACrD,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAA0B,EAC1B,KAA4D;IAE5D,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;IACtC,OAAO,IAAI,WAAW,CACpB,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,EACxB,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CACjE,CAAC;AACJ,CAAC"}
@@ -1,11 +0,0 @@
1
- /**
2
- * The exploration summary under the run summary: a `Reporter` whose
3
- * `onRunFinished` rows the list reporter prints, so the findings appear where
4
- * a reporter's own rows do, and the JSON reporter is left to the report.
5
- */
6
- import type { Reporter, ReporterSummary } from '../types.ts';
7
- import type { ExploreState } from './state.ts';
8
- export declare function exploreReporter(state: ExploreState): Reporter;
9
- /** The rows: what was explored, what was found, one row per finding, and the assessment. */
10
- export declare function summaryRows(state: ExploreState): ReporterSummary;
11
- //# sourceMappingURL=reporter.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/explore/reporter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,GAAG,QAAQ,CAK7D;AAED,4FAA4F;AAC5F,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,eAAe,CAgChE"}
@@ -1,54 +0,0 @@
1
- /**
2
- * The exploration summary under the run summary: a `Reporter` whose
3
- * `onRunFinished` rows the list reporter prints, so the findings appear where
4
- * a reporter's own rows do, and the JSON reporter is left to the report.
5
- */
6
- export function exploreReporter(state) {
7
- return {
8
- name: 'explore',
9
- onRunFinished: async () => summaryRows(state),
10
- };
11
- }
12
- /** The rows: what was explored, what was found, one row per finding, and the assessment. */
13
- export function summaryRows(state) {
14
- const counts = { passed: 0, failed: 0, blocked: 0, exhausted: 0 };
15
- for (const step of state.steps)
16
- counts[step.status] += 1;
17
- const parts = [
18
- `${counts.passed} passed`,
19
- ...(counts.failed > 0 ? [`${counts.failed} failed`] : []),
20
- ...(counts.exhausted > 0 ? [`${counts.exhausted} ended at their limit`] : []),
21
- ...(counts.blocked > 0 ? [`${counts.blocked} blocked`] : []),
22
- ];
23
- const issues = state.issues.length;
24
- const warnings = state.findings.length - issues;
25
- const rows = [
26
- {
27
- label: 'Explored',
28
- text: `${state.steps.length} step${state.steps.length === 1 ? '' : 's'} (${parts.join(', ')}); ended: ${ENDED_TEXT[state.ended]}`,
29
- },
30
- {
31
- label: 'Findings',
32
- text: state.findings.length === 0
33
- ? 'none'
34
- : `${issues} issue${issues === 1 ? '' : 's'}, ${warnings} warning${warnings === 1 ? '' : 's'}`,
35
- },
36
- ];
37
- for (const finding of [...state.findings].toSorted((a, b) => b.severity - a.severity || a.index - b.index)) {
38
- rows.push({
39
- label: ` S${finding.severity} ${finding.kind}`,
40
- text: `${finding.title}${finding.path === undefined ? '' : ` (${finding.path})`}`,
41
- });
42
- }
43
- if (state.summary !== undefined)
44
- rows.push({ label: 'Assessment', text: state.summary });
45
- return rows;
46
- }
47
- const ENDED_TEXT = {
48
- finished: 'the agent covered the goal',
49
- 'step-limit': 'the step limit was reached',
50
- time: 'the time budget ran out',
51
- stuck: 'steps kept failing',
52
- aborted: 'the run was cut short',
53
- };
54
- //# sourceMappingURL=reporter.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/explore/reporter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,MAAM,UAAU,eAAe,CAAC,KAAmB;IACjD,OAAO;QACL,IAAI,EAAE,SAAS;QACf,aAAa,EAAE,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,WAAW,CAAC,KAAmB;IAC7C,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAClE,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK;QAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG;QACZ,GAAG,MAAM,CAAC,MAAM,SAAS;QACzB,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,uBAAuB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7E,GAAG,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7D,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;IAChD,MAAM,IAAI,GAAsC;QAC9C;YACE,KAAK,EAAE,UAAU;YACjB,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,QAAQ,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;SAClI;QACD;YACE,KAAK,EAAE,UAAU;YACjB,IAAI,EACF,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACzB,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,GAAG,MAAM,SAAS,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,WAAW,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE;SACnG;KACF,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3G,IAAI,CAAC,IAAI,CAAC;YACR,KAAK,EAAE,MAAM,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE;YAC/C,IAAI,EAAE,GAAG,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,IAAI,GAAG,EAAE;SAClF,CAAC,CAAC;IACL,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACzF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,GAA0C;IACxD,QAAQ,EAAE,4BAA4B;IACtC,YAAY,EAAE,4BAA4B;IAC1C,IAAI,EAAE,yBAAyB;IAC/B,KAAK,EAAE,oBAAoB;IAC3B,OAAO,EAAE,uBAAuB;CACjC,CAAC"}