@e2edev/e2e 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (319) hide show
  1. package/README.md +4 -4
  2. package/dist/agent/act.d.ts +13 -7
  3. package/dist/agent/act.d.ts.map +1 -1
  4. package/dist/agent/act.js +125 -730
  5. package/dist/agent/act.js.map +1 -1
  6. package/dist/agent/action-dispatcher.d.ts +98 -0
  7. package/dist/agent/action-dispatcher.d.ts.map +1 -0
  8. package/dist/agent/action-dispatcher.js +319 -0
  9. package/dist/agent/action-dispatcher.js.map +1 -0
  10. package/dist/agent/actions.d.ts +16 -1
  11. package/dist/agent/actions.d.ts.map +1 -1
  12. package/dist/agent/actions.js +5 -1
  13. package/dist/agent/actions.js.map +1 -1
  14. package/dist/agent/default-agent.d.ts +1 -1
  15. package/dist/agent/default-agent.d.ts.map +1 -1
  16. package/dist/agent/default-agent.js +30 -9
  17. package/dist/agent/default-agent.js.map +1 -1
  18. package/dist/agent/error.d.ts.map +1 -1
  19. package/dist/agent/error.js +1 -0
  20. package/dist/agent/error.js.map +1 -1
  21. package/dist/agent/executor.d.ts +33 -5
  22. package/dist/agent/executor.d.ts.map +1 -1
  23. package/dist/agent/executor.js.map +1 -1
  24. package/dist/agent/index.js +1 -1
  25. package/dist/agent/index.js.map +1 -1
  26. package/dist/agent/invocation.d.ts +4 -9
  27. package/dist/agent/invocation.d.ts.map +1 -1
  28. package/dist/agent/invocation.js +1 -6
  29. package/dist/agent/invocation.js.map +1 -1
  30. package/dist/agent/model/sdk.d.ts +1 -1
  31. package/dist/agent/model/sdk.js +1 -1
  32. package/dist/agent/observation-feed.d.ts +123 -0
  33. package/dist/agent/observation-feed.d.ts.map +1 -0
  34. package/dist/agent/observation-feed.js +281 -0
  35. package/dist/agent/observation-feed.js.map +1 -0
  36. package/dist/agent/observation.d.ts +14 -1
  37. package/dist/agent/observation.d.ts.map +1 -1
  38. package/dist/agent/observation.js +58 -2
  39. package/dist/agent/observation.js.map +1 -1
  40. package/dist/agent/operation-queue.d.ts +21 -0
  41. package/dist/agent/operation-queue.d.ts.map +1 -0
  42. package/dist/agent/operation-queue.js +27 -0
  43. package/dist/agent/operation-queue.js.map +1 -0
  44. package/dist/agent/phases.js +1 -1
  45. package/dist/agent/phases.js.map +1 -1
  46. package/dist/agent/pixels.d.ts +21 -0
  47. package/dist/agent/pixels.d.ts.map +1 -0
  48. package/dist/agent/pixels.js +85 -0
  49. package/dist/agent/pixels.js.map +1 -0
  50. package/dist/agent/point-tap.d.ts +52 -0
  51. package/dist/agent/point-tap.d.ts.map +1 -0
  52. package/dist/agent/point-tap.js +88 -0
  53. package/dist/agent/point-tap.js.map +1 -0
  54. package/dist/agent/primitives.d.ts +7 -1
  55. package/dist/agent/primitives.d.ts.map +1 -1
  56. package/dist/agent/primitives.js +99 -27
  57. package/dist/agent/primitives.js.map +1 -1
  58. package/dist/agent/replay.d.ts +8 -3
  59. package/dist/agent/replay.d.ts.map +1 -1
  60. package/dist/agent/replay.js +44 -15
  61. package/dist/agent/replay.js.map +1 -1
  62. package/dist/agent/screen-update.d.ts +50 -18
  63. package/dist/agent/screen-update.d.ts.map +1 -1
  64. package/dist/agent/screen-update.js +76 -94
  65. package/dist/agent/screen-update.js.map +1 -1
  66. package/dist/agent/secrets.d.ts.map +1 -1
  67. package/dist/agent/secrets.js +13 -9
  68. package/dist/agent/secrets.js.map +1 -1
  69. package/dist/agent/step-accounting.d.ts +106 -0
  70. package/dist/agent/step-accounting.d.ts.map +1 -0
  71. package/dist/agent/step-accounting.js +216 -0
  72. package/dist/agent/step-accounting.js.map +1 -0
  73. package/dist/agent/step-cache.d.ts.map +1 -1
  74. package/dist/agent/step-cache.js +2 -2
  75. package/dist/agent/step-cache.js.map +1 -1
  76. package/dist/agent/tool-loop.d.ts.map +1 -1
  77. package/dist/agent/tool-loop.js +9 -2
  78. package/dist/agent/tool-loop.js.map +1 -1
  79. package/dist/agent/transcript-compaction.d.ts +38 -0
  80. package/dist/agent/transcript-compaction.d.ts.map +1 -0
  81. package/dist/agent/transcript-compaction.js +165 -0
  82. package/dist/agent/transcript-compaction.js.map +1 -0
  83. package/dist/cache/recorder.d.ts.map +1 -1
  84. package/dist/cache/recorder.js +20 -0
  85. package/dist/cache/recorder.js.map +1 -1
  86. package/dist/cache/trace.d.ts +28 -1
  87. package/dist/cache/trace.d.ts.map +1 -1
  88. package/dist/cache/trace.js +44 -0
  89. package/dist/cache/trace.js.map +1 -1
  90. package/dist/cli/index.d.ts.map +1 -1
  91. package/dist/cli/index.js +55 -32
  92. package/dist/cli/index.js.map +1 -1
  93. package/dist/cli/init/engine-versions.json +1 -1
  94. package/dist/cli/init/engines.d.ts +29 -10
  95. package/dist/cli/init/engines.d.ts.map +1 -1
  96. package/dist/cli/init/engines.js +34 -12
  97. package/dist/cli/init/engines.js.map +1 -1
  98. package/dist/cli/init/gateways.js +3 -3
  99. package/dist/cli/init/mcp-config.js +1 -1
  100. package/dist/cli/init/mcp-config.js.map +1 -1
  101. package/dist/cli/init/scaffold.d.ts.map +1 -1
  102. package/dist/cli/init/scaffold.js +1 -3
  103. package/dist/cli/init/scaffold.js.map +1 -1
  104. package/dist/cli/init.d.ts.map +1 -1
  105. package/dist/cli/init.js +3 -2
  106. package/dist/cli/init.js.map +1 -1
  107. package/dist/collect/collect.d.ts +8 -3
  108. package/dist/collect/collect.d.ts.map +1 -1
  109. package/dist/collect/collect.js +29 -5
  110. package/dist/collect/collect.js.map +1 -1
  111. package/dist/collect/registry.d.ts +20 -5
  112. package/dist/collect/registry.d.ts.map +1 -1
  113. package/dist/collect/registry.js +139 -54
  114. package/dist/collect/registry.js.map +1 -1
  115. package/dist/collect/select.js +3 -1
  116. package/dist/collect/select.js.map +1 -1
  117. package/dist/config/agent.d.ts +1 -7
  118. package/dist/config/agent.d.ts.map +1 -1
  119. package/dist/config/agent.js +5 -8
  120. package/dist/config/agent.js.map +1 -1
  121. package/dist/config/app.d.ts +24 -2
  122. package/dist/config/app.d.ts.map +1 -1
  123. package/dist/config/app.js +82 -15
  124. package/dist/config/app.js.map +1 -1
  125. package/dist/config/diagnose.d.ts.map +1 -1
  126. package/dist/config/diagnose.js +2 -0
  127. package/dist/config/diagnose.js.map +1 -1
  128. package/dist/config/resolve.d.ts +26 -3
  129. package/dist/config/resolve.d.ts.map +1 -1
  130. package/dist/config/resolve.js +66 -23
  131. package/dist/config/resolve.js.map +1 -1
  132. package/dist/config/secrets.d.ts +89 -0
  133. package/dist/config/secrets.d.ts.map +1 -0
  134. package/dist/config/secrets.js +67 -0
  135. package/dist/config/secrets.js.map +1 -0
  136. package/dist/engine/contract.d.ts +17 -2
  137. package/dist/engine/contract.d.ts.map +1 -1
  138. package/dist/engine/contract.js.map +1 -1
  139. package/dist/engine/index.d.ts +18 -4
  140. package/dist/engine/index.d.ts.map +1 -1
  141. package/dist/engine/index.js +8 -0
  142. package/dist/engine/index.js.map +1 -1
  143. package/dist/engine/session.d.ts.map +1 -1
  144. package/dist/engine/session.js +3 -0
  145. package/dist/engine/session.js.map +1 -1
  146. package/dist/engine/surface.d.ts +6 -2
  147. package/dist/engine/surface.d.ts.map +1 -1
  148. package/dist/engine/surface.js.map +1 -1
  149. package/dist/expect/attempt.d.ts +15 -0
  150. package/dist/expect/attempt.d.ts.map +1 -0
  151. package/dist/expect/attempt.js +33 -0
  152. package/dist/expect/attempt.js.map +1 -0
  153. package/dist/expect/index.d.ts +7 -5
  154. package/dist/expect/index.d.ts.map +1 -1
  155. package/dist/expect/index.js +8 -1
  156. package/dist/expect/index.js.map +1 -1
  157. package/dist/expect/poll.d.ts +5 -0
  158. package/dist/expect/poll.d.ts.map +1 -0
  159. package/dist/expect/poll.js +118 -0
  160. package/dist/expect/poll.js.map +1 -0
  161. package/dist/expect/values.d.ts.map +1 -1
  162. package/dist/expect/values.js +6 -1
  163. package/dist/expect/values.js.map +1 -1
  164. package/dist/explore/body.d.ts.map +1 -1
  165. package/dist/explore/body.js +3 -1
  166. package/dist/explore/body.js.map +1 -1
  167. package/dist/explore/executor.js +16 -11
  168. package/dist/explore/executor.js.map +1 -1
  169. package/dist/explore/index.d.ts +3 -2
  170. package/dist/explore/index.d.ts.map +1 -1
  171. package/dist/explore/index.js +5 -5
  172. package/dist/explore/index.js.map +1 -1
  173. package/dist/explore/plan.d.ts.map +1 -1
  174. package/dist/explore/plan.js +16 -5
  175. package/dist/explore/plan.js.map +1 -1
  176. package/dist/explore/progress.d.ts +43 -0
  177. package/dist/explore/progress.d.ts.map +1 -0
  178. package/dist/explore/progress.js +9 -0
  179. package/dist/explore/progress.js.map +1 -0
  180. package/dist/explore/state.d.ts +13 -12
  181. package/dist/explore/state.d.ts.map +1 -1
  182. package/dist/explore/state.js +28 -11
  183. package/dist/explore/state.js.map +1 -1
  184. package/dist/index.d.ts +3 -1
  185. package/dist/index.d.ts.map +1 -1
  186. package/dist/index.js +4 -1
  187. package/dist/index.js.map +1 -1
  188. package/dist/internal/brands.d.ts +2 -0
  189. package/dist/internal/brands.d.ts.map +1 -1
  190. package/dist/internal/brands.js +2 -0
  191. package/dist/internal/brands.js.map +1 -1
  192. package/dist/internal/geometry.d.ts +13 -0
  193. package/dist/internal/geometry.d.ts.map +1 -0
  194. package/dist/internal/geometry.js +16 -0
  195. package/dist/internal/geometry.js.map +1 -0
  196. package/dist/internal/urls.d.ts +6 -0
  197. package/dist/internal/urls.d.ts.map +1 -1
  198. package/dist/internal/urls.js +28 -0
  199. package/dist/internal/urls.js.map +1 -1
  200. package/dist/locator/engine.d.ts.map +1 -1
  201. package/dist/locator/engine.js +1 -0
  202. package/dist/locator/engine.js.map +1 -1
  203. package/dist/mcp/catalog.d.ts +5 -4
  204. package/dist/mcp/catalog.d.ts.map +1 -1
  205. package/dist/mcp/catalog.js +5 -36
  206. package/dist/mcp/catalog.js.map +1 -1
  207. package/dist/mcp/server.js +1 -1
  208. package/dist/mcp/server.js.map +1 -1
  209. package/dist/mcp/session.d.ts +1 -1
  210. package/dist/mcp/session.d.ts.map +1 -1
  211. package/dist/mcp/session.js +12 -6
  212. package/dist/mcp/session.js.map +1 -1
  213. package/dist/report/builtin.d.ts +14 -0
  214. package/dist/report/builtin.d.ts.map +1 -0
  215. package/dist/report/builtin.js +22 -0
  216. package/dist/report/builtin.js.map +1 -0
  217. package/dist/report/explore-text.d.ts +17 -0
  218. package/dist/report/explore-text.d.ts.map +1 -0
  219. package/dist/report/explore-text.js +37 -0
  220. package/dist/report/explore-text.js.map +1 -0
  221. package/dist/report/format.d.ts +2 -0
  222. package/dist/report/format.d.ts.map +1 -1
  223. package/dist/report/format.js +1 -1
  224. package/dist/report/format.js.map +1 -1
  225. package/dist/report/junit.d.ts +2 -4
  226. package/dist/report/junit.d.ts.map +1 -1
  227. package/dist/report/junit.js +7 -40
  228. package/dist/report/junit.js.map +1 -1
  229. package/dist/report/list-explore.d.ts +109 -0
  230. package/dist/report/list-explore.d.ts.map +1 -0
  231. package/dist/report/list-explore.js +368 -0
  232. package/dist/report/list-explore.js.map +1 -0
  233. package/dist/report/list.d.ts +21 -0
  234. package/dist/report/list.d.ts.map +1 -1
  235. package/dist/report/list.js +79 -3
  236. package/dist/report/list.js.map +1 -1
  237. package/dist/report/markdown.d.ts +39 -0
  238. package/dist/report/markdown.d.ts.map +1 -0
  239. package/dist/report/markdown.js +426 -0
  240. package/dist/report/markdown.js.map +1 -0
  241. package/dist/report/outcome.d.ts +29 -0
  242. package/dist/report/outcome.d.ts.map +1 -0
  243. package/dist/report/outcome.js +36 -0
  244. package/dist/report/outcome.js.map +1 -0
  245. package/dist/report/running-tree.d.ts +7 -7
  246. package/dist/report/running-tree.d.ts.map +1 -1
  247. package/dist/report/running-tree.js +12 -13
  248. package/dist/report/running-tree.js.map +1 -1
  249. package/dist/report/write.d.ts +10 -1
  250. package/dist/report/write.d.ts.map +1 -1
  251. package/dist/report/write.js +22 -1
  252. package/dist/report/write.js.map +1 -1
  253. package/dist/run/app-ports.d.ts +17 -0
  254. package/dist/run/app-ports.d.ts.map +1 -0
  255. package/dist/run/app-ports.js +74 -0
  256. package/dist/run/app-ports.js.map +1 -0
  257. package/dist/run/events.d.ts +10 -5
  258. package/dist/run/events.d.ts.map +1 -1
  259. package/dist/run/events.js.map +1 -1
  260. package/dist/run/execute.d.ts.map +1 -1
  261. package/dist/run/execute.js +24 -7
  262. package/dist/run/execute.js.map +1 -1
  263. package/dist/run/extended-fixtures.d.ts +33 -0
  264. package/dist/run/extended-fixtures.d.ts.map +1 -0
  265. package/dist/run/extended-fixtures.js +91 -0
  266. package/dist/run/extended-fixtures.js.map +1 -0
  267. package/dist/run/fixtures.d.ts.map +1 -1
  268. package/dist/run/fixtures.js +39 -11
  269. package/dist/run/fixtures.js.map +1 -1
  270. package/dist/run/managed-process.d.ts +18 -0
  271. package/dist/run/managed-process.d.ts.map +1 -1
  272. package/dist/run/managed-process.js +101 -6
  273. package/dist/run/managed-process.js.map +1 -1
  274. package/dist/run/realm.d.ts +5 -4
  275. package/dist/run/realm.d.ts.map +1 -1
  276. package/dist/run/realm.js +8 -7
  277. package/dist/run/realm.js.map +1 -1
  278. package/dist/run/runner.d.ts +9 -3
  279. package/dist/run/runner.d.ts.map +1 -1
  280. package/dist/run/runner.js +15 -12
  281. package/dist/run/runner.js.map +1 -1
  282. package/dist/run/secrecy.d.ts +3 -3
  283. package/dist/run/secrecy.d.ts.map +1 -1
  284. package/dist/run/secrecy.js +4 -4
  285. package/dist/run/secrecy.js.map +1 -1
  286. package/dist/run/standalone.js +6 -6
  287. package/dist/run/standalone.js.map +1 -1
  288. package/dist/run/worker/entry.js +4 -3
  289. package/dist/run/worker/entry.js.map +1 -1
  290. package/dist/run/worker/protocol.d.ts +7 -1
  291. package/dist/run/worker/protocol.d.ts.map +1 -1
  292. package/dist/run/worker/protocol.js.map +1 -1
  293. package/dist/secrets.d.ts +21 -0
  294. package/dist/secrets.d.ts.map +1 -0
  295. package/dist/secrets.js +66 -0
  296. package/dist/secrets.js.map +1 -0
  297. package/dist/types.d.ts +90 -50
  298. package/dist/types.d.ts.map +1 -1
  299. package/package.json +6 -3
  300. package/skills/e2e/SKILL.md +42 -25
  301. package/skills/e2e/references/agent.md +80 -24
  302. package/skills/e2e/references/debugging.md +6 -5
  303. package/skills/e2e/references/explore.md +13 -7
  304. package/skills/e2e/references/mcp.md +4 -4
  305. package/skills/e2e/references/running.md +32 -21
  306. package/skills/e2e/references/setup.md +42 -16
  307. package/skills/e2e/references/writing-tests.md +32 -16
  308. package/dist/agent/model/router.d.ts +0 -36
  309. package/dist/agent/model/router.d.ts.map +0 -1
  310. package/dist/agent/model/router.js +0 -45
  311. package/dist/agent/model/router.js.map +0 -1
  312. package/dist/credentials.d.ts +0 -13
  313. package/dist/credentials.d.ts.map +0 -1
  314. package/dist/credentials.js +0 -51
  315. package/dist/credentials.js.map +0 -1
  316. package/dist/explore/reporter.d.ts +0 -11
  317. package/dist/explore/reporter.d.ts.map +0 -1
  318. package/dist/explore/reporter.js +0 -54
  319. package/dist/explore/reporter.js.map +0 -1
@@ -12,28 +12,31 @@ test.describe('todos', { tags: ['todos'] }, () => {
12
12
  await app.open('/todos');
13
13
  });
14
14
 
15
- test('adds and completes a todo', async ({ screen, web }) => {
16
- await screen.getByLabel('New todo').fill('Write the release notes');
17
- await screen.getByRole('button', { name: 'Add' }).tap();
18
-
15
+ test('adds and completes a todo', async ({ agent, screen, web }) => {
16
+ await agent.act('add a todo named {title}', { params: { title: 'Write the release notes' } });
19
17
  await expect(screen.getByRole('listitem')).toHaveCount(1);
20
18
  await expect(screen.getByRole('status', { name: 'Remaining' })).toHaveText('1 remaining');
21
19
 
22
- await screen.getByRole('checkbox', { name: 'Write the release notes' }).check();
20
+ await agent.act('mark the todo as done');
23
21
  await expect(screen.getByRole('status', { name: 'Remaining' })).toHaveText('0 remaining');
24
22
  await expect(web).toHaveURL('/todos');
25
23
  });
26
24
 
27
25
  test('ignores an empty submission', async ({ screen }) => {
26
+ // An exact interaction: the empty submit is the point of the test.
28
27
  await screen.getByRole('button', { name: 'Add' }).tap();
29
28
  await expect(screen.getByRole('listitem')).toHaveCount(0);
30
29
  });
31
30
  });
32
31
  ```
33
32
 
34
- Files match the config `tests` glob, default `tests/**/*.e2e.ts`. Every test
35
- starts from clean state: a fresh browser context and no page open, so a test
36
- calls `app.open()` first (here in `beforeEach`).
33
+ The agent does the flow; `expect` pins what must be true after each goal,
34
+ and that check is what lets the trace cache replay the step on later runs.
35
+ `screen` actions are for exact interactions and values, like the empty
36
+ submit above or a sign-in form. Files match the config `tests` glob, default
37
+ `tests/**/*.e2e.ts`. Every test starts from clean state: a fresh browser
38
+ context and no page open, so a test calls `app.open()` first (here in
39
+ `beforeEach`).
37
40
 
38
41
  ## Registration
39
42
 
@@ -53,6 +56,8 @@ test.afterAll(async () => {});
53
56
  test.skip('later', async () => {});
54
57
  test.only('focus', async () => {}); // local only: CI fails with ONLY_IN_CI
55
58
  test.setup('sign in', { sessions: ['admin'] }, async ({ app, screen, session }) => {}); // see Sign-in sessions
59
+ const wsTest = test.extend<{ ws: Ws }>({ ws: async ({ web }, use) => { await use(await seed()); await drop(); } });
60
+ wsTest('uses the workspace', async ({ ws }) => {}); // code after use() is teardown, runs after failures too
56
61
  ```
57
62
 
58
63
  | Option | Default | Notes |
@@ -94,7 +99,7 @@ Fixtures are lazy; destructure them in the callback.
94
99
  | `back()` | One history step back. |
95
100
  | `restart()` | Recreates the context and keeps persisted state, including a restored session. |
96
101
  | `clearState()` | Clears cookies and storage, then relaunches. Not inside a serial group. |
97
- | `screenshot(label?)` | Saves a redacted screenshot as an artifact and returns its path. |
102
+ | `screenshot(label?)` | Saves a redacted screenshot as an artifact and returns its path. Fails with `POLICY_DENIED` after a secret fill. |
98
103
 
99
104
  ## Locators
100
105
 
@@ -166,7 +171,10 @@ read. Reading a password field's value is `POLICY_DENIED`.
166
171
 
167
172
  `expect(locator)` polls for up to `config.assertionTimeout` (5 s) or
168
173
  `{ timeout }`; `.not` inverts. `expect(web)` gives the browser matchers.
169
- `expect(value)` is synchronous.
174
+ `expect(value)` is synchronous. `expect.poll(read, { timeout?, interval?,
175
+ message? })` re-reads a value until a value matcher passes
176
+ (`assertionTimeout` and 100 ms by default, stopping with the attempt); a
177
+ throwing read keeps polling, and it is not a report step.
170
178
 
171
179
  ```ts
172
180
  await expect(screen.getByRole('status')).toHaveText('Saved');
@@ -175,6 +183,7 @@ await expect(screen.getByTestId('todo')).toHaveCount(3);
175
183
  await expect(web).toHaveURL('/dashboard'); // relative to the base URL, or a RegExp
176
184
  await expect(web).toHaveTitle(/Dashboard/);
177
185
  expect(await screen.getByTestId('total').textContent()).toContain('$');
186
+ await expect.poll(() => db.orders.count(), { timeout: 15_000 }).toBe(1); // re-reads until it holds
178
187
  ```
179
188
 
180
189
  | Locator matchers | Web matchers | Value matchers |
@@ -231,10 +240,16 @@ credentials: {
231
240
 
232
241
  `E2E_USER_ADMIN_USERNAME` and `E2E_USER_ADMIN_PASSWORD` override either
233
242
  field per run. `credentials.user('admin').password` is a `Secret` with no
234
- plaintext accessor; only `fill()` and `agent.act` params accept it. Once a
235
- secret is filled, screenshots stop being attached for the rest of that
236
- attempt, so sign in inside a setup test and keep the evidence in the tests
237
- that matter.
243
+ plaintext accessor; only `fill()` and `agent.act` params accept it. Any other
244
+ sensitive value (an API key, a token) is a `secrets` entry,
245
+ `secrets: { 'stripe-key': process.env.STRIPE_KEY ?? '' }`, overridable with
246
+ `E2E_SECRET_STRIPE_KEY`; `secrets.get('stripe-key')` is the same kind of
247
+ handle and fills any editable input, with the value redacted by name
248
+ everywhere the runner writes. Once a
249
+ secret is filled, model pixels and assertion screenshots are withheld for
250
+ the rest of that session, including later tests sharing a serial session.
251
+ `app.screenshot()` fails with `POLICY_DENIED` before capture. Sign in inside
252
+ a setup test and keep the evidence in the tests that matter.
238
253
 
239
254
  ## The web fixture (browser only)
240
255
 
@@ -275,8 +290,9 @@ expect(response.status).toBe(201);
275
290
  Otherwise tests are independent and may run on different workers.
276
291
  - Tag by area and by cost (`smoke`, `billing`, `agent`) and run subsets with
277
292
  `--tag`.
278
- - Mix agent steps in where the path varies and pin the outcome with
279
- `expect`; see the `agent` topic.
293
+ - Drive flows with `agent.act` and pin each outcome with `expect`. The
294
+ calls on this page are for exact values and exact checks; the `agent`
295
+ topic covers the steps that do the work.
280
296
 
281
297
  ## Mistakes to avoid
282
298
 
@@ -1,36 +0,0 @@
1
- /** Which model an invocation talks to. */
2
- import type { ResolvedAgentConfig } from '../../config/agent.ts';
3
- import type { ModelAdapter } from './adapter.ts';
4
- /**
5
- * Picks the adapter for one invocation.
6
- *
7
- * `agent.visionModel` exists because visual grounding is a much higher bar than
8
- * accepting an image, and pinning it separately keeps every text-only call on the
9
- * cheaper model. The vision adapter is built on first use rather than at fixture
10
- * acquisition: a project that pins a vision model must not fail a run that never
11
- * asks for pixels, for example because that model's credential is absent from
12
- * this environment.
13
- */
14
- export declare class ModelRouter {
15
- private readonly buildBase;
16
- private readonly buildVision;
17
- private baseAdapter;
18
- private visionAdapter;
19
- constructor(buildBase: () => ModelAdapter, buildVision: (() => ModelAdapter) | undefined);
20
- /**
21
- * The adapter for a call. An invocation that is sending pixels keeps the vision
22
- * model for the rest of its lifetime, including rounds whose pixels were
23
- * withheld, so a polling method never switches models between rounds.
24
- */
25
- select(vision: boolean): ModelAdapter;
26
- }
27
- /**
28
- * Builds the router for one attempt from resolved agent config.
29
- *
30
- * The base adapter is built on first use, like the vision adapter: a run whose
31
- * `agent.act()` steps go to a custom executor may legitimately have no model at
32
- * all, and must not fail on a MODEL_UNAVAILABLE it would never hit. Judgment
33
- * and locate methods still fail with MODEL_UNAVAILABLE on their first call.
34
- */
35
- export declare function createModelRouter(agent: ResolvedAgentConfig, build: (model: ResolvedAgentConfig['model']) => ModelAdapter): ModelRouter;
36
- //# sourceMappingURL=router.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../../../src/agent/model/router.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAE1C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD;;;;;;;;;GASG;AACH,qBAAa,WAAW;IAKpB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAL9B,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,aAAa,CAA2B;IAEhD,YACmB,SAAS,EAAE,MAAM,YAAY,EAC7B,WAAW,EAAE,CAAC,MAAM,YAAY,CAAC,GAAG,SAAS,EAC5D;IAEJ;;;;OAIG;IACH,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,YAAY,CAKpC;CACF;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,mBAAmB,EAC1B,KAAK,EAAE,CAAC,KAAK,EAAE,mBAAmB,CAAC,OAAO,CAAC,KAAK,YAAY,GAC3D,WAAW,CAMb"}
@@ -1,45 +0,0 @@
1
- /** Which model an invocation talks to. */
2
- /**
3
- * Picks the adapter for one invocation.
4
- *
5
- * `agent.visionModel` exists because visual grounding is a much higher bar than
6
- * accepting an image, and pinning it separately keeps every text-only call on the
7
- * cheaper model. The vision adapter is built on first use rather than at fixture
8
- * acquisition: a project that pins a vision model must not fail a run that never
9
- * asks for pixels, for example because that model's credential is absent from
10
- * this environment.
11
- */
12
- export class ModelRouter {
13
- buildBase;
14
- buildVision;
15
- baseAdapter;
16
- visionAdapter;
17
- constructor(buildBase, buildVision) {
18
- this.buildBase = buildBase;
19
- this.buildVision = buildVision;
20
- }
21
- /**
22
- * The adapter for a call. An invocation that is sending pixels keeps the vision
23
- * model for the rest of its lifetime, including rounds whose pixels were
24
- * withheld, so a polling method never switches models between rounds.
25
- */
26
- select(vision) {
27
- if (!vision || this.buildVision === undefined) {
28
- return (this.baseAdapter ??= this.buildBase());
29
- }
30
- return (this.visionAdapter ??= this.buildVision());
31
- }
32
- }
33
- /**
34
- * Builds the router for one attempt from resolved agent config.
35
- *
36
- * The base adapter is built on first use, like the vision adapter: a run whose
37
- * `agent.act()` steps go to a custom executor may legitimately have no model at
38
- * all, and must not fail on a MODEL_UNAVAILABLE it would never hit. Judgment
39
- * and locate methods still fail with MODEL_UNAVAILABLE on their first call.
40
- */
41
- export function createModelRouter(agent, build) {
42
- const visionModel = agent.visionModel;
43
- return new ModelRouter(() => build(agent.model), visionModel === undefined ? undefined : () => build(visionModel));
44
- }
45
- //# sourceMappingURL=router.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"router.js","sourceRoot":"","sources":["../../../src/agent/model/router.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAK1C;;;;;;;;;GASG;AACH,MAAM,OAAO,WAAW;IAKH,SAAS;IACT,WAAW;IALtB,WAAW,CAA2B;IACtC,aAAa,CAA2B;IAEhD,YACmB,SAA6B,EAC7B,WAA6C;yBAD7C,SAAS;2BACT,WAAW;IAC3B,CAAC;IAEJ;;;;OAIG;IACH,MAAM,CAAC,MAAe;QACpB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IACrD,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAA0B,EAC1B,KAA4D;IAE5D,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;IACtC,OAAO,IAAI,WAAW,CACpB,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,EACxB,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CACjE,CAAC;AACJ,CAAC"}
@@ -1,13 +0,0 @@
1
- /** Opaque credential handles. */
2
- import type { ResolvedCredential } from './config/resolve.ts';
3
- import type { Credentials } from './types.ts';
4
- /** Installed by the runner once config resolution completes. */
5
- export declare function setCredentialRegistry(map: ReadonlyMap<string, ResolvedCredential> | undefined): void;
6
- /**
7
- * Removes `map` from the slot if it is still the installed registry. A host
8
- * that closes one attempt while opening the next must not wipe what the new
9
- * attempt just installed.
10
- */
11
- export declare function releaseCredentialRegistry(map: ReadonlyMap<string, ResolvedCredential>): void;
12
- export declare const credentials: Credentials;
13
- //# sourceMappingURL=credentials.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../src/credentials.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAKjC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,KAAK,EAAc,WAAW,EAAU,MAAM,YAAY,CAAC;AAKlE,gEAAgE;AAChE,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,GAAG,SAAS,GAAG,IAAI,CAGpG;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,GAAG,EAAE,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,GAAG,IAAI,CAE5F;AAcD,eAAO,MAAM,WAAW,EAAE,WAuBzB,CAAC"}
@@ -1,51 +0,0 @@
1
- /** Opaque credential handles. */
2
- import { credentialBrand, secretBrand } from './internal/brands.js';
3
- import { ConfigurationError } from './internal/errors.js';
4
- import { realmSlot } from './internal/realm-slot.js';
5
- /** Global slot so test modules in an isolated realm reach the runner's registry. */
6
- const credentialsSlot = realmSlot('e2e.credentials.v1');
7
- /** Installed by the runner once config resolution completes. */
8
- export function setCredentialRegistry(map) {
9
- if (map === undefined)
10
- credentialsSlot.delete(globalThis);
11
- else
12
- credentialsSlot.set(globalThis, map);
13
- }
14
- /**
15
- * Removes `map` from the slot if it is still the installed registry. A host
16
- * that closes one attempt while opening the next must not wipe what the new
17
- * attempt just installed.
18
- */
19
- export function releaseCredentialRegistry(map) {
20
- if (credentialsSlot.get(globalThis) === map)
21
- credentialsSlot.delete(globalThis);
22
- }
23
- function getRegistry() {
24
- return credentialsSlot.get(globalThis);
25
- }
26
- function makeSecret(name) {
27
- return Object.freeze({
28
- name,
29
- purpose: 'password',
30
- [secretBrand]: true,
31
- });
32
- }
33
- export const credentials = {
34
- user(name) {
35
- const registry = getRegistry();
36
- if (registry === undefined) {
37
- throw new ConfigurationError('AUTH_CREDENTIAL_UNAVAILABLE', 'credentials.user() is only available while the e2e runner is active');
38
- }
39
- const resolved = registry.get(name);
40
- if (resolved === undefined) {
41
- throw new ConfigurationError('AUTH_CREDENTIAL_UNAVAILABLE', `credential "${name}" is not configured; add it to config.credentials or E2E_USER_* variables`);
42
- }
43
- return Object.freeze({
44
- name,
45
- username: resolved.username,
46
- password: makeSecret(name),
47
- [credentialBrand]: true,
48
- });
49
- },
50
- };
51
- //# sourceMappingURL=credentials.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"credentials.js","sourceRoot":"","sources":["../src/credentials.ts"],"names":[],"mappings":"AAAA,iCAAiC;AAEjC,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAIrD,oFAAoF;AACpF,MAAM,eAAe,GAAG,SAAS,CAA0C,oBAAoB,CAAC,CAAC;AAEjG,gEAAgE;AAChE,MAAM,UAAU,qBAAqB,CAAC,GAAwD;IAC5F,IAAI,GAAG,KAAK,SAAS;QAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;;QACrD,eAAe,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;AAC5C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,GAA4C;IACpF,IAAI,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,GAAG;QAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,WAAW;IAClB,OAAO,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI;QACJ,OAAO,EAAE,UAAmB;QAC5B,CAAC,WAAW,CAAC,EAAE,IAAa;KAC7B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAgB;IACtC,IAAI,CAAC,IAAY;QACf,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;QAC/B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,kBAAkB,CAC1B,6BAA6B,EAC7B,qEAAqE,CACtE,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,kBAAkB,CAC1B,6BAA6B,EAC7B,eAAe,IAAI,2EAA2E,CAC/F,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC;YACnB,IAAI;YACJ,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC;YAC1B,CAAC,eAAe,CAAC,EAAE,IAAa;SACjC,CAAC,CAAC;IACL,CAAC;CACF,CAAC"}
@@ -1,11 +0,0 @@
1
- /**
2
- * The exploration summary under the run summary: a `Reporter` whose
3
- * `onRunFinished` rows the list reporter prints, so the findings appear where
4
- * a reporter's own rows do, and the JSON reporter is left to the report.
5
- */
6
- import type { Reporter, ReporterSummary } from '../types.ts';
7
- import type { ExploreState } from './state.ts';
8
- export declare function exploreReporter(state: ExploreState): Reporter;
9
- /** The rows: what was explored, what was found, one row per finding, and the assessment. */
10
- export declare function summaryRows(state: ExploreState): ReporterSummary;
11
- //# sourceMappingURL=reporter.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/explore/reporter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,GAAG,QAAQ,CAK7D;AAED,4FAA4F;AAC5F,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,eAAe,CAgChE"}
@@ -1,54 +0,0 @@
1
- /**
2
- * The exploration summary under the run summary: a `Reporter` whose
3
- * `onRunFinished` rows the list reporter prints, so the findings appear where
4
- * a reporter's own rows do, and the JSON reporter is left to the report.
5
- */
6
- export function exploreReporter(state) {
7
- return {
8
- name: 'explore',
9
- onRunFinished: async () => summaryRows(state),
10
- };
11
- }
12
- /** The rows: what was explored, what was found, one row per finding, and the assessment. */
13
- export function summaryRows(state) {
14
- const counts = { passed: 0, failed: 0, blocked: 0, exhausted: 0 };
15
- for (const step of state.steps)
16
- counts[step.status] += 1;
17
- const parts = [
18
- `${counts.passed} passed`,
19
- ...(counts.failed > 0 ? [`${counts.failed} failed`] : []),
20
- ...(counts.exhausted > 0 ? [`${counts.exhausted} ended at their limit`] : []),
21
- ...(counts.blocked > 0 ? [`${counts.blocked} blocked`] : []),
22
- ];
23
- const issues = state.issues.length;
24
- const warnings = state.findings.length - issues;
25
- const rows = [
26
- {
27
- label: 'Explored',
28
- text: `${state.steps.length} step${state.steps.length === 1 ? '' : 's'} (${parts.join(', ')}); ended: ${ENDED_TEXT[state.ended]}`,
29
- },
30
- {
31
- label: 'Findings',
32
- text: state.findings.length === 0
33
- ? 'none'
34
- : `${issues} issue${issues === 1 ? '' : 's'}, ${warnings} warning${warnings === 1 ? '' : 's'}`,
35
- },
36
- ];
37
- for (const finding of [...state.findings].toSorted((a, b) => b.severity - a.severity || a.index - b.index)) {
38
- rows.push({
39
- label: ` S${finding.severity} ${finding.kind}`,
40
- text: `${finding.title}${finding.path === undefined ? '' : ` (${finding.path})`}`,
41
- });
42
- }
43
- if (state.summary !== undefined)
44
- rows.push({ label: 'Assessment', text: state.summary });
45
- return rows;
46
- }
47
- const ENDED_TEXT = {
48
- finished: 'the agent covered the goal',
49
- 'step-limit': 'the step limit was reached',
50
- time: 'the time budget ran out',
51
- stuck: 'steps kept failing',
52
- aborted: 'the run was cut short',
53
- };
54
- //# sourceMappingURL=reporter.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/explore/reporter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,MAAM,UAAU,eAAe,CAAC,KAAmB;IACjD,OAAO;QACL,IAAI,EAAE,SAAS;QACf,aAAa,EAAE,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,WAAW,CAAC,KAAmB;IAC7C,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAClE,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK;QAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG;QACZ,GAAG,MAAM,CAAC,MAAM,SAAS;QACzB,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,GAAG,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,uBAAuB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7E,GAAG,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7D,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;IAChD,MAAM,IAAI,GAAsC;QAC9C;YACE,KAAK,EAAE,UAAU;YACjB,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,QAAQ,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;SAClI;QACD;YACE,KAAK,EAAE,UAAU;YACjB,IAAI,EACF,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACzB,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,GAAG,MAAM,SAAS,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,WAAW,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE;SACnG;KACF,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3G,IAAI,CAAC,IAAI,CAAC;YACR,KAAK,EAAE,MAAM,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE;YAC/C,IAAI,EAAE,GAAG,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,IAAI,GAAG,EAAE;SAClF,CAAC,CAAC;IACL,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACzF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,GAA0C;IACxD,QAAQ,EAAE,4BAA4B;IACtC,YAAY,EAAE,4BAA4B;IAC1C,IAAI,EAAE,yBAAyB;IAC/B,KAAK,EAAE,oBAAoB;IAC3B,OAAO,EAAE,uBAAuB;CACjC,CAAC"}