e2e 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/README.md +10 -0
  2. package/dist/.build.json +1 -1
  3. package/dist/agent/act.js +7 -4
  4. package/dist/agent/action-dispatcher.js +4 -5
  5. package/dist/agent/actions.d.ts +32 -23
  6. package/dist/agent/actions.js +38 -24
  7. package/dist/agent/ai-sdk.js +5 -4
  8. package/dist/agent/error.js +3 -3
  9. package/dist/agent/invocation.js +3 -4
  10. package/dist/agent/model/sdk.js +2 -0
  11. package/dist/agent/model/tool-choice.d.ts +9 -0
  12. package/dist/agent/model/tool-choice.js +14 -0
  13. package/dist/agent/observation-feed.d.ts +5 -6
  14. package/dist/agent/observation-feed.js +8 -11
  15. package/dist/agent/observation.d.ts +40 -11
  16. package/dist/agent/observation.js +109 -59
  17. package/dist/agent/phases.d.ts +7 -1
  18. package/dist/agent/phases.js +12 -3
  19. package/dist/agent/point-tap.d.ts +8 -8
  20. package/dist/agent/point-tap.js +1 -1
  21. package/dist/agent/prompts.js +3 -0
  22. package/dist/agent/replay.d.ts +19 -8
  23. package/dist/agent/replay.js +29 -29
  24. package/dist/agent/scroll-target.d.ts +2 -3
  25. package/dist/agent/step-cache.d.ts +43 -15
  26. package/dist/agent/step-cache.js +156 -67
  27. package/dist/agent/tool-loop.js +94 -5
  28. package/dist/cache/anchors.d.ts +72 -23
  29. package/dist/cache/anchors.js +233 -60
  30. package/dist/cache/context.d.ts +25 -14
  31. package/dist/cache/context.js +50 -8
  32. package/dist/cache/decide.d.ts +2 -2
  33. package/dist/cache/decide.js +4 -6
  34. package/dist/cache/identity.d.ts +18 -4
  35. package/dist/cache/identity.js +9 -4
  36. package/dist/cache/recorder.d.ts +15 -1
  37. package/dist/cache/recorder.js +36 -23
  38. package/dist/cache/rekeyed.d.ts +46 -0
  39. package/dist/cache/rekeyed.js +89 -0
  40. package/dist/cache/relocate.d.ts +14 -14
  41. package/dist/cache/relocate.js +33 -25
  42. package/dist/cache/route.d.ts +32 -33
  43. package/dist/cache/route.js +94 -50
  44. package/dist/cache/store.d.ts +6 -0
  45. package/dist/cache/store.js +21 -1
  46. package/dist/cache/trace.d.ts +53 -14
  47. package/dist/cache/trace.js +76 -30
  48. package/dist/cli/bin.js +2 -2
  49. package/dist/cli/index.js +43 -32
  50. package/dist/cli/init/engines.d.ts +1 -3
  51. package/dist/cli/init/engines.js +2 -2
  52. package/dist/cli/init/gateways.d.ts +1 -1
  53. package/dist/cli/init/gateways.js +11 -1
  54. package/dist/cli/init/mcp-config.d.ts +2 -1
  55. package/dist/cli/init/mcp-config.js +8 -3
  56. package/dist/cli/init/sibling-versions.json +1 -1
  57. package/dist/cli/init/versions.d.ts +0 -8
  58. package/dist/cli/init/versions.js +1 -10
  59. package/dist/cli/mcp.d.ts +3 -2
  60. package/dist/cli/mcp.js +52 -16
  61. package/dist/cli/run-output.d.ts +17 -0
  62. package/dist/cli/run-output.js +45 -0
  63. package/dist/collect/collect.js +8 -4
  64. package/dist/collect/registry.js +27 -16
  65. package/dist/config/compiled-files.d.ts +36 -0
  66. package/dist/config/compiled-files.js +50 -0
  67. package/dist/config/diagnose.js +21 -19
  68. package/dist/config/esm-hooks.d.ts +39 -44
  69. package/dist/config/esm-hooks.js +238 -63
  70. package/dist/config/load.d.ts +1 -7
  71. package/dist/config/load.js +23 -66
  72. package/dist/config/register.d.ts +8 -0
  73. package/dist/config/register.js +9 -0
  74. package/dist/config/resolve.d.ts +1 -0
  75. package/dist/config/resolve.js +33 -1
  76. package/dist/config/specifiers.d.ts +19 -0
  77. package/dist/config/specifiers.js +33 -0
  78. package/dist/config/tsconfig.d.ts +56 -0
  79. package/dist/config/tsconfig.js +115 -0
  80. package/dist/config/typescript.d.ts +19 -0
  81. package/dist/config/typescript.js +253 -0
  82. package/dist/engine/contract.d.ts +1 -0
  83. package/dist/engine/index.d.ts +4 -2
  84. package/dist/engine/session.js +2 -2
  85. package/dist/expect/async.js +4 -2
  86. package/dist/expect/poll-scope.d.ts +52 -0
  87. package/dist/expect/poll-scope.js +113 -0
  88. package/dist/expect/poll.js +33 -6
  89. package/dist/expect/values.js +11 -4
  90. package/dist/internal/abandoned.d.ts +11 -0
  91. package/dist/internal/abandoned.js +23 -0
  92. package/dist/internal/brands.d.ts +2 -0
  93. package/dist/internal/brands.js +2 -0
  94. package/dist/internal/client-identity.d.ts +9 -0
  95. package/dist/internal/client-identity.js +14 -0
  96. package/dist/internal/errors.d.ts +10 -3
  97. package/dist/internal/errors.js +23 -8
  98. package/dist/internal/node-version.d.ts +16 -7
  99. package/dist/internal/node-version.js +50 -15
  100. package/dist/internal/package-manager.js +1 -1
  101. package/dist/internal/realm-slot.d.ts +5 -4
  102. package/dist/internal/redact.d.ts +9 -4
  103. package/dist/internal/redact.js +120 -23
  104. package/dist/internal/time.d.ts +26 -4
  105. package/dist/internal/time.js +58 -14
  106. package/dist/internal/urls.d.ts +9 -5
  107. package/dist/internal/urls.js +19 -10
  108. package/dist/locator/engine.d.ts +12 -2
  109. package/dist/locator/engine.js +55 -14
  110. package/dist/locator/screen.js +59 -9
  111. package/dist/mcp/catalog.d.ts +3 -2
  112. package/dist/mcp/catalog.js +6 -6
  113. package/dist/mcp/output.d.ts +59 -0
  114. package/dist/mcp/output.js +129 -0
  115. package/dist/mcp/server.d.ts +4 -0
  116. package/dist/mcp/server.js +9 -12
  117. package/dist/mcp/session.d.ts +8 -1
  118. package/dist/mcp/session.js +56 -15
  119. package/dist/mcp/tools.js +7 -5
  120. package/dist/oauth/chatgpt.js +3 -23
  121. package/dist/oauth/cli.js +1 -0
  122. package/dist/oauth/copilot.d.ts +5 -4
  123. package/dist/oauth/copilot.js +80 -7
  124. package/dist/oauth/grok.js +2 -1
  125. package/dist/oauth/models.js +2 -1
  126. package/dist/oauth/opencode-console.d.ts +10 -0
  127. package/dist/oauth/opencode-console.js +133 -0
  128. package/dist/oauth/providers/github-copilot.d.ts +12 -0
  129. package/dist/oauth/providers/github-copilot.js +77 -23
  130. package/dist/oauth/providers/opencode-console.d.ts +31 -0
  131. package/dist/oauth/providers/opencode-console.js +148 -0
  132. package/dist/oauth/providers.d.ts +2 -6
  133. package/dist/oauth/providers.js +3 -7
  134. package/dist/oauth/responses.d.ts +9 -0
  135. package/dist/oauth/responses.js +23 -0
  136. package/dist/report/build.d.ts +26 -0
  137. package/dist/report/build.js +16 -8
  138. package/dist/report/failure-text.d.ts +6 -1
  139. package/dist/report/failure-text.js +14 -4
  140. package/dist/report/format.d.ts +10 -5
  141. package/dist/report/format.js +12 -5
  142. package/dist/report/junit.js +11 -12
  143. package/dist/report/list.d.ts +12 -0
  144. package/dist/report/list.js +61 -24
  145. package/dist/report/markdown.d.ts +1 -1
  146. package/dist/report/markdown.js +37 -14
  147. package/dist/report/repeats.d.ts +6 -1
  148. package/dist/report/repeats.js +12 -3
  149. package/dist/run/artifacts.d.ts +22 -4
  150. package/dist/run/artifacts.js +71 -3
  151. package/dist/run/execute.d.ts +34 -5
  152. package/dist/run/execute.js +93 -44
  153. package/dist/run/extended-fixtures.js +6 -2
  154. package/dist/run/failure-evidence.js +10 -4
  155. package/dist/run/fixtures.js +6 -8
  156. package/dist/run/in-process.d.ts +2 -0
  157. package/dist/run/in-process.js +52 -10
  158. package/dist/run/last-run.d.ts +42 -8
  159. package/dist/run/last-run.js +121 -14
  160. package/dist/run/process-output.d.ts +73 -0
  161. package/dist/run/process-output.js +130 -0
  162. package/dist/run/provision.d.ts +2 -0
  163. package/dist/run/provision.js +1 -0
  164. package/dist/run/realm.d.ts +4 -1
  165. package/dist/run/realm.js +20 -6
  166. package/dist/run/records.d.ts +20 -4
  167. package/dist/run/records.js +26 -0
  168. package/dist/run/retry.d.ts +10 -1
  169. package/dist/run/retry.js +30 -12
  170. package/dist/run/runner.d.ts +9 -1
  171. package/dist/run/runner.js +101 -39
  172. package/dist/run/scheduler.js +255 -19
  173. package/dist/run/secrecy.d.ts +22 -19
  174. package/dist/run/secrecy.js +22 -18
  175. package/dist/run/serial.d.ts +75 -2
  176. package/dist/run/serial.js +243 -59
  177. package/dist/run/standalone.js +1 -1
  178. package/dist/run/steps.d.ts +0 -6
  179. package/dist/run/steps.js +3 -20
  180. package/dist/run/trace-redaction.d.ts +5 -6
  181. package/dist/run/trace-redaction.js +5 -6
  182. package/dist/run/worker/entry.js +40 -18
  183. package/dist/run/worker/handle.js +10 -5
  184. package/dist/run/worker/protocol.d.ts +85 -3
  185. package/dist/run/worker/session.d.ts +2 -0
  186. package/dist/run/worker/session.js +29 -0
  187. package/dist/telemetry/environment.d.ts +2 -1
  188. package/dist/telemetry/environment.js +12 -2
  189. package/dist/telemetry/events.js +1 -0
  190. package/dist/telemetry/project.d.ts +5 -6
  191. package/dist/telemetry/project.js +6 -11
  192. package/dist/telemetry/store.d.ts +5 -5
  193. package/dist/telemetry/store.js +5 -7
  194. package/dist/telemetry/telemetry.d.ts +2 -3
  195. package/dist/telemetry/telemetry.js +2 -4
  196. package/dist/types.d.ts +15 -6
  197. package/docs/agent-steps.mdx +1 -0
  198. package/docs/agents.mdx +7 -3
  199. package/docs/api-testing.mdx +2 -1
  200. package/docs/authentication.mdx +6 -3
  201. package/docs/browser.mdx +49 -12
  202. package/docs/bug-bash.mdx +2 -1
  203. package/docs/cache.mdx +66 -19
  204. package/docs/ci.mdx +14 -9
  205. package/docs/coding-agents.mdx +2 -1
  206. package/docs/core-concepts.mdx +208 -0
  207. package/docs/debugging.mdx +25 -12
  208. package/docs/decision-models.mdx +142 -0
  209. package/docs/example-projects.mdx +32 -0
  210. package/docs/executors.mdx +7 -2
  211. package/docs/explore.mdx +1 -0
  212. package/docs/github.mdx +14 -5
  213. package/docs/index.mdx +9 -5
  214. package/docs/integrations/eas.mdx +1 -0
  215. package/docs/integrations/index.mdx +1 -0
  216. package/docs/integrations/kernel.mdx +3 -2
  217. package/docs/migrate/cypress.mdx +26 -3
  218. package/docs/migrate/detox.mdx +24 -1
  219. package/docs/migrate/maestro.mdx +28 -4
  220. package/docs/migrate/playwright.mdx +44 -18
  221. package/docs/migrate/selenium.mdx +27 -3
  222. package/docs/mobile-ci.mdx +6 -5
  223. package/docs/mobile.mdx +96 -8
  224. package/docs/models.mdx +60 -9
  225. package/docs/quickstart.mdx +28 -16
  226. package/docs/reference/agent.mdx +13 -3
  227. package/docs/reference/app.mdx +2 -1
  228. package/docs/reference/cli.mdx +41 -18
  229. package/docs/reference/config.mdx +67 -13
  230. package/docs/reference/engine.mdx +25 -13
  231. package/docs/reference/environment.mdx +10 -2
  232. package/docs/reference/errors.mdx +13 -10
  233. package/docs/reference/expect.mdx +36 -13
  234. package/docs/reference/mcp.mdx +21 -8
  235. package/docs/reference/mobile.mdx +24 -13
  236. package/docs/reference/reporters.mdx +9 -2
  237. package/docs/reference/screen.mdx +49 -16
  238. package/docs/reference/test.mdx +32 -5
  239. package/docs/reference/web.mdx +133 -23
  240. package/docs/security.mdx +58 -42
  241. package/docs/subscriptions.mdx +92 -12
  242. package/docs/telemetry.mdx +9 -6
  243. package/docs/tools.mdx +1 -0
  244. package/docs/{starting-your-app.mdx → web.mdx} +157 -11
  245. package/docs/writing-an-engine.mdx +2 -1
  246. package/package.json +25 -8
  247. package/schema/report-v1.schema.json +14 -1
  248. package/skills/e2e/SKILL.md +5 -3
  249. package/skills/e2e/references/agent.md +33 -11
  250. package/skills/e2e/references/debugging.md +4 -4
  251. package/skills/e2e/references/mcp.md +8 -3
  252. package/skills/e2e/references/running.md +15 -9
  253. package/skills/e2e/references/setup.md +45 -16
  254. package/skills/e2e/references/writing-tests.md +35 -18
  255. package/docs/viewports.mdx +0 -146
  256. package/docs/writing-tests.mdx +0 -256
@@ -11,17 +11,17 @@
11
11
  * for the route precondition. The initial capture can also serve the executor
12
12
  * when replay performed no action.
13
13
  */
14
- import { anchorsPresent, describeAnchors } from '../cache/anchors.js';
14
+ import { deltaEvidenced, deltaHolds, describeDelta } from '../cache/anchors.js';
15
15
  import { decideTraceReplay, opensWithNavigate } from '../cache/decide.js';
16
- import { compareRoutes, routeOf } from '../cache/route.js';
17
- import { instructionDigest } from '../cache/identity.js';
18
- import { TraceRecorder } from '../cache/recorder.js';
16
+ import { sameRoute } from '../cache/route.js';
17
+ import { recordedProvenance, TraceRecorder } from '../cache/recorder.js';
19
18
  import { expandTrace, templateParams, templatesCollide, templateTrace } from '../cache/template.js';
20
19
  import { readTraceEntry } from '../cache/trace.js';
21
20
  import { sleep } from '../internal/time.js';
22
21
  import { AgentError, isAgentError, isModelUnreachable } from './error.js';
23
22
  import { isRuntimeHardStop } from './executor.js';
24
- import { replayTrace, verifyAnchors, } from './replay.js';
23
+ import { replayTrace, verifyEndState, } from './replay.js';
24
+ import { redactNodesAgain } from './observation.js';
25
25
  /** The outcome of a step that threw `cause`. */
26
26
  export function failedStepOutcome(cause) {
27
27
  if (isModelUnreachable(cause))
@@ -48,9 +48,8 @@ const STALE_REASONS = new Set([
48
48
  const END_WAIT_MARGIN_MS = 10_000;
49
49
  /**
50
50
  * How long a replay waits for a recorded destination path to be the current
51
- * one. A step that moved records no anchors: the path is its whole
52
- * postcondition, and the replayed tap that starts the navigation returns
53
- * before the new document commits. Reading the path once, right after the
51
+ * one. The replayed tap that starts a navigation returns before the new
52
+ * document commits. Reading the path once, right after the
54
53
  * tap, hands every navigation off as an end-mismatch; polling with the
55
54
  * settling backoff lets the destination arrive. Bounded like anchor polling.
56
55
  */
@@ -60,8 +59,11 @@ export class StepTraceSession {
60
59
  host;
61
60
  cache;
62
61
  keyHash;
62
+ /** The claimed key and the step it names, which an entry records as its provenance. */
63
+ claim;
63
64
  recorder;
64
65
  options;
66
+ redaction;
65
67
  info;
66
68
  prefix;
67
69
  startPath;
@@ -85,6 +87,8 @@ export class StepTraceSession {
85
87
  replayedWhole = false;
86
88
  /** True once a cached entry's actions were run this step, fully or partly. */
87
89
  consumedReplay = false;
90
+ /** True once the store returned an entry for this step, whether or not it replayed. */
91
+ readEntryHit = false;
88
92
  /** True once `cache.strict` failed the step on its recording, which is then kept for review rather than evicted. */
89
93
  failedStale = false;
90
94
  /** Grammar actions recorded so far when an end-mismatch hand-off happened. */
@@ -92,13 +96,15 @@ export class StepTraceSession {
92
96
  constructor(host, options) {
93
97
  this.host = host;
94
98
  this.options = options;
99
+ this.redaction = { redact: options.redact, redactCut: options.redactCut };
95
100
  this.cache = options.cache;
96
101
  // The key digests the params as the recording spells them, a placeholder
97
102
  // where each `unique()` value was, so every run's value finds one entry.
98
- this.keyHash = options.cache.claimKeyHash('act', options.instruction, templateParams(options.params, options.templates));
103
+ this.claim = options.cache.claimKey('act', options.instruction, templateParams(options.params, options.templates), options.agent);
104
+ this.keyHash = this.claim.keyHash;
99
105
  if (options.cache.mode === 'read-write') {
100
106
  this.recorder = new TraceRecorder({
101
- redact: options.redact,
107
+ ...this.redaction,
102
108
  maxActions: options.maxActions,
103
109
  });
104
110
  }
@@ -130,7 +136,8 @@ export class StepTraceSession {
130
136
  * self-finalized verdict on a full replay whose postcondition holds;
131
137
  * undefined dispatches the executor — after a miss from the top, after a
132
138
  * divergence mid-step with `replayedPrefix` set. Every failure to replay is
133
- * a miss, never an error; only runtime hard stops propagate.
139
+ * a miss, never an error; only runtime hard stops and a policy denial on
140
+ * the opening look propagate.
134
141
  */
135
142
  async begin() {
136
143
  this.startedMs = Date.now();
@@ -144,6 +151,8 @@ export class StepTraceSession {
144
151
  this.info = this.missed(read.reason, 0);
145
152
  if (read.unavailable !== true)
146
153
  this.failIfStale();
154
+ if (read.reason === 'no-entry')
155
+ await this.failIfRekeyed();
147
156
  return undefined;
148
157
  }
149
158
  // With an entry in hand the step is the cache's from its first moment: the
@@ -151,6 +160,7 @@ export class StepTraceSession {
151
160
  // replay finishes stays the cache's through its verdict and the re-stage,
152
161
  // so a reporter never shows a model turn that is not coming. Only a replay
153
162
  // that cannot finish the step hands it to the model.
163
+ this.readEntryHit = true;
154
164
  this.host.replaying(true);
155
165
  const verdict = await this.replayEntry(read.entry);
156
166
  if (verdict === undefined) {
@@ -173,6 +183,23 @@ export class StepTraceSession {
173
183
  this.failedStale = true;
174
184
  throw new AgentError('REPLAY_STALE', `the recording of this step no longer replays (${reason}), and cache.strict hands no step to the agent; ${strict.advice}`);
175
185
  }
186
+ /**
187
+ * Under `cache.strict`, ends a step whose key found no entry while the
188
+ * store holds a recording made for the same step under another key: the
189
+ * runner, the engine, the app, or the agent's context changed since, and
190
+ * the recording no longer replays as surely as one that diverged. A step
191
+ * whose instruction or params changed is a new step and still runs live.
192
+ */
193
+ async failIfRekeyed() {
194
+ const { strict } = this.cache;
195
+ if (strict === false || strict.recordings === undefined)
196
+ return;
197
+ const previous = await strict.recordings.underAnotherKey(this.keyHash, recordedProvenance(this.claim.step, this.options.redact));
198
+ if (previous === undefined)
199
+ return;
200
+ this.failedStale = true;
201
+ throw new AgentError('REPLAY_STALE', `the recording of this step no longer replays: the store holds it under another cache key (${previous}.json), since the runner, the engine, the app, or the agent's context changed after it was recorded, and cache.strict hands no step to the agent; ${strict.advice}`);
202
+ }
176
203
  /**
177
204
  * Closes the session once the step has settled. The whole write-side
178
205
  * decision lives here, in read-write mode only:
@@ -192,6 +219,10 @@ export class StepTraceSession {
192
219
  * anchor, or end wait, so drift is repaired only once a relocation fails
193
220
  * and the hand-off that follows re-records.
194
221
  * - passed otherwise: stage the recorded trace for attempt-end settlement.
222
+ * When this pass leaves nothing to stage (it changed nothing a replay
223
+ * could check) and an entry was read for the step, evict that entry: it
224
+ * did not serve this pass, and nothing recorded will replace it, so
225
+ * keeping it would hand every later run off the same way.
195
226
  * - failed after consuming a replay: evict. Without this, a diverged replay
196
227
  * whose step then fails stages nothing — and the poisoned entry would
197
228
  * replay its bad prefix on every future first attempt.
@@ -219,10 +250,15 @@ export class StepTraceSession {
219
250
  if (this.repairedAfterEndMismatch(recorder))
220
251
  await this.evict();
221
252
  else if (this.replayedWhole) {
222
- this.cache.staged.push({ kind: 'keep', keyHash: this.keyHash, stepIndex: this.options.stepIndex });
253
+ this.cache.staged.push({
254
+ kind: 'keep',
255
+ keyHash: this.keyHash,
256
+ stepIndex: this.options.stepIndex,
257
+ recordedFor: recordedProvenance(this.claim.step, this.options.redact),
258
+ });
223
259
  }
224
- else
225
- await this.stage(recorder, verdictSummary);
260
+ else if (!(await this.stage(recorder, verdictSummary)) && this.readEntryHit)
261
+ await this.evict();
226
262
  return;
227
263
  }
228
264
  }
@@ -262,7 +298,7 @@ export class StepTraceSession {
262
298
  * when this step may write. Settled as far as the capture's purpose asks.
263
299
  */
264
300
  async captureStart(purpose) {
265
- const observation = await probeScreen(this.host, purpose === 'path-only' ? 'raw' : 'held-still');
301
+ const observation = await probeScreen(this.host, purpose === 'path-only' ? 'raw' : 'held-still', true);
266
302
  this.startPath = observation?.path;
267
303
  if (this.recorder !== undefined && observation?.kind === 'semantic')
268
304
  this.startNodes = observation.nodes;
@@ -289,10 +325,33 @@ export class StepTraceSession {
289
325
  this.info = this.missed(decision.reason, trace.actions.length);
290
326
  return undefined;
291
327
  }
292
- const outcome = await replayTrace(this.host, trace, start?.kind === 'semantic' ? { initial: start } : {});
328
+ // Every screen the replay looks at, in order, from the start: the end
329
+ // check measures the recorded delta from the first of them on the route
330
+ // the replay ends on, so a free action is preceded by a look until one
331
+ // on the recorded end route has been seen.
332
+ const screens = start === undefined ? [] : [start];
333
+ const onEndRoute = (screen) => screen !== undefined && (trace.endPath === undefined || (screen.path !== undefined && sameRoute(screen.path, trace.endPath)));
334
+ const host = this.host;
335
+ const watched = {
336
+ get traceEligible() {
337
+ return host.traceEligible;
338
+ },
339
+ observe: async (mode) => {
340
+ const screen = await host.observe(mode);
341
+ screens.push(screen);
342
+ return screen;
343
+ },
344
+ actions: host.actions,
345
+ signal: host.signal,
346
+ remainingMs: () => host.remainingMs(),
347
+ };
348
+ const outcome = await replayTrace(watched, trace, {
349
+ ...(start?.kind === 'semantic' ? { initial: start } : {}),
350
+ looksBeforeFree: () => !onEndRoute(screens.at(-1)),
351
+ });
293
352
  this.consumedReplay = true;
294
353
  const stopReason = outcome.completed
295
- ? (await this.endStateMatches(trace))
354
+ ? start?.kind === 'semantic' && (await this.endStateMatches(trace, start, screens))
296
355
  ? undefined
297
356
  : 'end-mismatch'
298
357
  : (outcome.stopReason ?? 'action-failed');
@@ -309,57 +368,59 @@ export class StepTraceSession {
309
368
  }
310
369
  /**
311
370
  * The trace's postcondition against the live screen: the recorded end
312
- * route (when the live location is known) and every recorded end anchor
313
- * present again. A route the path leaves undecided is the recorded screen
314
- * only if the anchors are already on it, which then needs no second look.
371
+ * route when the live location is known, the recorded delta on the
372
+ * screen (`deltaHolds`), and evidence that the replay produced it rather
373
+ * than finding it there (`deltaEvidenced`). The evidence is measured from
374
+ * the first screen the replay saw on the route it ended on (`screens`, in
375
+ * order): the start for a step that stays on one screen, the page a
376
+ * recorded navigate opened before the switch on it was tapped. An outcome
377
+ * already showing there proves nothing, and neither does a recording with
378
+ * no delta at all.
315
379
  */
316
- async endStateMatches(trace) {
380
+ async endStateMatches(trace, start, screens) {
317
381
  if (!this.host.traceEligible)
318
382
  return false;
319
383
  const arrived = await this.endScreen(trace);
320
384
  if (arrived === undefined)
321
385
  return false;
322
- if (arrived.anchorsSeen)
323
- return true;
324
- return (await verifyAnchors(this.host, trace.endAnchors ?? [], {
325
- initial: arrived.screen,
386
+ // Both captures may predate a secret the replay resolved; read with the
387
+ // ledger as it is now, an unchanged node is no change.
388
+ const baseline = baselineScreen(screens, arrived.path);
389
+ const baselineNodes = baseline === undefined ? undefined : redactNodesAgain(baseline.nodes, this.redaction);
390
+ if (!deltaEvidenced(trace, baselineNodes, inputTargets(trace)))
391
+ return false;
392
+ const beforeNodes = baselineNodes ?? redactNodesAgain(start.nodes, this.redaction);
393
+ // Every look the wait takes must still be on the recorded end route: a
394
+ // screen that moved on after the route first matched is another screen.
395
+ const holds = (screen) => (trace.endPath === undefined || screen.path === undefined || sameRoute(screen.path, trace.endPath)) &&
396
+ deltaHolds(trace, screen.nodes, beforeNodes);
397
+ return (await verifyEndState(this.host, holds, {
398
+ initial: arrived,
326
399
  ...(trace.endWaitMs === undefined ? {} : { waitMs: trace.endWaitMs }),
327
400
  })) && this.host.traceEligible;
328
401
  }
329
402
  /**
330
403
  * Captures the semantic end state once its route matches the recording,
331
- * polling while a navigation the last action started commits. Reports
332
- * whether the anchors were what settled the route, so the caller does not
333
- * verify them again. A route still undecided when the poll runs out is
334
- * handed on with the anchors unseen: the caller's anchor wait, sized by
335
- * the recording, is the one that decides it, as for a route that matched.
404
+ * polling while a navigation the last action started commits. A route
405
+ * that never matches is another screen, whatever it shows: a link that
406
+ * now lands on a page sharing the recorded one's layout is exactly the
407
+ * flow that went wrong.
336
408
  */
337
409
  async endScreen(trace) {
338
410
  const startedMs = Date.now();
339
- const recorded = trace.endPath === undefined ? undefined : routeOf(trace.endPath);
340
- const anchors = trace.endAnchors ?? [];
411
+ const recorded = trace.endPath;
341
412
  for (let attempt = 0;; attempt += 1) {
342
413
  const observation = await probeScreen(this.host, 'raw');
343
414
  if (observation?.kind !== 'semantic' || !this.host.traceEligible)
344
415
  return undefined;
345
- if (recorded === undefined || observation.path === undefined)
346
- return { screen: observation, anchorsSeen: false };
347
- const verdict = compareRoutes(recorded, routeOf(observation.path));
348
- if (verdict === 'same')
349
- return { screen: observation, anchorsSeen: false };
350
- // A path the runner cannot recognize as the recorded route is the
351
- // recorded screen only if the recorded effect is visibly on it; with
352
- // no anchors recorded there is nothing to see, and it is another screen.
353
- const undecided = verdict === 'undecided' && anchors.length > 0;
354
- if (undecided && anchorsPresent(anchors, observation.nodes, { redact: this.options.redact })) {
355
- return { screen: observation, anchorsSeen: true };
356
- }
416
+ if (recorded === undefined || observation.path === undefined || sameRoute(recorded, observation.path))
417
+ return observation;
357
418
  const delay = END_PATH_DELAYS_MS[attempt];
358
419
  if (delay === undefined ||
359
420
  Date.now() - startedMs + delay > END_PATH_TIMEOUT_MS ||
360
421
  this.host.remainingMs() <= delay ||
361
422
  this.host.signal.aborted) {
362
- return undecided ? { screen: observation, anchorsSeen: false } : undefined;
423
+ return undefined;
363
424
  }
364
425
  await sleep(delay, this.host.signal);
365
426
  }
@@ -401,32 +462,36 @@ export class StepTraceSession {
401
462
  *
402
463
  * The end path and a fresh settled observation are the trace's
403
464
  * postcondition — the state the step passed in. The delta between the
404
- * baseline and the passing observation becomes the end anchors. When the
405
- * step moved to another pathname the whole new screen is the delta, and its
406
- * first stable anchors back the path check, which matches a created
407
- * record's page up to the id the app minted for it. A postcondition that
408
- * cannot be captured stages nothing: a trace without its check would
409
- * replay on mechanics alone.
465
+ * baseline and the passing observation becomes the anchors: what appeared
466
+ * and what vanished. When the step moved to another pathname the whole new
467
+ * screen is the delta, and its first stable anchors back the path check,
468
+ * which matches a created record's page up to the id the app minted for
469
+ * it. A postcondition that cannot be captured stages nothing, and neither
470
+ * does a step that changed nothing a replay could check, no node and no
471
+ * route: a trace without its check would replay on mechanics alone.
410
472
  */
411
473
  async stage(recorder, verdictSummary) {
412
474
  if (!this.host.traceEligible || this.startNodes === undefined || recorder.recordedCount === 0)
413
- return;
475
+ return false;
414
476
  const observation = await probeScreen(this.host, 'held-still');
415
477
  if (!this.host.traceEligible || observation?.kind !== 'semantic')
416
- return;
478
+ return false;
417
479
  const { nodes: endNodes, path: endPath } = observation;
418
- const endAnchors = describeAnchors(this.startNodes, endNodes, this.options);
480
+ // The start capture may predate a secret this step resolved; read with
481
+ // the ledger as it is now, an unchanged node is no delta.
482
+ const startNodes = redactNodesAgain(this.startNodes, this.redaction);
483
+ const routeMoved = this.startPath !== undefined && endPath !== undefined && !sameRoute(this.startPath, endPath);
484
+ const delta = describeDelta(startNodes, endNodes, routeMoved);
485
+ if (delta.appeared.length === 0 && delta.gone.length === 0 && !routeMoved)
486
+ return false;
419
487
  const trace = recorder.finalize({
420
488
  executor: this.options.executor,
421
- recordedFor: {
422
- testId: this.cache.identity.testId,
423
- targetId: this.cache.identity.targetId,
424
- instructionDigest: instructionDigest(this.options.instruction),
425
- },
489
+ recordedFor: this.claim.step,
426
490
  summary: verdictSummary ?? 'step passed',
427
491
  ...(this.startPath === undefined ? {} : { startPath: this.startPath }),
428
492
  ...(endPath === undefined ? {} : { endPath }),
429
- endAnchors,
493
+ endAnchors: delta.appeared,
494
+ goneAnchors: delta.gone,
430
495
  // How long the app took to show its end state after the last action,
431
496
  // plus room for a slower day: the budget a replay waits for the anchors
432
497
  // to return. Measured from the last action, not the step's start: the
@@ -435,12 +500,12 @@ export class StepTraceSession {
435
500
  endWaitMs: Date.now() - (recorder.lastActionAtMs ?? this.startedMs) + END_WAIT_MARGIN_MS,
436
501
  });
437
502
  if (trace === undefined)
438
- return;
503
+ return false;
439
504
  // A trace with no start anchor — no recorded path (a surface without a URL)
440
505
  // and no opening navigate — could never replay: `wrong-context` forever.
441
506
  // Writing it would be pure store traffic, so it is not written at all.
442
507
  if (trace.startPath === undefined && !opensWithNavigate(trace))
443
- return;
508
+ return false;
444
509
  // Stored with a slot where each `unique()` value appeared, so the next
445
510
  // run's values — a fresh timestamped name — replay the same flow. A
446
511
  // recording that cannot be templated safely is not written at all; when
@@ -448,12 +513,13 @@ export class StepTraceSession {
448
513
  if (templatesCollide(this.options.params, this.options.templates)) {
449
514
  if (this.info !== undefined)
450
515
  this.info = { ...this.info, notRecorded: 'param-collision' };
451
- return;
516
+ return false;
452
517
  }
453
518
  const templated = templateTrace(trace, this.options.templates);
454
519
  if (templated === undefined)
455
- return;
520
+ return false;
456
521
  this.cache.staged.push({ kind: 'write', keyHash: this.keyHash, trace: templated, stepIndex: this.options.stepIndex });
522
+ return true;
457
523
  }
458
524
  /**
459
525
  * Awaited so the step does not close — and a later read cannot be served
@@ -476,18 +542,41 @@ export class StepTraceSession {
476
542
  };
477
543
  }
478
544
  }
545
+ /** The controls a trace's input actions set a value or a state on: their changed anchors echo the input. */
546
+ function inputTargets(trace) {
547
+ return trace.actions.flatMap((action) => ('target' in action && action.target !== undefined ? [action.target] : []));
548
+ }
549
+ /**
550
+ * The first semantic screen of the replay's last stretch on the route it
551
+ * ended on (`endPath`), or undefined when its last screen before the end
552
+ * was on another route, so the last action moved it. A screen or an end
553
+ * without a location counts as on the route: there is nothing to compare.
554
+ */
555
+ function baselineScreen(screens, endPath) {
556
+ let baseline;
557
+ for (const screen of screens) {
558
+ if (endPath !== undefined && screen.path !== undefined && !sameRoute(screen.path, endPath))
559
+ baseline = undefined;
560
+ else if (screen.kind === 'semantic')
561
+ baseline ??= screen;
562
+ }
563
+ return baseline;
564
+ }
479
565
  /**
480
566
  * One cache capture, settled as far as `mode` asks, or undefined when the
481
567
  * surface cannot be observed right now, which is the executor's business,
482
568
  * not the cache's. Runtime hard stops (timeout, cancellation) are the step's
483
- * truth even when they land during cache bookkeeping, and propagate.
569
+ * truth even when they land during cache bookkeeping, and propagate. So does
570
+ * a policy denial on the step's opening look (`opening`): the executor's
571
+ * first look reads the same screen under the same policy, and would spend
572
+ * another operation budget to be denied again.
484
573
  */
485
- async function probeScreen(host, mode) {
574
+ async function probeScreen(host, mode, opening = false) {
486
575
  try {
487
576
  return await host.observe(mode);
488
577
  }
489
578
  catch (cause) {
490
- if (isRuntimeHardStop(cause))
579
+ if (isRuntimeHardStop(cause) || (opening && isAgentError(cause) && cause.code === 'POLICY_DENIED'))
491
580
  throw cause;
492
581
  return undefined;
493
582
  }
@@ -11,10 +11,11 @@
11
11
  * - model-call accounting and a step transcript for `--debug`.
12
12
  */
13
13
  import { asSdkLanguageModel, loadAiSdk } from './ai-sdk.js';
14
+ import { MODEL_REQUEST_HEADERS } from '../internal/client-identity.js';
14
15
  import { ConfigurationError, withHint } from '../internal/errors.js';
15
16
  import { failureHint, isAbort, TRANSPORT_RETRIES } from './model/sdk.js';
16
17
  import { isContextOverflow } from './model/overflow.js';
17
- import { isForcedToolCallSkipped, isForcedToolChoiceDowngraded, isForcedToolChoiceRejected } from './model/tool-choice.js';
18
+ import { isForcedToolCallMismatched, isForcedToolCallSkipped, isForcedToolChoiceDowngraded, isForcedToolChoiceRejected, } from './model/tool-choice.js';
18
19
  import { withStallGuard } from './model/stall.js';
19
20
  import { providerHints } from './model/provider-hints.js';
20
21
  import { isScreenOutput, toolResultTexts } from './screen-update.js';
@@ -192,6 +193,13 @@ class LoopRun {
192
193
  prompt = shrunk;
193
194
  continue;
194
195
  }
196
+ const corrected = this.correctForcedTurn(cause, tracker);
197
+ if (corrected === 'out-of-turns')
198
+ break;
199
+ if (corrected !== undefined) {
200
+ prompt = corrected;
201
+ continue;
202
+ }
195
203
  const freed = this.freeToolChoiceRetry(cause, tracker);
196
204
  if (freed !== undefined) {
197
205
  prompt = freed;
@@ -241,6 +249,7 @@ class LoopRun {
241
249
  toolChoice: this.toolChoice,
242
250
  ...(providerOptions === undefined ? {} : { providerOptions: providerOptions }),
243
251
  maxRetries: TRANSPORT_RETRIES,
252
+ headers: MODEL_REQUEST_HEADERS,
244
253
  stopWhen: [
245
254
  () => this.conclusion.concluded() || this.hardStop !== undefined,
246
255
  // Counted across every generate call of the step, so a retry after an
@@ -273,6 +282,49 @@ class LoopRun {
273
282
  { role: 'user', content: TEXT_REPLY_NOTICE },
274
283
  ];
275
284
  }
285
+ /**
286
+ * A turn forced to call `complete_step` that called other tools instead:
287
+ * the SDK throws `ToolChoiceViolationError` before running any of them. It
288
+ * is handled like a call to a tool the turn does not offer: the turn counts,
289
+ * each call goes back to the model as an error result, and the model is
290
+ * asked again on the same turn budget. Out of turns, the step ends as
291
+ * `STEP_NO_CONCLUSION`, like any step that never concluded.
292
+ */
293
+ correctForcedTurn(cause, tracker) {
294
+ if (this.lastRequest === undefined || this.context.signal.aborted || this.hardStop !== undefined)
295
+ return undefined;
296
+ if (!isForcedToolCallMismatched(cause))
297
+ return undefined;
298
+ const reply = thrownReply(cause.content);
299
+ const results = reply.calls.map((call) => ({
300
+ type: 'tool-result',
301
+ toolCallId: call.toolCallId,
302
+ toolName: call.toolName,
303
+ output: {
304
+ type: 'error-text',
305
+ value: `${call.toolName} did not run: only complete_step is available now. Call complete_step with your verdict.`,
306
+ },
307
+ }));
308
+ this.recordThrownTurn(tracker, reply.calls.map((call) => `${call.toolName}(${truncate(safeJson(call.input), MAX_TURN_CALL_CHARS)})`), [
309
+ skippedReplyText(cause.content),
310
+ ...results.map((result) => `[${result.toolName}] error: ${result.output.value}`),
311
+ ].filter((line) => line !== '').join('\n'));
312
+ if (this.turnsUsed >= this.maxTurns)
313
+ return 'out-of-turns';
314
+ this.note(`turn ${String(this.turnsUsed)} called other tools in place of complete_step: asking again`);
315
+ this.turnOffset = this.turnsUsed;
316
+ return [...this.lastRequest, reply.message, { role: 'tool', content: results }];
317
+ }
318
+ /**
319
+ * Counts a turn the provider answered, and billed, before the SDK threw on
320
+ * it. The SDK reports no usage for such a turn.
321
+ */
322
+ recordThrownTurn(tracker, calls, outcome) {
323
+ tracker.onStepEnd({ usage: UNREPORTED_USAGE, providerMetadata: undefined, reasoningText: undefined });
324
+ this.turnsUsed += 1;
325
+ this.turns.push({ index: this.turnsUsed, calls, outcome, notes: [] });
326
+ this.attachTranscript();
327
+ }
276
328
  /**
277
329
  * A model that refuses a forced tool choice (HTTP 400 naming
278
330
  * `tool_choice`) is asked again with `auto` and a tool-calls-only rule in
@@ -288,10 +340,7 @@ class LoopRun {
288
340
  if (this.context.signal.aborted || this.hardStop !== undefined)
289
341
  return undefined;
290
342
  if (isForcedToolCallSkipped(cause)) {
291
- // The provider answered, and billed, before the SDK threw: the turn counts.
292
- tracker.onStepEnd({ usage: UNREPORTED_USAGE, providerMetadata: undefined, reasoningText: undefined });
293
- this.turnsUsed += 1;
294
- this.turns.push({ index: this.turnsUsed, calls: [], outcome: skippedReplyText(cause.content), notes: [] });
343
+ this.recordThrownTurn(tracker, [], skippedReplyText(cause.content));
295
344
  this.note(`turn ${String(this.turnsUsed)} answered without the forced tool call: retrying with auto`);
296
345
  }
297
346
  else if (isForcedToolChoiceRejected(cause)) {
@@ -617,6 +666,46 @@ function skippedReplyText(content) {
617
666
  .trim();
618
667
  return text === '' ? '' : `assistant: ${truncate(text, MAX_TURN_TEXT_CHARS)}`;
619
668
  }
669
+ /**
670
+ * A reply the SDK threw on, as the assistant message the SDK's own history
671
+ * would carry: its text, reasoning, and tool calls, each with the provider's
672
+ * metadata (reasoning signatures and Gemini's thought signatures ride there),
673
+ * and the tool calls apart, to answer each with a result.
674
+ */
675
+ function thrownReply(content) {
676
+ const parts = [];
677
+ const calls = [];
678
+ for (const part of content) {
679
+ const metadata = part.providerMetadata === undefined ? {} : { providerOptions: part.providerMetadata };
680
+ if (part.type === 'text' && typeof part.text === 'string' && part.text !== '') {
681
+ parts.push({ type: 'text', text: part.text, ...metadata });
682
+ }
683
+ else if (part.type === 'reasoning' && typeof part.text === 'string') {
684
+ parts.push({ type: 'reasoning', text: part.text, ...metadata });
685
+ }
686
+ else if (part.type === 'tool-call' && typeof part.toolCallId === 'string' && typeof part.toolName === 'string') {
687
+ const call = {
688
+ type: 'tool-call',
689
+ toolCallId: part.toolCallId,
690
+ toolName: part.toolName,
691
+ input: typeof part.input === 'string' ? parseToolInput(part.input) : part.input,
692
+ ...metadata,
693
+ };
694
+ parts.push(call);
695
+ calls.push(call);
696
+ }
697
+ }
698
+ return { message: { role: 'assistant', content: parts }, calls };
699
+ }
700
+ /** A raw tool call's input for the history: the parsed JSON, or the text itself when it does not parse. */
701
+ function parseToolInput(input) {
702
+ try {
703
+ return JSON.parse(input);
704
+ }
705
+ catch {
706
+ return input;
707
+ }
708
+ }
620
709
  function describeOutput(output) {
621
710
  if (isScreenOutput(output))
622
711
  return `${output.text}\n[screenshot, ${String(output.pixels.data.byteLength)} bytes]`;