e2e 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/README.md +10 -0
  2. package/dist/.build.json +1 -1
  3. package/dist/agent/act.js +7 -4
  4. package/dist/agent/action-dispatcher.js +4 -5
  5. package/dist/agent/actions.d.ts +32 -23
  6. package/dist/agent/actions.js +38 -24
  7. package/dist/agent/ai-sdk.js +5 -4
  8. package/dist/agent/error.js +3 -3
  9. package/dist/agent/invocation.js +3 -4
  10. package/dist/agent/model/sdk.js +2 -0
  11. package/dist/agent/model/tool-choice.d.ts +9 -0
  12. package/dist/agent/model/tool-choice.js +14 -0
  13. package/dist/agent/observation-feed.d.ts +5 -6
  14. package/dist/agent/observation-feed.js +8 -11
  15. package/dist/agent/observation.d.ts +40 -11
  16. package/dist/agent/observation.js +109 -59
  17. package/dist/agent/phases.d.ts +7 -1
  18. package/dist/agent/phases.js +12 -3
  19. package/dist/agent/point-tap.d.ts +8 -8
  20. package/dist/agent/point-tap.js +1 -1
  21. package/dist/agent/prompts.js +3 -0
  22. package/dist/agent/replay.d.ts +19 -8
  23. package/dist/agent/replay.js +29 -29
  24. package/dist/agent/scroll-target.d.ts +2 -3
  25. package/dist/agent/step-cache.d.ts +43 -15
  26. package/dist/agent/step-cache.js +156 -67
  27. package/dist/agent/tool-loop.js +94 -5
  28. package/dist/cache/anchors.d.ts +72 -23
  29. package/dist/cache/anchors.js +233 -60
  30. package/dist/cache/context.d.ts +25 -14
  31. package/dist/cache/context.js +50 -8
  32. package/dist/cache/decide.d.ts +2 -2
  33. package/dist/cache/decide.js +4 -6
  34. package/dist/cache/identity.d.ts +18 -4
  35. package/dist/cache/identity.js +9 -4
  36. package/dist/cache/recorder.d.ts +15 -1
  37. package/dist/cache/recorder.js +36 -23
  38. package/dist/cache/rekeyed.d.ts +46 -0
  39. package/dist/cache/rekeyed.js +89 -0
  40. package/dist/cache/relocate.d.ts +14 -14
  41. package/dist/cache/relocate.js +33 -25
  42. package/dist/cache/route.d.ts +32 -33
  43. package/dist/cache/route.js +94 -50
  44. package/dist/cache/store.d.ts +6 -0
  45. package/dist/cache/store.js +21 -1
  46. package/dist/cache/trace.d.ts +53 -14
  47. package/dist/cache/trace.js +76 -30
  48. package/dist/cli/bin.js +2 -2
  49. package/dist/cli/index.js +43 -32
  50. package/dist/cli/init/engines.d.ts +1 -3
  51. package/dist/cli/init/engines.js +2 -2
  52. package/dist/cli/init/gateways.d.ts +1 -1
  53. package/dist/cli/init/gateways.js +11 -1
  54. package/dist/cli/init/mcp-config.d.ts +2 -1
  55. package/dist/cli/init/mcp-config.js +8 -3
  56. package/dist/cli/init/sibling-versions.json +1 -1
  57. package/dist/cli/init/versions.d.ts +0 -8
  58. package/dist/cli/init/versions.js +1 -10
  59. package/dist/cli/mcp.d.ts +3 -2
  60. package/dist/cli/mcp.js +52 -16
  61. package/dist/cli/run-output.d.ts +17 -0
  62. package/dist/cli/run-output.js +45 -0
  63. package/dist/collect/collect.js +8 -4
  64. package/dist/collect/registry.js +27 -16
  65. package/dist/config/compiled-files.d.ts +36 -0
  66. package/dist/config/compiled-files.js +50 -0
  67. package/dist/config/diagnose.js +21 -19
  68. package/dist/config/esm-hooks.d.ts +39 -44
  69. package/dist/config/esm-hooks.js +238 -63
  70. package/dist/config/load.d.ts +1 -7
  71. package/dist/config/load.js +23 -66
  72. package/dist/config/register.d.ts +8 -0
  73. package/dist/config/register.js +9 -0
  74. package/dist/config/resolve.d.ts +1 -0
  75. package/dist/config/resolve.js +33 -1
  76. package/dist/config/specifiers.d.ts +19 -0
  77. package/dist/config/specifiers.js +33 -0
  78. package/dist/config/tsconfig.d.ts +56 -0
  79. package/dist/config/tsconfig.js +115 -0
  80. package/dist/config/typescript.d.ts +19 -0
  81. package/dist/config/typescript.js +253 -0
  82. package/dist/engine/contract.d.ts +1 -0
  83. package/dist/engine/index.d.ts +4 -2
  84. package/dist/engine/session.js +2 -2
  85. package/dist/expect/async.js +4 -2
  86. package/dist/expect/poll-scope.d.ts +52 -0
  87. package/dist/expect/poll-scope.js +113 -0
  88. package/dist/expect/poll.js +33 -6
  89. package/dist/expect/values.js +11 -4
  90. package/dist/internal/abandoned.d.ts +11 -0
  91. package/dist/internal/abandoned.js +23 -0
  92. package/dist/internal/brands.d.ts +2 -0
  93. package/dist/internal/brands.js +2 -0
  94. package/dist/internal/client-identity.d.ts +9 -0
  95. package/dist/internal/client-identity.js +14 -0
  96. package/dist/internal/errors.d.ts +10 -3
  97. package/dist/internal/errors.js +23 -8
  98. package/dist/internal/node-version.d.ts +16 -7
  99. package/dist/internal/node-version.js +50 -15
  100. package/dist/internal/package-manager.js +1 -1
  101. package/dist/internal/realm-slot.d.ts +5 -4
  102. package/dist/internal/redact.d.ts +9 -4
  103. package/dist/internal/redact.js +120 -23
  104. package/dist/internal/time.d.ts +26 -4
  105. package/dist/internal/time.js +58 -14
  106. package/dist/internal/urls.d.ts +9 -5
  107. package/dist/internal/urls.js +19 -10
  108. package/dist/locator/engine.d.ts +12 -2
  109. package/dist/locator/engine.js +55 -14
  110. package/dist/locator/screen.js +59 -9
  111. package/dist/mcp/catalog.d.ts +3 -2
  112. package/dist/mcp/catalog.js +6 -6
  113. package/dist/mcp/output.d.ts +59 -0
  114. package/dist/mcp/output.js +129 -0
  115. package/dist/mcp/server.d.ts +4 -0
  116. package/dist/mcp/server.js +9 -12
  117. package/dist/mcp/session.d.ts +8 -1
  118. package/dist/mcp/session.js +56 -15
  119. package/dist/mcp/tools.js +7 -5
  120. package/dist/oauth/chatgpt.js +3 -23
  121. package/dist/oauth/cli.js +1 -0
  122. package/dist/oauth/copilot.d.ts +5 -4
  123. package/dist/oauth/copilot.js +80 -7
  124. package/dist/oauth/grok.js +2 -1
  125. package/dist/oauth/models.js +2 -1
  126. package/dist/oauth/opencode-console.d.ts +10 -0
  127. package/dist/oauth/opencode-console.js +133 -0
  128. package/dist/oauth/providers/github-copilot.d.ts +12 -0
  129. package/dist/oauth/providers/github-copilot.js +77 -23
  130. package/dist/oauth/providers/opencode-console.d.ts +31 -0
  131. package/dist/oauth/providers/opencode-console.js +148 -0
  132. package/dist/oauth/providers.d.ts +2 -6
  133. package/dist/oauth/providers.js +3 -7
  134. package/dist/oauth/responses.d.ts +9 -0
  135. package/dist/oauth/responses.js +23 -0
  136. package/dist/report/build.d.ts +26 -0
  137. package/dist/report/build.js +16 -8
  138. package/dist/report/failure-text.d.ts +6 -1
  139. package/dist/report/failure-text.js +14 -4
  140. package/dist/report/format.d.ts +10 -5
  141. package/dist/report/format.js +12 -5
  142. package/dist/report/junit.js +11 -12
  143. package/dist/report/list.d.ts +12 -0
  144. package/dist/report/list.js +61 -24
  145. package/dist/report/markdown.d.ts +1 -1
  146. package/dist/report/markdown.js +37 -14
  147. package/dist/report/repeats.d.ts +6 -1
  148. package/dist/report/repeats.js +12 -3
  149. package/dist/run/artifacts.d.ts +22 -4
  150. package/dist/run/artifacts.js +71 -3
  151. package/dist/run/execute.d.ts +34 -5
  152. package/dist/run/execute.js +93 -44
  153. package/dist/run/extended-fixtures.js +6 -2
  154. package/dist/run/failure-evidence.js +10 -4
  155. package/dist/run/fixtures.js +6 -8
  156. package/dist/run/in-process.d.ts +2 -0
  157. package/dist/run/in-process.js +52 -10
  158. package/dist/run/last-run.d.ts +42 -8
  159. package/dist/run/last-run.js +121 -14
  160. package/dist/run/process-output.d.ts +73 -0
  161. package/dist/run/process-output.js +130 -0
  162. package/dist/run/provision.d.ts +2 -0
  163. package/dist/run/provision.js +1 -0
  164. package/dist/run/realm.d.ts +4 -1
  165. package/dist/run/realm.js +20 -6
  166. package/dist/run/records.d.ts +20 -4
  167. package/dist/run/records.js +26 -0
  168. package/dist/run/retry.d.ts +10 -1
  169. package/dist/run/retry.js +30 -12
  170. package/dist/run/runner.d.ts +9 -1
  171. package/dist/run/runner.js +101 -39
  172. package/dist/run/scheduler.js +255 -19
  173. package/dist/run/secrecy.d.ts +22 -19
  174. package/dist/run/secrecy.js +22 -18
  175. package/dist/run/serial.d.ts +75 -2
  176. package/dist/run/serial.js +243 -59
  177. package/dist/run/standalone.js +1 -1
  178. package/dist/run/steps.d.ts +0 -6
  179. package/dist/run/steps.js +3 -20
  180. package/dist/run/trace-redaction.d.ts +5 -6
  181. package/dist/run/trace-redaction.js +5 -6
  182. package/dist/run/worker/entry.js +40 -18
  183. package/dist/run/worker/handle.js +10 -5
  184. package/dist/run/worker/protocol.d.ts +85 -3
  185. package/dist/run/worker/session.d.ts +2 -0
  186. package/dist/run/worker/session.js +29 -0
  187. package/dist/telemetry/environment.d.ts +2 -1
  188. package/dist/telemetry/environment.js +12 -2
  189. package/dist/telemetry/events.js +1 -0
  190. package/dist/telemetry/project.d.ts +5 -6
  191. package/dist/telemetry/project.js +6 -11
  192. package/dist/telemetry/store.d.ts +5 -5
  193. package/dist/telemetry/store.js +5 -7
  194. package/dist/telemetry/telemetry.d.ts +2 -3
  195. package/dist/telemetry/telemetry.js +2 -4
  196. package/dist/types.d.ts +15 -6
  197. package/docs/agent-steps.mdx +1 -0
  198. package/docs/agents.mdx +7 -3
  199. package/docs/api-testing.mdx +2 -1
  200. package/docs/authentication.mdx +6 -3
  201. package/docs/browser.mdx +49 -12
  202. package/docs/bug-bash.mdx +2 -1
  203. package/docs/cache.mdx +66 -19
  204. package/docs/ci.mdx +14 -9
  205. package/docs/coding-agents.mdx +2 -1
  206. package/docs/core-concepts.mdx +208 -0
  207. package/docs/debugging.mdx +25 -12
  208. package/docs/decision-models.mdx +142 -0
  209. package/docs/example-projects.mdx +32 -0
  210. package/docs/executors.mdx +7 -2
  211. package/docs/explore.mdx +1 -0
  212. package/docs/github.mdx +14 -5
  213. package/docs/index.mdx +9 -5
  214. package/docs/integrations/eas.mdx +1 -0
  215. package/docs/integrations/index.mdx +1 -0
  216. package/docs/integrations/kernel.mdx +3 -2
  217. package/docs/migrate/cypress.mdx +26 -3
  218. package/docs/migrate/detox.mdx +24 -1
  219. package/docs/migrate/maestro.mdx +28 -4
  220. package/docs/migrate/playwright.mdx +44 -18
  221. package/docs/migrate/selenium.mdx +27 -3
  222. package/docs/mobile-ci.mdx +6 -5
  223. package/docs/mobile.mdx +96 -8
  224. package/docs/models.mdx +60 -9
  225. package/docs/quickstart.mdx +28 -16
  226. package/docs/reference/agent.mdx +13 -3
  227. package/docs/reference/app.mdx +2 -1
  228. package/docs/reference/cli.mdx +41 -18
  229. package/docs/reference/config.mdx +67 -13
  230. package/docs/reference/engine.mdx +25 -13
  231. package/docs/reference/environment.mdx +10 -2
  232. package/docs/reference/errors.mdx +13 -10
  233. package/docs/reference/expect.mdx +36 -13
  234. package/docs/reference/mcp.mdx +21 -8
  235. package/docs/reference/mobile.mdx +24 -13
  236. package/docs/reference/reporters.mdx +9 -2
  237. package/docs/reference/screen.mdx +49 -16
  238. package/docs/reference/test.mdx +32 -5
  239. package/docs/reference/web.mdx +133 -23
  240. package/docs/security.mdx +58 -42
  241. package/docs/subscriptions.mdx +92 -12
  242. package/docs/telemetry.mdx +9 -6
  243. package/docs/tools.mdx +1 -0
  244. package/docs/{starting-your-app.mdx → web.mdx} +157 -11
  245. package/docs/writing-an-engine.mdx +2 -1
  246. package/package.json +25 -8
  247. package/schema/report-v1.schema.json +14 -1
  248. package/skills/e2e/SKILL.md +5 -3
  249. package/skills/e2e/references/agent.md +33 -11
  250. package/skills/e2e/references/debugging.md +4 -4
  251. package/skills/e2e/references/mcp.md +8 -3
  252. package/skills/e2e/references/running.md +15 -9
  253. package/skills/e2e/references/setup.md +45 -16
  254. package/skills/e2e/references/writing-tests.md +35 -18
  255. package/docs/viewports.mdx +0 -146
  256. package/docs/writing-tests.mdx +0 -256
@@ -3,56 +3,80 @@ import { classifyError, ConfigurationError, serializeError, } from '../internal/
3
3
  import { canonicalDigest, timestamp, uuidv7 } from '../internal/ids.js';
4
4
  import { groupTitles } from '../collect/registry.js';
5
5
  import { pairRecordings } from '../collect/select.js';
6
- import { createAttemptArtifacts, sanitizePathSegment } from './artifacts.js';
6
+ import { attemptSegments, createAttemptArtifacts, sanitizePathSegment } from './artifacts.js';
7
7
  import { findRegistered, RealmManager } from './realm.js';
8
8
  import { isFailedStatus } from './records.js';
9
- import { runWithRetries } from './retry.js';
9
+ import { isRetryEligible, retryVerdict, runWithRetries } from './retry.js';
10
10
  import { interruptedSkip, pairResult, repeatSegment } from './units.js';
11
+ /** The report id of one serial group variant: a group runs once per agent and per repeat. */
12
+ export function serialGroupId(serialId, targetName, agent, repeat) {
13
+ return canonicalDigest(repeat === 0 ? { serialId, targetId: targetName, agent } : { serialId, targetId: targetName, agent, repeat });
14
+ }
11
15
  /** Runs one serial unit to completion and emits every member result. */
12
16
  export async function runSerialUnit(host, members, file) {
17
+ const group = newSerialGroup(members, host.target);
18
+ const runs = [];
19
+ const finalStatus = await runWithRetries(members[0].options.retries + 1, host.interruptSignal, async (attemptIndex) => {
20
+ const run = await runSerialAttempt(host, group.id, members, file, attemptIndex);
21
+ runs.push(run);
22
+ group.attempts.push(run.record);
23
+ host.serialAttemptFinished(group.id, run);
24
+ // A beforeAll failure is not retry-eligible: the
25
+ // attempt stands as recorded and the retry loop stops here.
26
+ if (run.record.error?.code === 'HOOK_FAILED')
27
+ return undefined;
28
+ return run.record;
29
+ });
30
+ if (group.attempts.length === 0) {
31
+ group.status = 'skipped';
32
+ group.skip = interruptedSkip(host.interruptSignal);
33
+ }
34
+ else {
35
+ group.status = finalStatus;
36
+ }
37
+ host.emitSerialGroup(group);
38
+ for (const result of memberResults(group, members, runs))
39
+ host.emit(result);
40
+ return group;
41
+ }
42
+ /** A group record with no attempts yet. */
43
+ function newSerialGroup(members, target) {
13
44
  const first = members[0];
14
45
  const serialId = first.test.serialId;
15
46
  // Every member of a unit runs as the same agent (selection guarantees it),
16
47
  // so the group is one variant of the flow and is identified as such.
17
- const agent = first.agent;
18
- const { repeat } = first;
19
- const groupRecordId = canonicalDigest(repeat === 0 ? { serialId, targetId: host.target.name, agent } : { serialId, targetId: host.target.name, agent, repeat });
20
- const group = {
21
- id: groupRecordId,
48
+ const { agent, repeat } = first;
49
+ return {
50
+ id: serialGroupId(serialId, target.name, agent, repeat),
22
51
  serialId,
23
52
  declarationIndex: first.test.declarationIndex,
24
53
  file: first.test.file,
25
54
  titlePath: serialTitlePath(first.test),
26
- targetId: host.target.name,
27
- platform: host.target.platform,
55
+ targetId: target.name,
56
+ platform: target.platform,
28
57
  agent,
29
58
  repeat,
30
59
  memberTestIds: members.map((member) => member.test.id),
31
60
  status: 'failed',
32
61
  attempts: [],
33
62
  };
34
- const memberFinalStatus = new Map();
35
- const finalStatus = await runWithRetries(first.options.retries + 1, host.interruptSignal, async (attemptIndex) => {
36
- const attempt = await runSerialAttempt(host, members, file, attemptIndex);
37
- group.attempts.push(attempt);
38
- for (const member of attempt.members)
39
- memberFinalStatus.set(member.testId, member);
40
- // A beforeAll failure is not retry-eligible: the
41
- // attempt stands as recorded and the retry loop stops here.
42
- if (attempt.error?.code === 'HOOK_FAILED')
43
- return undefined;
44
- return attempt;
45
- });
46
- if (group.attempts.length === 0) {
47
- group.status = 'skipped';
48
- group.skip = interruptedSkip(host.interruptSignal);
49
- }
50
- else {
51
- group.status = finalStatus;
63
+ }
64
+ /**
65
+ * Each member's result once its group is decided. Members carry no attempts
66
+ * of their own; the verdict is the member's record in the last attempt that
67
+ * counts. A retry the run interrupted, or one that never reached its
68
+ * members, keeps the verdict before it.
69
+ */
70
+ function memberResults(group, members, runs) {
71
+ const verdicts = new Map();
72
+ for (const [attemptIndex, run] of runs.entries()) {
73
+ const cutRetry = attemptIndex > 0 && (run.record.status === 'interrupted' || !run.reachedMembers);
74
+ if (!cutRetry)
75
+ for (const member of run.record.members)
76
+ verdicts.set(member.testId, member);
52
77
  }
53
- host.emitSerialGroup(group);
54
- for (const member of members) {
55
- const memberRecord = memberFinalStatus.get(member.test.id);
78
+ return members.map((member) => {
79
+ const memberRecord = verdicts.get(member.test.id);
56
80
  let status;
57
81
  let skip;
58
82
  // A member that skipped itself stays skipped whatever the group did: a
@@ -71,30 +95,153 @@ export async function runSerialUnit(host, members, file) {
71
95
  else {
72
96
  status = memberRecord.status;
73
97
  }
74
- host.emit(pairResult(member, {
98
+ return pairResult(member, {
75
99
  status,
76
100
  selected: true,
77
101
  ...(skip !== undefined ? { skip } : {}),
78
102
  attempts: [],
79
- serialGroupId: groupRecordId,
80
- }));
103
+ serialGroupId: group.id,
104
+ });
105
+ });
106
+ }
107
+ /**
108
+ * What the runner heard of one serial group a worker is running: the
109
+ * attempts it finished, and the members finished so far in the attempt
110
+ * running now. When the worker dies before reporting the group, this is
111
+ * what the report keeps.
112
+ */
113
+ export class SerialGroupProgress {
114
+ finished = [];
115
+ current;
116
+ currentMembers = [];
117
+ /** The member whose body is running now, by test id, until its record arrives. */
118
+ running;
119
+ /** A member of the group announced its start. */
120
+ memberStarted(testId) {
121
+ this.running = testId;
122
+ }
123
+ /** A member finished in the attempt running now. */
124
+ memberFinished(attempt, member) {
125
+ this.current = attempt;
126
+ this.currentMembers.push(member);
127
+ if (this.running === member.testId)
128
+ this.running = undefined;
129
+ }
130
+ /** A group attempt finished; the next one starts empty. */
131
+ attemptFinished(run) {
132
+ this.finished.push(run);
133
+ this.current = undefined;
134
+ this.currentMembers = [];
135
+ this.running = undefined;
136
+ }
137
+ /**
138
+ * The group after its worker died with `error`, or undefined when the
139
+ * runner heard nothing of it running. The finished attempts stand, and the
140
+ * attempt in flight fails: the member whose body was running ends `status`
141
+ * with `error` (`timed-out` when the watchdog killed a worker that hung in
142
+ * it) and the ones after it skip. A crash between members (a hook,
143
+ * the session closing) fails the attempt and leaves its finished members
144
+ * as they were; one in a retry before any member ran fails the first, as a
145
+ * launch failure does. A crash after the last attempt, when no retry
146
+ * follows it, is recorded on that attempt.
147
+ */
148
+ crashed(members, target, error, status) {
149
+ const last = this.finished.at(-1);
150
+ if (!this.inFlight) {
151
+ if (last === undefined)
152
+ return undefined;
153
+ if (!retryFollows(last.record, this.finished.length, members)) {
154
+ const closed = {
155
+ ...last,
156
+ record: { ...last.record, secondaryErrors: [...last.record.secondaryErrors, error], cleanup: 'forced' },
157
+ };
158
+ return this.settled([...this.finished.slice(0, -1), closed], members, target);
159
+ }
160
+ const record = this.attemptInFlight('failed', error);
161
+ endBeforeMembers(record, members, error, NEVER_INTERRUPTED);
162
+ return this.settled([...this.finished, { record, reachedMembers: false }], members, target);
163
+ }
164
+ const next = this.currentMembers.length;
165
+ const wasRunning = this.running !== undefined && this.running === members[next]?.test.id;
166
+ const record = this.attemptInFlight(wasRunning ? status : 'failed', error);
167
+ const skip = wasRunning
168
+ ? predecessorFailed(next)
169
+ : { cause: 'infrastructure-unavailable', reason: 'worker process exited during this group attempt' };
170
+ for (const [memberIndex, member] of members.entries()) {
171
+ if (memberIndex < next)
172
+ continue;
173
+ record.members.push(wasRunning && memberIndex === next
174
+ ? memberFailedWith(record, memberIndex, member.test.id, error, status)
175
+ : skippedMember(record.id, memberIndex, member.test.id, skip));
176
+ }
177
+ return this.settled([...this.finished, { record, reachedMembers: true }], members, target);
178
+ }
179
+ /**
180
+ * The group after the run's interrupt stopped its worker, or undefined
181
+ * when the runner heard nothing of it running. The finished attempts
182
+ * stand, and an attempt in flight is interrupted: the members it finished
183
+ * keep their records and the rest skip with `skip`. The verdict is the one
184
+ * those attempts reach, as for a retry the interrupt cut short.
185
+ */
186
+ interrupted(members, target, skip) {
187
+ if (!this.inFlight)
188
+ return this.finished.length === 0 ? undefined : this.settled(this.finished, members, target);
189
+ const record = this.attemptInFlight('interrupted', undefined);
190
+ for (const [memberIndex, member] of members.entries()) {
191
+ if (memberIndex >= this.currentMembers.length)
192
+ record.members.push(skippedMember(record.id, memberIndex, member.test.id, skip));
193
+ }
194
+ return this.settled([...this.finished, { record, reachedMembers: true }], members, target);
195
+ }
196
+ /** Whether a member of the attempt in flight started or finished. */
197
+ get inFlight() {
198
+ return this.currentMembers.length > 0 || this.running !== undefined;
199
+ }
200
+ /** The record of the attempt in flight as it stood, with the members it finished. */
201
+ attemptInFlight(status, error) {
202
+ return {
203
+ id: this.current?.id ?? uuidv7(),
204
+ index: this.finished.length,
205
+ status,
206
+ startedAt: this.current?.startedAt ?? timestamp(),
207
+ durationMs: 0,
208
+ members: [...this.currentMembers],
209
+ artifacts: [],
210
+ ...(error === undefined ? {} : { error }),
211
+ secondaryErrors: [],
212
+ cleanup: 'forced',
213
+ };
214
+ }
215
+ settled(runs, members, target) {
216
+ const group = newSerialGroup(members, target);
217
+ group.status = retryVerdict(runs.map((run) => run.record));
218
+ group.attempts = runs.map((run) => run.record);
219
+ return { group, results: memberResults(group, members, runs) };
81
220
  }
82
- return group;
83
221
  }
84
- async function runSerialAttempt(host, members, file, attemptIndex) {
222
+ /**
223
+ * Whether the group retries after `record`, its attempt number `attempts`:
224
+ * the retry policy allows it, and a hook failure never retries.
225
+ */
226
+ function retryFollows(record, attempts, members) {
227
+ return attempts < members[0].options.retries + 1 && isRetryEligible(record) && record.error?.code !== 'HOOK_FAILED';
228
+ }
229
+ /** An abort signal that never fires. */
230
+ const NEVER_INTERRUPTED = new AbortController().signal;
231
+ async function runSerialAttempt(host, groupId, members, file, attemptIndex) {
85
232
  const attemptId = uuidv7();
86
233
  const startedAt = timestamp();
87
234
  const startedMs = Date.now();
88
235
  const memberRecords = [];
89
236
  const first = members[0];
90
237
  const recordings = pairRecordings(first, attemptIndex);
91
- const artifactSegments = [
238
+ const artifactSegments = attemptSegments(host.rerunDir, [
92
239
  host.target.name,
93
240
  sanitizePathSegment(first.test.serialId ?? first.test.id),
94
241
  first.agent,
95
242
  ...repeatSegment(first.repeat),
96
243
  `attempt-${attemptIndex}`,
97
- ];
244
+ ]);
98
245
  const artifacts = createAttemptArtifacts({
99
246
  artifactsRoot: host.artifactsRoot,
100
247
  segments: artifactSegments,
@@ -115,15 +262,20 @@ async function runSerialAttempt(host, members, file, attemptIndex) {
115
262
  secondaryErrors: [],
116
263
  cleanup: 'complete',
117
264
  };
265
+ const start = { id: attemptId, index: attemptIndex, startedAt };
266
+ /** Records one member and streams it, so a worker that dies later in the attempt does not take it along. */
267
+ const finishMember = (member) => {
268
+ memberRecords.push(member);
269
+ host.serialMemberFinished(groupId, start, member);
270
+ };
118
271
  let realm;
119
272
  try {
120
273
  realm = await host.realms.create(file);
121
274
  }
122
275
  catch (cause) {
123
- record.status = 'failed';
124
- record.error = serializeError(classifyError(cause), { phase: 'collection' });
276
+ endBeforeMembers(record, members, serializeError(classifyError(cause), { phase: 'collection' }), host.interruptSignal);
125
277
  record.durationMs = Date.now() - startedMs;
126
- return record;
278
+ return { record, reachedMembers: false };
127
279
  }
128
280
  let shared;
129
281
  try {
@@ -137,18 +289,10 @@ async function runSerialAttempt(host, members, file, attemptIndex) {
137
289
  };
138
290
  }
139
291
  catch (cause) {
140
- const error = classifyError(cause);
141
- record.status = 'failed';
142
- record.error = serializeError(error, { phase: 'launch' });
143
- for (let memberIndex = 0; memberIndex < members.length; memberIndex += 1) {
144
- memberRecords.push(skippedMember(attemptId, memberIndex, members[memberIndex].test.id, {
145
- cause: 'infrastructure-unavailable',
146
- reason: error.message,
147
- }));
148
- }
292
+ endBeforeMembers(record, members, serializeError(classifyError(cause), { phase: 'launch' }), host.interruptSignal);
149
293
  record.durationMs = Date.now() - startedMs;
150
294
  await host.realms.leave(realm);
151
- return record;
295
+ return { record, reachedMembers: false };
152
296
  }
153
297
  let skipRemaining;
154
298
  let hookFailure;
@@ -156,18 +300,15 @@ async function runSerialAttempt(host, members, file, attemptIndex) {
156
300
  const member = members[memberIndex];
157
301
  const memberId = `${attemptId}:member:${memberIndex}`;
158
302
  if (skipRemaining === undefined && host.interruptSignal.aborted) {
159
- skipRemaining = interruptedSkip(host.interruptSignal, {
160
- cause: 'infrastructure-unavailable',
161
- reason: 'run interrupted during this group attempt',
162
- });
303
+ skipRemaining = interruptedMemberSkip(host.interruptSignal);
163
304
  }
164
305
  if (skipRemaining !== undefined) {
165
- memberRecords.push(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
306
+ finishMember(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
166
307
  continue;
167
308
  }
168
309
  const registered = findRegistered(realm, member.test);
169
310
  if (registered === undefined) {
170
- memberRecords.push({
311
+ finishMember({
171
312
  id: memberId,
172
313
  index: memberIndex,
173
314
  testId: member.test.id,
@@ -187,7 +328,7 @@ async function runSerialAttempt(host, members, file, attemptIndex) {
187
328
  hookFailure = await host.realms.enterScopes(realm, registered);
188
329
  if (hookFailure !== undefined) {
189
330
  skipRemaining = { cause: 'hook-failed', reason: hookFailure.message };
190
- memberRecords.push(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
331
+ finishMember(skippedMember(attemptId, memberIndex, member.test.id, skipRemaining));
191
332
  continue;
192
333
  }
193
334
  host.pairStarted(member);
@@ -196,7 +337,7 @@ async function runSerialAttempt(host, members, file, attemptIndex) {
196
337
  shared,
197
338
  });
198
339
  shared.priorSteps.push(...memberAttempt.steps);
199
- memberRecords.push({
340
+ finishMember({
200
341
  id: memberId,
201
342
  index: memberIndex,
202
343
  testId: member.test.id,
@@ -247,11 +388,54 @@ async function runSerialAttempt(host, members, file, attemptIndex) {
247
388
  await host.closeSession(shared.session, { attemptId, recordings }, record, artifacts.sink, record.secondaryErrors);
248
389
  await artifacts.settle();
249
390
  record.durationMs = Date.now() - startedMs;
250
- return record;
391
+ return { record, reachedMembers: true };
392
+ }
393
+ /**
394
+ * Ends a group attempt that stopped before any member ran (its file would not
395
+ * load, its session would not launch). The first member fails with the
396
+ * attempt's error, as an ordinary test whose attempt failed the same way
397
+ * would, and the rest skip behind it. When the run's interrupt is what
398
+ * stopped it, the attempt is interrupted and every member skips.
399
+ */
400
+ function endBeforeMembers(record, members, error, interruptSignal) {
401
+ record.error = error;
402
+ if (interruptSignal.aborted) {
403
+ record.status = 'interrupted';
404
+ const skip = interruptedMemberSkip(interruptSignal);
405
+ members.forEach((member, memberIndex) => record.members.push(skippedMember(record.id, memberIndex, member.test.id, skip)));
406
+ return;
407
+ }
408
+ record.status = 'failed';
409
+ members.forEach((member, memberIndex) => {
410
+ record.members.push(memberIndex === 0
411
+ ? memberFailedWith(record, 0, member.test.id, error, 'failed')
412
+ : skippedMember(record.id, memberIndex, member.test.id, predecessorFailed(0)));
413
+ });
414
+ }
415
+ /** A member that failed with an error of its group attempt's, having run no step of its own. */
416
+ function memberFailedWith(record, index, testId, error, status) {
417
+ return {
418
+ id: `${record.id}:member:${index}`,
419
+ index,
420
+ testId,
421
+ status,
422
+ startedAt: record.startedAt,
423
+ durationMs: 0,
424
+ steps: [],
425
+ error,
426
+ secondaryErrors: [],
427
+ };
251
428
  }
252
429
  function isFailedMember(member) {
253
430
  return isFailedStatus(member.status);
254
431
  }
432
+ /** Why a member the run's interrupt kept from running in this group attempt skipped. */
433
+ function interruptedMemberSkip(interruptSignal) {
434
+ return interruptedSkip(interruptSignal, {
435
+ cause: 'infrastructure-unavailable',
436
+ reason: 'run interrupted during this group attempt',
437
+ });
438
+ }
255
439
  function predecessorFailed(memberIndex) {
256
440
  return {
257
441
  cause: 'serial-predecessor-failed',
@@ -49,7 +49,7 @@ export async function openStandaloneAttempt(options) {
49
49
  const finishEngines = (onFailure) => engines.finish({ runId, env: options.env, timeoutMs: config.cleanupTimeout, notice, onFailure });
50
50
  try {
51
51
  // One worker on one target: one slot to provision.
52
- prepared = await engines.prepare(target, 1, { runId, projectRoot: config.projectRoot, env: options.env, signal, notice });
52
+ prepared = await engines.prepare(target, 1, { runId, projectRoot: config.projectRoot, env: options.env, signal, headed: options.headed, notice });
53
53
  const hooks = { ci: config.ci, notice: (message) => notice('app', message) };
54
54
  processes = await startDeclaredProcesses([target], config.projectRoot, () => hooks, signal, debug, options.processes);
55
55
  if (signal.aborted) {
@@ -271,12 +271,6 @@ export interface StepRecorderOptions {
271
271
  /** Replaces secret values in a step's label, and in its error's message and details, before the record keeps them. */
272
272
  readonly redact?: (text: string) => string;
273
273
  }
274
- /**
275
- * Whether an unhandled rejection is an abandoned step's: already recorded as
276
- * `STEP_NOT_AWAITED`, reaching the process only because the promise the
277
- * test did not await rejected with it. Such a rejection is not a fault.
278
- */
279
- export declare function isAbandonedStepRejection(cause: unknown): boolean;
280
274
  export declare class StepRecorder {
281
275
  private readonly attemptId;
282
276
  private readonly attempt;
package/dist/run/steps.js CHANGED
@@ -1,6 +1,7 @@
1
1
  /** Attempt-scoped step timeline. */
2
2
  import { AsyncLocalStorage } from 'node:async_hooks';
3
3
  import { isAgentError } from '../agent/error.js';
4
+ import { markAbandonedRejection, relocateStack } from '../internal/abandoned.js';
4
5
  import { withAiTraceStep } from '../internal/ai-trace.js';
5
6
  import { classifyError, serializeError, TestError, withHint } from '../internal/errors.js';
6
7
  import { timestamp } from '../internal/ids.js';
@@ -9,16 +10,6 @@ import { sourceLocation } from '../internal/source.js';
9
10
  export const STEP_KINDS = ['agent', 'locator', 'assertion', 'screen', 'app', 'session', 'resource'];
10
11
  /** Frames kept when a step captures where it was called from; the user's line is a few frames up. */
11
12
  const STEP_STACK_FRAMES = 20;
12
- /** What abandoned steps rejected with. The promise the test holds rejects with the same value, and nobody awaits it. */
13
- const ABANDONED_REJECTIONS = new WeakSet();
14
- /**
15
- * Whether an unhandled rejection is an abandoned step's: already recorded as
16
- * `STEP_NOT_AWAITED`, reaching the process only because the promise the
17
- * test did not await rejected with it. Such a rejection is not a fault.
18
- */
19
- export function isAbandonedStepRejection(cause) {
20
- return typeof cause === 'object' && cause !== null && ABANDONED_REJECTIONS.has(cause);
21
- }
22
13
  /**
23
14
  * The stack at a step's start, or nothing without a project root to read it
24
15
  * against. Every step pays the capture, since the failing one is not known
@@ -38,13 +29,6 @@ function stepStack(projectRoot) {
38
29
  Error.stackTraceLimit = limit;
39
30
  }
40
31
  }
41
- /** Points `error` at the frames of `stack`, keeping its own name and message as the first line. */
42
- function relocate(error, stack) {
43
- if (stack === undefined)
44
- return;
45
- const frames = stack.split('\n').slice(1).join('\n');
46
- error.stack = `${error.name}: ${error.message}\n${frames}`;
47
- }
48
32
  /** Characters of a step label an error message quotes before clipping it. */
49
33
  const MAX_QUOTED_LABEL_CHARS = 80;
50
34
  function quoted(label) {
@@ -137,8 +121,7 @@ export class StepRecorder {
137
121
  }
138
122
  catch (cause) {
139
123
  if (this.abandoned.has(record.id)) {
140
- if (typeof cause === 'object' && cause !== null)
141
- ABANDONED_REJECTIONS.add(cause);
124
+ markAbandonedRejection(cause);
142
125
  throw cause;
143
126
  }
144
127
  record.durationMs = Date.now() - startedMs;
@@ -177,7 +160,7 @@ export class StepRecorder {
177
160
  for (const step of running) {
178
161
  const more = first === undefined && running.length > 1 ? ` and ${running.length - 1} more` : '';
179
162
  const error = new TestError('STEP_NOT_AWAITED', withHint(`the test body returned before ${step.api} ${quoted(step.label)}${more} finished`, 'put `await` in front of every step call'));
180
- relocate(error, this.stacks.get(step.id));
163
+ relocateStack(error, this.stacks.get(step.id));
181
164
  first ??= error;
182
165
  this.abandoned.add(step.id);
183
166
  step.status = 'failed';
@@ -12,12 +12,11 @@ import type { SecretLedger } from '../internal/redact.ts';
12
12
  * returned from `stopTrace`, relative to `dir`; one that resolves outside
13
13
  * `dir` is refused before anything is touched. An archive that cannot be
14
14
  * rewritten is deleted together with the rest of the trace, and the failure
15
- * is thrown: the runner keeps no trace it cannot vouch for. A trace comes
16
- * here from a session a secret was filled on, whose pixels are withheld, so
17
- * its screencast frames are dropped as a screenshot would be denied; or from
18
- * one whose engine holds a secret in its options and where nothing was
19
- * filled, which the caller passes `keepFrames` for: an engine-held secret is
20
- * protected as text only.
15
+ * is thrown: the runner keeps no trace it cannot vouch for. After a secret
16
+ * fill the session's pixels are withheld, so its screencast frames are
17
+ * dropped as a screenshot would be denied; with nothing filled the caller
18
+ * passes `keepFrames`: a value that reached the app otherwise (an engine
19
+ * option, a URL the test spelled it into) is protected as text only.
21
20
  */
22
21
  export declare function redactTraceArchives(dir: string, paths: readonly string[], ledger: SecretLedger, options?: {
23
22
  readonly keepFrames?: boolean;
@@ -16,12 +16,11 @@ import { inflateEntry, readZip, writeZip, zipEntry } from '../internal/zip.js';
16
16
  * returned from `stopTrace`, relative to `dir`; one that resolves outside
17
17
  * `dir` is refused before anything is touched. An archive that cannot be
18
18
  * rewritten is deleted together with the rest of the trace, and the failure
19
- * is thrown: the runner keeps no trace it cannot vouch for. A trace comes
20
- * here from a session a secret was filled on, whose pixels are withheld, so
21
- * its screencast frames are dropped as a screenshot would be denied; or from
22
- * one whose engine holds a secret in its options and where nothing was
23
- * filled, which the caller passes `keepFrames` for: an engine-held secret is
24
- * protected as text only.
19
+ * is thrown: the runner keeps no trace it cannot vouch for. After a secret
20
+ * fill the session's pixels are withheld, so its screencast frames are
21
+ * dropped as a screenshot would be denied; with nothing filled the caller
22
+ * passes `keepFrames`: a value that reached the app otherwise (an engine
23
+ * option, a URL the test spelled it into) is protected as text only.
25
24
  */
26
25
  export async function redactTraceArchives(dir, paths, ledger, options = {}) {
27
26
  const root = path.resolve(dir);
@@ -14,21 +14,31 @@ import { AiTraceRecorder, registerAiTraceRecorder } from '../../internal/ai-trac
14
14
  import { DebugTrace } from '../../internal/debug.js';
15
15
  import { classifyError, ConfigurationError, serializeError } from '../../internal/errors.js';
16
16
  import { StreamRedactor } from '../../internal/redact.js';
17
+ import { ConfigLoadHold } from '../process-output.js';
17
18
  import { processSecrets, registerStaticSecrets, staticSecretLedger } from '../secrecy.js';
18
19
  import { SessionStore } from '../sessions.js';
19
20
  import { TargetWorker } from './session.js';
20
- import { isAbandonedStepRejection } from '../steps.js';
21
+ import { isAbandonedRejection } from '../../internal/abandoned.js';
21
22
  /**
22
23
  * Outbound messages in flight. `process.send` is asynchronous and a
23
24
  * `process.exit` right behind it can drop the message, so exiting waits for
24
25
  * the queue to flush: the runner must see the worker's last word.
25
26
  */
26
27
  let outbox = Promise.resolve();
28
+ /** Holds what the process prints while `bootstrap` loads the config, until its secrets are known. */
29
+ const configLoad = new ConfigLoadHold();
27
30
  /** How long an exit waits for the outbox: a channel whose runner is gone may never acknowledge. */
28
31
  const FLUSH_GRACE_MS = 2_000;
29
- /** Queues one message; resolves once the channel has taken it (or is gone). */
32
+ /**
33
+ * Hands one message to the channel at once; resolves once the channel has
34
+ * taken it and every message before it (or is gone). The channel keeps the
35
+ * order. Waiting for the previous message's ack would take an event-loop
36
+ * turn: a `pair-start` sent right before a body that exits the process, or an
37
+ * `attempt-deadline` sent right before one that blocks the loop, would never
38
+ * leave.
39
+ */
30
40
  function send(message) {
31
- outbox = outbox.then(() => new Promise((resolve) => {
41
+ const taken = new Promise((resolve) => {
32
42
  try {
33
43
  if (process.send === undefined || !process.connected)
34
44
  resolve();
@@ -39,7 +49,8 @@ function send(message) {
39
49
  // channel already closed; nothing left to deliver
40
50
  resolve();
41
51
  }
42
- }));
52
+ });
53
+ outbox = outbox.then(() => taken);
43
54
  return outbox;
44
55
  }
45
56
  /**
@@ -79,6 +90,11 @@ function captureOutput(pairInFlight) {
79
90
  flushes.push(() => void output(redactor.flush(), heldFor));
80
91
  const write = (chunk, encoding, callback) => {
81
92
  const done = typeof encoding === 'function' ? encoding : callback;
93
+ if (configLoad.hold(name, chunk)) {
94
+ if (done !== undefined)
95
+ process.nextTick(done);
96
+ return true;
97
+ }
82
98
  const pair = pairInFlight();
83
99
  pending += 1;
84
100
  void output(redactor.push(chunk), pair).then(() => {
@@ -117,17 +133,22 @@ async function bootstrap(message, debug, aiTrace) {
117
133
  // process-wide, so calls from either module instance land in one trace.
118
134
  if (aiTrace !== undefined)
119
135
  await registerAiTraceRecorder(aiTrace, loadAiSdk);
120
- const raw = await loadConfigModule(message.configPath);
121
- const config = assignPorts(resolveConfig(raw, { projectRoot: message.projectRoot, configPath: message.configPath, env: process.env, cli: message.cli }), message.ports);
122
- if (config.configDigest !== message.configDigest) {
123
- throw new ConfigurationError('CONFIG_NOT_DETERMINISTIC', 'worker resolved a different config digest than the runner; config must be deterministic');
124
- }
125
- const target = config.targets.find((candidate) => candidate.name === message.targetName);
126
- if (target === undefined) {
127
- throw new ConfigurationError('UNKNOWN_TARGET', `unknown target "${message.targetName}"`);
128
- }
129
- setSecretRegistry(config);
130
- registerStaticSecrets(config.allSecrets);
136
+ // What the config's top-level code prints is held until its secrets are
137
+ // registered, so it is redacted like a test's output.
138
+ const { config, target } = await configLoad.during(async () => {
139
+ const raw = await loadConfigModule(message.configPath);
140
+ const resolved = assignPorts(resolveConfig(raw, { projectRoot: message.projectRoot, configPath: message.configPath, env: process.env, cli: message.cli }), message.ports);
141
+ if (resolved.configDigest !== message.configDigest) {
142
+ throw new ConfigurationError('CONFIG_NOT_DETERMINISTIC', 'worker resolved a different config digest than the runner; config must be deterministic');
143
+ }
144
+ const named = resolved.targets.find((candidate) => candidate.name === message.targetName);
145
+ if (named === undefined) {
146
+ throw new ConfigurationError('UNKNOWN_TARGET', `unknown target "${message.targetName}"`);
147
+ }
148
+ setSecretRegistry(resolved);
149
+ registerStaticSecrets(resolved.allSecrets);
150
+ return { config: resolved, target: named };
151
+ }, (stream, chunk) => process[stream].write(chunk), (line) => process.stderr.write(`e2e: [warning] ${line}\n`));
131
152
  let collectCounter = 0;
132
153
  const resolvePairs = async (unit) => {
133
154
  collectCounter += 1;
@@ -152,6 +173,7 @@ async function bootstrap(message, debug, aiTrace) {
152
173
  sessionStore: SessionStore.forWorker(message.runId, message.sessionsRoot, Buffer.from(message.sessionKeyBase64, 'base64')),
153
174
  runId: message.runId,
154
175
  artifactsRoot: message.artifactsRoot,
176
+ rerunDir: message.rerunDir,
155
177
  headed: message.headed,
156
178
  workerSlot: message.workerSlot,
157
179
  env: process.env,
@@ -169,13 +191,13 @@ function main() {
169
191
  process.on('SIGINT', () => undefined);
170
192
  process.on('SIGTERM', () => undefined);
171
193
  process.on('uncaughtException', (cause) => fatal(cause));
172
- // A step the body did not await rejects on a promise nobody holds once it
173
- // is cancelled; the attempt has already recorded it as STEP_NOT_AWAITED.
194
+ // A step or poll the test did not await rejects on a promise nobody holds
195
+ // once it is cancelled; the attempt has already recorded it as STEP_NOT_AWAITED.
174
196
  // Any other rejection nobody caught is charged to the attempt in flight,
175
197
  // or recorded against the last test that finished; only before any test
176
198
  // has finished is it the worker's.
177
199
  process.on('unhandledRejection', (cause) => {
178
- if (isAbandonedStepRejection(cause))
200
+ if (isAbandonedRejection(cause))
179
201
  return;
180
202
  if (worker?.strayRejection(cause) !== true)
181
203
  fatal(cause);