@itmetsam/samurai-framework 0.21.4 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/dist/api.d.ts +8 -1
  2. package/dist/api.js +1 -0
  3. package/dist/browser/browser-context.d.ts +18 -6
  4. package/dist/browser/browser-context.js +25 -7
  5. package/dist/browser/browser.d.ts +11 -2
  6. package/dist/browser/browser.js +31 -8
  7. package/dist/browser/cdp.d.ts +11 -0
  8. package/dist/browser/cdp.js +28 -0
  9. package/dist/browser/chrome-launch.d.ts +4 -0
  10. package/dist/browser/chrome-launch.js +11 -3
  11. package/dist/browser/context-cookie-store.d.ts +17 -0
  12. package/dist/browser/context-cookie-store.js +90 -0
  13. package/dist/browser/index.d.ts +8 -1
  14. package/dist/browser/index.js +5 -1
  15. package/dist/browser/keyboard.d.ts +35 -0
  16. package/dist/browser/keyboard.js +83 -0
  17. package/dist/browser/launch-firefox.js +1 -1
  18. package/dist/browser/launch-flags.d.ts +4 -1
  19. package/dist/browser/launch-flags.js +18 -5
  20. package/dist/browser/mouse.d.ts +42 -0
  21. package/dist/browser/mouse.js +66 -0
  22. package/dist/browser/page-navigation.d.ts +0 -2
  23. package/dist/browser/page-navigation.js +1 -6
  24. package/dist/browser/page-settings.d.ts +1 -1
  25. package/dist/browser/page.d.ts +20 -6
  26. package/dist/browser/page.js +24 -8
  27. package/dist/browser/session-capabilities.d.ts +10 -0
  28. package/dist/browser/session-capabilities.js +11 -0
  29. package/dist/browser/session.d.ts +2 -1
  30. package/dist/browser/session.js +3 -3
  31. package/dist/browser/storage-state.d.ts +4 -0
  32. package/dist/browser/storage-state.js +17 -0
  33. package/dist/browser/use-applier.d.ts +38 -0
  34. package/dist/browser/use-applier.js +121 -0
  35. package/dist/browser/use-commands.d.ts +36 -0
  36. package/dist/browser/use-commands.js +120 -0
  37. package/dist/browser/viewport.d.ts +3 -2
  38. package/dist/browser/viewport.js +15 -2
  39. package/dist/cli/args.d.ts +8 -4
  40. package/dist/cli/args.js +8 -5
  41. package/dist/cli/console-reporter.js +5 -1
  42. package/dist/cli/main.js +1 -0
  43. package/dist/cli/usage.d.ts +1 -1
  44. package/dist/cli/usage.js +4 -1
  45. package/dist/config/base-url.d.ts +4 -0
  46. package/dist/config/base-url.js +6 -0
  47. package/dist/config/config.js +8 -0
  48. package/dist/config/fully-parallel.d.ts +3 -0
  49. package/dist/config/fully-parallel.js +6 -0
  50. package/dist/config/group-use.d.ts +8 -0
  51. package/dist/config/group-use.js +41 -0
  52. package/dist/config/resolved-config.d.ts +2 -0
  53. package/dist/config/resolved-config.js +3 -0
  54. package/dist/config/storage-state-file.d.ts +3 -0
  55. package/dist/config/storage-state-file.js +20 -0
  56. package/dist/config/use-options.d.ts +18 -0
  57. package/dist/config/use-options.js +184 -0
  58. package/dist/config/web-server-options.d.ts +7 -0
  59. package/dist/config/web-server-options.js +111 -0
  60. package/dist/locator/check-actions.js +1 -1
  61. package/dist/locator/drag-actions.d.ts +17 -0
  62. package/dist/locator/drag-actions.js +80 -0
  63. package/dist/locator/element-actions.d.ts +10 -4
  64. package/dist/locator/element-actions.js +23 -16
  65. package/dist/locator/element-state.d.ts +5 -0
  66. package/dist/locator/element-state.js +2 -1
  67. package/dist/locator/event-actions.d.ts +10 -0
  68. package/dist/locator/event-actions.js +81 -0
  69. package/dist/locator/input-actions.d.ts +2 -11
  70. package/dist/locator/input-actions.js +3 -56
  71. package/dist/locator/key-actions.d.ts +11 -0
  72. package/dist/locator/key-actions.js +52 -0
  73. package/dist/locator/key-map.d.ts +16 -0
  74. package/dist/locator/key-map.js +76 -0
  75. package/dist/locator/locator.d.ts +35 -5
  76. package/dist/locator/locator.js +70 -16
  77. package/dist/locator/page-scripts.d.ts +2 -0
  78. package/dist/locator/page-scripts.js +7 -0
  79. package/dist/locator/pointer-actions.d.ts +44 -0
  80. package/dist/locator/pointer-actions.js +117 -0
  81. package/dist/locator/pointer-element-actions.d.ts +21 -0
  82. package/dist/locator/pointer-element-actions.js +42 -0
  83. package/dist/logger/warn-once.d.ts +4 -0
  84. package/dist/logger/warn-once.js +13 -0
  85. package/dist/request/api-request-context.d.ts +65 -0
  86. package/dist/request/api-request-context.js +198 -0
  87. package/dist/request/api-response.d.ts +34 -0
  88. package/dist/request/api-response.js +57 -0
  89. package/dist/request/cookie-jar.d.ts +48 -0
  90. package/dist/request/cookie-jar.js +164 -0
  91. package/dist/request/http-transport.d.ts +31 -0
  92. package/dist/request/http-transport.js +113 -0
  93. package/dist/request/redirect.d.ts +10 -0
  94. package/dist/request/redirect.js +21 -0
  95. package/dist/request/request-body.d.ts +27 -0
  96. package/dist/request/request-body.js +71 -0
  97. package/dist/request/request.d.ts +16 -0
  98. package/dist/request/request.js +20 -0
  99. package/dist/runner/all-hooks.d.ts +7 -1
  100. package/dist/runner/all-hooks.js +17 -1
  101. package/dist/runner/attachments.d.ts +16 -0
  102. package/dist/runner/attachments.js +60 -0
  103. package/dist/runner/fixture-args.d.ts +15 -0
  104. package/dist/runner/fixture-args.js +95 -0
  105. package/dist/runner/fixture-defs.d.ts +22 -0
  106. package/dist/runner/fixture-defs.js +104 -0
  107. package/dist/runner/fixture-scope.d.ts +33 -0
  108. package/dist/runner/fixture-scope.js +156 -0
  109. package/dist/runner/fixtures.d.ts +37 -0
  110. package/dist/runner/fixtures.js +40 -0
  111. package/dist/runner/global-hooks.d.ts +13 -0
  112. package/dist/runner/global-hooks.js +66 -0
  113. package/dist/runner/hooks.d.ts +6 -1
  114. package/dist/runner/hooks.js +15 -3
  115. package/dist/runner/options.d.ts +13 -2
  116. package/dist/runner/output-tail.d.ts +14 -0
  117. package/dist/runner/output-tail.js +30 -0
  118. package/dist/runner/pool-schedule.d.ts +22 -0
  119. package/dist/runner/pool-schedule.js +55 -0
  120. package/dist/runner/pool-units.d.ts +34 -0
  121. package/dist/runner/pool-units.js +85 -0
  122. package/dist/runner/pooled-run.d.ts +7 -5
  123. package/dist/runner/pooled-run.js +19 -9
  124. package/dist/runner/registry.d.ts +3 -39
  125. package/dist/runner/registry.js +99 -21
  126. package/dist/runner/reporter.d.ts +17 -1
  127. package/dist/runner/reporter.js +15 -2
  128. package/dist/runner/request-fixture.d.ts +14 -0
  129. package/dist/runner/request-fixture.js +25 -0
  130. package/dist/runner/run-hooks.d.ts +16 -0
  131. package/dist/runner/run-hooks.js +52 -0
  132. package/dist/runner/run-test.d.ts +1 -3
  133. package/dist/runner/run-test.js +98 -35
  134. package/dist/runner/run.d.ts +1 -0
  135. package/dist/runner/run.js +1 -0
  136. package/dist/runner/select.d.ts +15 -4
  137. package/dist/runner/select.js +38 -12
  138. package/dist/runner/serial-run.d.ts +23 -0
  139. package/dist/runner/serial-run.js +88 -0
  140. package/dist/runner/serial.d.ts +34 -0
  141. package/dist/runner/serial.js +63 -0
  142. package/dist/runner/shard.d.ts +22 -0
  143. package/dist/runner/shard.js +73 -0
  144. package/dist/runner/steps.d.ts +13 -0
  145. package/dist/runner/steps.js +67 -0
  146. package/dist/runner/suite.d.ts +8 -3
  147. package/dist/runner/suite.js +12 -3
  148. package/dist/runner/test-annotations.d.ts +16 -0
  149. package/dist/runner/test-annotations.js +47 -0
  150. package/dist/runner/test-function.d.ts +115 -0
  151. package/dist/runner/test-function.js +1 -0
  152. package/dist/runner/test-info.d.ts +104 -0
  153. package/dist/runner/test-info.js +170 -0
  154. package/dist/runner/test-runner.d.ts +1 -0
  155. package/dist/runner/test-runner.js +64 -19
  156. package/dist/runner/test-timeout.d.ts +22 -0
  157. package/dist/runner/test-timeout.js +45 -0
  158. package/dist/runner/use-options.d.ts +7 -0
  159. package/dist/runner/use-options.js +25 -0
  160. package/dist/runner/wait-for-url.d.ts +20 -0
  161. package/dist/runner/wait-for-url.js +65 -0
  162. package/dist/runner/web-server.d.ts +16 -0
  163. package/dist/runner/web-server.js +165 -0
  164. package/dist/runner/worker-pool.d.ts +13 -5
  165. package/dist/runner/worker-pool.js +85 -51
  166. package/dist/runner/worker-protocol.d.ts +8 -0
  167. package/dist/runner/worker-session.js +5 -2
  168. package/dist/runner/worker.d.ts +2 -0
  169. package/dist/runner/worker.js +7 -1
  170. package/dist/script/element-handle.d.ts +1 -1
  171. package/dist/script/element-handle.js +1 -1
  172. package/dist/types/config.d.ts +86 -2
  173. package/dist/types/extend.d.ts +30 -0
  174. package/dist/types/test.d.ts +138 -0
  175. package/dist/types/use.d.ts +85 -0
  176. package/package.json +1 -1
  177. package/dist/browser/emulation.d.ts +0 -61
  178. package/dist/browser/emulation.js +0 -180
@@ -0,0 +1,52 @@
1
+ import { resolveConfig } from "../config/resolved-config.js";
2
+ import { webServersOf } from "../config/web-server-options.js";
3
+ import logger from "../logger/index.js";
4
+ import { globalHooks } from "./global-hooks.js";
5
+ import { startWebServers } from "./web-server.js";
6
+ /**
7
+ * Runs `work` between the config's run-level steps: web servers start, then `globalSetup`, then `work`, then
8
+ * `globalTeardown`, then the servers stop. The teardown runs when the work or the setup throws (not when a server
9
+ * failed to start, since nothing was set up), and the servers are stopped whatever happens. When the config has no
10
+ * `baseURL` for the run, the first server's address becomes `settings.baseURL`.
11
+ * Errors: a failed start is `webServer(): ...`, a failed setup `globalSetup(): ...`; the work's own error wins over
12
+ * a teardown error, which is logged then.
13
+ */
14
+ export async function withRunHooks(run, work) {
15
+ const { config, settings, projectDir, signal } = run;
16
+ const servers = webServersOf(config);
17
+ if (servers.length === 0 &&
18
+ config.globalSetup === undefined &&
19
+ config.globalTeardown === undefined)
20
+ return work();
21
+ const running = await startWebServers(servers, { projectDir, signal });
22
+ let outcome;
23
+ try {
24
+ if (settings.baseURL === undefined && running.baseURL !== undefined)
25
+ settings.baseURL = running.baseURL;
26
+ const hooks = globalHooks(config, {
27
+ ...resolveConfig(config, { projectDir, env: process.env }),
28
+ environment: settings.environment,
29
+ ...(settings.baseURL !== undefined && { baseURL: settings.baseURL }),
30
+ }, projectDir);
31
+ try {
32
+ await hooks.setup();
33
+ outcome = { value: await work() };
34
+ }
35
+ catch (error) {
36
+ outcome = { error };
37
+ }
38
+ const teardownError = await hooks.teardown();
39
+ if (teardownError) {
40
+ if ("error" in outcome)
41
+ logger.error("%s", teardownError.message);
42
+ else
43
+ outcome = { error: teardownError };
44
+ }
45
+ }
46
+ finally {
47
+ await running.stop();
48
+ }
49
+ if (outcome && "error" in outcome)
50
+ throw outcome.error;
51
+ return outcome.value;
52
+ }
@@ -1,5 +1,5 @@
1
1
  import type { SupportedBrowser } from "../types/browser.js";
2
- import type { AllHookFixtures, RegisteredTestCase } from "../types/test.js";
2
+ import type { RegisteredTestCase } from "../types/test.js";
3
3
  import type { RunnerOptions } from "./options.js";
4
4
  import type { AllHooks } from "./all-hooks.js";
5
5
  import type { PreparedRun } from "./prepare-run.js";
@@ -17,7 +17,5 @@ export interface WorkerResources {
17
17
  /** The `beforeAll` / `afterAll` hooks of this worker: kept across tests, closed when the worker is done */
18
18
  allHooks?: AllHooks;
19
19
  }
20
- /** What `beforeAll` and `afterAll` hooks get: no page, only what a run knows */
21
- export declare function allHookFixtures(run: PreparedRun, browserName: SupportedBrowser, workerIndex?: number): AllHookFixtures;
22
20
  /** Runs one test in one browser and reports it: launch, run the body, judge the page's logs, close the browser */
23
21
  export declare function runOneTest(test: RegisteredTestCase, browserName: SupportedBrowser, run: PreparedRun, options: RunnerOptions, srcDir: string, reporter: TestSink, resources?: WorkerResources): Promise<void>;
@@ -1,9 +1,8 @@
1
+ import path from "node:path";
1
2
  import { takePendingAssertions } from "../assert/expect.js";
2
3
  import { Browser } from "../browser/browser.js";
3
- import { Evidence } from "../browser/evidence.js";
4
+ import { cleanName, Evidence } from "../browser/evidence.js";
4
5
  import { projectDir, readConfig } from "../config/config.js";
5
- import { createSecretsFixture } from "../config/secrets.js";
6
- import { createEnvFixture } from "../config/variables.js";
7
6
  import logger from "../logger/index.js";
8
7
  import { ConnectionClosedError } from "../transport/bidi-connection.js";
9
8
  import { judgePageLogs } from "./page-logs-report.js";
@@ -13,7 +12,14 @@ import { isSkipSignal, staticSkip } from "./skip.js";
13
12
  import { runOutput } from "./output.js";
14
13
  import { testDirOf } from "./test-dir.js";
15
14
  import { toTestError } from "./test-error.js";
16
- import { createInputsFixture, createOutputsFixture, ioOfAttempt, missingOutputs, } from "./test-io.js";
15
+ import { runInTestScope, TestInfoImpl } from "./test-info.js";
16
+ import { applyDeclaredMarks, judgeExpectation } from "./test-annotations.js";
17
+ import { TestTimeout } from "./test-timeout.js";
18
+ import { ioOfAttempt, missingOutputs } from "./test-io.js";
19
+ import { buildTestFixtures } from "./fixtures.js";
20
+ import { FixtureScope } from "./fixture-scope.js";
21
+ import { RequestFixture } from "./request-fixture.js";
22
+ import { effectiveUse } from "./use-options.js";
17
23
  import { beginOutputs, endOutputs } from "../wait/output-sink.js";
18
24
  /**
19
25
  * The state of one test run. A test ends once, whichever of finishing, failing, timing out, an abort or a
@@ -46,6 +52,10 @@ class TestOutcome {
46
52
  reason: signal.reason,
47
53
  });
48
54
  }
55
+ /** The test body ended by itself: a test expected to fail is passed by a failure and failed by a pass */
56
+ conclude(error, logs) {
57
+ this.finish(judgeExpectation(this.info, error), logs);
58
+ }
49
59
  finish(error, logs) {
50
60
  if (this.ended)
51
61
  return;
@@ -53,20 +63,11 @@ class TestOutcome {
53
63
  this.reporter.onTestEnd(this.test, this.browser, error, {
54
64
  ...logs,
55
65
  screenshots: this.evidence?.screenshots,
66
+ ...this.info?.reportExtras(),
56
67
  ...this.io(),
57
68
  });
58
69
  }
59
70
  }
60
- /** What `beforeAll` and `afterAll` hooks get: no page, only what a run knows */
61
- export function allHookFixtures(run, browserName, workerIndex = 0) {
62
- const { settings, secrets } = run;
63
- return {
64
- browserName,
65
- workerIndex,
66
- env: createEnvFixture(settings.environment, settings.variables),
67
- secrets: createSecretsFixture(settings.environment, secrets),
68
- };
69
- }
70
71
  /** Runs one test in one browser and reports it: launch, run the body, judge the page's logs, close the browser */
71
72
  export async function runOneTest(test, browserName, run, options, srcDir, reporter, resources = {}) {
72
73
  // A test marked skip or fixme never runs, whoever asked for it (a flow's test node, a worker)
@@ -78,25 +79,29 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
78
79
  logger.verbose("Starting test: %s in %s", test.name, browserName);
79
80
  reporter.onTestStart(test, browserName);
80
81
  const outcome = new TestOutcome(reporter, test, browserName);
81
- const { settings, secrets } = run;
82
+ const { settings } = run;
82
83
  const timeout = settings.timeout;
83
84
  const { resultDir } = runOutput(options, projectDir());
84
85
  let browser;
85
86
  let page;
86
- let timeoutTimer;
87
+ let customFixtures;
88
+ let requestFixture;
87
89
  let onRunAborted;
88
90
  const timeoutController = new AbortController();
89
91
  let browserGone = () => { };
92
+ let testTimeout;
90
93
  /** Resolves when the time is up or the run is aborted; the test is reported failed then */
91
94
  const limit = new Promise((resolve) => {
92
- timeoutTimer = setTimeout(() => {
95
+ // The test can change its own limit (`test.setTimeout`, `test.slow`): the message names the one in force
96
+ testTimeout = new TestTimeout(timeout, () => {
93
97
  outcome.finish({
94
- message: `Test timed out after ${timeout / 1000} seconds`,
98
+ message: `Test timed out after ${(testTimeout?.ms ?? timeout) / 1000} seconds`,
95
99
  type: "timeout",
96
100
  });
97
101
  timeoutController.abort(new Error("Test timed out"));
98
102
  resolve();
99
- }, timeout);
103
+ });
104
+ testTimeout.start();
100
105
  onRunAborted = () => {
101
106
  if (timeoutController.signal.aborted)
102
107
  return;
@@ -113,10 +118,12 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
113
118
  try {
114
119
  // The `beforeAll` hooks come first, before a browser exists
115
120
  await resources.allHooks?.ensure(test, browserName);
121
+ const use = await effectiveUse(test);
116
122
  const launched = await Browser.launch(browserName, {
117
123
  port: resources.port ?? options.port ?? DEFAULT_PORT,
118
124
  ...(resources.profileDir && { profileDir: resources.profileDir }),
119
125
  headless: options.headless ?? settings.headless,
126
+ use,
120
127
  }, timeoutController.signal);
121
128
  browser = launched.browser;
122
129
  // A person closing the window, or the browser crashing: end the test now instead of waiting for a
@@ -133,25 +140,70 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
133
140
  browserGone();
134
141
  });
135
142
  page = launched.page;
136
- outcome.evidence = new Evidence(resultDir, `${browserName}/${testDirOf(test, srcDir)}${resources.attempt ? `/retry-${resources.attempt}` : ""}`);
143
+ // An empty `use` leaves the browser exactly as it was: no applier, nothing sent
144
+ const applied = Object.keys(use).length > 0
145
+ ? await launched.browser.setUse(use, projectDir())
146
+ : undefined;
147
+ const testFolder = `${browserName}${test.group === undefined ? "" : `/${cleanName(test.group)}`}/${testDirOf(test, srcDir)}${resources.attempt ? `/retry-${resources.attempt}` : ""}`;
148
+ outcome.evidence = new Evidence(resultDir, testFolder);
137
149
  launched.page.setEvidence(outcome.evidence);
138
- const fixtures = {
139
- page: launched.page,
140
- browser: launched.browser,
150
+ const info = new TestInfoImpl({
151
+ test,
141
152
  browserName,
142
153
  workerIndex: resources.workerIndex ?? 0,
143
- env: createEnvFixture(settings.environment, settings.variables),
144
- secrets: createSecretsFixture(settings.environment, secrets),
145
- inputs: createInputsFixture(test.inputs, options.inputs),
146
- outputs: createOutputsFixture(sink),
147
- };
148
- await runWithEachHooks(test.suite, fixtures, timeoutController.signal, () => test.function(fixtures));
154
+ retry: resources.attempt ?? 0,
155
+ settings: { baseURL: settings.baseURL, timeout },
156
+ ...(testTimeout && { timeout: testTimeout }),
157
+ resultDir,
158
+ outputDir: path.join(resultDir, "attachments", ...testFolder.split("/")),
159
+ });
160
+ outcome.info = info;
161
+ applyDeclaredMarks(info, test);
162
+ requestFixture = new RequestFixture(settings.baseURL, use);
163
+ const built = buildTestFixtures({
164
+ request: requestFixture,
165
+ info,
166
+ launched,
167
+ browserName,
168
+ workerIndex: resources.workerIndex ?? 0,
169
+ run,
170
+ test,
171
+ options,
172
+ sink,
173
+ device: {
174
+ isMobile: applied?.isMobile ?? false,
175
+ hasTouch: applied?.hasTouch ?? false,
176
+ },
177
+ });
178
+ // A test written with `test.extend()` gets its custom fixtures set up on demand, and torn down below
179
+ const custom = test.extension
180
+ ? new FixtureScope(test.extension, built)
181
+ : undefined;
182
+ customFixtures = custom;
183
+ const fixtures = custom?.fixtures ?? built;
184
+ let failure;
185
+ let failed = false;
186
+ try {
187
+ await runInTestScope({ info, fixtures }, () => runWithEachHooks(test.suite, fixtures, timeoutController.signal, async () => {
188
+ await custom?.prepare(test.function);
189
+ return test.function(fixtures);
190
+ }, (error) => info.bodyDone(error), custom));
191
+ }
192
+ catch (err) {
193
+ failure = err;
194
+ failed = true;
195
+ }
196
+ const cleanup = await custom?.teardown(timeout);
197
+ if (cleanup && (!failed || isSkipSignal(failure)))
198
+ throw cleanup;
199
+ if (failed)
200
+ throw failure;
149
201
  // Already reported as timed out
150
202
  if (timeoutController.signal.aborted)
151
203
  return;
152
204
  const unawaited = takePendingAssertions();
153
205
  if (unawaited.length > 0) {
154
- outcome.finish({
206
+ outcome.conclude({
155
207
  message: unawaited
156
208
  .map((label) => `${label} was not awaited`)
157
209
  .join("\n"),
@@ -162,23 +214,29 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
162
214
  // Checked only after a pass: a test that failed first reports its own failure
163
215
  const missing = missingOutputs(test.outputs, sink);
164
216
  if (missing) {
165
- outcome.finish(missing, await judgeLogs(page, true));
217
+ outcome.conclude(missing, await judgeLogs(page, true));
166
218
  return;
167
219
  }
168
220
  const verdict = await judgeLogs(page, false);
169
- outcome.finish(verdict.error, verdict);
221
+ outcome.conclude(verdict.error, verdict);
170
222
  }
171
223
  catch (err) {
172
224
  if (timeoutController.signal.aborted)
173
225
  return;
174
226
  if (isSkipSignal(err))
175
227
  return outcome.skip(err);
176
- outcome.finish(outcome.browserClosed || err instanceof ConnectionClosedError
228
+ const closed = outcome.browserClosed || err instanceof ConnectionClosedError;
229
+ const error = closed
177
230
  ? {
178
231
  message: "The browser was closed while the test was running",
179
232
  type: "error",
180
233
  }
181
- : toTestError(err), await judgeLogs(page, true));
234
+ : toTestError(err);
235
+ const logs = await judgeLogs(page, true);
236
+ if (closed)
237
+ outcome.finish(error, logs);
238
+ else
239
+ outcome.conclude(error, logs);
182
240
  }
183
241
  };
184
242
  // Open for this attempt only: whatever a capture still saves after the attempt ended is ignored
@@ -188,9 +246,14 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
188
246
  await Promise.race([limit, gone, runBody()]);
189
247
  }
190
248
  finally {
249
+ // A test cut short (timeout, abort) did not get to its own cleanup: its fixtures are torn down before the browser
250
+ const late = await customFixtures?.teardown(timeout);
251
+ if (late)
252
+ logger.debug("A fixture failed after the test ended", { late });
253
+ await requestFixture?.dispose();
191
254
  endOutputs();
192
255
  }
193
- clearTimeout(timeoutTimer);
256
+ testTimeout?.stop();
194
257
  if (onRunAborted)
195
258
  options.signal?.removeEventListener("abort", onRunAborted);
196
259
  takePendingAssertions();
@@ -48,6 +48,7 @@ export declare function listGroups(options?: RunTestsOptions): Promise<ResolvedG
48
48
  export { BROWSER_NAMES, checkBrowserList } from "../config/browsers.js";
49
49
  export { parseWorkers } from "../config/workers.js";
50
50
  export { parseRetries } from "../config/retries.js";
51
+ export { parseShard, type Shard } from "./shard.js";
51
52
  export type { RunEvent } from "./reporter.js";
52
53
  export type { Modifier, TestError, TestResult, TestSummary, } from "../types/test.js";
53
54
  export type { ResolvedGroup } from "./groups.js";
@@ -70,4 +70,5 @@ export async function listGroups(options = {}) {
70
70
  export { BROWSER_NAMES, checkBrowserList } from "../config/browsers.js";
71
71
  export { parseWorkers } from "../config/workers.js";
72
72
  export { parseRetries } from "../config/retries.js";
73
+ export { parseShard } from "./shard.js";
73
74
  export { DEFAULT_PORT } from "./ports.js";
@@ -15,10 +15,21 @@ export declare function isSelected(name: string, { grep, testNames }: Pick<Runne
15
15
  export declare function idOf(test: RegisteredTestCase, srcDir: string): string;
16
16
  /** Imports the spec files once and resolves the config's groups against what they registered */
17
17
  export declare function loadSpecs(testFiles: string[]): Promise<LoadedSpecs>;
18
- /** The tests the options select: by `group`, `testNames` and `grep` */
18
+ /** The group names the options select, once each and in the order given; none when no group is selected */
19
+ export declare function groupNamesOf(group: RunnerOptions["group"]): string[];
20
+ /** The tests the options select: by `group` (in any of them), `testNames` and `grep`; each test once */
19
21
  export declare function selectTests({ all, groups, srcDir }: LoadedSpecs, options: RunnerOptions): RegisteredTestCase[];
20
22
  /**
21
- * The browsers the run uses: the options' (`--browser`), else the selected group's, else the project's, else
22
- * Firefox
23
+ * The runs of the selected tests: without a selected group, the tests as they are; else one copy per selected group
24
+ * that holds the test (its `group` set), the first group's tests first
23
25
  */
24
- export declare function browsersOfRun(groups: ResolvedGroup[], options: RunnerOptions): Promise<SupportedBrowser[]>;
26
+ export declare function expandByGroup(selected: RegisteredTestCase[], { groups, srcDir }: LoadedSpecs, options: RunnerOptions): RegisteredTestCase[];
27
+ /**
28
+ * The browsers a run of a test uses: the options' (`--browser`), else its group's, else the project's, else Firefox
29
+ */
30
+ export declare function browserPlan(groups: ResolvedGroup[], options: RunnerOptions): Promise<{
31
+ /** Every browser of the run, in the order they run */
32
+ browsers: SupportedBrowser[];
33
+ /** The browsers one run of a test uses */
34
+ browsersOf(test: RegisteredTestCase): SupportedBrowser[];
35
+ }>;
@@ -53,25 +53,51 @@ export async function loadSpecs(testFiles) {
53
53
  const all = await registerAll(testFiles);
54
54
  return { all, ...(await resolveAll(all)) };
55
55
  }
56
- /** The tests the options select: by `group`, `testNames` and `grep` */
56
+ /** The group names the options select, once each and in the order given; none when no group is selected */
57
+ export function groupNamesOf(group) {
58
+ return [...new Set(group === undefined ? [] : [group].flat())];
59
+ }
60
+ /** The tests the options select: by `group` (in any of them), `testNames` and `grep`; each test once */
57
61
  export function selectTests({ all, groups, srcDir }, options) {
58
- const { group } = options;
62
+ const names = groupNamesOf(options.group);
59
63
  let inGroup = () => true;
60
- if (group !== undefined) {
61
- const members = new Set(findGroup(groups, group).testIds);
64
+ if (names.length > 0) {
65
+ const members = new Set(names.flatMap((name) => findGroup(groups, name).testIds));
62
66
  inGroup = (test) => members.has(idOf(test, srcDir));
63
67
  }
64
68
  return all.filter((test) => inGroup(test) && isSelected(test.name, options));
65
69
  }
66
70
  /**
67
- * The browsers the run uses: the options' (`--browser`), else the selected group's, else the project's, else
68
- * Firefox
71
+ * The runs of the selected tests: without a selected group, the tests as they are; else one copy per selected group
72
+ * that holds the test (its `group` set), the first group's tests first
73
+ */
74
+ export function expandByGroup(selected, { groups, srcDir }, options) {
75
+ const names = groupNamesOf(options.group);
76
+ if (names.length === 0)
77
+ return selected;
78
+ return names.flatMap((name) => {
79
+ const members = new Set(findGroup(groups, name).testIds);
80
+ return selected
81
+ .filter((test) => members.has(idOf(test, srcDir)))
82
+ .map((test) => ({ ...test, group: name }));
83
+ });
84
+ }
85
+ /**
86
+ * The browsers a run of a test uses: the options' (`--browser`), else its group's, else the project's, else Firefox
69
87
  */
70
- export async function browsersOfRun(groups, options) {
71
- const { browsers: flag, group } = options;
88
+ export async function browserPlan(groups, options) {
89
+ const { browsers: flag } = options;
72
90
  const config = await loadRawConfig();
73
- const ofGroup = group !== undefined && !(flag && flag.length > 0)
74
- ? findGroup(groups, group).browsers
75
- : undefined;
76
- return browsersFor({ flag, group: ofGroup, project: config.browsers });
91
+ const names = groupNamesOf(options.group);
92
+ const flagged = flag && flag.length > 0;
93
+ const ofGroup = new Map(names.map((name) => [name, findGroup(groups, name).browsers]));
94
+ const project = browsersFor({ flag, project: config.browsers });
95
+ const forGroup = (name) => flagged ? project : (ofGroup.get(name) ?? project);
96
+ const browsers = [
97
+ ...new Set(names.length > 0 ? names.flatMap(forGroup) : project),
98
+ ];
99
+ return {
100
+ browsers,
101
+ browsersOf: (test) => test.group === undefined ? project : forGroup(test.group),
102
+ };
77
103
  }
@@ -0,0 +1,23 @@
1
+ import type { SupportedBrowser } from "../types/browser.js";
2
+ import type { RegisteredTestCase } from "../types/test.js";
3
+ import type { TestSink } from "./reporter.js";
4
+ /** One attempt at one test of the block, reporting to `sink` */
5
+ export type BlockTest = (test: RegisteredTestCase, sink: TestSink, attempt: number) => Promise<void>;
6
+ export interface SerialBlockRun {
7
+ tests: RegisteredTestCase[];
8
+ browser: SupportedBrowser;
9
+ retries: number;
10
+ signal: AbortSignal | undefined;
11
+ runTest: BlockTest;
12
+ /** Called before the block starts again after a failure: closes what the block's tests share (`afterAll`) */
13
+ restart(): Promise<void>;
14
+ sink: TestSink;
15
+ }
16
+ /**
17
+ * Runs the tests of a serial block in order. After a failure the rest of the block is reported as skipped; with
18
+ * `retries`, the whole block then runs again from its first test (after `restart`), up to `retries` times.
19
+ * Without retries every test is reported as it ends. With them the ends are held until the block's last attempt is
20
+ * known, as `runWithRetries` does for one test: each test ends once, with `retry` saying which attempt that was.
21
+ * An aborted run is never run again.
22
+ */
23
+ export declare function serialBlockRun(run: SerialBlockRun): Promise<void>;
@@ -0,0 +1,88 @@
1
+ import { SERIAL_SKIP_REASON } from "./serial.js";
2
+ import { staticSkip } from "./skip.js";
3
+ /**
4
+ * Runs the tests of a serial block in order. After a failure the rest of the block is reported as skipped; with
5
+ * `retries`, the whole block then runs again from its first test (after `restart`), up to `retries` times.
6
+ * Without retries every test is reported as it ends. With them the ends are held until the block's last attempt is
7
+ * known, as `runWithRetries` does for one test: each test ends once, with `retry` saying which attempt that was.
8
+ * An aborted run is never run again.
9
+ */
10
+ export async function serialBlockRun(run) {
11
+ const { tests, browser, retries, signal, sink } = run;
12
+ const live = retries === 0;
13
+ const previous = tests.map(() => []);
14
+ for (let attempt = 0;; attempt++) {
15
+ const ended = [];
16
+ let failedAt = -1;
17
+ for (const [at, test] of tests.entries()) {
18
+ const skip = staticSkip(test);
19
+ if (skip) {
20
+ sink.onTestSkipped(test, browser, skip);
21
+ continue;
22
+ }
23
+ if (signal?.aborted) {
24
+ for (const rest of tests.slice(at)) {
25
+ if (staticSkip(rest))
26
+ continue;
27
+ sink.onTestStart(rest, browser);
28
+ sink.onTestEnd(rest, browser, {
29
+ message: "Run aborted",
30
+ type: "error",
31
+ });
32
+ }
33
+ return;
34
+ }
35
+ const attemptSink = {
36
+ onTestStart: (t, b) => sink.onTestStart(t, b, attempt > 0 ? attempt : undefined),
37
+ onTestEnd: (t, b, error, extras) => {
38
+ ended[at] = { error, extras };
39
+ if (live)
40
+ sink.onTestEnd(t, b, error, extras);
41
+ },
42
+ onTestSkipped: (t, b, why) => sink.onTestSkipped(t, b, why),
43
+ };
44
+ await run.runTest(test, attemptSink, attempt);
45
+ if (ended[at]?.error) {
46
+ failedAt = at;
47
+ break;
48
+ }
49
+ }
50
+ const error = ended[failedAt]?.error;
51
+ if (error &&
52
+ attempt < retries &&
53
+ !signal?.aborted &&
54
+ error.message !== "Run aborted") {
55
+ previous[failedAt].push(error);
56
+ await run.restart();
57
+ continue;
58
+ }
59
+ if (!live)
60
+ flush(run, ended, previous, attempt);
61
+ if (failedAt >= 0)
62
+ for (const rest of tests.slice(failedAt + 1)) {
63
+ const skip = staticSkip(rest) ?? {
64
+ annotation: "skip",
65
+ reason: SERIAL_SKIP_REASON,
66
+ };
67
+ sink.onTestSkipped(rest, browser, skip);
68
+ }
69
+ return;
70
+ }
71
+ }
72
+ /** Reports the held ends of the last attempt, each saying which attempt it was and what failed before */
73
+ function flush({ tests, browser, sink }, ended, previous, attempt) {
74
+ for (const [at, test] of tests.entries()) {
75
+ const end = ended[at];
76
+ if (!end)
77
+ continue;
78
+ const before = previous[at];
79
+ sink.onTestEnd(test, browser, end.error, {
80
+ ...end.extras,
81
+ ...(attempt > 0 && {
82
+ retry: attempt,
83
+ previousAttempts: [...before],
84
+ ...(!end.error && before.length > 0 && { flaky: true }),
85
+ }),
86
+ });
87
+ }
88
+ }
@@ -0,0 +1,34 @@
1
+ import type { RegisteredTestCase, Suite } from "../types/test.js";
2
+ /** Why a test of a serial block did not run: an earlier test of the block failed */
3
+ export declare const SERIAL_SKIP_REASON = "serial: an earlier test failed";
4
+ /** `describe.configure(options)`: sets the mode of the suite the call is written in */
5
+ export declare function configureSuite(suite: Suite, options: unknown): void;
6
+ /** A group of tests that run in order on one worker; if one fails, the rest of the block is skipped */
7
+ export interface SerialBlock {
8
+ /** The file and the `describe` titles of the block's outermost serial suite; the same in every process */
9
+ key: string;
10
+ /** The outermost suite in serial mode */
11
+ suite: Suite;
12
+ }
13
+ /** The serial block a test belongs to: the outermost suite around it in `"serial"` mode, else none */
14
+ export declare function serialBlockOf(test: RegisteredTestCase): SerialBlock | undefined;
15
+ /** The tests that must run one after the other on one worker, in the order they are listed */
16
+ export interface RunUnit {
17
+ /** The same for every test of the unit, in every process */
18
+ key: string;
19
+ /** A serial block: a failure skips the rest and a retry runs the whole block again */
20
+ serial: boolean;
21
+ }
22
+ /**
23
+ * The unit a test runs in. A serial block is one. With `fullyParallel: false` every other test is in the unit of
24
+ * its file, unless a suite around it is in `"parallel"` mode. Otherwise (`true`, or not set: the tests of a file
25
+ * go to workers one by one) there is no unit.
26
+ */
27
+ export declare function unitOf(test: RegisteredTestCase, fullyParallel: boolean | undefined): RunUnit | undefined;
28
+ /** Tests that follow each other and belong to one serial block (`block`), or one test outside any block */
29
+ export interface Chunk {
30
+ block?: SerialBlock;
31
+ tests: RegisteredTestCase[];
32
+ }
33
+ /** Splits a list of tests into serial blocks and the free tests between them, keeping the order */
34
+ export declare function chunkBySerialBlock(tests: RegisteredTestCase[]): Chunk[];
@@ -0,0 +1,63 @@
1
+ import { suiteChain } from "./suite.js";
2
+ /** Why a test of a serial block did not run: an earlier test of the block failed */
3
+ export const SERIAL_SKIP_REASON = "serial: an earlier test failed";
4
+ const MODES = ["default", "parallel", "serial"];
5
+ /** `describe.configure(options)`: sets the mode of the suite the call is written in */
6
+ export function configureSuite(suite, options) {
7
+ if (typeof options !== "object" || options === null)
8
+ throw new TypeError("describe.configure(): expected an options object like { mode: 'serial' }");
9
+ const { mode, ...rest } = options;
10
+ const unknown = Object.keys(rest);
11
+ if (unknown.length > 0)
12
+ throw new TypeError(`describe.configure(): unknown option "${unknown[0]}" (only mode is supported)`);
13
+ if (!MODES.includes(mode))
14
+ throw new TypeError(`describe.configure(): mode must be "default", "parallel" or "serial", got ${JSON.stringify(mode)}`);
15
+ if (mode === "default")
16
+ delete suite.mode;
17
+ else
18
+ suite.mode = mode;
19
+ }
20
+ const groupSuffix = (test) => test.group === undefined ? "" : `@${test.group}`;
21
+ /** The serial block a test belongs to: the outermost suite around it in `"serial"` mode, else none */
22
+ export function serialBlockOf(test) {
23
+ const suite = suiteChain(test.suite).find((s) => s.mode === "serial");
24
+ if (!suite)
25
+ return undefined;
26
+ const titles = suiteChain(suite).flatMap((s) => s.title === undefined ? [] : [s.title]);
27
+ // The same block run for two groups is two blocks
28
+ return {
29
+ key: `serial:${test.file}::${titles.join(" > ")}${groupSuffix(test)}`,
30
+ suite,
31
+ };
32
+ }
33
+ /**
34
+ * The unit a test runs in. A serial block is one. With `fullyParallel: false` every other test is in the unit of
35
+ * its file, unless a suite around it is in `"parallel"` mode. Otherwise (`true`, or not set: the tests of a file
36
+ * go to workers one by one) there is no unit.
37
+ */
38
+ export function unitOf(test, fullyParallel) {
39
+ const block = serialBlockOf(test);
40
+ if (block)
41
+ return { key: block.key, serial: true };
42
+ if (fullyParallel !== false)
43
+ return undefined;
44
+ const nearest = suiteChain(test.suite)
45
+ .reverse()
46
+ .find((s) => s.mode === "parallel");
47
+ return nearest
48
+ ? undefined
49
+ : { key: `file:${test.file}${groupSuffix(test)}`, serial: false };
50
+ }
51
+ /** Splits a list of tests into serial blocks and the free tests between them, keeping the order */
52
+ export function chunkBySerialBlock(tests) {
53
+ const chunks = [];
54
+ for (const test of tests) {
55
+ const block = serialBlockOf(test);
56
+ const last = chunks[chunks.length - 1];
57
+ if (block && last?.block?.key === block.key)
58
+ last.tests.push(test);
59
+ else
60
+ chunks.push({ ...(block && { block }), tests: [test] });
61
+ }
62
+ return chunks;
63
+ }
@@ -0,0 +1,22 @@
1
+ import type { RegisteredTestCase } from "../types/test.js";
2
+ /** One slice of a run split over several machines: slice `current` of `total`, counting from 1 */
3
+ export interface Shard {
4
+ current: number;
5
+ total: number;
6
+ }
7
+ /**
8
+ * Reads a shard: `"2/4"` (the command line's `--shard`) or `{ current: 2, total: 4 }` (the `shard` run option).
9
+ * `where` names the source in the error.
10
+ */
11
+ export declare function parseShard(value: unknown, where: string): Shard;
12
+ /**
13
+ * What a shard keeps together: a serial block is never split, and so is a file, unless `fullyParallel` is on or a
14
+ * `describe` around the test is in `"parallel"` mode: then the test goes by itself.
15
+ */
16
+ export declare function shardGroupOf(test: RegisteredTestCase, fullyParallel: boolean | undefined): string;
17
+ /**
18
+ * The tests of one shard. The groups (in the order their first test comes) are dealt out in order, so the shards are
19
+ * as equal in tests as whole groups allow: a group goes to the shard that holds its first test when all tests are
20
+ * cut into `total` equal ranges. The shards are disjoint and together hold every test; the order stays.
21
+ */
22
+ export declare function shardTests(tests: RegisteredTestCase[], { current, total }: Shard, groupOf: (test: RegisteredTestCase) => string): RegisteredTestCase[];