@itmetsam/samurai-framework 0.21.3 → 0.21.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +3 -1
- package/dist/assert/asymmetric.d.ts +29 -0
- package/dist/assert/asymmetric.js +166 -0
- package/dist/assert/element-probes.d.ts +20 -0
- package/dist/assert/element-probes.js +45 -0
- package/dist/assert/expect.d.ts +20 -9
- package/dist/assert/expect.js +12 -4
- package/dist/assert/locator-assertion-base.d.ts +52 -0
- package/dist/assert/locator-assertion-base.js +78 -0
- package/dist/assert/locator-assertions.d.ts +26 -49
- package/dist/assert/locator-assertions.js +35 -78
- package/dist/assert/more-locator-assertions.d.ts +30 -0
- package/dist/assert/more-locator-assertions.js +86 -0
- package/dist/assert/retry-assertions.d.ts +28 -0
- package/dist/assert/retry-assertions.js +113 -0
- package/dist/assert/text-match.d.ts +2 -0
- package/dist/assert/text-match.js +8 -0
- package/dist/assert/value-assertions.d.ts +29 -1
- package/dist/assert/value-assertions.js +84 -5
- package/dist/assert/value-matchers.d.ts +19 -0
- package/dist/assert/value-matchers.js +53 -0
- package/dist/browser/browser.d.ts +9 -0
- package/dist/browser/browser.js +25 -5
- package/dist/browser/cdp.d.ts +11 -0
- package/dist/browser/cdp.js +28 -0
- package/dist/browser/chrome-launch.d.ts +4 -0
- package/dist/browser/chrome-launch.js +2 -2
- package/dist/browser/index.d.ts +5 -0
- package/dist/browser/index.js +4 -0
- package/dist/browser/launch-firefox.js +1 -1
- package/dist/browser/launch-flags.d.ts +4 -1
- package/dist/browser/launch-flags.js +18 -5
- package/dist/browser/navigation-error.d.ts +5 -1
- package/dist/browser/navigation-error.js +10 -2
- package/dist/browser/page-content.d.ts +5 -0
- package/dist/browser/page-content.js +43 -0
- package/dist/browser/page-evaluate.d.ts +9 -0
- package/dist/browser/page-evaluate.js +20 -0
- package/dist/browser/page-locators.d.ts +1 -1
- package/dist/browser/page-locators.js +1 -1
- package/dist/browser/page-navigation.d.ts +18 -0
- package/dist/browser/page-navigation.js +52 -2
- package/dist/browser/page-settings.d.ts +1 -1
- package/dist/browser/page.d.ts +52 -1
- package/dist/browser/page.js +66 -2
- package/dist/browser/session-capabilities.d.ts +10 -0
- package/dist/browser/session-capabilities.js +11 -0
- package/dist/browser/session.d.ts +2 -1
- package/dist/browser/session.js +3 -3
- package/dist/browser/storage-state.d.ts +6 -0
- package/dist/browser/storage-state.js +37 -0
- package/dist/browser/use-applier.d.ts +27 -0
- package/dist/browser/use-applier.js +94 -0
- package/dist/browser/viewport.d.ts +3 -2
- package/dist/browser/viewport.js +15 -2
- package/dist/config/config.js +2 -0
- package/dist/config/use-options.d.ts +16 -0
- package/dist/config/use-options.js +128 -0
- package/dist/locator/element-access.d.ts +2 -0
- package/dist/locator/element-reads.d.ts +17 -0
- package/dist/locator/element-reads.js +31 -0
- package/dist/locator/evaluate.d.ts +12 -0
- package/dist/locator/evaluate.js +36 -0
- package/dist/locator/locator.d.ts +23 -0
- package/dist/locator/locator.js +36 -1
- package/dist/logger/warn-once.d.ts +4 -0
- package/dist/logger/warn-once.js +13 -0
- package/dist/runner/attachments.d.ts +16 -0
- package/dist/runner/attachments.js +60 -0
- package/dist/runner/fixture-args.d.ts +15 -0
- package/dist/runner/fixture-args.js +95 -0
- package/dist/runner/fixture-defs.d.ts +22 -0
- package/dist/runner/fixture-defs.js +103 -0
- package/dist/runner/fixture-scope.d.ts +33 -0
- package/dist/runner/fixture-scope.js +156 -0
- package/dist/runner/fixtures.d.ts +34 -0
- package/dist/runner/fixtures.js +39 -0
- package/dist/runner/hooks.d.ts +6 -1
- package/dist/runner/hooks.js +15 -3
- package/dist/runner/registry.d.ts +3 -39
- package/dist/runner/registry.js +93 -19
- package/dist/runner/reporter.d.ts +7 -1
- package/dist/runner/reporter.js +3 -0
- package/dist/runner/run-test.d.ts +1 -3
- package/dist/runner/run-test.js +92 -34
- package/dist/runner/steps.d.ts +13 -0
- package/dist/runner/steps.js +67 -0
- package/dist/runner/suite.d.ts +5 -0
- package/dist/runner/suite.js +8 -0
- package/dist/runner/test-annotations.d.ts +16 -0
- package/dist/runner/test-annotations.js +47 -0
- package/dist/runner/test-function.d.ts +103 -0
- package/dist/runner/test-function.js +1 -0
- package/dist/runner/test-info.d.ts +104 -0
- package/dist/runner/test-info.js +169 -0
- package/dist/runner/test-runner.js +2 -1
- package/dist/runner/test-timeout.d.ts +22 -0
- package/dist/runner/test-timeout.js +45 -0
- package/dist/runner/use-options.d.ts +7 -0
- package/dist/runner/use-options.js +18 -0
- package/dist/runner/worker-session.js +2 -1
- package/dist/script/call-function.d.ts +7 -0
- package/dist/script/call-function.js +21 -3
- package/dist/script/element-handle.d.ts +1 -1
- package/dist/script/element-handle.js +1 -1
- package/dist/types/config.d.ts +3 -2
- package/dist/types/extend.d.ts +30 -0
- package/dist/types/test.d.ts +120 -0
- package/dist/types/use.d.ts +47 -0
- package/dist/wait/wait-until.d.ts +3 -1
- package/dist/wait/wait-until.js +7 -2
- package/package.json +1 -1
package/dist/runner/run-test.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
+
import path from "node:path";
|
|
1
2
|
import { takePendingAssertions } from "../assert/expect.js";
|
|
2
3
|
import { Browser } from "../browser/browser.js";
|
|
3
4
|
import { Evidence } from "../browser/evidence.js";
|
|
4
5
|
import { projectDir, readConfig } from "../config/config.js";
|
|
5
|
-
import { createSecretsFixture } from "../config/secrets.js";
|
|
6
|
-
import { createEnvFixture } from "../config/variables.js";
|
|
7
6
|
import logger from "../logger/index.js";
|
|
8
7
|
import { ConnectionClosedError } from "../transport/bidi-connection.js";
|
|
9
8
|
import { judgePageLogs } from "./page-logs-report.js";
|
|
@@ -13,7 +12,13 @@ import { isSkipSignal, staticSkip } from "./skip.js";
|
|
|
13
12
|
import { runOutput } from "./output.js";
|
|
14
13
|
import { testDirOf } from "./test-dir.js";
|
|
15
14
|
import { toTestError } from "./test-error.js";
|
|
16
|
-
import {
|
|
15
|
+
import { runInTestScope, TestInfoImpl } from "./test-info.js";
|
|
16
|
+
import { applyDeclaredMarks, judgeExpectation } from "./test-annotations.js";
|
|
17
|
+
import { TestTimeout } from "./test-timeout.js";
|
|
18
|
+
import { ioOfAttempt, missingOutputs } from "./test-io.js";
|
|
19
|
+
import { buildTestFixtures } from "./fixtures.js";
|
|
20
|
+
import { FixtureScope } from "./fixture-scope.js";
|
|
21
|
+
import { effectiveUse } from "./use-options.js";
|
|
17
22
|
import { beginOutputs, endOutputs } from "../wait/output-sink.js";
|
|
18
23
|
/**
|
|
19
24
|
* The state of one test run. A test ends once, whichever of finishing, failing, timing out, an abort or a
|
|
@@ -46,6 +51,10 @@ class TestOutcome {
|
|
|
46
51
|
reason: signal.reason,
|
|
47
52
|
});
|
|
48
53
|
}
|
|
54
|
+
/** The test body ended by itself: a test expected to fail is passed by a failure and failed by a pass */
|
|
55
|
+
conclude(error, logs) {
|
|
56
|
+
this.finish(judgeExpectation(this.info, error), logs);
|
|
57
|
+
}
|
|
49
58
|
finish(error, logs) {
|
|
50
59
|
if (this.ended)
|
|
51
60
|
return;
|
|
@@ -53,20 +62,11 @@ class TestOutcome {
|
|
|
53
62
|
this.reporter.onTestEnd(this.test, this.browser, error, {
|
|
54
63
|
...logs,
|
|
55
64
|
screenshots: this.evidence?.screenshots,
|
|
65
|
+
...this.info?.reportExtras(),
|
|
56
66
|
...this.io(),
|
|
57
67
|
});
|
|
58
68
|
}
|
|
59
69
|
}
|
|
60
|
-
/** What `beforeAll` and `afterAll` hooks get: no page, only what a run knows */
|
|
61
|
-
export function allHookFixtures(run, browserName, workerIndex = 0) {
|
|
62
|
-
const { settings, secrets } = run;
|
|
63
|
-
return {
|
|
64
|
-
browserName,
|
|
65
|
-
workerIndex,
|
|
66
|
-
env: createEnvFixture(settings.environment, settings.variables),
|
|
67
|
-
secrets: createSecretsFixture(settings.environment, secrets),
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
70
|
/** Runs one test in one browser and reports it: launch, run the body, judge the page's logs, close the browser */
|
|
71
71
|
export async function runOneTest(test, browserName, run, options, srcDir, reporter, resources = {}) {
|
|
72
72
|
// A test marked skip or fixme never runs, whoever asked for it (a flow's test node, a worker)
|
|
@@ -78,25 +78,28 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
|
|
|
78
78
|
logger.verbose("Starting test: %s in %s", test.name, browserName);
|
|
79
79
|
reporter.onTestStart(test, browserName);
|
|
80
80
|
const outcome = new TestOutcome(reporter, test, browserName);
|
|
81
|
-
const { settings
|
|
81
|
+
const { settings } = run;
|
|
82
82
|
const timeout = settings.timeout;
|
|
83
83
|
const { resultDir } = runOutput(options, projectDir());
|
|
84
84
|
let browser;
|
|
85
85
|
let page;
|
|
86
|
-
let
|
|
86
|
+
let customFixtures;
|
|
87
87
|
let onRunAborted;
|
|
88
88
|
const timeoutController = new AbortController();
|
|
89
89
|
let browserGone = () => { };
|
|
90
|
+
let testTimeout;
|
|
90
91
|
/** Resolves when the time is up or the run is aborted; the test is reported failed then */
|
|
91
92
|
const limit = new Promise((resolve) => {
|
|
92
|
-
|
|
93
|
+
// The test can change its own limit (`test.setTimeout`, `test.slow`): the message names the one in force
|
|
94
|
+
testTimeout = new TestTimeout(timeout, () => {
|
|
93
95
|
outcome.finish({
|
|
94
|
-
message: `Test timed out after ${timeout / 1000} seconds`,
|
|
96
|
+
message: `Test timed out after ${(testTimeout?.ms ?? timeout) / 1000} seconds`,
|
|
95
97
|
type: "timeout",
|
|
96
98
|
});
|
|
97
99
|
timeoutController.abort(new Error("Test timed out"));
|
|
98
100
|
resolve();
|
|
99
|
-
}
|
|
101
|
+
});
|
|
102
|
+
testTimeout.start();
|
|
100
103
|
onRunAborted = () => {
|
|
101
104
|
if (timeoutController.signal.aborted)
|
|
102
105
|
return;
|
|
@@ -113,10 +116,12 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
|
|
|
113
116
|
try {
|
|
114
117
|
// The `beforeAll` hooks come first, before a browser exists
|
|
115
118
|
await resources.allHooks?.ensure(test, browserName);
|
|
119
|
+
const use = await effectiveUse(test);
|
|
116
120
|
const launched = await Browser.launch(browserName, {
|
|
117
121
|
port: resources.port ?? options.port ?? DEFAULT_PORT,
|
|
118
122
|
...(resources.profileDir && { profileDir: resources.profileDir }),
|
|
119
123
|
headless: options.headless ?? settings.headless,
|
|
124
|
+
use,
|
|
120
125
|
}, timeoutController.signal);
|
|
121
126
|
browser = launched.browser;
|
|
122
127
|
// A person closing the window, or the browser crashing: end the test now instead of waiting for a
|
|
@@ -133,25 +138,68 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
|
|
|
133
138
|
browserGone();
|
|
134
139
|
});
|
|
135
140
|
page = launched.page;
|
|
136
|
-
|
|
141
|
+
// An empty `use` leaves the browser exactly as it was: no applier, nothing sent
|
|
142
|
+
const applied = Object.keys(use).length > 0
|
|
143
|
+
? await launched.browser.setUse(use, projectDir())
|
|
144
|
+
: undefined;
|
|
145
|
+
const testFolder = `${browserName}/${testDirOf(test, srcDir)}${resources.attempt ? `/retry-${resources.attempt}` : ""}`;
|
|
146
|
+
outcome.evidence = new Evidence(resultDir, testFolder);
|
|
137
147
|
launched.page.setEvidence(outcome.evidence);
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
browser: launched.browser,
|
|
148
|
+
const info = new TestInfoImpl({
|
|
149
|
+
test,
|
|
141
150
|
browserName,
|
|
142
151
|
workerIndex: resources.workerIndex ?? 0,
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
152
|
+
retry: resources.attempt ?? 0,
|
|
153
|
+
settings: { baseURL: settings.baseURL, timeout },
|
|
154
|
+
...(testTimeout && { timeout: testTimeout }),
|
|
155
|
+
resultDir,
|
|
156
|
+
outputDir: path.join(resultDir, "attachments", ...testFolder.split("/")),
|
|
157
|
+
});
|
|
158
|
+
outcome.info = info;
|
|
159
|
+
applyDeclaredMarks(info, test);
|
|
160
|
+
const built = buildTestFixtures({
|
|
161
|
+
info,
|
|
162
|
+
launched,
|
|
163
|
+
browserName,
|
|
164
|
+
workerIndex: resources.workerIndex ?? 0,
|
|
165
|
+
run,
|
|
166
|
+
test,
|
|
167
|
+
options,
|
|
168
|
+
sink,
|
|
169
|
+
device: {
|
|
170
|
+
isMobile: applied?.isMobile ?? false,
|
|
171
|
+
hasTouch: applied?.hasTouch ?? false,
|
|
172
|
+
},
|
|
173
|
+
});
|
|
174
|
+
// A test written with `test.extend()` gets its custom fixtures set up on demand, and torn down below
|
|
175
|
+
const custom = test.extension
|
|
176
|
+
? new FixtureScope(test.extension, built)
|
|
177
|
+
: undefined;
|
|
178
|
+
customFixtures = custom;
|
|
179
|
+
const fixtures = custom?.fixtures ?? built;
|
|
180
|
+
let failure;
|
|
181
|
+
let failed = false;
|
|
182
|
+
try {
|
|
183
|
+
await runInTestScope({ info, fixtures }, () => runWithEachHooks(test.suite, fixtures, timeoutController.signal, async () => {
|
|
184
|
+
await custom?.prepare(test.function);
|
|
185
|
+
return test.function(fixtures);
|
|
186
|
+
}, (error) => info.bodyDone(error), custom));
|
|
187
|
+
}
|
|
188
|
+
catch (err) {
|
|
189
|
+
failure = err;
|
|
190
|
+
failed = true;
|
|
191
|
+
}
|
|
192
|
+
const cleanup = await custom?.teardown(timeout);
|
|
193
|
+
if (cleanup && (!failed || isSkipSignal(failure)))
|
|
194
|
+
throw cleanup;
|
|
195
|
+
if (failed)
|
|
196
|
+
throw failure;
|
|
149
197
|
// Already reported as timed out
|
|
150
198
|
if (timeoutController.signal.aborted)
|
|
151
199
|
return;
|
|
152
200
|
const unawaited = takePendingAssertions();
|
|
153
201
|
if (unawaited.length > 0) {
|
|
154
|
-
outcome.
|
|
202
|
+
outcome.conclude({
|
|
155
203
|
message: unawaited
|
|
156
204
|
.map((label) => `${label} was not awaited`)
|
|
157
205
|
.join("\n"),
|
|
@@ -162,23 +210,29 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
|
|
|
162
210
|
// Checked only after a pass: a test that failed first reports its own failure
|
|
163
211
|
const missing = missingOutputs(test.outputs, sink);
|
|
164
212
|
if (missing) {
|
|
165
|
-
outcome.
|
|
213
|
+
outcome.conclude(missing, await judgeLogs(page, true));
|
|
166
214
|
return;
|
|
167
215
|
}
|
|
168
216
|
const verdict = await judgeLogs(page, false);
|
|
169
|
-
outcome.
|
|
217
|
+
outcome.conclude(verdict.error, verdict);
|
|
170
218
|
}
|
|
171
219
|
catch (err) {
|
|
172
220
|
if (timeoutController.signal.aborted)
|
|
173
221
|
return;
|
|
174
222
|
if (isSkipSignal(err))
|
|
175
223
|
return outcome.skip(err);
|
|
176
|
-
outcome.
|
|
224
|
+
const closed = outcome.browserClosed || err instanceof ConnectionClosedError;
|
|
225
|
+
const error = closed
|
|
177
226
|
? {
|
|
178
227
|
message: "The browser was closed while the test was running",
|
|
179
228
|
type: "error",
|
|
180
229
|
}
|
|
181
|
-
: toTestError(err)
|
|
230
|
+
: toTestError(err);
|
|
231
|
+
const logs = await judgeLogs(page, true);
|
|
232
|
+
if (closed)
|
|
233
|
+
outcome.finish(error, logs);
|
|
234
|
+
else
|
|
235
|
+
outcome.conclude(error, logs);
|
|
182
236
|
}
|
|
183
237
|
};
|
|
184
238
|
// Open for this attempt only: whatever a capture still saves after the attempt ended is ignored
|
|
@@ -188,9 +242,13 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
|
|
|
188
242
|
await Promise.race([limit, gone, runBody()]);
|
|
189
243
|
}
|
|
190
244
|
finally {
|
|
245
|
+
// A test cut short (timeout, abort) did not get to its own cleanup: its fixtures are torn down before the browser
|
|
246
|
+
const late = await customFixtures?.teardown(timeout);
|
|
247
|
+
if (late)
|
|
248
|
+
logger.debug("A fixture failed after the test ended", { late });
|
|
191
249
|
endOutputs();
|
|
192
250
|
}
|
|
193
|
-
|
|
251
|
+
testTimeout?.stop();
|
|
194
252
|
if (onRunAborted)
|
|
195
253
|
options.signal?.removeEventListener("abort", onRunAborted);
|
|
196
254
|
takePendingAssertions();
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { TestStepEntry } from "../types/test.js";
|
|
2
|
+
/** The steps a test has run, in the order they started */
|
|
3
|
+
export declare class StepLog {
|
|
4
|
+
readonly entries: TestStepEntry[];
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* `test.step`: runs `body` as a step of `log`, nested under the step it is called in. The step is listed with how
|
|
8
|
+
* it ended; a throw fails the step and goes on. A step that outlasts `options.timeout` fails with a timeout error
|
|
9
|
+
* (its body keeps running, as nothing can stop it).
|
|
10
|
+
*/
|
|
11
|
+
export declare function runStep<T>(log: StepLog, title: unknown, body: unknown, options?: {
|
|
12
|
+
timeout?: number;
|
|
13
|
+
}): Promise<T>;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { checkTimeout } from "./test-timeout.js";
|
|
3
|
+
import { isSkipSignal } from "./skip.js";
|
|
4
|
+
/** The steps a test has run, in the order they started */
|
|
5
|
+
export class StepLog {
|
|
6
|
+
constructor() {
|
|
7
|
+
this.entries = [];
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
/** The step being run: a step started inside it is listed under it */
|
|
11
|
+
const running = new AsyncLocalStorage();
|
|
12
|
+
const UNFINISHED = "the test ended before the step finished";
|
|
13
|
+
/**
|
|
14
|
+
* `test.step`: runs `body` as a step of `log`, nested under the step it is called in. The step is listed with how
|
|
15
|
+
* it ended; a throw fails the step and goes on. A step that outlasts `options.timeout` fails with a timeout error
|
|
16
|
+
* (its body keeps running, as nothing can stop it).
|
|
17
|
+
*/
|
|
18
|
+
export async function runStep(log, title, body, options) {
|
|
19
|
+
if (typeof title !== "string" || title === "")
|
|
20
|
+
throw new TypeError("step(): the title must be a non-empty string");
|
|
21
|
+
if (typeof body !== "function")
|
|
22
|
+
throw new TypeError("step(): needs a function as its second argument");
|
|
23
|
+
const limit = options?.timeout === undefined
|
|
24
|
+
? 0
|
|
25
|
+
: checkTimeout("step()", options.timeout);
|
|
26
|
+
// Listed as unfinished until it ends, so a test cut short still shows where it stopped
|
|
27
|
+
const entry = {
|
|
28
|
+
title,
|
|
29
|
+
status: "failed",
|
|
30
|
+
duration: 0,
|
|
31
|
+
error: UNFINISHED,
|
|
32
|
+
};
|
|
33
|
+
const parent = running.getStore();
|
|
34
|
+
if (parent)
|
|
35
|
+
(parent.steps ??= []).push(entry);
|
|
36
|
+
else
|
|
37
|
+
log.entries.push(entry);
|
|
38
|
+
const started = Date.now();
|
|
39
|
+
let timer;
|
|
40
|
+
try {
|
|
41
|
+
const call = Promise.resolve().then(() => running.run(entry, body));
|
|
42
|
+
const value = limit === 0
|
|
43
|
+
? await call
|
|
44
|
+
: await Promise.race([
|
|
45
|
+
call,
|
|
46
|
+
new Promise((_, reject) => {
|
|
47
|
+
timer = setTimeout(() => reject(new Error(`step ${JSON.stringify(title)} timed out after ${limit} ms`)), limit);
|
|
48
|
+
}),
|
|
49
|
+
]);
|
|
50
|
+
entry.status = "passed";
|
|
51
|
+
delete entry.error;
|
|
52
|
+
return value;
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
if (isSkipSignal(error)) {
|
|
56
|
+
entry.status = "skipped";
|
|
57
|
+
delete entry.error;
|
|
58
|
+
}
|
|
59
|
+
else
|
|
60
|
+
entry.error = error instanceof Error ? error.message : String(error);
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
clearTimeout(timer);
|
|
65
|
+
entry.duration = Date.now() - started;
|
|
66
|
+
}
|
|
67
|
+
}
|
package/dist/runner/suite.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { EachHook, Modifier, Suite } from "../types/test.js";
|
|
2
|
+
import type { UseOptions } from "../types/use.js";
|
|
2
3
|
/** The four hook lists a suite can hold */
|
|
3
4
|
export type HookKind = "beforeEach" | "afterEach" | "beforeAll" | "afterAll";
|
|
4
5
|
export declare function newSuite(title?: string, parent?: Suite, modifier?: Modifier, reason?: string): Suite;
|
|
@@ -28,3 +29,7 @@ export declare function beforeEachOf(suite: Suite): EachHook[];
|
|
|
28
29
|
export declare function afterEachOf(suite: Suite): EachHook[];
|
|
29
30
|
/** Adds a hook to the suite being registered */
|
|
30
31
|
export declare function addHook<K extends HookKind>(suite: Suite, kind: K, hook: NonNullable<Suite[K]>[number]): void;
|
|
32
|
+
/** Adds the options of a `test.use()` to the suite being registered */
|
|
33
|
+
export declare function addUse(suite: Suite, options: UseOptions): void;
|
|
34
|
+
/** The `test.use()` options that apply to a test, outermost suite first: merge them in this order */
|
|
35
|
+
export declare function useOf(suite: Suite): UseOptions[];
|
package/dist/runner/suite.js
CHANGED
|
@@ -80,3 +80,11 @@ export function afterEachOf(suite) {
|
|
|
80
80
|
export function addHook(suite, kind, hook) {
|
|
81
81
|
(suite[kind] ??= []).push(hook);
|
|
82
82
|
}
|
|
83
|
+
/** Adds the options of a `test.use()` to the suite being registered */
|
|
84
|
+
export function addUse(suite, options) {
|
|
85
|
+
(suite.use ??= []).push(options);
|
|
86
|
+
}
|
|
87
|
+
/** The `test.use()` options that apply to a test, outermost suite first: merge them in this order */
|
|
88
|
+
export function useOf(suite) {
|
|
89
|
+
return suiteChain(suite).flatMap((s) => s.use ?? []);
|
|
90
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { RegisteredTestCase, Suite, TestError, TestFixtures, TestInfo } from "../types/test.js";
|
|
2
|
+
type Condition = boolean | ((fixtures: TestFixtures) => boolean);
|
|
3
|
+
/**
|
|
4
|
+
* `test.fail(...)` and `test.slow(...)` without a title. In a running test or hook it marks that test when the
|
|
5
|
+
* condition holds (no condition: always). In a `describe` block, or at the top of a file, it marks every test of
|
|
6
|
+
* `suite` instead, and the condition must be a boolean as no test runs yet.
|
|
7
|
+
*/
|
|
8
|
+
export declare function markTests(type: "fail" | "slow", suite: () => Suite, condition?: Condition, description?: string): void;
|
|
9
|
+
/** Applies what the test and the suites around it declared (`test.fail("t", fn)`, `test.slow(true)` in a `describe`) */
|
|
10
|
+
export declare function applyDeclaredMarks(info: TestInfo, test: Pick<RegisteredTestCase, "suite" | "expectedFail">): void;
|
|
11
|
+
/**
|
|
12
|
+
* What a test that ends by itself is reported as, given what it is expected to do: a test expected to fail is
|
|
13
|
+
* passed by a failure and failed by a pass. A timeout is never the failure it was waiting for.
|
|
14
|
+
*/
|
|
15
|
+
export declare function judgeExpectation(info: TestInfo | undefined, error: TestError | undefined): TestError | undefined;
|
|
16
|
+
export {};
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { conditionHolds, infoFor, runningScope } from "./test-info.js";
|
|
2
|
+
/**
|
|
3
|
+
* `test.fail(...)` and `test.slow(...)` without a title. In a running test or hook it marks that test when the
|
|
4
|
+
* condition holds (no condition: always). In a `describe` block, or at the top of a file, it marks every test of
|
|
5
|
+
* `suite` instead, and the condition must be a boolean as no test runs yet.
|
|
6
|
+
*/
|
|
7
|
+
export function markTests(type, suite, condition = true, description) {
|
|
8
|
+
const label = `${type}()`;
|
|
9
|
+
if (description !== undefined && typeof description !== "string")
|
|
10
|
+
throw new TypeError(`${label}: the reason must be a string`);
|
|
11
|
+
if (runningScope()) {
|
|
12
|
+
if (conditionHolds(label, condition))
|
|
13
|
+
infoFor(label)[type](true, description);
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
if (typeof condition === "function")
|
|
17
|
+
throw new Error(`${label}: a condition function needs a running test`);
|
|
18
|
+
if (!condition)
|
|
19
|
+
return;
|
|
20
|
+
const target = suite();
|
|
21
|
+
(target.marks ??= []).push({
|
|
22
|
+
type,
|
|
23
|
+
...(description !== undefined && { description }),
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
/** Applies what the test and the suites around it declared (`test.fail("t", fn)`, `test.slow(true)` in a `describe`) */
|
|
27
|
+
export function applyDeclaredMarks(info, test) {
|
|
28
|
+
const chain = [];
|
|
29
|
+
for (let s = test.suite; s; s = s.parent)
|
|
30
|
+
chain.unshift(s);
|
|
31
|
+
for (const suite of chain)
|
|
32
|
+
for (const mark of suite.marks ?? [])
|
|
33
|
+
info[mark.type](true, mark.description);
|
|
34
|
+
if (test.expectedFail)
|
|
35
|
+
info.fail(true, test.expectedFail.description);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* What a test that ends by itself is reported as, given what it is expected to do: a test expected to fail is
|
|
39
|
+
* passed by a failure and failed by a pass. A timeout is never the failure it was waiting for.
|
|
40
|
+
*/
|
|
41
|
+
export function judgeExpectation(info, error) {
|
|
42
|
+
if (info?.expectedStatus !== "failed")
|
|
43
|
+
return error;
|
|
44
|
+
if (!error)
|
|
45
|
+
return { message: "expected to fail, but passed", type: "error" };
|
|
46
|
+
return error.type === "timeout" ? error : undefined;
|
|
47
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import type { AllHook, TestCaseBase, TestDetails, TestInfo, TestOptions } from "../types/test.js";
|
|
2
|
+
import type { UseOptions } from "../types/use.js";
|
|
3
|
+
import type { FixtureDefinitions, FixturesOf, MergeFixtures } from "../types/extend.js";
|
|
4
|
+
/** What the registry keeps of a test body; the fixtures it gets are those of the test it was written for */
|
|
5
|
+
export type Body = TestCaseBase["function"];
|
|
6
|
+
/** The fixtures of tests made by `test.extend()`: none for the plain `test` */
|
|
7
|
+
export type NoFixtures = Record<never, never>;
|
|
8
|
+
/** A test body or hook that gets the built-in fixtures and the custom ones `X` */
|
|
9
|
+
export type BodyOf<X> = (fixtures: FixturesOf<X>) => Promise<void>;
|
|
10
|
+
export type HookOf<X> = (fixtures: FixturesOf<X>) => Promise<void> | void;
|
|
11
|
+
/** `test(title, fn)`, `test.skip(title, fn)`, `test.skip(condition, reason)` and the other forms */
|
|
12
|
+
export interface TestFunction<X = NoFixtures> {
|
|
13
|
+
(title: string, fn: BodyOf<X>): void;
|
|
14
|
+
(title: string, options: TestOptions, fn: BodyOf<X>): void;
|
|
15
|
+
/** Registers a test that is not run */
|
|
16
|
+
skip: SkipFunction<X>;
|
|
17
|
+
/** Registers a test that is not run because it is known to be broken */
|
|
18
|
+
fixme: SkipFunction<X>;
|
|
19
|
+
/**
|
|
20
|
+
* Registers a test that is expected to fail (`test.fail("title", fn)`): it is reported as passed when it fails and
|
|
21
|
+
* as failed when it passes. Inside a test or hook, `test.fail()` or `test.fail(condition, reason)` marks the running
|
|
22
|
+
* test the same way; in a `describe` block, `test.fail(true, reason)` marks every test in it.
|
|
23
|
+
*/
|
|
24
|
+
fail: FailFunction<X>;
|
|
25
|
+
/**
|
|
26
|
+
* Triples the timeout of the running test (`test.slow()`, `test.slow(condition, reason)`); in a `describe` block
|
|
27
|
+
* `test.slow(true, reason)` does it for every test in it
|
|
28
|
+
*/
|
|
29
|
+
slow: ConditionFunction<X>;
|
|
30
|
+
/** Sets the timeout, in milliseconds, of the running test (0: none); the time already spent counts */
|
|
31
|
+
setTimeout: (ms: number) => void;
|
|
32
|
+
/**
|
|
33
|
+
* Runs `body` as a named step of the running test and returns its value; steps nest, and the report lists them
|
|
34
|
+
* with their status, duration and error. A step that throws fails and rethrows.
|
|
35
|
+
*/
|
|
36
|
+
step: StepFunction;
|
|
37
|
+
/** Runs only this test (and the other `only` ones) */
|
|
38
|
+
only: {
|
|
39
|
+
(title: string, fn: BodyOf<X>): void;
|
|
40
|
+
(title: string, options: TestOptions, fn: BodyOf<X>): void;
|
|
41
|
+
};
|
|
42
|
+
/** The running test's info, in the test and its hooks; throws outside one */
|
|
43
|
+
info: () => TestInfo;
|
|
44
|
+
/**
|
|
45
|
+
* Sets the `use` options (locale, colorScheme, storageState, ...) for the `describe` block it is written in, or for
|
|
46
|
+
* the whole file at the top level, whether before or after the tests. An inner block wins over an outer one, and
|
|
47
|
+
* both win over config `use`.
|
|
48
|
+
*/
|
|
49
|
+
use: (options: UseOptions) => void;
|
|
50
|
+
/** `describe`, under the name other frameworks use */
|
|
51
|
+
describe: DescribeFunction;
|
|
52
|
+
/** `beforeEach`, as `test.beforeEach` */
|
|
53
|
+
beforeEach: (hook: HookOf<X>) => void;
|
|
54
|
+
/** `afterEach`, as `test.afterEach` */
|
|
55
|
+
afterEach: (hook: HookOf<X>) => void;
|
|
56
|
+
/** `beforeAll`, as `test.beforeAll` */
|
|
57
|
+
beforeAll: (hook: AllHook) => void;
|
|
58
|
+
/** `afterAll`, as `test.afterAll` */
|
|
59
|
+
afterAll: (hook: AllHook) => void;
|
|
60
|
+
/**
|
|
61
|
+
* A `test` that also has the fixtures you define: each is a value, or a function `async ({ page }, use) => {
|
|
62
|
+
* ...set up; await use(value); ...clean up }` that may use the built-in fixtures and the other ones. A fixture is
|
|
63
|
+
* set up when a test or hook destructures it, and cleaned up after the test and its `afterEach` hooks. Write
|
|
64
|
+
* `[fixture, { auto: true }]` to set it up for every test. A fixture with the name of a built-in one replaces it.
|
|
65
|
+
*/
|
|
66
|
+
extend<T extends object>(fixtures: FixtureDefinitions<T, X>): TestFunction<MergeFixtures<X, T>>;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* With a title and a body it registers a skipped test, `{ reason }` saying why; with a condition it skips the
|
|
70
|
+
* running test when the condition is true
|
|
71
|
+
*/
|
|
72
|
+
export interface SkipFunction<X> {
|
|
73
|
+
(title: string, fn: BodyOf<X>): void;
|
|
74
|
+
(title: string, options: TestOptions, fn: BodyOf<X>): void;
|
|
75
|
+
/** A function gets the test's fixtures (`({ browserName }) => browserName === "firefox"`) */
|
|
76
|
+
(condition: boolean | ((fixtures: FixturesOf<X>) => boolean), reason?: string): void;
|
|
77
|
+
}
|
|
78
|
+
/** `describe.skip` and `describe.fixme`: every test inside gets the modifier, and `{ reason }` says why */
|
|
79
|
+
export interface DescribeSkipFunction {
|
|
80
|
+
(title: string, fn: () => void): void;
|
|
81
|
+
(title: string, details: TestDetails, fn: () => void): void;
|
|
82
|
+
}
|
|
83
|
+
/** `describe(title, fn)` and the same with a modifier on every test inside */
|
|
84
|
+
export interface DescribeFunction {
|
|
85
|
+
(title: string, fn: () => void): void;
|
|
86
|
+
skip: DescribeSkipFunction;
|
|
87
|
+
fixme: DescribeSkipFunction;
|
|
88
|
+
only(title: string, fn: () => void): void;
|
|
89
|
+
}
|
|
90
|
+
/** `test.fail()` and `test.slow()`: with a condition (a function gets the test's fixtures), or none for "always" */
|
|
91
|
+
export interface ConditionFunction<X> {
|
|
92
|
+
(): void;
|
|
93
|
+
(condition: boolean | ((fixtures: FixturesOf<X>) => boolean), reason?: string): void;
|
|
94
|
+
}
|
|
95
|
+
/** `test.fail("title", fn)` registers a test expected to fail; the other forms mark the running test */
|
|
96
|
+
export interface FailFunction<X> extends ConditionFunction<X> {
|
|
97
|
+
(title: string, fn: BodyOf<X>): void;
|
|
98
|
+
(title: string, options: TestOptions, fn: BodyOf<X>): void;
|
|
99
|
+
}
|
|
100
|
+
/** `test.step(title, body, { timeout })` */
|
|
101
|
+
export type StepFunction = <T>(title: string, body: () => Promise<T> | T, options?: {
|
|
102
|
+
timeout?: number;
|
|
103
|
+
}) => Promise<T>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import type { SupportedBrowser } from "../types/browser.js";
|
|
2
|
+
import type { RegisteredTestCase, TestAnnotation, TestAttachment, TestError, TestFixtures, TestInfo, TestStepEntry } from "../types/test.js";
|
|
3
|
+
import { type AttachInput } from "./attachments.js";
|
|
4
|
+
import { StepLog } from "./steps.js";
|
|
5
|
+
import { TestTimeout } from "./test-timeout.js";
|
|
6
|
+
/** What the running test's info needs of the run */
|
|
7
|
+
export interface TestInfoSources {
|
|
8
|
+
test: RegisteredTestCase;
|
|
9
|
+
browserName: SupportedBrowser;
|
|
10
|
+
workerIndex: number;
|
|
11
|
+
/** The attempt, counting from 0 */
|
|
12
|
+
retry: number;
|
|
13
|
+
settings: {
|
|
14
|
+
baseURL: string | undefined;
|
|
15
|
+
timeout: number;
|
|
16
|
+
};
|
|
17
|
+
/** The run's result folder: attachment paths in the report are relative to it */
|
|
18
|
+
resultDir: string;
|
|
19
|
+
outputDir: string;
|
|
20
|
+
/** The test's time limit, which the info can change; without it the info only keeps the number */
|
|
21
|
+
timeout?: TestTimeout;
|
|
22
|
+
}
|
|
23
|
+
/** What a report entry lists from the info, besides the rest of its extras */
|
|
24
|
+
export interface InfoReportExtras {
|
|
25
|
+
attachments?: {
|
|
26
|
+
name: string;
|
|
27
|
+
contentType: string;
|
|
28
|
+
path: string;
|
|
29
|
+
}[];
|
|
30
|
+
annotations?: TestAnnotation[];
|
|
31
|
+
steps?: TestStepEntry[];
|
|
32
|
+
}
|
|
33
|
+
/** The `TestInfo` of one attempt of a test. The runner creates it, the test only reads it. */
|
|
34
|
+
export declare class TestInfoImpl implements TestInfo {
|
|
35
|
+
readonly title: string;
|
|
36
|
+
readonly titlePath: string[];
|
|
37
|
+
readonly file: string;
|
|
38
|
+
readonly line?: number;
|
|
39
|
+
readonly column?: number;
|
|
40
|
+
readonly retry: number;
|
|
41
|
+
readonly workerIndex: number;
|
|
42
|
+
readonly project: {
|
|
43
|
+
name: string;
|
|
44
|
+
};
|
|
45
|
+
readonly config: {
|
|
46
|
+
baseURL: string | undefined;
|
|
47
|
+
timeout: number;
|
|
48
|
+
};
|
|
49
|
+
annotations: TestAnnotation[];
|
|
50
|
+
expectedStatus: TestInfo["expectedStatus"];
|
|
51
|
+
/** The steps the test ran with `test.step` */
|
|
52
|
+
readonly stepLog: StepLog;
|
|
53
|
+
readonly errors: TestError[];
|
|
54
|
+
readonly outputDir: string;
|
|
55
|
+
private readonly saved;
|
|
56
|
+
private readonly used;
|
|
57
|
+
private readonly started;
|
|
58
|
+
private ended;
|
|
59
|
+
private result;
|
|
60
|
+
private readonly resultDir;
|
|
61
|
+
private readonly limit;
|
|
62
|
+
constructor(from: TestInfoSources);
|
|
63
|
+
get timeout(): number;
|
|
64
|
+
setTimeout(ms: number): void;
|
|
65
|
+
slow(condition?: boolean, description?: string): void;
|
|
66
|
+
fail(condition?: boolean, description?: string): void;
|
|
67
|
+
skip(condition?: boolean, description?: string): void;
|
|
68
|
+
fixme(condition?: boolean, description?: string): void;
|
|
69
|
+
private annotated;
|
|
70
|
+
private note;
|
|
71
|
+
get status(): TestInfo["status"];
|
|
72
|
+
get duration(): number;
|
|
73
|
+
get attachments(): readonly TestAttachment[];
|
|
74
|
+
attach(name: string, options: AttachInput): Promise<void>;
|
|
75
|
+
/** The body (with its `beforeEach` hooks) ended: `error` is what it threw, if anything. Runs before the `afterEach` hooks. */
|
|
76
|
+
bodyDone(error: unknown): void;
|
|
77
|
+
/** The fields a report entry adds for this attempt: nothing when there is nothing to list */
|
|
78
|
+
reportExtras(): InfoReportExtras;
|
|
79
|
+
}
|
|
80
|
+
/** The running test: its info and its fixtures (a condition function of `test.skip` gets them) */
|
|
81
|
+
export interface TestScope {
|
|
82
|
+
info: TestInfo;
|
|
83
|
+
fixtures: TestFixtures;
|
|
84
|
+
}
|
|
85
|
+
/** Runs `fn` (the body and hooks of one test) so that `test.info()` and `test.skip(fn)` find the test */
|
|
86
|
+
export declare function runInTestScope<T>(scope: TestScope, fn: () => T): T;
|
|
87
|
+
/** `test.info()` */
|
|
88
|
+
export declare function currentTestInfo(): TestInfo;
|
|
89
|
+
/** The running test's scope, if a test (or one of its hooks) is running */
|
|
90
|
+
export declare function runningScope(): TestScope | undefined;
|
|
91
|
+
/** The running test's info for `label` (`setTimeout()`); throws when no test is running */
|
|
92
|
+
export declare function infoFor(label: string): TestInfo;
|
|
93
|
+
/** The running test's steps; throws when no test is running */
|
|
94
|
+
export declare function currentStepLog(): StepLog;
|
|
95
|
+
/**
|
|
96
|
+
* Whether a condition of `test.skip`, `test.fail` or `test.slow` holds: a boolean as it is, a function called with
|
|
97
|
+
* the running test's fixtures (it must be synchronous). `label` starts the errors (`test.skip()`).
|
|
98
|
+
*/
|
|
99
|
+
export declare function conditionHolds(label: string, condition: boolean | ((fixtures: TestFixtures) => boolean)): boolean;
|
|
100
|
+
/**
|
|
101
|
+
* `test.skip(condition, reason)` and `test.fixme(...)` from inside a test or its hooks: throws the skip signal
|
|
102
|
+
* when `condition` is true, or when it is a function and returns something truthy.
|
|
103
|
+
*/
|
|
104
|
+
export declare function skipFromCondition(annotation: "skip" | "fixme", condition: boolean | ((fixtures: TestFixtures) => boolean), reason?: string): void;
|