@itmetsam/samurai-framework 0.12.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -29
- package/dist/api.d.ts +2 -2
- package/dist/api.js +1 -1
- package/dist/cli/console-reporter.js +6 -2
- package/dist/cli/main.js +1 -1
- package/dist/config/config.js +3 -0
- package/dist/config/forbid-only.d.ts +5 -0
- package/dist/config/forbid-only.js +9 -0
- package/dist/flows/run/project.js +22 -9
- package/dist/logger/index.d.ts +6 -0
- package/dist/logger/index.js +47 -6
- package/dist/runner/all-hooks.d.ts +26 -0
- package/dist/runner/all-hooks.js +61 -0
- package/dist/runner/focus.d.ts +5 -0
- package/dist/runner/focus.js +19 -0
- package/dist/runner/hooks.d.ts +24 -0
- package/dist/runner/hooks.js +79 -0
- package/dist/runner/pooled-run.d.ts +7 -1
- package/dist/runner/pooled-run.js +38 -10
- package/dist/runner/project.js +5 -0
- package/dist/runner/registry.d.ts +35 -4
- package/dist/runner/registry.js +66 -11
- package/dist/runner/reporter.d.ts +8 -0
- package/dist/runner/reporter.js +22 -1
- package/dist/runner/run-test.d.ts +6 -1
- package/dist/runner/run-test.js +27 -1
- package/dist/runner/run.d.ts +4 -2
- package/dist/runner/run.js +5 -1
- package/dist/runner/skip.d.ts +10 -0
- package/dist/runner/skip.js +18 -0
- package/dist/runner/suite.d.ts +24 -0
- package/dist/runner/suite.js +73 -0
- package/dist/runner/test-error.js +4 -0
- package/dist/runner/test-runner.js +43 -18
- package/dist/runner/worker-pool.d.ts +8 -1
- package/dist/runner/worker-pool.js +34 -2
- package/dist/runner/worker-session.js +11 -1
- package/dist/steps/parse.js +16 -0
- package/dist/types/config.d.ts +8 -1
- package/dist/types/test.d.ts +43 -1
- package/package.json +4 -4
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
import { idOf } from "./select.js";
|
|
2
|
+
import { staticSkip } from "./skip.js";
|
|
2
3
|
/**
|
|
3
4
|
* Lists the run: every selected test in the first browser, then every selected test in the next one. The
|
|
4
5
|
* position in the list is the item's `index`. `serialKey` says which group makes a test run alone.
|
|
5
6
|
*/
|
|
6
7
|
export function planItems(selected, browsers, serialKey) {
|
|
7
8
|
const planned = browsers.flatMap((browser) => selected.map((test) => ({ test, browser })));
|
|
8
|
-
|
|
9
|
+
// A test that does not run (skip, fixme) is in the plan but not in the queue; `index` is its place in the plan
|
|
10
|
+
const items = planned.flatMap(({ test, browser }, index) => {
|
|
11
|
+
if (staticSkip(test))
|
|
12
|
+
return [];
|
|
9
13
|
const key = serialKey(test);
|
|
10
|
-
return
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
return [
|
|
15
|
+
{
|
|
16
|
+
index,
|
|
17
|
+
file: test.file,
|
|
18
|
+
testName: test.name,
|
|
19
|
+
browser,
|
|
20
|
+
...(key !== undefined && { serialKey: key }),
|
|
21
|
+
},
|
|
22
|
+
];
|
|
17
23
|
});
|
|
18
24
|
return { items, planned };
|
|
19
25
|
}
|
|
@@ -32,6 +38,21 @@ export function serialKeyOf(groups, configGroups, srcDir) {
|
|
|
32
38
|
});
|
|
33
39
|
return (test) => keys.get(idOf(test, srcDir));
|
|
34
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* How many items can run at the same time: each item alone counts one, and the items of one serial key together
|
|
43
|
+
* count one. More workers than this would only sit idle.
|
|
44
|
+
*/
|
|
45
|
+
export function concurrency(items) {
|
|
46
|
+
const keys = new Set();
|
|
47
|
+
let free = 0;
|
|
48
|
+
for (const item of items) {
|
|
49
|
+
if (item.serialKey === undefined)
|
|
50
|
+
free++;
|
|
51
|
+
else
|
|
52
|
+
keys.add(item.serialKey);
|
|
53
|
+
}
|
|
54
|
+
return free + keys.size;
|
|
55
|
+
}
|
|
35
56
|
/**
|
|
36
57
|
* How a run goes: through the pool, in this process, or in this process because the workers could not read the
|
|
37
58
|
* config (it was given as an object; a worker reads `samurai.config.ts` from the project folder)
|
|
@@ -42,11 +63,17 @@ export function poolMode(input) {
|
|
|
42
63
|
return input.inlineConfig ? "inline-config" : "pool";
|
|
43
64
|
}
|
|
44
65
|
/**
|
|
45
|
-
* Runs the planned items on the pool and reports them as they happen.
|
|
66
|
+
* Runs the planned items on the pool and reports them as they happen. Tests that do not run (skip, fixme) are
|
|
67
|
+
* reported first, here, and never reach a worker. An item the pool answers without events
|
|
46
68
|
* (a queued one cut short by an abort, or one whose worker died before it began) is reported here at the end.
|
|
47
69
|
*/
|
|
48
70
|
export async function runPooled(pool, planned, items, reporter, signal) {
|
|
49
71
|
reporter.setOrder(planned);
|
|
72
|
+
for (const { test, browser } of planned) {
|
|
73
|
+
const annotation = staticSkip(test);
|
|
74
|
+
if (annotation)
|
|
75
|
+
reporter.onTestSkipped(test, browser, { annotation });
|
|
76
|
+
}
|
|
50
77
|
const started = new Set();
|
|
51
78
|
const ended = new Set();
|
|
52
79
|
const begin = (index) => {
|
|
@@ -68,7 +95,8 @@ export async function runPooled(pool, planned, items, reporter, signal) {
|
|
|
68
95
|
? begin(event.item.index)
|
|
69
96
|
: end(event.item.index, event.result),
|
|
70
97
|
});
|
|
71
|
-
results.forEach((result,
|
|
98
|
+
results.forEach((result, position) => {
|
|
99
|
+
const index = items[position].index;
|
|
72
100
|
if (!ended.has(index))
|
|
73
101
|
end(index, result);
|
|
74
102
|
});
|
package/dist/runner/project.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import path from "node:path";
|
|
1
2
|
import { removeGeneratedFiles } from "../browser/generated-files.js";
|
|
2
3
|
import { loadConfig, projectDir, setActiveProject } from "../config/config.js";
|
|
3
4
|
import { resetSecrets } from "../config/mask.js";
|
|
4
5
|
import { setRunSettings } from "../config/run-settings.js";
|
|
6
|
+
import { keepLogFiles } from "../logger/index.js";
|
|
5
7
|
import { prepareRun } from "./prepare-run.js";
|
|
6
8
|
let active = false;
|
|
7
9
|
/**
|
|
@@ -14,6 +16,8 @@ export async function withProject(options, work) {
|
|
|
14
16
|
active = true;
|
|
15
17
|
const { projectDir: dir, config, dataDir, environment, timeout, expectTimeout, headless, } = options;
|
|
16
18
|
const previousDir = projectDir();
|
|
19
|
+
// The log goes to the project's `logs/` folder, not to wherever this process was started
|
|
20
|
+
const releaseLogs = keepLogFiles(path.resolve(dir ?? previousDir));
|
|
17
21
|
try {
|
|
18
22
|
setActiveProject({ dir: dir ?? previousDir, config, dataDir });
|
|
19
23
|
const loaded = config ?? (await loadConfig());
|
|
@@ -32,5 +36,6 @@ export async function withProject(options, work) {
|
|
|
32
36
|
setRunSettings(undefined);
|
|
33
37
|
setActiveProject(undefined);
|
|
34
38
|
active = false;
|
|
39
|
+
releaseLogs();
|
|
35
40
|
}
|
|
36
41
|
}
|
|
@@ -1,7 +1,38 @@
|
|
|
1
|
-
import type { RegisteredTestCase, TestCaseBase } from "../types/test.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
export
|
|
1
|
+
import type { AllHook, EachHook, RegisteredTestCase, TestCaseBase } from "../types/test.js";
|
|
2
|
+
type Body = TestCaseBase["function"];
|
|
3
|
+
/** `test(title, fn)`, `test.skip(title, fn)`, `test.skip(condition, reason)` and the other forms */
|
|
4
|
+
export interface TestFunction {
|
|
5
|
+
(title: string, fn: Body): void;
|
|
6
|
+
/** Registers a test that is not run */
|
|
7
|
+
skip: SkipFunction;
|
|
8
|
+
/** Registers a test that is not run because it is known to be broken */
|
|
9
|
+
fixme: SkipFunction;
|
|
10
|
+
/** Runs only this test (and the other `only` ones) */
|
|
11
|
+
only(title: string, fn: Body): void;
|
|
12
|
+
}
|
|
13
|
+
/** With a title and a body it registers a skipped test; with a condition it skips the running test when the condition is true */
|
|
14
|
+
interface SkipFunction {
|
|
15
|
+
(title: string, fn: Body): void;
|
|
16
|
+
(condition: boolean, reason?: string): void;
|
|
17
|
+
}
|
|
18
|
+
/** `describe(title, fn)` and the same with a modifier on every test inside */
|
|
19
|
+
export interface DescribeFunction {
|
|
20
|
+
(title: string, fn: () => void): void;
|
|
21
|
+
skip(title: string, fn: () => void): void;
|
|
22
|
+
fixme(title: string, fn: () => void): void;
|
|
23
|
+
only(title: string, fn: () => void): void;
|
|
24
|
+
}
|
|
25
|
+
export declare const test: TestFunction;
|
|
26
|
+
export declare const describe: DescribeFunction;
|
|
27
|
+
/** Runs before each test of the `describe` block it is written in (of the file, at the top level) */
|
|
28
|
+
export declare function beforeEach(hook: EachHook): void;
|
|
29
|
+
/** Runs after each test of the `describe` block it is written in, also when the test failed */
|
|
30
|
+
export declare function afterEach(hook: EachHook): void;
|
|
31
|
+
/** Runs once per worker, before the first test of its `describe` block that the worker runs. It gets no page. */
|
|
32
|
+
export declare function beforeAll(hook: AllHook): void;
|
|
33
|
+
/** Runs once per worker, when the worker is done, for every `describe` block it ran a `beforeAll` for */
|
|
34
|
+
export declare function afterAll(hook: AllHook): void;
|
|
5
35
|
export declare function registeredTests(): readonly RegisteredTestCase[];
|
|
6
36
|
/** Forgets every registered test: a run registers its specs afresh */
|
|
7
37
|
export declare function clearRegistry(): void;
|
|
38
|
+
export {};
|
package/dist/runner/registry.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { fileURLToPath } from "node:url";
|
|
2
2
|
import logger from "../logger/index.js";
|
|
3
|
+
import { SkipSignal } from "./skip.js";
|
|
4
|
+
import { addHook, currentSuite, enterSuite, leaveSuite, modifierOf, resetSuites, } from "./suite.js";
|
|
3
5
|
const registered = [];
|
|
4
|
-
/**
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
export function describe(title, fn) {
|
|
8
|
-
titleStack.push(title);
|
|
6
|
+
/** Runs `fn` as the `describe` block `title`, under `modifier` when there is one. `fn` runs immediately and must be synchronous. */
|
|
7
|
+
function openDescribe(file, title, fn, modifier) {
|
|
8
|
+
enterSuite(file, title, modifier);
|
|
9
9
|
try {
|
|
10
10
|
const result = fn();
|
|
11
11
|
if (result instanceof Promise) {
|
|
@@ -14,24 +14,79 @@ export function describe(title, fn) {
|
|
|
14
14
|
}
|
|
15
15
|
}
|
|
16
16
|
finally {
|
|
17
|
-
|
|
17
|
+
leaveSuite();
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
|
-
|
|
21
|
-
const
|
|
20
|
+
function add(file, title, fn, modifier) {
|
|
21
|
+
const suite = currentSuite(file);
|
|
22
|
+
const titlePath = [...titlesOf(suite), title];
|
|
22
23
|
const name = titlePath.join(" > ");
|
|
23
|
-
|
|
24
|
+
const effective = modifierOf(modifier, suite);
|
|
25
|
+
registered.push({
|
|
26
|
+
name,
|
|
27
|
+
titlePath,
|
|
28
|
+
function: fn,
|
|
29
|
+
file,
|
|
30
|
+
suite,
|
|
31
|
+
...(effective && { modifier: effective }),
|
|
32
|
+
});
|
|
24
33
|
logger.debug("Registered test %s", name);
|
|
25
34
|
}
|
|
35
|
+
/** The `describe` titles from the file down to `suite` */
|
|
36
|
+
function titlesOf(suite) {
|
|
37
|
+
const titles = [];
|
|
38
|
+
for (let s = suite; s; s = s.parent)
|
|
39
|
+
if (s.title !== undefined)
|
|
40
|
+
titles.unshift(s.title);
|
|
41
|
+
return titles;
|
|
42
|
+
}
|
|
43
|
+
/** `test.skip` and `test.fixme`: a title and a body register a test with the modifier; a condition skips the running test */
|
|
44
|
+
function skipping(annotation) {
|
|
45
|
+
return ((first, second) => {
|
|
46
|
+
if (typeof first === "string") {
|
|
47
|
+
// Frames: Error, callerFile, this function, the spec
|
|
48
|
+
add(callerFile(), first, second, annotation);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
if (first)
|
|
52
|
+
throw new SkipSignal(annotation, second);
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
export const test = Object.assign((title, fn) => add(callerFile(), title, fn), {
|
|
56
|
+
skip: skipping("skip"),
|
|
57
|
+
fixme: skipping("fixme"),
|
|
58
|
+
only: (title, fn) => add(callerFile(), title, fn, "only"),
|
|
59
|
+
});
|
|
60
|
+
export const describe = Object.assign((title, fn) => openDescribe(callerFile(), title, fn), {
|
|
61
|
+
skip: (title, fn) => openDescribe(callerFile(), title, fn, "skip"),
|
|
62
|
+
fixme: (title, fn) => openDescribe(callerFile(), title, fn, "fixme"),
|
|
63
|
+
only: (title, fn) => openDescribe(callerFile(), title, fn, "only"),
|
|
64
|
+
});
|
|
65
|
+
/** Runs before each test of the `describe` block it is written in (of the file, at the top level) */
|
|
66
|
+
export function beforeEach(hook) {
|
|
67
|
+
addHook(currentSuite(callerFile()), "beforeEach", hook);
|
|
68
|
+
}
|
|
69
|
+
/** Runs after each test of the `describe` block it is written in, also when the test failed */
|
|
70
|
+
export function afterEach(hook) {
|
|
71
|
+
addHook(currentSuite(callerFile()), "afterEach", hook);
|
|
72
|
+
}
|
|
73
|
+
/** Runs once per worker, before the first test of its `describe` block that the worker runs. It gets no page. */
|
|
74
|
+
export function beforeAll(hook) {
|
|
75
|
+
addHook(currentSuite(callerFile()), "beforeAll", hook);
|
|
76
|
+
}
|
|
77
|
+
/** Runs once per worker, when the worker is done, for every `describe` block it ran a `beforeAll` for */
|
|
78
|
+
export function afterAll(hook) {
|
|
79
|
+
addHook(currentSuite(callerFile()), "afterAll", hook);
|
|
80
|
+
}
|
|
26
81
|
export function registeredTests() {
|
|
27
82
|
return registered;
|
|
28
83
|
}
|
|
29
84
|
/** Forgets every registered test: a run registers its specs afresh */
|
|
30
85
|
export function clearRegistry() {
|
|
31
86
|
registered.length = 0;
|
|
32
|
-
|
|
87
|
+
resetSuites();
|
|
33
88
|
}
|
|
34
|
-
/** File of the code that called
|
|
89
|
+
/** File of the code that called the entry point: stack lines are Error, callerFile, the entry point, the caller */
|
|
35
90
|
function callerFile() {
|
|
36
91
|
const caller = new Error().stack?.split("\n")[3] ?? "";
|
|
37
92
|
const match = caller.match(/\((.+?):\d+:\d+\)$/) ?? caller.match(/at (.+?):\d+:\d+$/);
|
|
@@ -38,6 +38,13 @@ export interface ReporterOptions {
|
|
|
38
38
|
export interface TestSink {
|
|
39
39
|
onTestStart(test: RegisteredTestCase, browser: SupportedBrowser): void;
|
|
40
40
|
onTestEnd(test: RegisteredTestCase, browser: SupportedBrowser, error?: TestError, extras?: TestEndExtras): void;
|
|
41
|
+
/** A test that does not run: reported as started and ended at once */
|
|
42
|
+
onTestSkipped(test: RegisteredTestCase, browser: SupportedBrowser, skip: TestSkip): void;
|
|
43
|
+
}
|
|
44
|
+
/** Why a test was skipped */
|
|
45
|
+
export interface TestSkip {
|
|
46
|
+
annotation: "skip" | "fixme";
|
|
47
|
+
reason?: string | undefined;
|
|
41
48
|
}
|
|
42
49
|
/** What a test's report entry lists besides its status */
|
|
43
50
|
export interface TestEndExtras {
|
|
@@ -67,6 +74,7 @@ export default class TestReporter implements TestSink {
|
|
|
67
74
|
onStart(total: number, workers?: number): void;
|
|
68
75
|
onEnd(): Promise<TestSummary>;
|
|
69
76
|
onTestStart(test: RegisteredTestCase, browser: SupportedBrowser): void;
|
|
77
|
+
onTestSkipped(test: RegisteredTestCase, browser: SupportedBrowser, { annotation, reason }: TestSkip): void;
|
|
70
78
|
/**
|
|
71
79
|
* Ends a test that ran in a worker: the worker built the result (masked, with its own duration), this keeps the
|
|
72
80
|
* start time from this process's clock, as the worker's clock has another origin
|
package/dist/runner/reporter.js
CHANGED
|
@@ -62,7 +62,7 @@ export default class TestReporter {
|
|
|
62
62
|
environment: this.environment,
|
|
63
63
|
browsers: this.browsers,
|
|
64
64
|
tests,
|
|
65
|
-
status: tests.every((r) => r.status === "success")
|
|
65
|
+
status: tests.every((r) => r.status === "success" || r.status === "skipped")
|
|
66
66
|
? "success"
|
|
67
67
|
: "failed",
|
|
68
68
|
};
|
|
@@ -92,6 +92,27 @@ export default class TestReporter {
|
|
|
92
92
|
browser,
|
|
93
93
|
});
|
|
94
94
|
}
|
|
95
|
+
onTestSkipped(test, browser, { annotation, reason }) {
|
|
96
|
+
const key = keyOf(test, browser);
|
|
97
|
+
// A test skipped from inside already started; one that never ran starts and ends at once
|
|
98
|
+
if (this.testResults.get(key)?.status !== "started")
|
|
99
|
+
this.onTestStart(test, browser);
|
|
100
|
+
const result = maskDeep({
|
|
101
|
+
...this.testResults.get(key),
|
|
102
|
+
status: "skipped",
|
|
103
|
+
duration: 0,
|
|
104
|
+
annotation,
|
|
105
|
+
...(reason !== undefined && { reason }),
|
|
106
|
+
});
|
|
107
|
+
this.testResults.set(key, result);
|
|
108
|
+
this.emit({
|
|
109
|
+
type: "test-end",
|
|
110
|
+
name: test.name,
|
|
111
|
+
file: test.file,
|
|
112
|
+
browser,
|
|
113
|
+
result,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
95
116
|
/**
|
|
96
117
|
* Ends a test that ran in a worker: the worker built the result (masked, with its own duration), this keeps the
|
|
97
118
|
* start time from this process's clock, as the worker's clock has another origin
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { SupportedBrowser } from "../types/browser.js";
|
|
2
|
-
import type { RegisteredTestCase } from "../types/test.js";
|
|
2
|
+
import type { AllHookFixtures, RegisteredTestCase } from "../types/test.js";
|
|
3
3
|
import type { RunnerOptions } from "./options.js";
|
|
4
|
+
import type { AllHooks } from "./all-hooks.js";
|
|
4
5
|
import type { PreparedRun } from "./prepare-run.js";
|
|
5
6
|
import type { TestSink } from "./reporter.js";
|
|
6
7
|
/** What a worker gives its tests that the main process does not: its own port and profile, and its number */
|
|
@@ -11,6 +12,10 @@ export interface WorkerResources {
|
|
|
11
12
|
profileDir?: string;
|
|
12
13
|
/** The `workerIndex` fixture. @default 0 */
|
|
13
14
|
workerIndex?: number;
|
|
15
|
+
/** The `beforeAll` / `afterAll` hooks of this worker: kept across tests, closed when the worker is done */
|
|
16
|
+
allHooks?: AllHooks;
|
|
14
17
|
}
|
|
18
|
+
/** What `beforeAll` and `afterAll` hooks get: no page, only what a run knows */
|
|
19
|
+
export declare function allHookFixtures(run: PreparedRun, browserName: SupportedBrowser, workerIndex?: number): AllHookFixtures;
|
|
15
20
|
/** Runs one test in one browser and reports it: launch, run the body, judge the page's logs, close the browser */
|
|
16
21
|
export declare function runOneTest(test: RegisteredTestCase, browserName: SupportedBrowser, run: PreparedRun, options: RunnerOptions, srcDir: string, reporter: TestSink, resources?: WorkerResources): Promise<void>;
|
package/dist/runner/run-test.js
CHANGED
|
@@ -9,6 +9,8 @@ import logger from "../logger/index.js";
|
|
|
9
9
|
import { ConnectionClosedError } from "../transport/bidi-connection.js";
|
|
10
10
|
import { judgePageLogs } from "./page-logs-report.js";
|
|
11
11
|
import { DEFAULT_PORT } from "./ports.js";
|
|
12
|
+
import { runWithEachHooks } from "./hooks.js";
|
|
13
|
+
import { isSkipSignal } from "./skip.js";
|
|
12
14
|
import { testDirOf } from "./test-dir.js";
|
|
13
15
|
import { toTestError } from "./test-error.js";
|
|
14
16
|
/**
|
|
@@ -30,6 +32,16 @@ class TestOutcome {
|
|
|
30
32
|
get isEnded() {
|
|
31
33
|
return this.ended;
|
|
32
34
|
}
|
|
35
|
+
/** The test skipped itself (`test.skip(condition)`): reported as skipped, not as a failure */
|
|
36
|
+
skip(signal) {
|
|
37
|
+
if (this.ended)
|
|
38
|
+
return;
|
|
39
|
+
this.ended = true;
|
|
40
|
+
this.reporter.onTestSkipped(this.test, this.browser, {
|
|
41
|
+
annotation: signal.annotation,
|
|
42
|
+
reason: signal.reason,
|
|
43
|
+
});
|
|
44
|
+
}
|
|
33
45
|
finish(error, logs) {
|
|
34
46
|
if (this.ended)
|
|
35
47
|
return;
|
|
@@ -40,6 +52,16 @@ class TestOutcome {
|
|
|
40
52
|
});
|
|
41
53
|
}
|
|
42
54
|
}
|
|
55
|
+
/** What `beforeAll` and `afterAll` hooks get: no page, only what a run knows */
|
|
56
|
+
export function allHookFixtures(run, browserName, workerIndex = 0) {
|
|
57
|
+
const { settings, secrets } = run;
|
|
58
|
+
return {
|
|
59
|
+
browserName,
|
|
60
|
+
workerIndex,
|
|
61
|
+
env: createEnvFixture(settings.environment, settings.variables),
|
|
62
|
+
secrets: createSecretsFixture(settings.environment, secrets),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
43
65
|
/** Runs one test in one browser and reports it: launch, run the body, judge the page's logs, close the browser */
|
|
44
66
|
export async function runOneTest(test, browserName, run, options, srcDir, reporter, resources = {}) {
|
|
45
67
|
logger.verbose("Starting test: %s in %s", test.name, browserName);
|
|
@@ -81,6 +103,8 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
|
|
|
81
103
|
/** Launches the browser, runs the test body and reports how it went */
|
|
82
104
|
const runBody = async () => {
|
|
83
105
|
try {
|
|
106
|
+
// The `beforeAll` hooks come first, before a browser exists
|
|
107
|
+
await resources.allHooks?.ensure(test, browserName);
|
|
84
108
|
const launched = await Browser.launch(browserName, {
|
|
85
109
|
port: resources.port ?? options.port ?? DEFAULT_PORT,
|
|
86
110
|
...(resources.profileDir && { profileDir: resources.profileDir }),
|
|
@@ -111,7 +135,7 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
|
|
|
111
135
|
env: createEnvFixture(settings.environment, settings.variables),
|
|
112
136
|
secrets: createSecretsFixture(settings.environment, secrets),
|
|
113
137
|
};
|
|
114
|
-
await test.function(fixtures);
|
|
138
|
+
await runWithEachHooks(test.suite, fixtures, timeoutController.signal, () => test.function(fixtures));
|
|
115
139
|
// Already reported as timed out
|
|
116
140
|
if (timeoutController.signal.aborted)
|
|
117
141
|
return;
|
|
@@ -131,6 +155,8 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
|
|
|
131
155
|
catch (err) {
|
|
132
156
|
if (timeoutController.signal.aborted)
|
|
133
157
|
return;
|
|
158
|
+
if (isSkipSignal(err))
|
|
159
|
+
return outcome.skip(err);
|
|
134
160
|
outcome.finish(outcome.browserClosed || err instanceof ConnectionClosedError
|
|
135
161
|
? {
|
|
136
162
|
message: "The browser was closed while the test was running",
|
package/dist/runner/run.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { TestSummary } from "../types/test.js";
|
|
1
|
+
import type { Modifier, TestSummary } from "../types/test.js";
|
|
2
2
|
import { type ProjectOptions } from "./project.js";
|
|
3
3
|
import type { ResolvedGroup } from "./groups.js";
|
|
4
4
|
import type { RunnerOptions } from "./options.js";
|
|
@@ -19,6 +19,8 @@ export interface FoundTest {
|
|
|
19
19
|
/** Full name: `describe` titles and the test title joined with " > " */
|
|
20
20
|
name: string;
|
|
21
21
|
file: string;
|
|
22
|
+
/** `skip` or `fixme` (not run), or `only` (focused); absent for a plain test. A modifier on a `describe` counts. */
|
|
23
|
+
modifier?: Modifier;
|
|
22
24
|
}
|
|
23
25
|
/** Imports the project's specs without running anything and lists the tests the options select */
|
|
24
26
|
export declare function listTests(options?: RunTestsOptions): Promise<FoundTest[]>;
|
|
@@ -27,7 +29,7 @@ export declare function listGroups(options?: RunTestsOptions): Promise<ResolvedG
|
|
|
27
29
|
export { BROWSER_NAMES, checkBrowserList } from "../config/browsers.js";
|
|
28
30
|
export { parseWorkers } from "../config/workers.js";
|
|
29
31
|
export type { RunEvent } from "./reporter.js";
|
|
30
|
-
export type { TestError, TestResult, TestSummary } from "../types/test.js";
|
|
32
|
+
export type { Modifier, TestError, TestResult, TestSummary, } from "../types/test.js";
|
|
31
33
|
export type { ResolvedGroup } from "./groups.js";
|
|
32
34
|
export type { RunnerOptions } from "./options.js";
|
|
33
35
|
export type { ProjectOptions } from "./project.js";
|
package/dist/runner/run.js
CHANGED
|
@@ -35,7 +35,11 @@ export async function runTests(options = {}) {
|
|
|
35
35
|
}
|
|
36
36
|
/** Imports the project's specs without running anything and lists the tests the options select */
|
|
37
37
|
export async function listTests(options = {}) {
|
|
38
|
-
return inProject(options, async (runner) => (await runner.register()).map(({ name, file }) => ({
|
|
38
|
+
return inProject(options, async (runner) => (await runner.register()).map(({ name, file, modifier }) => ({
|
|
39
|
+
name,
|
|
40
|
+
file,
|
|
41
|
+
...(modifier && { modifier }),
|
|
42
|
+
})));
|
|
39
43
|
}
|
|
40
44
|
/** Imports the project's specs without running anything and lists the config's groups with the tests each holds */
|
|
41
45
|
export async function listGroups(options = {}) {
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { RegisteredTestCase } from "../types/test.js";
|
|
2
|
+
/** Thrown by `test.skip(condition, reason)` and `test.fixme(condition, reason)` inside a test: the test ends as skipped */
|
|
3
|
+
export declare class SkipSignal extends Error {
|
|
4
|
+
readonly annotation: "skip" | "fixme";
|
|
5
|
+
readonly reason?: string | undefined;
|
|
6
|
+
constructor(annotation: "skip" | "fixme", reason?: string | undefined);
|
|
7
|
+
}
|
|
8
|
+
export declare function isSkipSignal(error: unknown): error is SkipSignal;
|
|
9
|
+
/** Why a test does not run at all: it is marked `skip` or `fixme`, itself or by a `describe` around it */
|
|
10
|
+
export declare function staticSkip(test: Pick<RegisteredTestCase, "modifier">): "skip" | "fixme" | undefined;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** Thrown by `test.skip(condition, reason)` and `test.fixme(condition, reason)` inside a test: the test ends as skipped */
|
|
2
|
+
export class SkipSignal extends Error {
|
|
3
|
+
constructor(annotation, reason) {
|
|
4
|
+
super(reason ?? "skipped");
|
|
5
|
+
this.annotation = annotation;
|
|
6
|
+
this.reason = reason;
|
|
7
|
+
this.name = "SkipSignal";
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
export function isSkipSignal(error) {
|
|
11
|
+
return error instanceof SkipSignal;
|
|
12
|
+
}
|
|
13
|
+
/** Why a test does not run at all: it is marked `skip` or `fixme`, itself or by a `describe` around it */
|
|
14
|
+
export function staticSkip(test) {
|
|
15
|
+
return test.modifier === "skip" || test.modifier === "fixme"
|
|
16
|
+
? test.modifier
|
|
17
|
+
: undefined;
|
|
18
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { EachHook, Modifier, Suite } from "../types/test.js";
|
|
2
|
+
/** The four hook lists a suite can hold */
|
|
3
|
+
export type HookKind = "beforeEach" | "afterEach" | "beforeAll" | "afterAll";
|
|
4
|
+
export declare function newSuite(title?: string, parent?: Suite, modifier?: Modifier): Suite;
|
|
5
|
+
/** The suite a test or hook written now belongs to: the innermost open `describe`, else the file's own */
|
|
6
|
+
export declare function currentSuite(file: string): Suite;
|
|
7
|
+
/** Opens a `describe` block under the current suite; `leaveSuite` closes it */
|
|
8
|
+
export declare function enterSuite(file: string, title: string, modifier?: Modifier): Suite;
|
|
9
|
+
export declare function leaveSuite(): void;
|
|
10
|
+
/** Forgets every suite: a run registers its specs afresh */
|
|
11
|
+
export declare function resetSuites(): void;
|
|
12
|
+
/**
|
|
13
|
+
* What applies to a test: the innermost `skip` or `fixme` (the test's own, then its suites', inside out), else
|
|
14
|
+
* `only` when the test or any suite around it is focused. A skip beats a focus.
|
|
15
|
+
*/
|
|
16
|
+
export declare function modifierOf(own: Modifier | undefined, suite: Suite): Modifier | undefined;
|
|
17
|
+
/** The suites from the file's root down to `suite` */
|
|
18
|
+
export declare function suiteChain(suite: Suite): Suite[];
|
|
19
|
+
/** The `beforeEach` hooks of a test: outermost suite first, in the order they were written */
|
|
20
|
+
export declare function beforeEachOf(suite: Suite): EachHook[];
|
|
21
|
+
/** The `afterEach` hooks of a test: innermost suite first, in the order they were written */
|
|
22
|
+
export declare function afterEachOf(suite: Suite): EachHook[];
|
|
23
|
+
/** Adds a hook to the suite being registered */
|
|
24
|
+
export declare function addHook<K extends HookKind>(suite: Suite, kind: K, hook: NonNullable<Suite[K]>[number]): void;
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/** One root suite per spec file; a `describe` block is a child of the suite it is written in */
|
|
2
|
+
const roots = new Map();
|
|
3
|
+
/** The `describe` blocks being registered, outermost first */
|
|
4
|
+
const open = [];
|
|
5
|
+
export function newSuite(title, parent, modifier) {
|
|
6
|
+
return {
|
|
7
|
+
...(title !== undefined && { title }),
|
|
8
|
+
...(parent && { parent }),
|
|
9
|
+
...(modifier && { modifier }),
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
/** The suite a test or hook written now belongs to: the innermost open `describe`, else the file's own */
|
|
13
|
+
export function currentSuite(file) {
|
|
14
|
+
const inner = open[open.length - 1];
|
|
15
|
+
if (inner)
|
|
16
|
+
return inner;
|
|
17
|
+
let root = roots.get(file);
|
|
18
|
+
if (!root) {
|
|
19
|
+
root = newSuite();
|
|
20
|
+
roots.set(file, root);
|
|
21
|
+
}
|
|
22
|
+
return root;
|
|
23
|
+
}
|
|
24
|
+
/** Opens a `describe` block under the current suite; `leaveSuite` closes it */
|
|
25
|
+
export function enterSuite(file, title, modifier) {
|
|
26
|
+
const suite = newSuite(title, currentSuite(file), modifier);
|
|
27
|
+
open.push(suite);
|
|
28
|
+
return suite;
|
|
29
|
+
}
|
|
30
|
+
export function leaveSuite() {
|
|
31
|
+
open.pop();
|
|
32
|
+
}
|
|
33
|
+
/** Forgets every suite: a run registers its specs afresh */
|
|
34
|
+
export function resetSuites() {
|
|
35
|
+
roots.clear();
|
|
36
|
+
open.length = 0;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* What applies to a test: the innermost `skip` or `fixme` (the test's own, then its suites', inside out), else
|
|
40
|
+
* `only` when the test or any suite around it is focused. A skip beats a focus.
|
|
41
|
+
*/
|
|
42
|
+
export function modifierOf(own, suite) {
|
|
43
|
+
let focused = own === "only";
|
|
44
|
+
let found = own === "only" ? undefined : own;
|
|
45
|
+
for (let s = suite; s && !found; s = s.parent) {
|
|
46
|
+
if (s.modifier === "only")
|
|
47
|
+
focused = true;
|
|
48
|
+
else
|
|
49
|
+
found = s.modifier;
|
|
50
|
+
}
|
|
51
|
+
return found ?? (focused ? "only" : undefined);
|
|
52
|
+
}
|
|
53
|
+
/** The suites from the file's root down to `suite` */
|
|
54
|
+
export function suiteChain(suite) {
|
|
55
|
+
const chain = [];
|
|
56
|
+
for (let s = suite; s; s = s.parent)
|
|
57
|
+
chain.unshift(s);
|
|
58
|
+
return chain;
|
|
59
|
+
}
|
|
60
|
+
/** The `beforeEach` hooks of a test: outermost suite first, in the order they were written */
|
|
61
|
+
export function beforeEachOf(suite) {
|
|
62
|
+
return suiteChain(suite).flatMap((s) => s.beforeEach ?? []);
|
|
63
|
+
}
|
|
64
|
+
/** The `afterEach` hooks of a test: innermost suite first, in the order they were written */
|
|
65
|
+
export function afterEachOf(suite) {
|
|
66
|
+
return suiteChain(suite)
|
|
67
|
+
.reverse()
|
|
68
|
+
.flatMap((s) => s.afterEach ?? []);
|
|
69
|
+
}
|
|
70
|
+
/** Adds a hook to the suite being registered */
|
|
71
|
+
export function addHook(suite, kind, hook) {
|
|
72
|
+
(suite[kind] ??= []).push(hook);
|
|
73
|
+
}
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { AssertionError, toReportValue } from "../assert/assertion-error.js";
|
|
2
|
+
import { HookError } from "./hooks.js";
|
|
2
3
|
/** Maps anything a test threw to the error stored in the report */
|
|
3
4
|
export function toTestError(err) {
|
|
5
|
+
// A hook's failure keeps what it threw (an assertion stays one) under the hook's own message
|
|
6
|
+
if (err instanceof HookError)
|
|
7
|
+
return { ...toTestError(err.cause), message: err.message };
|
|
4
8
|
if (err instanceof AssertionError) {
|
|
5
9
|
return {
|
|
6
10
|
message: err.message,
|