@itmetsam/samurai-framework 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/browser/chrome-launch.d.ts +12 -0
- package/dist/browser/chrome-launch.js +44 -1
- package/dist/cli/args.d.ts +7 -2
- package/dist/cli/args.js +7 -1
- package/dist/cli/console-reporter.d.ts +4 -0
- package/dist/cli/console-reporter.js +13 -6
- package/dist/cli/init.js +1 -1
- package/dist/cli/main.d.ts +1 -1
- package/dist/cli/main.js +7 -0
- package/dist/config/config.js +5 -1
- package/dist/flows/run/handlers/support.d.ts +4 -1
- package/dist/flows/run/handlers/support.js +1 -1
- package/dist/flows/run/project.js +9 -1
- package/dist/flows/run/run.d.ts +5 -0
- package/dist/flows/run/run.js +7 -1
- package/dist/recorder/session.d.ts +3 -0
- package/dist/recorder/session.js +4 -1
- package/dist/runner/browsers.d.ts +22 -0
- package/dist/runner/browsers.js +44 -0
- package/dist/runner/groups.d.ts +5 -1
- package/dist/runner/groups.js +9 -2
- package/dist/runner/reporter.d.ts +10 -2
- package/dist/runner/reporter.js +30 -10
- package/dist/runner/run.d.ts +1 -0
- package/dist/runner/run.js +1 -0
- package/dist/runner/test-runner.d.ts +9 -1
- package/dist/runner/test-runner.js +46 -19
- package/dist/transport/bidi-connection.d.ts +13 -0
- package/dist/transport/bidi-connection.js +21 -1
- package/dist/types/config.d.ts +5 -3
- package/dist/types/test.d.ts +8 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -33,6 +33,18 @@ export declare const chromeCapabilities: (launch: ChromeLaunch) => {
|
|
|
33
33
|
};
|
|
34
34
|
};
|
|
35
35
|
};
|
|
36
|
+
/** Another Chrome has the profile open: a second one would wait for it until the start timeout */
|
|
37
|
+
export declare class ChromeProfileInUseError extends Error {
|
|
38
|
+
readonly profileDir: string;
|
|
39
|
+
readonly pid: number;
|
|
40
|
+
constructor(profileDir: string, pid: number);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The process of a Chrome that has `profileDir` open, if there is one. Chrome marks a profile it uses with a
|
|
44
|
+
* `SingletonLock` link named `<host>-<pid>` and removes it when it closes; a link left by a process that is gone, or
|
|
45
|
+
* by another computer sharing the folder, does not count. Windows keeps no such link: nothing is found there.
|
|
46
|
+
*/
|
|
47
|
+
export declare function chromeUsingProfile(profileDir: string, host?: string, alive?: (pid: number) => boolean): number | undefined;
|
|
36
48
|
/**
|
|
37
49
|
* Chrome speaks BiDi through its chromedriver: this starts one, asks it for a session with `webSocketUrl`, and
|
|
38
50
|
* connects to the socket it hands back. The session exists when this resolves, so there is no `session.new`.
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import EventEmitter from "node:events";
|
|
3
|
-
import { mkdirSync } from "node:fs";
|
|
3
|
+
import { mkdirSync, readlinkSync } from "node:fs";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
4
6
|
import logger from "../logger/index.js";
|
|
5
7
|
import { BiDiConnector } from "../transport/bidi-connection.js";
|
|
6
8
|
import { SESSION_CAPABILITIES } from "./session-capabilities.js";
|
|
@@ -27,6 +29,44 @@ export const chromeCapabilities = (launch) => ({
|
|
|
27
29
|
},
|
|
28
30
|
},
|
|
29
31
|
});
|
|
32
|
+
/** Another Chrome has the profile open: a second one would wait for it until the start timeout */
|
|
33
|
+
export class ChromeProfileInUseError extends Error {
|
|
34
|
+
constructor(profileDir, pid) {
|
|
35
|
+
super(`Chrome is already running on the profile ${profileDir} (process ${pid}). Close that Chrome, or end the process if a previous run left it behind.`);
|
|
36
|
+
this.profileDir = profileDir;
|
|
37
|
+
this.pid = pid;
|
|
38
|
+
this.name = "ChromeProfileInUseError";
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The process of a Chrome that has `profileDir` open, if there is one. Chrome marks a profile it uses with a
|
|
43
|
+
* `SingletonLock` link named `<host>-<pid>` and removes it when it closes; a link left by a process that is gone, or
|
|
44
|
+
* by another computer sharing the folder, does not count. Windows keeps no such link: nothing is found there.
|
|
45
|
+
*/
|
|
46
|
+
export function chromeUsingProfile(profileDir, host = os.hostname(), alive = isAlive) {
|
|
47
|
+
let target;
|
|
48
|
+
try {
|
|
49
|
+
target = readlinkSync(path.join(profileDir, "SingletonLock"));
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
const match = /^(.*)-(\d+)$/.exec(target);
|
|
55
|
+
if (!match || match[1] !== host)
|
|
56
|
+
return undefined;
|
|
57
|
+
const pid = Number(match[2]);
|
|
58
|
+
return alive(pid) ? pid : undefined;
|
|
59
|
+
}
|
|
60
|
+
function isAlive(pid) {
|
|
61
|
+
try {
|
|
62
|
+
process.kill(pid, 0);
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
// No permission to signal it still means it exists
|
|
67
|
+
return err.code === "EPERM";
|
|
68
|
+
}
|
|
69
|
+
}
|
|
30
70
|
/**
|
|
31
71
|
* Chrome speaks BiDi through its chromedriver: this starts one, asks it for a session with `webSocketUrl`, and
|
|
32
72
|
* connects to the socket it hands back. The session exists when this resolves, so there is no `session.new`.
|
|
@@ -34,6 +74,9 @@ export const chromeCapabilities = (launch) => ({
|
|
|
34
74
|
*/
|
|
35
75
|
export async function launchChrome(launch) {
|
|
36
76
|
mkdirSync(launch.profileDir, { recursive: true });
|
|
77
|
+
const running = chromeUsingProfile(launch.profileDir);
|
|
78
|
+
if (running !== undefined)
|
|
79
|
+
throw new ChromeProfileInUseError(launch.profileDir, running);
|
|
37
80
|
launch.signal?.throwIfAborted();
|
|
38
81
|
const driver = spawn(launch.driverPath, ["--port=0"], {
|
|
39
82
|
stdio: ["ignore", "pipe", "pipe"],
|
package/dist/cli/args.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type ProjectOptions, type RunnerOptions } from "../runner/run.js";
|
|
2
|
+
/** `firefox` or `chrome` */
|
|
3
|
+
type SupportedBrowser = NonNullable<RunnerOptions["browsers"]>[number];
|
|
2
4
|
/** The values a command line can override for one run */
|
|
3
5
|
type RunOverrides = Pick<ProjectOptions, "environment" | "timeout" | "expectTimeout">;
|
|
4
6
|
/** Flags and positional arguments of a command line; an unknown flag is an error */
|
|
@@ -11,6 +13,7 @@ export declare function parseCommandLine(argv: string[]): {
|
|
|
11
13
|
port?: string | undefined;
|
|
12
14
|
grep?: string | undefined;
|
|
13
15
|
group?: string | undefined;
|
|
16
|
+
browser?: string[] | undefined;
|
|
14
17
|
file?: string[] | undefined;
|
|
15
18
|
json?: boolean | undefined;
|
|
16
19
|
all?: boolean | undefined;
|
|
@@ -33,8 +36,10 @@ export interface RunnerFlags {
|
|
|
33
36
|
grep?: string;
|
|
34
37
|
group?: string;
|
|
35
38
|
files?: string[];
|
|
39
|
+
/** `--browser`, repeated for more than one; replaces the config's `browsers` for this run */
|
|
40
|
+
browsers?: SupportedBrowser[];
|
|
36
41
|
}
|
|
37
|
-
/** Reads `--headless`, `--port <n>`, `--grep <text>`, `--group <name>` and `--
|
|
42
|
+
/** Reads `--headless`, `--port <n>`, `--grep <text>`, `--group <name>`, `--file <spec>` and `--browser <name>` (both repeatable) from `argv` */
|
|
38
43
|
export declare function parseRunnerFlags(argv: string[]): RunnerFlags;
|
|
39
44
|
/** Reads `--env`, `--timeout` and `--expect-timeout` from `argv`; the environment falls back to SAMURAI_ENV */
|
|
40
45
|
export declare function parseRunOverrides(argv: string[], env?: Partial<NodeJS.ProcessEnv>): RunOverrides;
|
package/dist/cli/args.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { parseArgs } from "node:util";
|
|
2
|
+
import { checkBrowserList, } from "../runner/run.js";
|
|
2
3
|
const FLAGS = {
|
|
3
4
|
env: { type: "string" },
|
|
4
5
|
timeout: { type: "string" },
|
|
@@ -7,6 +8,7 @@ const FLAGS = {
|
|
|
7
8
|
port: { type: "string" },
|
|
8
9
|
grep: { type: "string" },
|
|
9
10
|
group: { type: "string" },
|
|
11
|
+
browser: { type: "string", multiple: true },
|
|
10
12
|
file: { type: "string", multiple: true },
|
|
11
13
|
json: { type: "boolean" },
|
|
12
14
|
all: { type: "boolean" },
|
|
@@ -33,7 +35,7 @@ export function parseCommandLine(argv) {
|
|
|
33
35
|
function parseFlags(argv) {
|
|
34
36
|
return parseCommandLine(argv).values;
|
|
35
37
|
}
|
|
36
|
-
/** Reads `--headless`, `--port <n>`, `--grep <text>`, `--group <name>` and `--
|
|
38
|
+
/** Reads `--headless`, `--port <n>`, `--grep <text>`, `--group <name>`, `--file <spec>` and `--browser <name>` (both repeatable) from `argv` */
|
|
37
39
|
export function parseRunnerFlags(argv) {
|
|
38
40
|
const values = parseFlags(argv);
|
|
39
41
|
const flags = {};
|
|
@@ -52,6 +54,10 @@ export function parseRunnerFlags(argv) {
|
|
|
52
54
|
flags.group = values.group;
|
|
53
55
|
if (values.file?.length)
|
|
54
56
|
flags.files = values.file;
|
|
57
|
+
if (values.browser?.length) {
|
|
58
|
+
checkBrowserList(values.browser, "--browser");
|
|
59
|
+
flags.browsers = values.browser;
|
|
60
|
+
}
|
|
55
61
|
return flags;
|
|
56
62
|
}
|
|
57
63
|
/** Reads `--env`, `--timeout` and `--expect-timeout` from `argv`; the environment falls back to SAMURAI_ENV */
|
|
@@ -8,8 +8,12 @@ export interface ConsoleReporterOptions {
|
|
|
8
8
|
export declare class ConsoleReporter {
|
|
9
9
|
private readonly write;
|
|
10
10
|
private readonly paint;
|
|
11
|
+
/** Whether the run has more than one browser: only then is each test labelled with its browser */
|
|
12
|
+
private labelled;
|
|
11
13
|
constructor({ write, color }: ConsoleReporterOptions);
|
|
12
14
|
handle(event: RunEvent): void;
|
|
15
|
+
/** ` [chrome]` in a run with more than one browser, else nothing */
|
|
16
|
+
private tag;
|
|
13
17
|
private line;
|
|
14
18
|
private summary;
|
|
15
19
|
}
|
|
@@ -2,16 +2,19 @@ const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`;
|
|
|
2
2
|
/** Prints a run for a person: one line per test as it ends, then every failure in full and a count */
|
|
3
3
|
export class ConsoleReporter {
|
|
4
4
|
constructor({ write, color }) {
|
|
5
|
+
/** Whether the run has more than one browser: only then is each test labelled with its browser */
|
|
6
|
+
this.labelled = false;
|
|
5
7
|
this.write = write;
|
|
6
8
|
this.paint = (code, text) => color ? `\u001b[${code}m${text}\u001b[0m` : text;
|
|
7
9
|
}
|
|
8
10
|
handle(event) {
|
|
9
11
|
switch (event.type) {
|
|
10
12
|
case "run-start":
|
|
11
|
-
this.
|
|
13
|
+
this.labelled = event.browsers.length > 1;
|
|
14
|
+
this.write(`Running ${event.total} ${event.total === 1 ? "test" : "tests"} against ${event.environment}${this.labelled ? ` in ${event.browsers.join(", ")}` : ""}\n`);
|
|
12
15
|
return;
|
|
13
16
|
case "test-end":
|
|
14
|
-
this.write(` ${this.line(event.name, event.result)}\n`);
|
|
17
|
+
this.write(` ${this.line(event.name, event.result, event.browser)}\n`);
|
|
15
18
|
return;
|
|
16
19
|
case "run-end":
|
|
17
20
|
this.summary(event.summary);
|
|
@@ -20,13 +23,17 @@ export class ConsoleReporter {
|
|
|
20
23
|
return;
|
|
21
24
|
}
|
|
22
25
|
}
|
|
23
|
-
|
|
26
|
+
/** ` [chrome]` in a run with more than one browser, else nothing */
|
|
27
|
+
tag(browser) {
|
|
28
|
+
return this.labelled ? ` ${this.paint(90, `[${browser}]`)}` : "";
|
|
29
|
+
}
|
|
30
|
+
line(name, result, browser) {
|
|
24
31
|
const took = "duration" in result
|
|
25
32
|
? ` ${this.paint(90, `(${seconds(result.duration)})`)}`
|
|
26
33
|
: "";
|
|
27
34
|
return result.status === "success"
|
|
28
|
-
? `${this.paint(32, "✔")} ${name}${took}`
|
|
29
|
-
: `${this.paint(31, "✖")} ${name}${took}`;
|
|
35
|
+
? `${this.paint(32, "✔")} ${name}${this.tag(browser)}${took}`
|
|
36
|
+
: `${this.paint(31, "✖")} ${name}${this.tag(browser)}${took}`;
|
|
30
37
|
}
|
|
31
38
|
summary(summary) {
|
|
32
39
|
const failed = summary.tests.filter((test) => test.status !== "success");
|
|
@@ -34,7 +41,7 @@ export class ConsoleReporter {
|
|
|
34
41
|
if (!("name" in test))
|
|
35
42
|
continue;
|
|
36
43
|
const message = "message" in test ? test.message : "did not finish";
|
|
37
|
-
this.write(`\n${this.paint(31, `${test.name}`)}\n`);
|
|
44
|
+
this.write(`\n${this.paint(31, `${test.name}`)}${"browser" in test ? this.tag(test.browser) : ""}\n`);
|
|
38
45
|
this.write(`${message
|
|
39
46
|
.split("\n")
|
|
40
47
|
.map((line) => ` ${line}`)
|
package/dist/cli/init.js
CHANGED
package/dist/cli/main.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export declare const USAGE = "samurai: browser tests over WebDriver BiDi\n\nUsage:\n samurai run [options] Run the tests (the default command)\n samurai list [options] List the tests without running them\n samurai record <spec> Open a browser and record what you do into a spec file\n samurai init [folder] Scaffold a project: config, an example test, .env.example\n samurai flow <command> List, check and run the flows in ./flows (see below)\n samurai browsers [command] List, install and remove the browsers SAMURAI launches\n\nOptions for run and list:\n --env <name> Environment to run against (or SAMURAI_ENV)\n --file <spec> Only this spec file; repeatable\n --grep <text> Only tests whose full name contains the text\n --group <name> Only the tests of this group of the config's groups\n --json Machine-readable output: JSON lines for run, one array for list\n\nOptions for run:\n --timeout <ms> Time one test may take\n --expect-timeout <ms> Time actions and assertions retry\n --headless, --no-headless Run the browser without a window, or with one.\n Default: the config's use.headless, else headless when CI is set\n --port <n> Browser debugging port (default 9223)\n\nOptions for record (also --env, --port, --json, --headless, --no-headless):\n --new <title> Add a test with this title (creates the spec file if needed)\n --test <name|index> Record into this test; not needed when the file has one\n --at <n> Insert from step n (default: after the last step)\n --url <url> Page to start on (default: the environment's baseURL)\n Stop with Ctrl+C or by closing the browser window.\n\n -h, --help Show this help\n -v, --version Show the version\n\nExit code: 0 when every test passed, 1 when a test failed, 2 for a usage or configuration error.\n\nsamurai flow:\n list The flows in ./flows\n check [<flow>...] Check flows without running them\n run <flow>... | --all Run flows; each writes result/flows/<id>.json\n Options for run: --env, --headless, --timeout, --expect-timeout, --port, --json, --allow-unsupported\n Run `samurai flow` alone for the details.\n\nsamurai browsers:\n list The pinned browsers and every build in the cache (the default)\n install [firefox|chrome|all]\n Download the pinned builds (default: all)\n uninstall <name> [--build <id>] | --unused\n\nRun it in the project folder, the one holding samurai.config.ts.\n";
|
|
1
|
+
export declare const USAGE = "samurai: browser tests over WebDriver BiDi\n\nUsage:\n samurai run [options] Run the tests (the default command)\n samurai list [options] List the tests without running them\n samurai record <spec> Open a browser and record what you do into a spec file\n samurai init [folder] Scaffold a project: config, an example test, .env.example\n samurai flow <command> List, check and run the flows in ./flows (see below)\n samurai browsers [command] List, install and remove the browsers SAMURAI launches\n\nOptions for run and list:\n --env <name> Environment to run against (or SAMURAI_ENV)\n --file <spec> Only this spec file; repeatable\n --grep <text> Only tests whose full name contains the text\n --group <name> Only the tests of this group of the config's groups\n --browser <name> Run in this browser (firefox or chrome) instead of the config's\n browsers; repeat it for more than one\n --json Machine-readable output: JSON lines for run, one array for list\n\nOptions for run:\n --timeout <ms> Time one test may take\n --expect-timeout <ms> Time actions and assertions retry\n --headless, --no-headless Run the browser without a window, or with one.\n Default: the config's use.headless, else headless when CI is set\n --port <n> Browser debugging port (default 9223)\n\nOptions for record (also --env, --port, --json, --headless, --no-headless):\n --browser <name> Record in this browser (default: the first of the config's browsers)\n --new <title> Add a test with this title (creates the spec file if needed)\n --test <name|index> Record into this test; not needed when the file has one\n --at <n> Insert from step n (default: after the last step)\n --url <url> Page to start on (default: the environment's baseURL)\n Stop with Ctrl+C or by closing the browser window.\n\n -h, --help Show this help\n -v, --version Show the version\n\nExit code: 0 when every test passed, 1 when a test failed, 2 for a usage or configuration error.\n\nsamurai flow:\n list The flows in ./flows\n check [<flow>...] Check flows without running them\n run <flow>... | --all Run flows; each writes result/flows/<id>.json\n Options for run: --env, --headless, --timeout, --expect-timeout, --port, --json, --allow-unsupported\n Run `samurai flow` alone for the details.\n\nsamurai browsers:\n list The pinned browsers and every build in the cache (the default)\n install [firefox|chrome|all]\n Download the pinned builds (default: all)\n uninstall <name> [--build <id>] | --unused\n\nRun it in the project folder, the one holding samurai.config.ts.\n";
|
|
2
2
|
export interface Io {
|
|
3
3
|
out: (text: string) => void;
|
|
4
4
|
err: (text: string) => void;
|
package/dist/cli/main.js
CHANGED
|
@@ -22,6 +22,8 @@ Options for run and list:
|
|
|
22
22
|
--file <spec> Only this spec file; repeatable
|
|
23
23
|
--grep <text> Only tests whose full name contains the text
|
|
24
24
|
--group <name> Only the tests of this group of the config's groups
|
|
25
|
+
--browser <name> Run in this browser (firefox or chrome) instead of the config's
|
|
26
|
+
browsers; repeat it for more than one
|
|
25
27
|
--json Machine-readable output: JSON lines for run, one array for list
|
|
26
28
|
|
|
27
29
|
Options for run:
|
|
@@ -32,6 +34,7 @@ Options for run:
|
|
|
32
34
|
--port <n> Browser debugging port (default 9223)
|
|
33
35
|
|
|
34
36
|
Options for record (also --env, --port, --json, --headless, --no-headless):
|
|
37
|
+
--browser <name> Record in this browser (default: the first of the config's browsers)
|
|
35
38
|
--new <title> Add a test with this title (creates the spec file if needed)
|
|
36
39
|
--test <name|index> Record into this test; not needed when the file has one
|
|
37
40
|
--at <n> Insert from step n (default: after the last step)
|
|
@@ -99,6 +102,8 @@ export async function main(argv, io) {
|
|
|
99
102
|
if (at !== undefined && (!Number.isInteger(at) || at < 0))
|
|
100
103
|
throw new Error(`--at must be a step number, got "${values.at}"`);
|
|
101
104
|
const flags = parseRunnerFlags(rest);
|
|
105
|
+
if (flags.browsers && flags.browsers.length > 1)
|
|
106
|
+
throw new Error("--browser: a recording uses one browser");
|
|
102
107
|
const test = values.test === undefined
|
|
103
108
|
? undefined
|
|
104
109
|
: /^\d+$/.test(values.test)
|
|
@@ -116,6 +121,7 @@ export async function main(argv, io) {
|
|
|
116
121
|
...(values.url !== undefined && { url: values.url }),
|
|
117
122
|
...(flags.headless !== undefined && { headless: flags.headless }),
|
|
118
123
|
...(flags.port !== undefined && { port: flags.port }),
|
|
124
|
+
...(flags.browsers && { browser: flags.browsers[0] }),
|
|
119
125
|
...(io.signal && { signal: io.signal }),
|
|
120
126
|
onStarted: () => {
|
|
121
127
|
if (!values.json)
|
|
@@ -178,6 +184,7 @@ export async function main(argv, io) {
|
|
|
178
184
|
...(flags.grep !== undefined && { grep: flags.grep }),
|
|
179
185
|
...(flags.group !== undefined && { group: flags.group }),
|
|
180
186
|
...(flags.files && { files: flags.files }),
|
|
187
|
+
...(flags.browsers && { browsers: flags.browsers }),
|
|
181
188
|
};
|
|
182
189
|
if (command === "list") {
|
|
183
190
|
const tests = await listTests(options);
|
package/dist/config/config.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import logger from "../logger/index.js";
|
|
3
3
|
import { pathToFileURL } from "url";
|
|
4
|
+
import { validateBrowserConfig } from "../runner/browsers.js";
|
|
4
5
|
let activeProject;
|
|
5
6
|
/** Makes `project` the one config, secrets, reports and downloads are resolved for; `undefined` ends it */
|
|
6
7
|
export function setActiveProject(project) {
|
|
@@ -22,14 +23,17 @@ export function browsersDir() {
|
|
|
22
23
|
}
|
|
23
24
|
/** The whole `samurai.config.ts` of the project (the working directory unless a project is active) */
|
|
24
25
|
export async function loadConfig() {
|
|
25
|
-
if (activeProject?.config)
|
|
26
|
+
if (activeProject?.config) {
|
|
27
|
+
validateBrowserConfig(activeProject.config);
|
|
26
28
|
return activeProject.config;
|
|
29
|
+
}
|
|
27
30
|
const configUrl = path.join(projectDir(), "samurai.config.ts");
|
|
28
31
|
logger.debug("Reading config from: %s", configUrl);
|
|
29
32
|
const config = (await import(pathToFileURL(configUrl).href));
|
|
30
33
|
if (!config.default) {
|
|
31
34
|
throw new Error("samurai.config.ts does not export defineConfig as default!");
|
|
32
35
|
}
|
|
36
|
+
validateBrowserConfig(config.default);
|
|
33
37
|
return config.default;
|
|
34
38
|
}
|
|
35
39
|
/** A project's config without keeping the project active (`withProject` is one at a time): the given one, else its samurai.config.ts */
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type AnyNodeDefinition, type ExpressionContext, type FlowFile, type FlowNodeModel } from "../../core/index.js";
|
|
2
2
|
import type { RunEvent } from "../../../runner/reporter.js";
|
|
3
|
+
import type { SupportedBrowser } from "../../../types/browser.js";
|
|
3
4
|
import type { NodeResult, NodeTest } from "../events.js";
|
|
4
5
|
/** What a handler can use of the run */
|
|
5
6
|
export interface HandlerRun {
|
|
@@ -10,8 +11,10 @@ export interface HandlerRun {
|
|
|
10
11
|
};
|
|
11
12
|
context: ExpressionContext;
|
|
12
13
|
nodeTests: Record<string, NodeTest[]>;
|
|
14
|
+
/** The browsers of the nodes that have their own (a Group node's group); the others use the project's */
|
|
15
|
+
nodeBrowsers?: Record<string, SupportedBrowser[]>;
|
|
13
16
|
signal: AbortSignal;
|
|
14
|
-
runTests: (tests: NodeTest[], onEvent: (e: RunEvent) => void) => Promise<"passed" | "failed" | "cancelled">;
|
|
17
|
+
runTests: (tests: NodeTest[], onEvent: (e: RunEvent) => void, browsers?: SupportedBrowser[]) => Promise<"passed" | "failed" | "cancelled">;
|
|
15
18
|
sleep: (ms: number, signal: AbortSignal) => Promise<void>;
|
|
16
19
|
onTestEvent: (nodeId: string, e: RunEvent) => void;
|
|
17
20
|
/** Definitions of kinds core doesn't have (steps), for expression scopes */
|
|
@@ -7,7 +7,7 @@ export const fail = (e) => ({
|
|
|
7
7
|
export async function runNodeTests(node, run, list) {
|
|
8
8
|
let result;
|
|
9
9
|
try {
|
|
10
|
-
result = await run.runTests(list, (e) => run.onTestEvent(node.id, e));
|
|
10
|
+
result = await run.runTests(list, (e) => run.onTestEvent(node.id, e), run.nodeBrowsers?.[node.id]);
|
|
11
11
|
}
|
|
12
12
|
catch (e) {
|
|
13
13
|
return fail(e);
|
|
@@ -109,6 +109,7 @@ export async function runFlowInProject(options) {
|
|
|
109
109
|
// Phase 3: walk it
|
|
110
110
|
const byId = new Map(tests.map((t) => [t.id, t]));
|
|
111
111
|
const nodeTests = {};
|
|
112
|
+
const nodeBrowsers = {};
|
|
112
113
|
for (const node of flow.nodes) {
|
|
113
114
|
const ids = node.kind === "test"
|
|
114
115
|
? [node.ref?.testId ?? ""]
|
|
@@ -117,6 +118,11 @@ export async function runFlowInProject(options) {
|
|
|
117
118
|
: undefined;
|
|
118
119
|
if (!ids)
|
|
119
120
|
continue;
|
|
121
|
+
if (node.kind === "group") {
|
|
122
|
+
const browsers = groups.find((g) => g.name === node.ref?.group)?.browsers;
|
|
123
|
+
if (browsers)
|
|
124
|
+
nodeBrowsers[node.id] = browsers;
|
|
125
|
+
}
|
|
120
126
|
nodeTests[node.id] = ids.flatMap((id) => {
|
|
121
127
|
const t = byId.get(id);
|
|
122
128
|
return t ? [{ file: t.file, name: t.name }] : [];
|
|
@@ -126,6 +132,7 @@ export async function runFlowInProject(options) {
|
|
|
126
132
|
return runFlow({
|
|
127
133
|
flow,
|
|
128
134
|
nodeTests,
|
|
135
|
+
nodeBrowsers,
|
|
129
136
|
environment,
|
|
130
137
|
...(options.signal && { signal: options.signal }),
|
|
131
138
|
...(options.onEvent && { onEvent: options.onEvent }),
|
|
@@ -133,7 +140,7 @@ export async function runFlowInProject(options) {
|
|
|
133
140
|
unsupported,
|
|
134
141
|
handlers: { ...HANDLERS, ...steps.handlers },
|
|
135
142
|
nodes: steps.nodes,
|
|
136
|
-
runTests: async ({ tests, onEvent, signal }) => {
|
|
143
|
+
runTests: async ({ tests, onEvent, signal, browsers }) => {
|
|
137
144
|
const summary = await run({
|
|
138
145
|
projectDir,
|
|
139
146
|
config,
|
|
@@ -148,6 +155,7 @@ export async function runFlowInProject(options) {
|
|
|
148
155
|
expectTimeout: options.expectTimeout,
|
|
149
156
|
}),
|
|
150
157
|
...(options.port !== undefined && { port: options.port }),
|
|
158
|
+
...(browsers && { browsers }),
|
|
151
159
|
files: [...new Set(tests.map((t) => t.file))],
|
|
152
160
|
testNames: tests.map((t) => t.name),
|
|
153
161
|
reportPath: false,
|
package/dist/flows/run/run.d.ts
CHANGED
|
@@ -2,10 +2,13 @@ import { type AnyNodeDefinition, type FlowFile } from "../core/index.js";
|
|
|
2
2
|
import type { RunEvent } from "../../runner/reporter.js";
|
|
3
3
|
import type { FlowEvent, FlowSummary, NodeTest } from "./events.js";
|
|
4
4
|
import { type Handler } from "./handlers/index.js";
|
|
5
|
+
import type { SupportedBrowser } from "../../types/browser.js";
|
|
5
6
|
export interface RunFlowOptions {
|
|
6
7
|
flow: FlowFile;
|
|
7
8
|
/** The tests of each Test and Group node, by node id */
|
|
8
9
|
nodeTests: Record<string, NodeTest[]>;
|
|
10
|
+
/** The browsers a Group node runs in (its group's); a node not listed uses the project's */
|
|
11
|
+
nodeBrowsers?: Record<string, SupportedBrowser[]>;
|
|
9
12
|
environment: {
|
|
10
13
|
name: string;
|
|
11
14
|
variables: Record<string, string | number | boolean>;
|
|
@@ -20,6 +23,8 @@ export interface RunFlowOptions {
|
|
|
20
23
|
tests: NodeTest[];
|
|
21
24
|
onEvent: (e: RunEvent) => void;
|
|
22
25
|
signal: AbortSignal;
|
|
26
|
+
/** Set for a node that runs in browsers of its own */
|
|
27
|
+
browsers?: SupportedBrowser[];
|
|
23
28
|
}) => Promise<"passed" | "failed" | "cancelled">;
|
|
24
29
|
sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
|
|
25
30
|
/**
|
package/dist/flows/run/run.js
CHANGED
|
@@ -65,9 +65,15 @@ export async function runFlow(options) {
|
|
|
65
65
|
environment: options.environment,
|
|
66
66
|
context,
|
|
67
67
|
nodeTests: options.nodeTests,
|
|
68
|
+
...(options.nodeBrowsers && { nodeBrowsers: options.nodeBrowsers }),
|
|
68
69
|
signal,
|
|
69
70
|
sleep: options.sleep ?? defaultSleep,
|
|
70
|
-
runTests: (tests, onTestEvent) => options.runTests({
|
|
71
|
+
runTests: (tests, onTestEvent, browsers) => options.runTests({
|
|
72
|
+
tests,
|
|
73
|
+
onEvent: onTestEvent,
|
|
74
|
+
signal,
|
|
75
|
+
...(browsers && { browsers }),
|
|
76
|
+
}),
|
|
71
77
|
onTestEvent: (nodeId, event) => onEvent({ type: "test", nodeId, event }),
|
|
72
78
|
nodes,
|
|
73
79
|
log: (nodeId, level, text) => logger[level](`[${nodeId}] ${text}`),
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Browser } from "../browser/browser.js";
|
|
2
2
|
import type Page from "../browser/page.js";
|
|
3
|
+
import type { SupportedBrowser } from "../types/browser.js";
|
|
3
4
|
import type { PreparedRun } from "../runner/prepare-run.js";
|
|
4
5
|
import { type ProjectOptions } from "../runner/project.js";
|
|
5
6
|
import type { SamuraiTestConfig } from "../types/config.js";
|
|
@@ -20,6 +21,8 @@ export interface RecordSpecOptions extends ProjectOptions {
|
|
|
20
21
|
at?: number;
|
|
21
22
|
/** The page to start on. @default the environment's base URL; without one the browser starts blank */
|
|
22
23
|
url?: string;
|
|
24
|
+
/** The browser to record in. @default the first of the config's `browsers`, else Firefox */
|
|
25
|
+
browser?: SupportedBrowser;
|
|
23
26
|
/** The browser's debugging port. @default 9223 */
|
|
24
27
|
port?: number;
|
|
25
28
|
/** Run the browser without a window. A person can't use it then; for tests and machines without a display. @default the config's `use.headless`, else headless on CI and a window otherwise */
|
package/dist/recorder/session.js
CHANGED
|
@@ -2,6 +2,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { Browser } from "../browser/browser.js";
|
|
4
4
|
import { ensureWarm } from "../browser/warm-up.js";
|
|
5
|
+
import { browsersFor } from "../runner/browsers.js";
|
|
5
6
|
import { loadConfig, projectDir, readConfig } from "../config/config.js";
|
|
6
7
|
import { loadSecrets } from "../config/secrets.js";
|
|
7
8
|
import { filesFolder, isFolderIgnored, isGitIgnored, } from "../project/index.js";
|
|
@@ -101,7 +102,9 @@ export async function recordSpec(options) {
|
|
|
101
102
|
throw new Error(`${options.file} does not exist; give a title with --new to create it`);
|
|
102
103
|
}
|
|
103
104
|
// Before the spec is touched: a recording without its browser changes nothing
|
|
104
|
-
|
|
105
|
+
// Recording in two browsers at once makes no sense: the first of the project's, or the one asked for
|
|
106
|
+
const browserName = options.browser ??
|
|
107
|
+
browsersFor({ project: await readConfig("browsers") })[0];
|
|
105
108
|
await ensureWarm(browserName, { port: options.port ?? 9223 }, options.signal);
|
|
106
109
|
options.signal?.throwIfAborted();
|
|
107
110
|
source ??= "";
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { SupportedBrowser } from "../types/browser.js";
|
|
2
|
+
import type { SamuraiTestConfig } from "../types/config.js";
|
|
3
|
+
/** Every browser a config may name, in the order the messages list them */
|
|
4
|
+
export declare const BROWSER_NAMES: readonly SupportedBrowser[];
|
|
5
|
+
/** The browsers of a run that names none */
|
|
6
|
+
export declare const DEFAULT_BROWSERS: readonly SupportedBrowser[];
|
|
7
|
+
/** Throws when `list` is not a non-empty list of distinct browser names; `where` names the key in the message */
|
|
8
|
+
export declare function checkBrowserList(list: unknown, where: string): void;
|
|
9
|
+
/**
|
|
10
|
+
* Checks the browser keys of a config: the removed `browser`, and `browsers` on the project and on each group.
|
|
11
|
+
* Throws an Error that says what to write instead.
|
|
12
|
+
*/
|
|
13
|
+
export declare function validateBrowserConfig(config: SamuraiTestConfig): void;
|
|
14
|
+
/**
|
|
15
|
+
* The browsers a run uses, the first that is set winning: the command line, the group being run, the project, then
|
|
16
|
+
* Firefox. An empty list counts as not given.
|
|
17
|
+
*/
|
|
18
|
+
export declare function browsersFor(sources: {
|
|
19
|
+
flag?: readonly SupportedBrowser[] | undefined;
|
|
20
|
+
group?: readonly SupportedBrowser[] | undefined;
|
|
21
|
+
project?: readonly SupportedBrowser[] | undefined;
|
|
22
|
+
}): SupportedBrowser[];
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** Every browser a config may name, in the order the messages list them */
|
|
2
|
+
export const BROWSER_NAMES = ["firefox", "chrome"];
|
|
3
|
+
/** The browsers of a run that names none */
|
|
4
|
+
export const DEFAULT_BROWSERS = ["firefox"];
|
|
5
|
+
const isBrowser = (value) => typeof value === "string" &&
|
|
6
|
+
BROWSER_NAMES.includes(value);
|
|
7
|
+
/** Throws when `list` is not a non-empty list of distinct browser names; `where` names the key in the message */
|
|
8
|
+
export function checkBrowserList(list, where) {
|
|
9
|
+
if (!Array.isArray(list) || list.length === 0)
|
|
10
|
+
throw new Error(`${where} must list at least one browser.`);
|
|
11
|
+
const seen = new Set();
|
|
12
|
+
for (const name of list) {
|
|
13
|
+
if (!isBrowser(name))
|
|
14
|
+
throw new Error(`Unknown browser "${String(name)}" in ${where}. Choose ${BROWSER_NAMES.join(" or ")}.`);
|
|
15
|
+
if (seen.has(name))
|
|
16
|
+
throw new Error(`Browser "${name}" is listed twice in ${where}.`);
|
|
17
|
+
seen.add(name);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Checks the browser keys of a config: the removed `browser`, and `browsers` on the project and on each group.
|
|
22
|
+
* Throws an Error that says what to write instead.
|
|
23
|
+
*/
|
|
24
|
+
export function validateBrowserConfig(config) {
|
|
25
|
+
const legacy = config.browser;
|
|
26
|
+
if (legacy !== undefined)
|
|
27
|
+
throw new Error(`"browser" was replaced by "browsers": use browsers: [${JSON.stringify(legacy)}]`);
|
|
28
|
+
if (config.browsers !== undefined)
|
|
29
|
+
checkBrowserList(config.browsers, "browsers");
|
|
30
|
+
config.groups?.forEach((group, index) => {
|
|
31
|
+
if (group.browsers !== undefined)
|
|
32
|
+
checkBrowserList(group.browsers, `groups[${index}].browsers`);
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The browsers a run uses, the first that is set winning: the command line, the group being run, the project, then
|
|
37
|
+
* Firefox. An empty list counts as not given.
|
|
38
|
+
*/
|
|
39
|
+
export function browsersFor(sources) {
|
|
40
|
+
for (const list of [sources.flag, sources.group, sources.project])
|
|
41
|
+
if (list && list.length > 0)
|
|
42
|
+
return [...list];
|
|
43
|
+
return [...DEFAULT_BROWSERS];
|
|
44
|
+
}
|
package/dist/runner/groups.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { SupportedBrowser } from "../types/browser.js";
|
|
1
2
|
import type { SamuraiGroup } from "../types/config.js";
|
|
2
3
|
/** The test folder when the config names none */
|
|
3
4
|
export declare const DEFAULT_SRC_DIR = "./src";
|
|
@@ -22,6 +23,8 @@ export interface ResolvedGroup {
|
|
|
22
23
|
file: string;
|
|
23
24
|
title: string;
|
|
24
25
|
}[];
|
|
26
|
+
/** The browsers the group's tests run in: its own `browsers`, else the project's, else Firefox */
|
|
27
|
+
browsers: SupportedBrowser[];
|
|
25
28
|
}
|
|
26
29
|
/**
|
|
27
30
|
* The real path of `p` (symlinks resolved), or `p` itself when it does not exist. Registered test files come
|
|
@@ -33,10 +36,11 @@ export declare const toPosix: (p: string) => string;
|
|
|
33
36
|
/** The id of a test: its spec file relative to `srcDir` with `/` separators, then `::` and its full name */
|
|
34
37
|
export declare const testId: (file: string, name: string) => string;
|
|
35
38
|
/** Works out which of `tests` (files relative to `srcDir`) belong to the group */
|
|
36
|
-
export declare function resolveGroup(group: SamuraiGroup, tests: GroupTest[], projectDir: string, srcDir: string): ResolvedGroup;
|
|
39
|
+
export declare function resolveGroup(group: SamuraiGroup, tests: GroupTest[], projectDir: string, srcDir: string, projectBrowsers?: readonly SupportedBrowser[]): ResolvedGroup;
|
|
37
40
|
/** Resolves every group of the config */
|
|
38
41
|
export declare function resolveGroups(config: {
|
|
39
42
|
groups?: SamuraiGroup[];
|
|
43
|
+
browsers?: SupportedBrowser[];
|
|
40
44
|
}, tests: GroupTest[], projectDir: string, srcDir: string): ResolvedGroup[];
|
|
41
45
|
/** The group called `name`, or an error naming the groups there are */
|
|
42
46
|
export declare function findGroup<T extends {
|
package/dist/runner/groups.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { realpathSync } from "node:fs";
|
|
2
2
|
import { matchesGlob, relative, resolve } from "node:path";
|
|
3
|
+
import { browsersFor } from "./browsers.js";
|
|
3
4
|
/** The test folder when the config names none */
|
|
4
5
|
export const DEFAULT_SRC_DIR = "./src";
|
|
5
6
|
/** The spec pattern when a group names none */
|
|
@@ -21,7 +22,11 @@ export const toPosix = (p) => p.replace(/\\/g, "/");
|
|
|
21
22
|
/** The id of a test: its spec file relative to `srcDir` with `/` separators, then `::` and its full name */
|
|
22
23
|
export const testId = (file, name) => `${toPosix(file)}::${name}`;
|
|
23
24
|
/** Works out which of `tests` (files relative to `srcDir`) belong to the group */
|
|
24
|
-
export function resolveGroup(group, tests, projectDir, srcDir) {
|
|
25
|
+
export function resolveGroup(group, tests, projectDir, srcDir, projectBrowsers) {
|
|
26
|
+
const browsers = browsersFor({
|
|
27
|
+
group: group.browsers,
|
|
28
|
+
project: projectBrowsers,
|
|
29
|
+
});
|
|
25
30
|
const found = tests.map((t) => ({ ...t, id: testId(t.file, t.name) }));
|
|
26
31
|
// Tests picked by hand override src and testMatch
|
|
27
32
|
if (group.tests) {
|
|
@@ -38,6 +43,7 @@ export function resolveGroup(group, tests, projectDir, srcDir) {
|
|
|
38
43
|
missing: picked
|
|
39
44
|
.filter((t) => !known.has(t.id))
|
|
40
45
|
.map(({ file, title }) => ({ file, title })),
|
|
46
|
+
browsers,
|
|
41
47
|
};
|
|
42
48
|
}
|
|
43
49
|
const base = group.src ? resolve(projectDir, group.src) : srcDir;
|
|
@@ -54,11 +60,12 @@ export function resolveGroup(group, tests, projectDir, srcDir) {
|
|
|
54
60
|
pattern: [group.src, group.testMatch].filter(Boolean).join(" · ") || pattern,
|
|
55
61
|
testIds,
|
|
56
62
|
missing: [],
|
|
63
|
+
browsers,
|
|
57
64
|
};
|
|
58
65
|
}
|
|
59
66
|
/** Resolves every group of the config */
|
|
60
67
|
export function resolveGroups(config, tests, projectDir, srcDir) {
|
|
61
|
-
return (config.groups ?? []).map((group) => resolveGroup(group, tests, projectDir, srcDir));
|
|
68
|
+
return (config.groups ?? []).map((group) => resolveGroup(group, tests, projectDir, srcDir, config.browsers));
|
|
62
69
|
}
|
|
63
70
|
/** The group called `name`, or an error naming the groups there are */
|
|
64
71
|
export function findGroup(groups, name) {
|
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
import type { RegisteredTestCase, TestError, TestLogEntry, TestResult, TestSummary } from "../types/test.js";
|
|
2
|
+
import type { SupportedBrowser } from "../types/browser.js";
|
|
2
3
|
/** What a run reports while it happens */
|
|
3
4
|
export type RunEvent = {
|
|
4
5
|
type: "run-start";
|
|
5
6
|
environment: string;
|
|
7
|
+
/** Every selected test in every browser */
|
|
6
8
|
total: number;
|
|
9
|
+
browsers: SupportedBrowser[];
|
|
7
10
|
} | {
|
|
8
11
|
type: "test-start";
|
|
9
12
|
name: string;
|
|
10
13
|
file: string;
|
|
14
|
+
browser: SupportedBrowser;
|
|
11
15
|
} | {
|
|
12
16
|
type: "test-end";
|
|
13
17
|
name: string;
|
|
14
18
|
file: string;
|
|
19
|
+
browser: SupportedBrowser;
|
|
15
20
|
result: TestResult;
|
|
16
21
|
} | {
|
|
17
22
|
type: "run-end";
|
|
@@ -20,6 +25,8 @@ export type RunEvent = {
|
|
|
20
25
|
export interface ReporterOptions {
|
|
21
26
|
/** The environment the run uses, recorded in the report */
|
|
22
27
|
environment: string;
|
|
28
|
+
/** The browsers the run uses, in the order they run; recorded in the report */
|
|
29
|
+
browsers: SupportedBrowser[];
|
|
23
30
|
/** Where the report is written. @default <cwd>/result/report.json */
|
|
24
31
|
output?: string | false;
|
|
25
32
|
/** Called with every event, in order. A throwing listener is ignored */
|
|
@@ -29,14 +36,15 @@ export default class TestReporter {
|
|
|
29
36
|
private testResults;
|
|
30
37
|
private summary;
|
|
31
38
|
private environment;
|
|
39
|
+
private browsers;
|
|
32
40
|
private output;
|
|
33
41
|
private listener;
|
|
34
42
|
constructor(options: ReporterOptions);
|
|
35
43
|
private emit;
|
|
36
44
|
onStart(total?: number): void;
|
|
37
45
|
onEnd(): Promise<TestSummary>;
|
|
38
|
-
onTestStart(test: RegisteredTestCase): void;
|
|
39
|
-
onTestEnd(test: RegisteredTestCase, error?: TestError, extras?: {
|
|
46
|
+
onTestStart(test: RegisteredTestCase, browser: SupportedBrowser): void;
|
|
47
|
+
onTestEnd(test: RegisteredTestCase, browser: SupportedBrowser, error?: TestError, extras?: {
|
|
40
48
|
logs?: TestLogEntry[];
|
|
41
49
|
logsDropped?: number;
|
|
42
50
|
screenshots?: {
|
package/dist/runner/reporter.js
CHANGED
|
@@ -2,14 +2,15 @@ import { mkdir, writeFile } from "fs/promises";
|
|
|
2
2
|
import path from "path";
|
|
3
3
|
import { projectDir } from "../config/config.js";
|
|
4
4
|
import { maskDeep } from "../config/mask.js";
|
|
5
|
-
/** Results are keyed by file
|
|
6
|
-
function keyOf(test) {
|
|
7
|
-
return `${test.file}::${test.name}`;
|
|
5
|
+
/** Results are keyed by file, full title and browser, so equal titles in different files or describe blocks, and the same test in two browsers, stay apart */
|
|
6
|
+
function keyOf(test, browser) {
|
|
7
|
+
return `${test.file}::${test.name}::${browser}`;
|
|
8
8
|
}
|
|
9
9
|
export default class TestReporter {
|
|
10
10
|
constructor(options) {
|
|
11
11
|
this.testResults = new Map();
|
|
12
12
|
this.environment = options.environment;
|
|
13
|
+
this.browsers = [...options.browsers];
|
|
13
14
|
this.output =
|
|
14
15
|
options.output ?? path.join(projectDir(), "result/report.json");
|
|
15
16
|
this.listener = options.onEvent;
|
|
@@ -29,9 +30,15 @@ export default class TestReporter {
|
|
|
29
30
|
status: "failed",
|
|
30
31
|
startTime: t.startTime,
|
|
31
32
|
environment: this.environment,
|
|
33
|
+
browsers: this.browsers,
|
|
32
34
|
tests: [],
|
|
33
35
|
};
|
|
34
|
-
this.emit({
|
|
36
|
+
this.emit({
|
|
37
|
+
type: "run-start",
|
|
38
|
+
environment: this.environment,
|
|
39
|
+
total,
|
|
40
|
+
browsers: this.browsers,
|
|
41
|
+
});
|
|
35
42
|
}
|
|
36
43
|
async onEnd() {
|
|
37
44
|
performance.mark(`group-finish`);
|
|
@@ -40,6 +47,7 @@ export default class TestReporter {
|
|
|
40
47
|
...this.summary,
|
|
41
48
|
duration,
|
|
42
49
|
environment: this.environment,
|
|
50
|
+
browsers: this.browsers,
|
|
43
51
|
tests: Array.from(this.testResults.values()),
|
|
44
52
|
status: Array.from(this.testResults.values()).every((r) => r.status === "success")
|
|
45
53
|
? "success"
|
|
@@ -54,19 +62,25 @@ export default class TestReporter {
|
|
|
54
62
|
this.emit({ type: "run-end", summary: this.summary });
|
|
55
63
|
return this.summary;
|
|
56
64
|
}
|
|
57
|
-
onTestStart(test) {
|
|
58
|
-
const key = keyOf(test);
|
|
65
|
+
onTestStart(test, browser) {
|
|
66
|
+
const key = keyOf(test, browser);
|
|
59
67
|
const t = performance.mark(`${key}-start`);
|
|
60
68
|
this.testResults.set(key, {
|
|
61
69
|
name: test.name,
|
|
62
70
|
file: test.file,
|
|
71
|
+
browser,
|
|
63
72
|
status: "started",
|
|
64
73
|
startTime: t.startTime,
|
|
65
74
|
});
|
|
66
|
-
this.emit({
|
|
75
|
+
this.emit({
|
|
76
|
+
type: "test-start",
|
|
77
|
+
name: test.name,
|
|
78
|
+
file: test.file,
|
|
79
|
+
browser,
|
|
80
|
+
});
|
|
67
81
|
}
|
|
68
|
-
onTestEnd(test, error, extras) {
|
|
69
|
-
const key = keyOf(test);
|
|
82
|
+
onTestEnd(test, browser, error, extras) {
|
|
83
|
+
const key = keyOf(test, browser);
|
|
70
84
|
performance.mark(`${key}-finish`);
|
|
71
85
|
const duration = performance.measure("test-duration", `${key}-start`, `${key}-finish`).duration;
|
|
72
86
|
const result = maskDeep({
|
|
@@ -79,6 +93,12 @@ export default class TestReporter {
|
|
|
79
93
|
...(extras?.screenshots?.length && { screenshots: extras.screenshots }),
|
|
80
94
|
});
|
|
81
95
|
this.testResults.set(key, result);
|
|
82
|
-
this.emit({
|
|
96
|
+
this.emit({
|
|
97
|
+
type: "test-end",
|
|
98
|
+
name: test.name,
|
|
99
|
+
file: test.file,
|
|
100
|
+
browser,
|
|
101
|
+
result,
|
|
102
|
+
});
|
|
83
103
|
}
|
|
84
104
|
}
|
package/dist/runner/run.d.ts
CHANGED
|
@@ -24,6 +24,7 @@ export interface FoundTest {
|
|
|
24
24
|
export declare function listTests(options?: RunTestsOptions): Promise<FoundTest[]>;
|
|
25
25
|
/** Imports the project's specs without running anything and lists the config's groups with the tests each holds */
|
|
26
26
|
export declare function listGroups(options?: RunTestsOptions): Promise<ResolvedGroup[]>;
|
|
27
|
+
export { BROWSER_NAMES, checkBrowserList } from "./browsers.js";
|
|
27
28
|
export type { RunEvent } from "./reporter.js";
|
|
28
29
|
export type { TestError, TestResult, TestSummary } from "../types/test.js";
|
|
29
30
|
export type { ResolvedGroup } from "./groups.js";
|
package/dist/runner/run.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { SupportedBrowser } from "../types/browser.js";
|
|
1
2
|
import type { RegisteredTestCase } from "../types/test.js";
|
|
2
3
|
import type { PreparedRun } from "./prepare-run.js";
|
|
3
4
|
import { type RunEvent } from "./reporter.js";
|
|
@@ -13,6 +14,8 @@ export interface RunnerOptions {
|
|
|
13
14
|
testNames?: string[];
|
|
14
15
|
/** Only the tests of this group of the config's `groups` (unknown name: an error). Combines with the other filters */
|
|
15
16
|
group?: string;
|
|
17
|
+
/** The browsers to run in, instead of the group's or the project's `browsers` (the command line's `--browser`) */
|
|
18
|
+
browsers?: SupportedBrowser[];
|
|
16
19
|
/** Run the browser without a window. @default false */
|
|
17
20
|
headless?: boolean;
|
|
18
21
|
/** Remote debugging port of the browser. @default 9223 */
|
|
@@ -49,7 +52,12 @@ export default class TestRunner {
|
|
|
49
52
|
groups(): Promise<ResolvedGroup[]>;
|
|
50
53
|
/** Imports the spec files and returns the tests the options select: by `group`, `testNames` and `grep` */
|
|
51
54
|
register(): Promise<RegisteredTestCase[]>;
|
|
52
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* The browsers the run uses: the options' (`--browser`), else the selected group's, else the project's, else
|
|
57
|
+
* Firefox. A group is only looked up when there is something to look up
|
|
58
|
+
*/
|
|
59
|
+
private browsersOfRun;
|
|
60
|
+
/** Runs the selected tests in every browser of the run, one browser after the other, and returns the summary that was also written to the report */
|
|
53
61
|
start(): Promise<import("../types/test.js").TestSummary>;
|
|
54
62
|
private executeTestCase;
|
|
55
63
|
}
|
|
@@ -4,6 +4,8 @@ import path from "path";
|
|
|
4
4
|
import { pathToFileURL } from "url";
|
|
5
5
|
import { Browser } from "../browser/browser.js";
|
|
6
6
|
import { ensureWarm } from "../browser/warm-up.js";
|
|
7
|
+
import { projectBrowser } from "../browsers/resolve.js";
|
|
8
|
+
import { browsersFor } from "./browsers.js";
|
|
7
9
|
import { loadConfig, projectDir, readConfig } from "../config/config.js";
|
|
8
10
|
import logger from "../logger/index.js";
|
|
9
11
|
import { assertModuleProject } from "./module-type.js";
|
|
@@ -108,34 +110,59 @@ export default class TestRunner {
|
|
|
108
110
|
}
|
|
109
111
|
return all.filter((test) => inGroup(test) && isSelected(test.name, this.options));
|
|
110
112
|
}
|
|
111
|
-
/**
|
|
113
|
+
/**
|
|
114
|
+
* The browsers the run uses: the options' (`--browser`), else the selected group's, else the project's, else
|
|
115
|
+
* Firefox. A group is only looked up when there is something to look up
|
|
116
|
+
*/
|
|
117
|
+
async browsersOfRun() {
|
|
118
|
+
const { browsers: flag, group } = this.options;
|
|
119
|
+
const config = await loadConfig();
|
|
120
|
+
let ofGroup;
|
|
121
|
+
if (group !== undefined && !(flag && flag.length > 0)) {
|
|
122
|
+
const { groups } = await this.resolveAll(await this.registerAll());
|
|
123
|
+
ofGroup = findGroup(groups, group).browsers;
|
|
124
|
+
}
|
|
125
|
+
return browsersFor({ flag, group: ofGroup, project: config.browsers });
|
|
126
|
+
}
|
|
127
|
+
/** Runs the selected tests in every browser of the run, one browser after the other, and returns the summary that was also written to the report */
|
|
112
128
|
async start() {
|
|
113
129
|
const { reportPath, onEvent, signal } = this.options;
|
|
130
|
+
const selected = await this.register();
|
|
131
|
+
const browsers = await this.browsersOfRun();
|
|
114
132
|
const reporter = new TestReporter({
|
|
115
133
|
environment: this.run.settings.environment,
|
|
134
|
+
browsers,
|
|
116
135
|
...(reportPath !== undefined && { output: reportPath }),
|
|
117
136
|
...(onEvent && { onEvent }),
|
|
118
137
|
});
|
|
119
|
-
const selected = await this.register();
|
|
120
138
|
this.srcDir = realDirs((await loadConfig()).srcDir).srcDir;
|
|
121
|
-
// A missing browser is one clear error before the run, not a failure per test
|
|
122
|
-
// first start happens here, outside any test's timeout
|
|
123
|
-
if (selected.length > 0)
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
139
|
+
// A missing browser is one clear error before the run, not a failure per test, and before any browser has run a
|
|
140
|
+
// test; a fresh build's slow first start happens here too, outside any test's timeout
|
|
141
|
+
if (selected.length > 0) {
|
|
142
|
+
for (const browser of browsers)
|
|
143
|
+
await projectBrowser(browser);
|
|
144
|
+
for (const browser of browsers)
|
|
145
|
+
await ensureWarm(browser, { port: this.options.port ?? 9223 }, signal);
|
|
146
|
+
}
|
|
147
|
+
reporter.onStart(selected.length * browsers.length);
|
|
148
|
+
for (const browser of browsers) {
|
|
149
|
+
for (const test of selected) {
|
|
150
|
+
if (signal?.aborted) {
|
|
151
|
+
reporter.onTestStart(test, browser);
|
|
152
|
+
reporter.onTestEnd(test, browser, {
|
|
153
|
+
message: "Run aborted",
|
|
154
|
+
type: "error",
|
|
155
|
+
});
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
await this.executeTestCase(test, browser, reporter);
|
|
131
159
|
}
|
|
132
|
-
await this.executeTestCase(test, reporter);
|
|
133
160
|
}
|
|
134
161
|
return reporter.onEnd();
|
|
135
162
|
}
|
|
136
|
-
async executeTestCase(test, reporter) {
|
|
137
|
-
logger.verbose("Starting test: %s", test.name);
|
|
138
|
-
reporter.onTestStart(test);
|
|
163
|
+
async executeTestCase(test, selectedBrowser, reporter) {
|
|
164
|
+
logger.verbose("Starting test: %s in %s", test.name, selectedBrowser);
|
|
165
|
+
reporter.onTestStart(test, selectedBrowser);
|
|
139
166
|
// A test ends once, whichever of finishing, failing, timing out, an abort or a closed browser comes first
|
|
140
167
|
let ended = false;
|
|
141
168
|
let evidence;
|
|
@@ -143,7 +170,7 @@ export default class TestRunner {
|
|
|
143
170
|
if (ended)
|
|
144
171
|
return;
|
|
145
172
|
ended = true;
|
|
146
|
-
reporter.onTestEnd(test, error, {
|
|
173
|
+
reporter.onTestEnd(test, selectedBrowser, error, {
|
|
147
174
|
...logs,
|
|
148
175
|
screenshots: evidence?.screenshots,
|
|
149
176
|
});
|
|
@@ -151,7 +178,6 @@ export default class TestRunner {
|
|
|
151
178
|
let browserClosed = false;
|
|
152
179
|
/** Set once the runner itself closes the browser */
|
|
153
180
|
let closing = false;
|
|
154
|
-
const selectedBrowser = (await readConfig("browser")) ?? "firefox";
|
|
155
181
|
const { settings, secrets } = this.run;
|
|
156
182
|
const timeout = settings.timeout;
|
|
157
183
|
const { reportPath } = this.options;
|
|
@@ -205,11 +231,12 @@ export default class TestRunner {
|
|
|
205
231
|
resolve();
|
|
206
232
|
});
|
|
207
233
|
page = launched.page;
|
|
208
|
-
evidence = new Evidence(resultDir, testDirOf(test, this.srcDir));
|
|
234
|
+
evidence = new Evidence(resultDir, `${selectedBrowser}/${testDirOf(test, this.srcDir)}`);
|
|
209
235
|
launched.page.setEvidence(evidence);
|
|
210
236
|
const fixtures = {
|
|
211
237
|
page: launched.page,
|
|
212
238
|
browser: launched.browser,
|
|
239
|
+
browserName: selectedBrowser,
|
|
213
240
|
env: createEnvFixture(settings.environment, settings.variables),
|
|
214
241
|
secrets: createSecretsFixture(settings.environment, secrets),
|
|
215
242
|
};
|
|
@@ -65,3 +65,16 @@ export declare class BiDiConnector {
|
|
|
65
65
|
private messageListener;
|
|
66
66
|
kill(): void;
|
|
67
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* The message as the debug log gets it: a `script.message` on a private channel is logged by channel and
|
|
70
|
+
* action only, so a value the tester types or a secret added from the recorder toolbar never reaches the log
|
|
71
|
+
* (an added secret is only registered for masking after it is saved).
|
|
72
|
+
*/
|
|
73
|
+
export declare function forLog(message: {
|
|
74
|
+
type?: string;
|
|
75
|
+
method?: string;
|
|
76
|
+
params?: {
|
|
77
|
+
channel?: unknown;
|
|
78
|
+
data?: unknown;
|
|
79
|
+
};
|
|
80
|
+
}): unknown;
|
|
@@ -216,7 +216,27 @@ const PRIVATE_CHANNEL = /^samurai-(recorder|toolbar)-/;
|
|
|
216
216
|
* action only, so a value the tester types or a secret added from the recorder toolbar never reaches the log
|
|
217
217
|
* (an added secret is only registered for masking after it is saved).
|
|
218
218
|
*/
|
|
219
|
-
function forLog(message) {
|
|
219
|
+
export function forLog(message) {
|
|
220
|
+
return shortened(redacted(message));
|
|
221
|
+
}
|
|
222
|
+
/** A string longer than this is cut in the log: a file's bytes travel in 1 MB chunks, 50 MB makes 70 MB of debug log */
|
|
223
|
+
const LOG_STRING_LIMIT = 1000;
|
|
224
|
+
/** A copy of `value` with every long string cut to its start and its length; short values come back as they are */
|
|
225
|
+
function shortened(value, depth = 0) {
|
|
226
|
+
if (typeof value === "string")
|
|
227
|
+
return value.length > LOG_STRING_LIMIT
|
|
228
|
+
? `${value.slice(0, 200)}… (${value.length} characters)`
|
|
229
|
+
: value;
|
|
230
|
+
if (value === null || typeof value !== "object" || depth > 30)
|
|
231
|
+
return value;
|
|
232
|
+
if (Array.isArray(value))
|
|
233
|
+
return value.map((item) => shortened(item, depth + 1));
|
|
234
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [
|
|
235
|
+
key,
|
|
236
|
+
shortened(item, depth + 1),
|
|
237
|
+
]));
|
|
238
|
+
}
|
|
239
|
+
function redacted(message) {
|
|
220
240
|
const channel = message.params?.channel;
|
|
221
241
|
if (message.type !== "event" ||
|
|
222
242
|
message.method !== "script.message" ||
|
package/dist/types/config.d.ts
CHANGED
|
@@ -24,6 +24,8 @@ export interface SamuraiGroup {
|
|
|
24
24
|
* Overrides 'src' and 'testMatch' options.
|
|
25
25
|
*/
|
|
26
26
|
tests?: SamuraiTest[];
|
|
27
|
+
/** The browsers this group's tests run in, instead of the project's `browsers` */
|
|
28
|
+
browsers?: SupportedBrowser[];
|
|
27
29
|
}
|
|
28
30
|
|
|
29
31
|
export interface SamuraiEnvironment {
|
|
@@ -45,10 +47,10 @@ export interface SamuraiTestConfig {
|
|
|
45
47
|
*/
|
|
46
48
|
srcDir?: string;
|
|
47
49
|
/**
|
|
48
|
-
* The
|
|
49
|
-
* @default "firefox"
|
|
50
|
+
* The browsers the tests run in, one after the other: every selected test runs once in each. A group can name its own.
|
|
51
|
+
* @default ["firefox"]
|
|
50
52
|
*/
|
|
51
|
-
|
|
53
|
+
browsers?: SupportedBrowser[];
|
|
52
54
|
/**
|
|
53
55
|
* Browser binaries to launch instead of the builds SAMURAI downloads, relative to the project.
|
|
54
56
|
* `SAMURAI_FIREFOX_PATH` / `SAMURAI_CHROME_PATH` win over these.
|
package/dist/types/test.d.ts
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import type { Browser } from "../browser/browser.ts";
|
|
2
2
|
import type Page from "../browser/page.ts";
|
|
3
|
+
import type { SupportedBrowser } from "./browser.ts";
|
|
3
4
|
|
|
4
5
|
/** What a test function receives */
|
|
5
6
|
export interface TestFixtures {
|
|
6
7
|
page: Page;
|
|
7
8
|
browser: Browser;
|
|
9
|
+
/** The browser this run of the test is in */
|
|
10
|
+
browserName: SupportedBrowser;
|
|
8
11
|
/** The chosen environment's variables; reading an unknown name throws */
|
|
9
12
|
env: Readonly<Record<string, string | number | boolean>>;
|
|
10
13
|
/** Secrets by name (SAMURAI_SECRET_<NAME>); reading an unset name throws */
|
|
@@ -66,12 +69,15 @@ export interface TestLogEntry {
|
|
|
66
69
|
interface TestResultIdentity {
|
|
67
70
|
name: string;
|
|
68
71
|
file: string;
|
|
72
|
+
/** The browser the test ran in */
|
|
73
|
+
browser: SupportedBrowser;
|
|
69
74
|
startTime: number;
|
|
70
75
|
}
|
|
71
76
|
|
|
72
77
|
interface TestResultBase {
|
|
73
78
|
name: string;
|
|
74
79
|
file: string;
|
|
80
|
+
browser: SupportedBrowser;
|
|
75
81
|
startTime: number;
|
|
76
82
|
status: "started";
|
|
77
83
|
}
|
|
@@ -102,4 +108,6 @@ interface TestSummaryBase {
|
|
|
102
108
|
tests: TestResult[];
|
|
103
109
|
/** The environment the run used */
|
|
104
110
|
environment: string;
|
|
111
|
+
/** The browsers the run used, in the order they ran */
|
|
112
|
+
browsers: SupportedBrowser[];
|
|
105
113
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@itmetsam/samurai-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "Browser test framework over WebDriver BiDi: auto-waiting locators, environments and secrets, a recorder, and a command line.",
|
|
5
5
|
"main": "dist/api.js",
|
|
6
6
|
"exports": {
|