@itmetsam/samurai-framework 0.10.0 → 0.12.1

package/dist/runner/run-test.js

@@ -0,0 +1,178 @@
+import path from "path";
+import { takePendingAssertions } from "../assert/expect.js";
+import { Browser } from "../browser/browser.js";
+import { Evidence } from "../browser/evidence.js";
+import { projectDir, readConfig } from "../config/config.js";
+import { createSecretsFixture } from "../config/secrets.js";
+import { createEnvFixture } from "../config/variables.js";
+import logger from "../logger/index.js";
+import { ConnectionClosedError } from "../transport/bidi-connection.js";
+import { judgePageLogs } from "./page-logs-report.js";
+import { DEFAULT_PORT } from "./ports.js";
+import { testDirOf } from "./test-dir.js";
+import { toTestError } from "./test-error.js";
+/**
+ * The state of one test run. A test ends once, whichever of finishing, failing, timing out, an abort or a
+ * closed browser comes first: that one is reported and the rest are ignored.
+ */
+class TestOutcome {
+    constructor(reporter, test, browser) {
+        this.reporter = reporter;
+        this.test = test;
+        this.browser = browser;
+        /** The test has been reported: later calls to `finish` do nothing */
+        this.ended = false;
+        /** The runner itself is closing the browser: its exit is expected and is not a failure */
+        this.closing = false;
+        /** A person closed the browser, or it crashed, while the test ran */
+        this.browserClosed = false;
+    }
+    get isEnded() {
+        return this.ended;
+    }
+    finish(error, logs) {
+        if (this.ended)
+            return;
+        this.ended = true;
+        this.reporter.onTestEnd(this.test, this.browser, error, {
+            ...logs,
+            screenshots: this.evidence?.screenshots,
+        });
+    }
+}
+/** Runs one test in one browser and reports it: launch, run the body, judge the page's logs, close the browser */
+export async function runOneTest(test, browserName, run, options, srcDir, reporter, resources = {}) {
+    logger.verbose("Starting test: %s in %s", test.name, browserName);
+    reporter.onTestStart(test, browserName);
+    const outcome = new TestOutcome(reporter, test, browserName);
+    const { settings, secrets } = run;
+    const timeout = settings.timeout;
+    const { reportPath } = options;
+    const resultDir = typeof reportPath === "string"
+        ? path.dirname(reportPath)
+        : path.join(projectDir(), "result");
+    let browser;
+    let page;
+    let timeoutTimer;
+    let onRunAborted;
+    const timeoutController = new AbortController();
+    let browserGone = () => { };
+    /** Resolves when the time is up or the run is aborted; the test is reported failed then */
+    const limit = new Promise((resolve) => {
+        timeoutTimer = setTimeout(() => {
+            outcome.finish({
+                message: `Test timed out after ${timeout / 1000} seconds`,
+                type: "timeout",
+            });
+            timeoutController.abort(new Error("Test timed out"));
+            resolve();
+        }, timeout);
+        onRunAborted = () => {
+            if (timeoutController.signal.aborted)
+                return;
+            outcome.finish({ message: "Run aborted", type: "error" });
+            timeoutController.abort(new Error("Run aborted"));
+            resolve();
+        };
+        options.signal?.addEventListener("abort", onRunAborted, { once: true });
+    });
+    /** Resolves when the browser exits on its own while the test runs */
+    const gone = new Promise((resolve) => (browserGone = resolve));
+    /** Launches the browser, runs the test body and reports how it went */
+    const runBody = async () => {
+        try {
+            const launched = await Browser.launch(browserName, {
+                port: resources.port ?? options.port ?? DEFAULT_PORT,
+                ...(resources.profileDir && { profileDir: resources.profileDir }),
+                headless: options.headless ?? settings.headless,
+            }, timeoutController.signal);
+            browser = launched.browser;
+            // A person closing the window, or the browser crashing: end the test now instead of waiting for a
+            // command to time out
+            void launched.browser.exited.then(() => {
+                if (outcome.isEnded || outcome.closing)
+                    return;
+                outcome.browserClosed = true;
+                outcome.finish({
+                    message: "The browser was closed while the test was running",
+                    type: "error",
+                });
+                timeoutController.abort(new Error("Browser closed"));
+                browserGone();
+            });
+            page = launched.page;
+            outcome.evidence = new Evidence(resultDir, `${browserName}/${testDirOf(test, srcDir)}`);
+            launched.page.setEvidence(outcome.evidence);
+            const fixtures = {
+                page: launched.page,
+                browser: launched.browser,
+                browserName,
+                workerIndex: resources.workerIndex ?? 0,
+                env: createEnvFixture(settings.environment, settings.variables),
+                secrets: createSecretsFixture(settings.environment, secrets),
+            };
+            await test.function(fixtures);
+            // Already reported as timed out
+            if (timeoutController.signal.aborted)
+                return;
+            const unawaited = takePendingAssertions();
+            if (unawaited.length > 0) {
+                outcome.finish({
+                    message: unawaited
+                        .map((matcher) => `expect(locator).${matcher}() was not awaited`)
+                        .join("\n"),
+                    type: "assertion",
+                }, await judgeLogs(page, true));
+                return;
+            }
+            const verdict = await judgeLogs(page, false);
+            outcome.finish(verdict.error, verdict);
+        }
+        catch (err) {
+            if (timeoutController.signal.aborted)
+                return;
+            outcome.finish(outcome.browserClosed || err instanceof ConnectionClosedError
+                ? {
+                    message: "The browser was closed while the test was running",
+                    type: "error",
+                }
+                : toTestError(err), await judgeLogs(page, true));
+        }
+    };
+    await Promise.race([limit, gone, runBody()]);
+    clearTimeout(timeoutTimer);
+    if (onRunAborted)
+        options.signal?.removeEventListener("abort", onRunAborted);
+    takePendingAssertions();
+    logger.verbose("Closing browser for test: %s", test.name);
+    outcome.closing = true;
+    await browser?.close();
+}
+/** Reads the page's log once the test body is done and applies config `logs` (see judgePageLogs) */
+async function judgeLogs(page, testFailed) {
+    if (!page)
+        return {};
+    try {
+        // Events and command replies share one ordered connection: after this, everything logged so far is in
+        await page.syncLogs();
+    }
+    catch (err) {
+        logger.debug("Could not sync the page's logs", { err });
+    }
+    let config;
+    try {
+        config = await readConfig("logs");
+    }
+    catch (err) {
+        logger.debug("Could not read logs config, using defaults", { err });
+    }
+    return judgePageLogs({
+        config,
+        entries: page.getLogs(),
+        errors: page.pageErrors(),
+        dropped: page.logsDropped,
+        testFailed,
+        allowPageErrors: page.pageErrorsAllowed,
+        routeErrors: page.routeErrors(),
+    });
+}

package/dist/runner/run.d.ts

@@ -1,7 +1,7 @@
 import type { TestSummary } from "../types/test.js";
 import { type ProjectOptions } from "./project.js";
 import type { ResolvedGroup } from "./groups.js";
-import { type RunnerOptions } from "./test-runner.js";
+import type { RunnerOptions } from "./options.js";
 export interface RunTestsOptions extends RunnerOptions, ProjectOptions {
 }
 /**
@@ -24,9 +24,11 @@ export interface FoundTest {
 export declare function listTests(options?: RunTestsOptions): Promise<FoundTest[]>;
 /** Imports the project's specs without running anything and lists the config's groups with the tests each holds */
 export declare function listGroups(options?: RunTestsOptions): Promise<ResolvedGroup[]>;
-export { BROWSER_NAMES, checkBrowserList } from "./browsers.js";
+export { BROWSER_NAMES, checkBrowserList } from "../config/browsers.js";
+export { parseWorkers } from "../config/workers.js";
 export type { RunEvent } from "./reporter.js";
 export type { TestError, TestResult, TestSummary } from "../types/test.js";
 export type { ResolvedGroup } from "./groups.js";
-export type { RunnerOptions } from "./test-runner.js";
+export type { RunnerOptions } from "./options.js";
 export type { ProjectOptions } from "./project.js";
+export { DEFAULT_PORT } from "./ports.js";

package/dist/runner/run.js

@@ -1,5 +1,26 @@
 import { withProject } from "./project.js";
-import TestRunner, {} from "./test-runner.js";
+import TestRunner from "./test-runner.js";
+/**
+ * Sets the project up (config, environment, secrets), builds a runner from the rest of the options and hands both to
+ * `work`. `headless` stays with the runner options on purpose: the runner reads it, and the project's run settings
+ * (`withProject`'s own `headless`) are left to the config.
+ */
+function inProject(options, work) {
+    const { projectDir, config, dataDir, environment, timeout, expectTimeout, ...runner } = options;
+    return withProject(
+    // The cast is for `exactOptionalPropertyTypes`: these may be undefined, which the project options leave out
+    {
+        projectDir,
+        config,
+        dataDir,
+        environment,
+        timeout,
+        expectTimeout,
+    }, async (run) => work(await TestRunner.init(run, runner, {
+        inlineConfig: config !== undefined,
+        ...(dataDir !== undefined && { dataDir }),
+    })));
+}
 /**
  * Runs the project's tests, one after the other, and resolves with the summary (also written to the report).
  * Spec files are TypeScript: the process must be able to import them (run under `tsx`, or with Node's own
@@ -10,38 +31,16 @@ import TestRunner, {} from "./test-runner.js";
  *    onEvent: (event) => console.log(event.type) });
  */
 export async function runTests(options = {}) {
-    const { projectDir, config, dataDir, environment, timeout, expectTimeout, ...runner } = options;
-    return withProject({
-        projectDir,
-        config,
-        dataDir,
-        environment,
-        timeout,
-        expectTimeout,
-    }, async (run) => (await TestRunner.init(run, undefined, runner)).start());
+    return inProject(options, (runner) => runner.start());
 }
 /** Imports the project's specs without running anything and lists the tests the options select */
 export async function listTests(options = {}) {
-    const { projectDir, config, dataDir, environment, timeout, expectTimeout, ...runner } = options;
-    return withProject({
-        projectDir,
-        config,
-        dataDir,
-        environment,
-        timeout,
-        expectTimeout,
-    }, async (run) => (await (await TestRunner.init(run, undefined, runner)).register()).map(({ name, file }) => ({ name, file })));
+    return inProject(options, async (runner) => (await runner.register()).map(({ name, file }) => ({ name, file })));
 }
 /** Imports the project's specs without running anything and lists the config's groups with the tests each holds */
 export async function listGroups(options = {}) {
-    const { projectDir, config, dataDir, environment, timeout, expectTimeout, ...runner } = options;
-    return withProject({
-        projectDir,
-        config,
-        dataDir,
-        environment,
-        timeout,
-        expectTimeout,
-    }, async (run) => (await TestRunner.init(run, undefined, runner)).groups());
+    return inProject(options, (runner) => runner.groups());
 }
-export { BROWSER_NAMES, checkBrowserList } from "./browsers.js";
+export { BROWSER_NAMES, checkBrowserList } from "../config/browsers.js";
+export { parseWorkers } from "../config/workers.js";
+export { DEFAULT_PORT } from "./ports.js";

package/dist/runner/select.d.ts

@@ -0,0 +1,24 @@
+import type { SupportedBrowser } from "../types/browser.js";
+import type { RegisteredTestCase } from "../types/test.js";
+import { type ResolvedGroup } from "./groups.js";
+import type { RunnerOptions } from "./options.js";
+/** The specs a run imported, with the config's groups resolved against them */
+export interface LoadedSpecs {
+    all: RegisteredTestCase[];
+    groups: ResolvedGroup[];
+    /** The real `srcDir` the groups were resolved against */
+    srcDir: string;
+}
+/** Whether a test with this full name is selected by the options */
+export declare function isSelected(name: string, { grep, testNames }: Pick<RunnerOptions, "grep" | "testNames">): boolean;
+/** Id of a registered test: its spec file relative to `srcDir`, then its name */
+export declare function idOf(test: RegisteredTestCase, srcDir: string): string;
+/** Imports the spec files once and resolves the config's groups against what they registered */
+export declare function loadSpecs(testFiles: string[]): Promise<LoadedSpecs>;
+/** The tests the options select: by `group`, `testNames` and `grep` */
+export declare function selectTests({ all, groups, srcDir }: LoadedSpecs, options: RunnerOptions): RegisteredTestCase[];
+/**
+ * The browsers the run uses: the options' (`--browser`), else the selected group's, else the project's, else
+ * Firefox
+ */
+export declare function browsersOfRun(groups: ResolvedGroup[], options: RunnerOptions): Promise<SupportedBrowser[]>;

package/dist/runner/select.js

@@ -0,0 +1,77 @@
+import path from "path";
+import { pathToFileURL } from "url";
+import { loadConfig } from "../config/config.js";
+import logger from "../logger/index.js";
+import { browsersFor } from "./browsers.js";
+import { findGroup, resolveGroups, testId, toPosix, } from "./groups.js";
+import { assertModuleProject } from "./module-type.js";
+import { clearRegistry, registeredTests } from "./registry.js";
+import { realDirs } from "./test-dir.js";
+/** Whether a test with this full name is selected by the options */
+export function isSelected(name, { grep, testNames }) {
+    if (testNames && !testNames.includes(name))
+        return false;
+    if (grep === undefined)
+        return true;
+    if (typeof grep === "string")
+        return name.includes(grep);
+    grep.lastIndex = 0;
+    return grep.test(name);
+}
+/**
+ * Imports the spec files and returns every test they register. A fresh registry and a fresh import of each
+ * file, so a second run in this process registers its tests again (and sees edited specs, for files loaded as
+ * ES modules; CommonJS files stay cached until the process ends)
+ */
+async function registerAll(testFiles) {
+    clearRegistry();
+    const stamp = `?run=${Date.now().toString(36)}`;
+    for (const file of testFiles)
+        assertModuleProject(file);
+    for (const file of testFiles) {
+        logger.verbose("Trying to register file: %s", file);
+        await import(pathToFileURL(file).href + stamp);
+    }
+    return [...registeredTests()];
+}
+/** Id of a registered test: its spec file relative to `srcDir`, then its name */
+export function idOf(test, srcDir) {
+    return testId(toPosix(path.relative(srcDir, test.file)), test.name);
+}
+/** The config's groups, resolved against the tests the specs register */
+async function resolveAll(tests) {
+    const config = await loadConfig();
+    const { dir, srcDir } = realDirs(config.srcDir);
+    const groups = resolveGroups(config, tests.map(({ file, name }) => ({
+        file: toPosix(path.relative(srcDir, file)),
+        name,
+    })), dir, srcDir);
+    return { groups, srcDir };
+}
+/** Imports the spec files once and resolves the config's groups against what they registered */
+export async function loadSpecs(testFiles) {
+    const all = await registerAll(testFiles);
+    return { all, ...(await resolveAll(all)) };
+}
+/** The tests the options select: by `group`, `testNames` and `grep` */
+export function selectTests({ all, groups, srcDir }, options) {
+    const { group } = options;
+    let inGroup = () => true;
+    if (group !== undefined) {
+        const members = new Set(findGroup(groups, group).testIds);
+        inGroup = (test) => members.has(idOf(test, srcDir));
+    }
+    return all.filter((test) => inGroup(test) && isSelected(test.name, options));
+}
+/**
+ * The browsers the run uses: the options' (`--browser`), else the selected group's, else the project's, else
+ * Firefox
+ */
+export async function browsersOfRun(groups, options) {
+    const { browsers: flag, group } = options;
+    const config = await loadConfig();
+    const ofGroup = group !== undefined && !(flag && flag.length > 0)
+        ? findGroup(groups, group).browsers
+        : undefined;
+    return browsersFor({ flag, group: ofGroup, project: config.browsers });
+}

package/dist/runner/test-dir.d.ts

@@ -0,0 +1,13 @@
+import type { RegisteredTestCase } from "../types/test.js";
+/** The project and source folders as real paths: registered test files come from stack traces, which resolve symlinks */
+export declare function realDirs(configuredSrcDir: string | undefined): {
+    dir: string;
+    srcDir: string;
+};
+/**
+ * The folder of a test's screenshots below `screenshots/`: its spec file without extension, then its cleaned
+ * name. A spec outside srcDir loses the `..` (and a drive or root) its relative path starts with, so it stays
+ * below `screenshots/`.
+ * @internal Exported for tests
+ */
+export declare function testDirOf(test: RegisteredTestCase, srcDir: string): string;

package/dist/runner/test-dir.js

@@ -0,0 +1,27 @@
+import path from "path";
+import { projectDir } from "../config/config.js";
+import { cleanName } from "../browser/evidence.js";
+import { DEFAULT_SRC_DIR, realPath, toPosix } from "./groups.js";
+/** The project and source folders as real paths: registered test files come from stack traces, which resolve symlinks */
+export function realDirs(configuredSrcDir) {
+    const dir = realPath(projectDir());
+    const srcDir = realPath(path.resolve(dir, configuredSrcDir ?? DEFAULT_SRC_DIR));
+    return { dir, srcDir };
+}
+/**
+ * The folder of a test's screenshots below `screenshots/`: its spec file without extension, then its cleaned
+ * name. A spec outside srcDir loses the `..` (and a drive or root) its relative path starts with, so it stays
+ * below `screenshots/`.
+ * @internal Exported for tests
+ */
+export function testDirOf(test, srcDir) {
+    const file = toPosix(path.relative(srcDir, test.file))
+        .replace(/\.[cm]?[jt]sx?$/, "")
+        .split("/")
+        .filter((segment) => segment !== "" &&
+        segment !== "." &&
+        segment !== ".." &&
+        !/^[A-Za-z]:$/.test(segment))
+        .join("/");
+    return `${file}/${cleanName(test.name)}`;
+}

package/dist/runner/test-runner.d.ts

@@ -1,71 +1,40 @@
-import type { SupportedBrowser } from "../types/browser.js";
 import type { RegisteredTestCase } from "../types/test.js";
-import type { PreparedRun } from "./prepare-run.js";
-import { type RunEvent } from "./reporter.js";
-import type { SamuraiGroup } from "../types/config.js";
 import type { ResolvedGroup } from "./groups.js";
-/** What decides which tests of the found files run, and how the browser starts */
-export interface RunnerOptions {
-    /** Spec files to run, absolute or relative to the project folder. Default: every `*.spec.ts` under `srcDir` */
-    files?: string[];
-    /** Only tests whose full name matches (a string must be contained in it) */
-    grep?: string | RegExp;
-    /** Only tests with exactly one of these full names (`describe` titles joined with " > ") */
-    testNames?: string[];
-    /** Only the tests of this group of the config's `groups` (unknown name: an error). Combines with the other filters */
-    group?: string;
-    /** The browsers to run in, instead of the group's or the project's `browsers` (the command line's `--browser`) */
-    browsers?: SupportedBrowser[];
-    /** Run the browser without a window. @default false */
-    headless?: boolean;
-    /** Remote debugging port of the browser. @default 9223 */
-    port?: number;
-    /** Where the report is written; `false` writes none. @default `<project>/result/report.json` */
-    reportPath?: string | false;
-    /** Receives start and end of the run and of every test, as they happen */
-    onEvent?: (event: RunEvent) => void;
-    /** Aborting stops the running test (its browser is closed), skips the rest and reports the run as failed */
-    signal?: AbortSignal;
+import type { RunnerOptions } from "./options.js";
+import type { PreparedRun } from "./prepare-run.js";
+import { type PooledRunner } from "./pooled-run.js";
+import { type PoolOptions } from "./worker-pool.js";
+export type { RunnerOptions } from "./options.js";
+/** What a runner knows about the project it runs in, besides the options of the run */
+export interface ProjectInfo {
+    /** The config was given as an object, not read from `samurai.config.ts`: a worker process cannot get it */
+    inlineConfig?: boolean;
+    /** Where the framework keeps browser profiles; a worker needs the same folder */
+    dataDir?: string;
+    /** Makes the pool of a run with more than one worker; tests give a fake */
+    createPool?: (options: PoolOptions) => PooledRunner & {
+        dispose(): Promise<void>;
+    };
 }
-/** Whether a test with this full name is selected by the options */
-export declare function isSelected(name: string, { grep, testNames }: Pick<RunnerOptions, "grep" | "testNames">): boolean;
+/** Finds the spec files, picks the tests and browsers of a run and runs each test in each browser */
 export default class TestRunner {
     private testFiles;
     private run;
-    private group?;
     private options;
-    /** The real `srcDir` of this run, set by `start`; test screenshots are saved under paths relative to it */
-    private srcDir;
-    constructor(testFiles: string[], run: PreparedRun, group?: SamuraiGroup, options?: RunnerOptions);
-    static init(run: PreparedRun, group?: SamuraiGroup, options?: RunnerOptions): Promise<TestRunner>;
-    /**
-     * Imports the spec files and returns every test they register. A fresh registry and a fresh import of each
-     * file, so a second run in this process registers its tests again (and sees edited specs, for files loaded as
-     * ES modules; CommonJS files stay cached until the process ends)
-     */
-    private registerAll;
-    /** Id of a registered test: its spec file relative to `srcDir`, then its name */
-    private idOf;
-    /** The config's groups, resolved against the tests the specs register */
-    private resolveAll;
+    private project;
+    constructor(testFiles: string[], run: PreparedRun, options?: RunnerOptions, project?: ProjectInfo);
+    static init(run: PreparedRun, options?: RunnerOptions, project?: ProjectInfo): Promise<TestRunner>;
     /** The groups of the config with the tests each holds; see `listGroups` */
     groups(): Promise<ResolvedGroup[]>;
     /** Imports the spec files and returns the tests the options select: by `group`, `testNames` and `grep` */
     register(): Promise<RegisteredTestCase[]>;
     /**
-     * The browsers the run uses: the options' (`--browser`), else the selected group's, else the project's, else
-     * Firefox. A group is only looked up when there is something to look up
+     * Runs the selected tests in every browser of the run, one browser after the other, and returns the summary that
+     * was also written to the report. With more than one worker the tests run at the same time in worker processes.
      */
-    private browsersOfRun;
-    /** Runs the selected tests in every browser of the run, one browser after the other, and returns the summary that was also written to the report */
     start(): Promise<import("../types/test.js").TestSummary>;
-    private executeTestCase;
+    /** Whether anything but the default says how many workers to use */
+    private workersAsked;
+    /** The pool of a run: up to `workers` worker processes that read the same project and settings as this run */
+    private createPool;
 }
-/**
- * The folder of a test's screenshots below `screenshots/`: its spec file without extension, then its cleaned
- * name. A spec outside srcDir loses the `..` (and a drive or root) its relative path starts with, so it stays
- * below `screenshots/`.
- * @internal Exported for tests
- */
-export declare function testDirOf(test: RegisteredTestCase, srcDir: string): string;
-export { describe, test } from "./registry.js";