@itmetsam/samurai-framework 0.9.0 → 0.12.0

package/dist/runner/test-runner.js

@@ -1,307 +1,145 @@
-import assert from "assert";
 import { glob } from "node:fs/promises";
 import path from "path";
-import { pathToFileURL } from "url";
-import { Browser } from "../browser/browser.js";
 import { ensureWarm } from "../browser/warm-up.js";
+import { projectBrowser } from "../browsers/resolve.js";
 import { loadConfig, projectDir, readConfig } from "../config/config.js";
 import logger from "../logger/index.js";
-import { assertModuleProject } from "./module-type.js";
-import { clearRegistry, registeredTests } from "./registry.js";
-import { createEnvFixture } from "../config/variables.js";
-import { createSecretsFixture } from "../config/secrets.js";
-import TestReporter, {} from "./reporter.js";
-import Page from "../browser/page.js";
-import { cleanName, Evidence } from "../browser/evidence.js";
-import { takePendingAssertions } from "../assert/expect.js";
-import { toTestError } from "./test-error.js";
-import { ConnectionClosedError } from "../transport/bidi-connection.js";
-import { DEFAULT_SRC_DIR, findGroup, realPath, resolveGroups, testId, toPosix, } from "./groups.js";
-import { judgePageLogs } from "./page-logs-report.js";
-/** Whether a test with this full name is selected by the options */
-export function isSelected(name, { grep, testNames }) {
-    if (testNames && !testNames.includes(name))
-        return false;
-    if (grep === undefined)
-        return true;
-    if (typeof grep === "string")
-        return name.includes(grep);
-    grep.lastIndex = 0;
-    return grep.test(name);
-}
+import { DEFAULT_SRC_DIR, DEFAULT_TEST_MATCH } from "./groups.js";
+import { DEFAULT_PORT } from "./ports.js";
+import TestReporter from "./reporter.js";
+import { planItems, poolMode, runPooled, serialKeyOf, } from "./pooled-run.js";
+import { runOneTest } from "./run-test.js";
+import { browsersOfRun, loadSpecs, selectTests } from "./select.js";
+import { resolveWorkers } from "./workers-option.js";
+import { WorkerPool } from "./worker-pool.js";
+import { processWorkerFactory } from "./worker-process.js";
+/** Finds the spec files, picks the tests and browsers of a run and runs each test in each browser */
 export default class TestRunner {
-    constructor(testFiles, run, group, options = {}) {
-        /** The real `srcDir` of this run, set by `start`; test screenshots are saved under paths relative to it */
-        this.srcDir = "";
+    constructor(testFiles, run, options = {}, project = {}) {
         this.testFiles = testFiles;
         this.run = run;
-        this.group = group;
         this.options = options;
+        this.project = project;
     }
-    static async init(run, group, options = {}) {
+    static async init(run, options = {}, project = {}) {
         logger.verbose("Initializing test runner");
-        let srcDir = "";
-        if (group && group.src) {
-            srcDir = group.src;
-        }
-        else {
-            srcDir = (await readConfig("srcDir")) ?? "./src";
-        }
-        assert(srcDir, "No srcDir declared!");
+        const srcDir = (await readConfig("srcDir")) ?? DEFAULT_SRC_DIR;
         const cwd = path.resolve(projectDir(), srcDir);
         if (options.files) {
-            return new TestRunner(options.files.map((file) => path.resolve(projectDir(), file)), run, group, options);
+            return new TestRunner(options.files.map((file) => path.resolve(projectDir(), file)), run, options, project);
         }
         logger.verbose("Reading test files from: %s", cwd);
         const testFiles = [];
-        for await (const entry of glob("**/*.spec.ts", {
+        for await (const entry of glob(DEFAULT_TEST_MATCH, {
             cwd,
         })) {
             testFiles.push(entry);
         }
         logger.debug("Found %d tests in srcDir", testFiles.length);
-        return new TestRunner(testFiles.map((file) => path.join(cwd, file)), run, group, options);
-    }
-    /**
-     * Imports the spec files and returns every test they register. A fresh registry and a fresh import of each
-     * file, so a second run in this process registers its tests again (and sees edited specs, for files loaded as
-     * ES modules; CommonJS files stay cached until the process ends)
-     */
-    async registerAll() {
-        clearRegistry();
-        const stamp = `?run=${Date.now().toString(36)}`;
-        for (const file of this.testFiles)
-            assertModuleProject(file);
-        for (const file of this.testFiles) {
-            logger.verbose("Trying to register file: %s", file);
-            await import(pathToFileURL(file).href + stamp);
-        }
-        return [...registeredTests()];
-    }
-    /** Id of a registered test: its spec file relative to `srcDir`, then its name */
-    idOf(test, srcDir) {
-        return testId(toPosix(path.relative(srcDir, test.file)), test.name);
-    }
-    /** The config's groups, resolved against the tests the specs register */
-    async resolveAll(tests) {
-        const config = await loadConfig();
-        const { dir, srcDir } = realDirs(config.srcDir);
-        const groups = resolveGroups(config, tests.map(({ file, name }) => ({
-            file: toPosix(path.relative(srcDir, file)),
-            name,
-        })), dir, srcDir);
-        return { groups, srcDir };
+        return new TestRunner(testFiles.map((file) => path.join(cwd, file)), run, options, project);
     }
     /** The groups of the config with the tests each holds; see `listGroups` */
     async groups() {
-        return (await this.resolveAll(await this.registerAll())).groups;
+        return (await loadSpecs(this.testFiles)).groups;
     }
     /** Imports the spec files and returns the tests the options select: by `group`, `testNames` and `grep` */
     async register() {
-        const all = await this.registerAll();
-        const { group } = this.options;
-        let inGroup = () => true;
-        if (group !== undefined) {
-            const { groups, srcDir } = await this.resolveAll(all);
-            const members = new Set(findGroup(groups, group).testIds);
-            inGroup = (test) => members.has(this.idOf(test, srcDir));
-        }
-        return all.filter((test) => inGroup(test) && isSelected(test.name, this.options));
+        return selectTests(await loadSpecs(this.testFiles), this.options);
     }
-    /** Runs the selected tests and returns the summary that was also written to the report */
+    /**
+     * Runs the selected tests in every browser of the run, one browser after the other, and returns the summary that
+     * was also written to the report. With more than one worker the tests run at the same time in worker processes.
+     */
     async start() {
         const { reportPath, onEvent, signal } = this.options;
+        // The specs are imported once: their top-level code runs once per run
+        const loaded = await loadSpecs(this.testFiles);
+        const selected = selectTests(loaded, this.options);
+        const browsers = await browsersOfRun(loaded.groups, this.options);
         const reporter = new TestReporter({
             environment: this.run.settings.environment,
+            browsers,
             ...(reportPath !== undefined && { output: reportPath }),
             ...(onEvent && { onEvent }),
         });
-        const selected = await this.register();
-        this.srcDir = realDirs((await loadConfig()).srcDir).srcDir;
-        // A missing browser is one clear error before the run, not a failure per test; a fresh build's slow
-        // first start happens here, outside any test's timeout
-        if (selected.length > 0)
-            await ensureWarm((await readConfig("browser")) ?? "firefox", { port: this.options.port ?? 9223 }, signal);
-        reporter.onStart(selected.length);
-        for (const test of selected) {
-            if (signal?.aborted) {
-                reporter.onTestStart(test);
-                reporter.onTestEnd(test, { message: "Run aborted", type: "error" });
-                continue;
+        // How the run goes is settled first: a bad `workers` is an error before any browser starts
+        const config = await loadConfig();
+        const requested = resolveWorkers({
+            flag: this.options.workers,
+            env: process.env.SAMURAI_WORKERS,
+            config: config.workers,
+        });
+        const plan = planItems(selected, browsers, serialKeyOf(loaded.groups, config.groups, loaded.srcDir));
+        const mode = poolMode({
+            workers: requested,
+            items: plan.items.length,
+            inlineConfig: this.project.inlineConfig ?? false,
+        });
+        if (mode === "inline-config" && this.workersAsked(config.workers))
+            logger.warn("workers needs samurai.config.ts in the project folder; running on one worker");
+        // A missing browser is one clear error before the run, not a failure per test, and before any browser has run a
+        // test; a fresh build's slow first start happens here too, outside any test's timeout
+        if (selected.length > 0) {
+            for (const browser of browsers)
+                await projectBrowser(browser);
+            for (const browser of browsers)
+                await ensureWarm(browser, { port: this.options.port ?? DEFAULT_PORT }, signal);
+        }
+        if (mode === "pool") {
+            const workers = Math.min(requested, plan.items.length);
+            reporter.onStart(plan.items.length, workers);
+            const pool = this.createPool(workers);
+            try {
+                await runPooled(pool, plan.planned, plan.items, reporter, signal);
+            }
+            finally {
+                await pool.dispose();
             }
-            await this.executeTestCase(test, reporter);
+            return reporter.onEnd();
         }
-        return reporter.onEnd();
-    }
-    async executeTestCase(test, reporter) {
-        logger.verbose("Starting test: %s", test.name);
-        reporter.onTestStart(test);
-        // A test ends once, whichever of finishing, failing, timing out, an abort or a closed browser comes first
-        let ended = false;
-        let evidence;
-        const finish = (error, logs) => {
-            if (ended)
-                return;
-            ended = true;
-            reporter.onTestEnd(test, error, {
-                ...logs,
-                screenshots: evidence?.screenshots,
-            });
-        };
-        let browserClosed = false;
-        /** Set once the runner itself closes the browser */
-        let closing = false;
-        const selectedBrowser = (await readConfig("browser")) ?? "firefox";
-        const { settings, secrets } = this.run;
-        const timeout = settings.timeout;
-        const { reportPath } = this.options;
-        const resultDir = typeof reportPath === "string"
-            ? path.dirname(reportPath)
-            : path.join(projectDir(), "result");
-        let browser;
-        let page;
-        let timeoutTimer;
-        let onRunAborted;
-        const timeoutController = new AbortController();
-        await Promise.race([
-            new Promise((resolve) => {
-                timeoutTimer = setTimeout(() => {
-                    finish({
-                        message: `Test timed out after ${timeout / 1000} seconds`,
-                        type: "timeout",
-                    });
-                    timeoutController.abort(new Error("Test timed out"));
-                    resolve();
-                }, timeout);
-                onRunAborted = () => {
-                    if (timeoutController.signal.aborted)
-                        return;
-                    finish({ message: "Run aborted", type: "error" });
-                    timeoutController.abort(new Error("Run aborted"));
-                    resolve();
-                };
-                this.options.signal?.addEventListener("abort", onRunAborted, {
-                    once: true,
-                });
-            }),
-            new Promise(async (resolve, reject) => {
-                try {
-                    const launched = await Browser.launch(selectedBrowser, {
-                        port: this.options.port ?? 9223,
-                        headless: this.options.headless ?? settings.headless,
-                    }, timeoutController.signal);
-                    browser = launched.browser;
-                    // A person closing the window, or the browser crashing: end the test now instead of waiting for a
-                    // command to time out
-                    void launched.browser.exited.then(() => {
-                        if (ended || closing)
-                            return;
-                        browserClosed = true;
-                        finish({
-                            message: "The browser was closed while the test was running",
-                            type: "error",
-                        });
-                        timeoutController.abort(new Error("Browser closed"));
-                        resolve();
+        reporter.onStart(selected.length * browsers.length);
+        for (const browser of browsers) {
+            for (const test of selected) {
+                if (signal?.aborted) {
+                    reporter.onTestStart(test, browser);
+                    reporter.onTestEnd(test, browser, {
+                        message: "Run aborted",
+                        type: "error",
                     });
-                    page = launched.page;
-                    evidence = new Evidence(resultDir, testDirOf(test, this.srcDir));
-                    launched.page.setEvidence(evidence);
-                    const fixtures = {
-                        page: launched.page,
-                        browser: launched.browser,
-                        env: createEnvFixture(settings.environment, settings.variables),
-                        secrets: createSecretsFixture(settings.environment, secrets),
-                    };
-                    await test.function(fixtures);
-                    // Already reported as timed out
-                    if (timeoutController.signal.aborted)
-                        return resolve();
-                    const unawaited = takePendingAssertions();
-                    if (unawaited.length > 0) {
-                        finish({
-                            message: unawaited
-                                .map((matcher) => `expect(locator).${matcher}() was not awaited`)
-                                .join("\n"),
-                            type: "assertion",
-                        }, await judgeLogs(page, true));
-                        return resolve();
-                    }
-                    const verdict = await judgeLogs(page, false);
-                    finish(verdict.error, verdict);
-                    resolve();
+                    continue;
                 }
-                catch (err) {
-                    if (timeoutController.signal.aborted)
-                        return resolve();
-                    finish(browserClosed || err instanceof ConnectionClosedError
-                        ? {
-                            message: "The browser was closed while the test was running",
-                            type: "error",
-                        }
-                        : toTestError(err), await judgeLogs(page, true));
-                    resolve();
-                }
-            }),
-        ]);
-        clearTimeout(timeoutTimer);
-        if (onRunAborted)
-            this.options.signal?.removeEventListener("abort", onRunAborted);
-        takePendingAssertions();
-        logger.verbose("Closing browser for test: %s", test.name);
-        closing = true;
-        await browser?.close();
-    }
-}
-/** The project and source folders as real paths: registered test files come from stack traces, which resolve symlinks */
-function realDirs(configuredSrcDir) {
-    const dir = realPath(projectDir());
-    const srcDir = realPath(path.resolve(dir, configuredSrcDir ?? DEFAULT_SRC_DIR));
-    return { dir, srcDir };
-}
-/**
- * The folder of a test's screenshots below `screenshots/`: its spec file without extension, then its cleaned
- * name. A spec outside srcDir loses the `..` (and a drive or root) its relative path starts with, so it stays
- * below `screenshots/`.
- * @internal Exported for tests
- */
-export function testDirOf(test, srcDir) {
-    const file = toPosix(path.relative(srcDir, test.file))
-        .replace(/\.[cm]?[jt]sx?$/, "")
-        .split("/")
-        .filter((segment) => segment !== "" &&
-        segment !== "." &&
-        segment !== ".." &&
-        !/^[A-Za-z]:$/.test(segment))
-        .join("/");
-    return `${file}/${cleanName(test.name)}`;
-}
-/** Reads the page's log once the test body is done and applies config `logs` (see judgePageLogs) */
-async function judgeLogs(page, testFailed) {
-    if (!page)
-        return {};
-    try {
-        // Events and command replies share one ordered connection: after this, everything logged so far is in
-        await page.syncLogs();
-    }
-    catch (err) {
-        logger.debug("Could not sync the page's logs", { err });
+                await runOneTest(test, browser, this.run, this.options, loaded.srcDir, reporter);
+            }
+        }
+        return reporter.onEnd();
     }
-    let config;
-    try {
-        config = await readConfig("logs");
+    /** Whether anything but the default says how many workers to use */
+    workersAsked(fromConfig) {
+        return (this.options.workers !== undefined ||
+            Boolean(process.env.SAMURAI_WORKERS?.trim()) ||
+            fromConfig !== undefined);
     }
-    catch (err) {
-        logger.debug("Could not read logs config, using defaults", { err });
+    /** The pool of a run: up to `workers` worker processes that read the same project and settings as this run */
+    createPool(workers) {
+        const { settings } = this.run;
+        const { reportPath, headless, port } = this.options;
+        const options = {
+            workers,
+            factory: processWorkerFactory(),
+            init: {
+                projectDir: projectDir(),
+                ...(this.project.dataDir !== undefined && {
+                    dataDir: this.project.dataDir,
+                }),
+                environment: settings.environment,
+                timeout: settings.timeout,
+                expectTimeout: settings.expectTimeout,
+                ...(headless !== undefined && { headless }),
+                ...(typeof reportPath === "string" && { reportPath }),
+                files: this.testFiles,
+                basePort: port ?? DEFAULT_PORT,
+            },
+        };
+        return (this.project.createPool ?? ((o) => new WorkerPool(o)))(options);
     }
-    return judgePageLogs({
-        config,
-        entries: page.getLogs(),
-        errors: page.pageErrors(),
-        dropped: page.logsDropped,
-        testFailed,
-        allowPageErrors: page.pageErrorsAllowed,
-        routeErrors: page.routeErrors(),
-    });
 }
-export { describe, test } from "./registry.js";

package/dist/runner/worker-pool.d.ts

@@ -0,0 +1,67 @@
+import type { TestResult } from "../types/test.js";
+import type { WorkerFactory, WorkerInit, WorkItem } from "./worker-protocol.js";
+export type PoolEvent = {
+    type: "test-start";
+    item: WorkItem;
+    worker: number;
+} | {
+    type: "test-end";
+    item: WorkItem;
+    worker: number;
+    result: TestResult;
+};
+export interface PoolOptions {
+    /** The cap on workers running at the same time */
+    workers: number;
+    init: Omit<WorkerInit, "workerIndex">;
+    factory: WorkerFactory;
+    /** The abort grace period in ms; 2000 by default */
+    graceMs?: number;
+}
+export interface RunHooks {
+    onEvent?: (event: PoolEvent) => void;
+    signal?: AbortSignal;
+}
+/**
+ * A fixed cap of worker processes that take items from one shared queue.
+ * Workers start when there is work; `run()` can be called several times at
+ * once and all calls share the cap.
+ */
+export declare class WorkerPool {
+    private readonly options;
+    private readonly queue;
+    private readonly slots;
+    private readonly calls;
+    private readonly timers;
+    private nextId;
+    /** Workers in a row that exited before they were ready */
+    private startupFailures;
+    private disposed;
+    constructor(options: PoolOptions);
+    /** Results in item order; may be called several times at once (flows): all calls share the cap */
+    run(items: WorkItem[], hooks?: RunHooks): Promise<TestResult[]>;
+    /** Tells every worker to exit and kills it; safe to call twice */
+    dispose(): Promise<void>;
+    /** Hands queued items to ready workers, then starts workers for what is left */
+    private pump;
+    private busyKeys;
+    /** The queue position of the first item whose serial key is not running */
+    private nextEligible;
+    /** How many queued items could run at the same time right now */
+    private eligibleCount;
+    private freeIndex;
+    private start;
+    private dispatch;
+    private onMessage;
+    private onExit;
+    /** Records an item's result, tells the listener, and gives the worker its next item */
+    private finish;
+    private settle;
+    private emit;
+    /** Fails the queued items of a call, asks its busy workers to stop, and kills the ones that do not */
+    private abort;
+    private fatal;
+    private rejectAll;
+    /** Forgets every worker, then kills it, so their exits are not taken for crashes */
+    private stopWorkers;
+}