@itmetsam/samurai-framework 0.21.4 → 0.22.0

package/dist/runner/test-runner.d.ts

@@ -39,6 +39,7 @@ export default class TestRunner {
      * was also written to the report. With more than one worker the tests run at the same time in worker processes.
      */
     start(): Promise<import("../types/test.js").TestSummary>;
+    private execute;
     /** Whether anything but the default says how many workers to use */
     private workersAsked;
     /** The pool of a run: up to `workers` worker processes that read the same project and settings as this run */

package/dist/runner/test-runner.js

@@ -11,13 +11,18 @@ import { DEFAULT_SRC_DIR, DEFAULT_TEST_MATCH } from "../config/defaults.js";
 import { DEFAULT_PORT } from "./ports.js";
 import TestReporter from "./reporter.js";
 import { concurrency, planItems, poolMode, runPooled, serialKeyOf, } from "./pooled-run.js";
-import { allHookFixtures, runOneTest } from "./run-test.js";
+import { allHookFixtures } from "./fixtures.js";
+import { runOneTest } from "./run-test.js";
 import { staticSkip } from "./skip.js";
-import { browsersOfRun, loadSpecs, selectTests } from "./select.js";
+import { browserPlan, expandByGroup, loadSpecs, selectTests, } from "./select.js";
 import { resolveRetries } from "./retries-option.js";
 import { runWithRetries } from "./retry.js";
+import { chunkBySerialBlock, unitOf } from "./serial.js";
+import { parseShard, shardGroupOf, shardTests } from "./shard.js";
+import { serialBlockRun } from "./serial-run.js";
 import { resolveWorkers } from "../config/workers-option.js";
 import { runOutput } from "./output.js";
+import { withRunHooks } from "./run-hooks.js";
 import { WorkerPool } from "./worker-pool.js";
 import { processWorkerFactory } from "./worker-process.js";
 /** Finds the spec files, picks the tests and browsers of a run and runs each test in each browser */
@@ -62,18 +67,37 @@ export default class TestRunner {
      * was also written to the report. With more than one worker the tests run at the same time in worker processes.
      */
     async start() {
+        const config = await loadRawConfig();
+        // Web servers and `globalSetup` come first and `globalTeardown` last, around everything below
+        return withRunHooks({
+            config,
+            settings: this.run.settings,
+            projectDir: projectDir(),
+            signal: this.options.signal,
+        }, () => this.execute(config));
+    }
+    async execute(config) {
         const { onEvent, signal } = this.options;
         // The specs are imported once: their top-level code runs once per run
         const loaded = await loadSpecs(this.testFiles);
-        const config = await loadRawConfig();
         // A focused test the config refuses stops the run before anything starts; else only the focused ones run
         assertNoOnly(loaded.all, forbidsOnly(config), loaded.srcDir);
-        const selected = narrowToOnly(selectTests(loaded, this.options));
-        const browsers = await browsersOfRun(loaded.groups, this.options);
+        const shard = this.options.shard
+            ? parseShard(this.options.shard, "shard")
+            : undefined;
+        const focused = narrowToOnly(selectTests(loaded, this.options));
+        const sharded = shard
+            ? shardTests(focused, shard, (test) => shardGroupOf(test, config.fullyParallel))
+            : focused;
+        // A test of two selected groups is two runs, each with its group's `use` and `browsers`
+        const selected = expandByGroup(sharded, loaded, this.options);
+        const { browsers, browsersOf } = await browserPlan(loaded.groups, this.options);
+        const runsIn = (test, browser) => browsersOf(test).includes(browser);
         const reporter = new TestReporter({
             environment: this.run.settings.environment,
             browsers,
             output: runOutput(this.options, projectDir()).report,
+            ...(shard && { shard }),
             ...(onEvent && { onEvent }),
         });
         // How the run goes is settled first: a bad `workers` is an error before any browser starts
@@ -88,7 +112,7 @@ export default class TestRunner {
             env: process.env.SAMURAI_RETRIES,
             config: config.retries,
         });
-        const plan = planItems(selected, browsers, serialKeyOf(loaded.groups, config.groups, loaded.srcDir), this.options.inputs);
+        const plan = planItems(selected, browsers, serialKeyOf(loaded.groups, config.groups, loaded.srcDir), this.options.inputs, (test) => unitOf(test, config.fullyParallel), runsIn);
         // A pool larger than what can overlap would only hold idle workers: a group that runs one at a time needs one
         const lanes = concurrency(plan.items);
         const mode = poolMode({
@@ -119,7 +143,7 @@ export default class TestRunner {
             }
             return reporter.onEnd();
         }
-        reporter.onStart(selected.length * browsers.length);
+        reporter.onStart(plan.planned.length);
         // The `afterAll` hooks run when the last test is done, before the run ends
         const allHooks = new AllHooks({
             timeout: this.run.settings.timeout,
@@ -127,21 +151,41 @@ export default class TestRunner {
         });
         try {
             for (const browser of browsers) {
-                for (const test of selected) {
-                    const skip = staticSkip(test);
-                    if (skip) {
-                        reporter.onTestSkipped(test, browser, skip);
-                        continue;
-                    }
-                    if (signal?.aborted) {
-                        reporter.onTestStart(test, browser);
-                        reporter.onTestEnd(test, browser, {
-                            message: "Run aborted",
-                            type: "error",
+                const forBrowser = selected.filter((test) => runsIn(test, browser));
+                for (const chunk of chunkBySerialBlock(forBrowser)) {
+                    const runTest = (test, sink, attempt) => runOneTest(test, browser, this.run, this.options, loaded.srcDir, sink, { allHooks, attempt });
+                    if (chunk.block) {
+                        const { suite } = chunk.block;
+                        await serialBlockRun({
+                            tests: chunk.tests,
+                            browser,
+                            retries,
+                            signal,
+                            sink: reporter,
+                            runTest,
+                            restart: async () => {
+                                for (const error of await allHooks.closeWithin(suite))
+                                    logger.error("%s", error.message);
+                            },
                         });
                         continue;
                     }
-                    await runWithRetries(test, browser, retries, signal, (sink, attempt) => runOneTest(test, browser, this.run, this.options, loaded.srcDir, sink, { allHooks, attempt }), reporter);
+                    for (const test of chunk.tests) {
+                        const skip = staticSkip(test);
+                        if (skip) {
+                            reporter.onTestSkipped(test, browser, skip);
+                            continue;
+                        }
+                        if (signal?.aborted) {
+                            reporter.onTestStart(test, browser);
+                            reporter.onTestEnd(test, browser, {
+                                message: "Run aborted",
+                                type: "error",
+                            });
+                            continue;
+                        }
+                        await runWithRetries(test, browser, retries, signal, (sink, attempt) => runTest(test, sink, attempt), reporter);
+                    }
                 }
             }
         }
@@ -170,6 +214,7 @@ export default class TestRunner {
                     dataDir: this.project.dataDir,
                 }),
                 environment: settings.environment,
+                ...(settings.baseURL !== undefined && { baseURL: settings.baseURL }),
                 timeout: settings.timeout,
                 expectTimeout: settings.expectTimeout,
                 ...(headless !== undefined && { headless }),

package/dist/runner/test-timeout.d.ts

@@ -0,0 +1,22 @@
+/**
+ * The time limit of one test. It starts once, and can be changed while the test runs: the new limit counts from
+ * the start of the test, so the time already spent counts against it (a limit already passed ends the test now).
+ */
+export declare class TestTimeout {
+    private limit;
+    private readonly onExpire;
+    private timer;
+    private startedAt;
+    private expired;
+    constructor(limit: number, onExpire: () => void);
+    /** The limit in milliseconds; 0 means none */
+    get ms(): number;
+    /** Starts the clock; nothing can expire before this */
+    start(): void;
+    /** Changes the limit, re-arming the timer from the start of the test */
+    set(ms: number): void;
+    stop(): void;
+    private arm;
+}
+/** Checks a timeout in milliseconds; `label` starts the error (`setTimeout()`) */
+export declare function checkTimeout(label: string, ms: unknown): number;

package/dist/runner/test-timeout.js

@@ -0,0 +1,45 @@
+/**
+ * The time limit of one test. It starts once, and can be changed while the test runs: the new limit counts from
+ * the start of the test, so the time already spent counts against it (a limit already passed ends the test now).
+ */
+export class TestTimeout {
+    constructor(limit, onExpire) {
+        this.limit = limit;
+        this.onExpire = onExpire;
+        this.expired = false;
+    }
+    /** The limit in milliseconds; 0 means none */
+    get ms() {
+        return this.limit;
+    }
+    /** Starts the clock; nothing can expire before this */
+    start() {
+        this.startedAt = Date.now();
+        this.arm();
+    }
+    /** Changes the limit, re-arming the timer from the start of the test */
+    set(ms) {
+        this.limit = ms;
+        if (this.startedAt !== undefined && !this.expired)
+            this.arm();
+    }
+    stop() {
+        clearTimeout(this.timer);
+    }
+    arm() {
+        clearTimeout(this.timer);
+        if (this.limit === 0 || this.startedAt === undefined)
+            return;
+        const left = Math.max(0, this.limit - (Date.now() - this.startedAt));
+        this.timer = setTimeout(() => {
+            this.expired = true;
+            this.onExpire();
+        }, left);
+    }
+}
+/** Checks a timeout in milliseconds; `label` starts the error (`setTimeout()`) */
+export function checkTimeout(label, ms) {
+    if (typeof ms !== "number" || !Number.isFinite(ms) || ms < 0)
+        throw new Error(`${label}: the timeout must be a number of milliseconds, 0 or more`);
+    return ms;
+}

package/dist/runner/use-options.d.ts

@@ -0,0 +1,7 @@
+import type { GroupUse } from "../types/config.js";
+import type { RegisteredTestCase } from "../types/test.js";
+/**
+ * The `use` options one test runs with: config `use`, then the `use` of the group the test runs for (if any), then
+ * the `test.use()` calls around the test, outermost first, a later one winning key by key
+ */
+export declare function effectiveUse(test: RegisteredTestCase): Promise<GroupUse>;

package/dist/runner/use-options.js

@@ -0,0 +1,25 @@
+import { readConfig } from "../config/config.js";
+import { groupUseOf } from "../config/group-use.js";
+import { mergeUse, useOptionsOf } from "../config/use-options.js";
+import logger from "../logger/index.js";
+import { useOf } from "./suite.js";
+/**
+ * The `use` options one test runs with: config `use`, then the `use` of the group the test runs for (if any), then
+ * the `test.use()` calls around the test, outermost first, a later one winning key by key
+ */
+export async function effectiveUse(test) {
+    let config;
+    let groups;
+    try {
+        config = await readConfig("use");
+        groups = await readConfig("groups");
+    }
+    catch (err) {
+        logger.debug("Could not read use config, using defaults", { err });
+    }
+    return mergeUse([
+        useOptionsOf(config),
+        test.group === undefined ? {} : groupUseOf(groups, test.group),
+        ...useOf(test.suite),
+    ]);
+}

package/dist/runner/wait-for-url.d.ts

@@ -0,0 +1,20 @@
+/** What to probe: an `http`/`https` URL, or a local port */
+export type ReadyTarget = {
+    url: string;
+    ignoreHTTPSErrors?: boolean;
+} | {
+    port: number;
+};
+/** One probe of `target` */
+export declare function answers(target: ReadyTarget): Promise<boolean>;
+/** The target as text for messages: the URL, or `localhost:<port>` */
+export declare function describeTarget(target: ReadyTarget): string;
+/**
+ * Probes `target` until it answers. Resolves "ready", or "timeout" after `timeout` ms, or "aborted" when `signal`
+ * fires or `stopped()` turns true (the caller saw the server die)
+ */
+export declare function waitUntilReady(target: ReadyTarget, options: {
+    timeout: number;
+    signal?: AbortSignal;
+    stopped?: () => boolean;
+}): Promise<"ready" | "timeout" | "aborted">;

package/dist/runner/wait-for-url.js

@@ -0,0 +1,65 @@
+import http from "node:http";
+import https from "node:https";
+import net from "node:net";
+/** Pause between two probes */
+const POLL_MS = 100;
+/** Resolves true once `url` answers with a status from 200 to 403 (a server that is up but wants a login counts) */
+function urlAnswers(url, options = {}) {
+    return new Promise((resolve) => {
+        const secure = new URL(url).protocol === "https:";
+        const request = (secure ? https : http).get(url, {
+            timeout: 2000,
+            headers: { Accept: "text/html,application/json,*/*;q=0.8" },
+            ...(secure && { rejectUnauthorized: !options.ignoreHTTPSErrors }),
+        }, (response) => {
+            response.resume();
+            const status = response.statusCode ?? 0;
+            resolve(status >= 200 && status <= 403);
+        });
+        request.on("timeout", () => request.destroy());
+        request.on("error", () => resolve(false));
+    });
+}
+/** Resolves true when something on this machine accepts a TCP connection on `port` (IPv4 or IPv6 `localhost`) */
+function portAnswers(port) {
+    return new Promise((resolve) => {
+        const socket = net.connect({ port, host: "localhost", timeout: 2000 });
+        socket.once("connect", () => {
+            socket.destroy();
+            resolve(true);
+        });
+        socket.once("timeout", () => {
+            socket.destroy();
+            resolve(false);
+        });
+        socket.once("error", () => resolve(false));
+    });
+}
+/** One probe of `target` */
+export function answers(target) {
+    return "url" in target
+        ? urlAnswers(target.url, {
+            ...(target.ignoreHTTPSErrors && { ignoreHTTPSErrors: true }),
+        })
+        : portAnswers(target.port);
+}
+/** The target as text for messages: the URL, or `localhost:<port>` */
+export function describeTarget(target) {
+    return "url" in target ? target.url : `localhost:${target.port}`;
+}
+/**
+ * Probes `target` until it answers. Resolves "ready", or "timeout" after `timeout` ms, or "aborted" when `signal`
+ * fires or `stopped()` turns true (the caller saw the server die)
+ */
+export async function waitUntilReady(target, options) {
+    const deadline = Date.now() + options.timeout;
+    for (;;) {
+        if (options.signal?.aborted || options.stopped?.())
+            return "aborted";
+        if (await answers(target))
+            return "ready";
+        if (Date.now() >= deadline)
+            return "timeout";
+        await new Promise((resolve) => setTimeout(resolve, POLL_MS));
+    }
+}

package/dist/runner/web-server.d.ts

@@ -0,0 +1,16 @@
+import type { WebServerConfig } from "../types/config.js";
+/** The servers of a run, started */
+export interface RunningServers {
+    /** Where the first server with a `url` or `port` answers: the base URL when none is configured */
+    baseURL?: string;
+    /** Stops every server this run started (not the ones it reused). Safe to call twice */
+    stop(): Promise<void>;
+}
+/**
+ * Starts the servers one after the other and waits for each. A server that fails to start stops the ones already
+ * started and throws `webServer(): what is wrong`, with the last lines of its output.
+ */
+export declare function startWebServers(servers: readonly WebServerConfig[], context: {
+    projectDir: string;
+    signal?: AbortSignal | undefined;
+}): Promise<RunningServers>;

package/dist/runner/web-server.js

@@ -0,0 +1,165 @@
+import { spawn, spawnSync } from "node:child_process";
+import path from "node:path";
+import { DEFAULT_GRACEFUL_TIMEOUT, DEFAULT_WEB_SERVER_TIMEOUT, } from "../config/web-server-options.js";
+import logger from "../logger/index.js";
+import { OutputTail } from "./output-tail.js";
+import { answers, describeTarget, waitUntilReady, } from "./wait-for-url.js";
+const fail = (problem) => {
+    throw new Error(`webServer(): ${problem}`);
+};
+const windows = process.platform === "win32";
+function targetOf(server) {
+    return server.url !== undefined
+        ? {
+            url: server.url,
+            ...(server.ignoreHTTPSErrors && { ignoreHTTPSErrors: true }),
+        }
+        : { port: server.port };
+}
+/** The base URL a server stands for: its URL's origin, or `http://localhost:<port>` */
+function baseURLOf(server) {
+    return server.url !== undefined
+        ? new URL(server.url).origin
+        : `http://localhost:${server.port}`;
+}
+function signalGroup(pid, signal, child) {
+    try {
+        if (windows) {
+            if (signal !== 0)
+                spawnSync("taskkill", ["/pid", String(pid), "/T", "/F"]);
+            return child?.exitCode === null;
+        }
+        process.kill(-pid, signal);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+class StartedServer {
+    constructor(child, server) {
+        this.child = child;
+        this.server = server;
+        this.exited = false;
+        this.exitText = "";
+        this.onProcessExit = () => {
+            if (this.child.pid !== undefined)
+                signalGroup(this.child.pid, "SIGKILL", this.child);
+        };
+        this.tail = new OutputTail(20, Object.values(server.env ?? {}));
+        for (const [stream, mode, out] of [
+            [child.stdout, server.stdout ?? "ignore", process.stdout],
+            [child.stderr, server.stderr ?? "pipe", process.stderr],
+        ])
+            stream?.on("data", (chunk) => {
+                this.tail.add(chunk);
+                if (mode === "pipe")
+                    out.write(chunk);
+            });
+        this.exitPromise = new Promise((resolve) => {
+            child.once("exit", (code, signal) => {
+                this.exited = true;
+                this.exitText =
+                    code !== null ? `code ${code}` : `signal ${String(signal)}`;
+                resolve();
+            });
+            child.once("error", (error) => {
+                this.exited = true;
+                this.exitText = `error: ${error.message}`;
+                resolve();
+            });
+        });
+        process.once("exit", this.onProcessExit);
+    }
+    hasExited() {
+        return this.exited;
+    }
+    describeExit() {
+        return this.exitText;
+    }
+    /** SIGTERM-style signal to the group, then SIGKILL for what is left after the grace period */
+    stop() {
+        this.stopping ??= this.stopNow();
+        return this.stopping;
+    }
+    async stopNow() {
+        const pid = this.child.pid;
+        const { signal = "SIGTERM", timeout = DEFAULT_GRACEFUL_TIMEOUT } = this.server.gracefulShutdown ?? {};
+        if (pid !== undefined) {
+            if (timeout > 0 && !windows) {
+                signalGroup(pid, signal);
+                const deadline = Date.now() + timeout;
+                while (Date.now() < deadline &&
+                    (!this.exited || signalGroup(pid, 0, this.child)))
+                    await new Promise((resolve) => setTimeout(resolve, 50));
+            }
+            signalGroup(pid, "SIGKILL", this.child);
+            await Promise.race([
+                this.exitPromise,
+                new Promise((resolve) => setTimeout(resolve, 2000)),
+            ]);
+        }
+        process.off("exit", this.onProcessExit);
+    }
+}
+async function startOne(server, context) {
+    const target = targetOf(server);
+    const name = describeTarget(target);
+    if (await answers(target)) {
+        if (server.reuseExistingServer)
+            return undefined;
+        fail(`${name} is already in use: stop what runs there, or set reuseExistingServer: true`);
+    }
+    logger.verbose("Starting web server for %s", name);
+    const child = spawn(server.command, {
+        shell: true,
+        cwd: path.resolve(context.projectDir, server.cwd ?? "."),
+        env: { ...process.env, ...server.env },
+        stdio: ["ignore", "pipe", "pipe"],
+        detached: !windows,
+    });
+    const started = new StartedServer(child, server);
+    const timeout = server.timeout ?? DEFAULT_WEB_SERVER_TIMEOUT;
+    const verdict = await waitUntilReady(target, {
+        timeout,
+        ...(context.signal && { signal: context.signal }),
+        stopped: () => started.hasExited(),
+    });
+    if (verdict === "ready")
+        return started;
+    // Capture what the server printed before it is stopped
+    const exited = started.hasExited();
+    const how = exited ? started.describeExit() : "";
+    await started.stop();
+    const output = started.tail.text();
+    const log = output === "" ? "" : `\nLast output:\n${output}`;
+    if (verdict === "aborted" && !exited)
+        return fail(`aborted while waiting for ${name}`);
+    if (exited)
+        fail(`the process exited (${how}) before ${name} was ready${log}`);
+    return fail(`timed out after ${timeout}ms waiting for ${name}${log}`);
+}
+/**
+ * Starts the servers one after the other and waits for each. A server that fails to start stops the ones already
+ * started and throws `webServer(): what is wrong`, with the last lines of its output.
+ */
+export async function startWebServers(servers, context) {
+    const started = [];
+    const stop = async () => {
+        await Promise.all([...started].reverse().map((s) => s.stop()));
+    };
+    let baseURL;
+    try {
+        for (const server of servers) {
+            const running = await startOne(server, context);
+            if (running)
+                started.push(running);
+            baseURL ??= baseURLOf(server);
+        }
+    }
+    catch (error) {
+        await stop();
+        throw error;
+    }
+    return { ...(baseURL !== undefined && { baseURL }), stop };
+}

package/dist/runner/worker-pool.d.ts

@@ -36,6 +36,7 @@ export declare class WorkerPool {
     private readonly slots;
     private readonly calls;
     private readonly timers;
+    private readonly units;
     private nextId;
     /** Workers in a row that exited before they were ready */
     private startupFailures;
@@ -52,18 +53,25 @@ export declare class WorkerPool {
     private sendHome;
     /** Hands queued items to ready workers, then starts workers for what is left */
     private pump;
-    private busyKeys;
-    /** The queue position of the first item whose serial key is not running */
-    private nextEligible;
-    /** How many queued items could run at the same time right now */
-    private eligibleCount;
     private freeIndex;
     private start;
     private dispatch;
     private onMessage;
     private onExit;
+    /** The attempt a started item is on: the worker's own retry, or the attempt at its serial block */
+    private retryOf;
     /** Records an item's result, tells the listener, and gives the worker its next item */
     private finish;
+    private report;
+    /**
+     * An item of a serial block ended. After a failure the rest of the block is skipped; with retries the block
+     * starts over on a new worker first, and what its tests ended with is held until the last attempt is known.
+     */
+    private finishInBlock;
+    /** Reports what a block's tests ended with in its last attempt */
+    private release;
+    /** Puts the block back in the queue, from its first test, and tells the worker that ran it to leave */
+    private restartBlock;
     private settle;
     private emit;
     /** Fails the queued items of a call, asks its busy workers to stop, and kills the ones that do not */