@itmetsam/samurai-framework 0.21.4 → 0.22.0

package/dist/runner/worker-pool.js

@@ -1,3 +1,5 @@
+import { failedResult, serialSkipped, UnitState } from "./pool-units.js";
+import { eligibleCount, nextEligible } from "./pool-schedule.js";
 /** How long a busy worker may take to wind down after an abort before it is killed */
 const GRACE_MS = 2000;
 const SHUTDOWN_MS = 60000;
@@ -15,6 +17,7 @@ export class WorkerPool {
         this.slots = new Map();
         this.calls = new Set();
         this.timers = new Set();
+        this.units = new Map();
         this.nextId = 0;
         /** Workers in a row that exited before they were ready */
         this.startupFailures = 0;
@@ -95,51 +98,23 @@ export class WorkerPool {
         if (this.disposed)
             return;
         for (const slot of [...this.slots.values()]) {
-            if (!slot.ready || slot.busy)
+            if (!slot.ready || slot.busy || slot.retiring)
                 continue;
-            const at = this.nextEligible();
+            const at = nextEligible(this.queue, [...this.slots.values()], slot);
             if (at < 0)
-                break;
+                continue;
             const [entry] = this.queue.splice(at, 1);
             if (entry)
                 this.dispatch(slot, entry);
         }
         let starting = [...this.slots.values()].filter((s) => !s.ready).length;
-        const wanted = this.eligibleCount();
+        const wanted = eligibleCount(this.queue, [...this.slots.values()]);
         while (this.slots.size < this.options.workers && starting < wanted) {
             if (!this.start(this.freeIndex()))
                 return;
             starting++;
         }
     }
-    busyKeys() {
-        const keys = new Set();
-        for (const { busy } of this.slots.values()) {
-            const key = busy?.entry.item.serialKey;
-            if (key)
-                keys.add(key);
-        }
-        return keys;
-    }
-    /** The queue position of the first item whose serial key is not running */
-    nextEligible() {
-        const busy = this.busyKeys();
-        return this.queue.findIndex(({ item }) => !item.serialKey || !busy.has(item.serialKey));
-    }
-    /** How many queued items could run at the same time right now */
-    eligibleCount() {
-        const taken = this.busyKeys();
-        let count = 0;
-        for (const { item } of this.queue) {
-            if (item.serialKey) {
-                if (taken.has(item.serialKey))
-                    continue;
-                taken.add(item.serialKey);
-            }
-            count++;
-        }
-        return count;
-    }
     freeIndex() {
         let index = 0;
         while (this.slots.has(index))
@@ -165,6 +140,13 @@ export class WorkerPool {
     }
     dispatch(slot, entry) {
         const id = this.nextId++;
+        const { unit, serialKey } = entry.item;
+        if (unit) {
+            slot.unit = unit;
+            slot.unitKey = serialKey;
+            if (!this.units.has(unit))
+                this.units.set(unit, new UnitState(entry.call));
+        }
         slot.busy = { id, entry, startTime: Date.now() };
         slot.handle.send({ type: "run", id, item: entry.item });
     }
@@ -187,7 +169,7 @@ export class WorkerPool {
                     type: "test-start",
                     item: busy.entry.item,
                     worker: slot.index,
-                    ...(message.retry && { retry: message.retry }),
+                    ...this.retryOf(busy.entry, message.retry),
                 });
             case "test-end":
                 if (busy?.id !== message.id)
@@ -209,23 +191,84 @@ export class WorkerPool {
             const message = call.aborted
                 ? "Run aborted"
                 : `The worker process exited (code ${code})`;
-            this.finish(slot, busy.entry, failure(item, message, busy.startTime));
+            this.finish(slot, busy.entry, failedResult(item, message, busy.startTime));
         }
         else {
             this.pump();
         }
     }
+    /** The attempt a started item is on: the worker's own retry, or the attempt at its serial block */
+    retryOf({ item }, workerRetry) {
+        const retry = workerRetry || (item.serial ? this.units.get(item.unit)?.attempt : 0);
+        return retry ? { retry } : {};
+    }
     /** Records an item's result, tells the listener, and gives the worker its next item */
     finish(slot, entry, result) {
         const stamped = { ...result, worker: slot.index };
+        if (entry.item.serial)
+            this.finishInBlock(slot, entry, stamped);
+        else
+            this.report(entry, slot.index, stamped);
+        this.pump();
+    }
+    report(entry, worker, result) {
         this.emit(entry.call, {
             type: "test-end",
             item: entry.item,
-            worker: slot.index,
-            result: stamped,
+            worker,
+            result,
         });
-        this.settle(entry, stamped);
-        this.pump();
+        this.settle(entry, result);
+    }
+    /**
+     * An item of a serial block ended. After a failure the rest of the block is skipped; with retries the block
+     * starts over on a new worker first, and what its tests ended with is held until the last attempt is known.
+     */
+    finishInBlock(slot, entry, result) {
+        const state = this.units.get(entry.item.unit);
+        const retries = this.options.init.retries ?? 0;
+        if (retries === 0)
+            this.report(entry, slot.index, result);
+        else
+            state.hold(entry, result);
+        if (result.status !== "failed") {
+            const last = !this.queue.some((e) => e.item.unit === entry.item.unit);
+            if (last)
+                this.release(state);
+            return;
+        }
+        const aborted = entry.call.aborted || result.message === "Run aborted";
+        if (state.attempt < retries && !aborted && !this.disposed) {
+            return this.restartBlock(slot, state, entry, result);
+        }
+        this.release(state);
+        for (let i = 0; i < this.queue.length; i++) {
+            const next = this.queue[i];
+            if (next.item.unit !== entry.item.unit)
+                continue;
+            this.queue.splice(i--, 1);
+            this.report(next, slot.index, serialSkipped(next.item));
+        }
+    }
+    /** Reports what a block's tests ended with in its last attempt */
+    release(state) {
+        for (const { entry, result } of state.release())
+            this.report(entry, result.worker ?? 0, result);
+    }
+    /** Puts the block back in the queue, from its first test, and tells the worker that ran it to leave */
+    restartBlock(slot, state, failed, result) {
+        this.queue.unshift(...state.restart(failed, result));
+        slot.unit = slot.unitKey = undefined;
+        if (this.slots.get(slot.index) !== slot)
+            return;
+        slot.retiring = true;
+        slot.handle.send({ type: "exit" });
+        const timer = setTimeout(() => {
+            this.timers.delete(timer);
+            slot.handle.kill();
+        }, this.options.shutdownMs ?? SHUTDOWN_MS);
+        timer.unref();
+        this.timers.add(timer);
     }
     settle({ call, at }, result) {
         call.results[at] = result;
@@ -246,12 +289,15 @@ export class WorkerPool {
     /** Fails the queued items of a call, asks its busy workers to stop, and kills the ones that do not */
     abort(call) {
         call.aborted = true;
+        for (const state of this.units.values())
+            if (state.call === call)
+                this.release(state);
         for (let i = this.queue.length - 1; i >= 0; i--) {
             const entry = this.queue[i];
             if (entry?.call !== call)
                 continue;
             this.queue.splice(i, 1);
-            this.settle(entry, failure(entry.item, "Run aborted"));
+            this.settle(entry, failedResult(entry.item, "Run aborted"));
         }
         for (const slot of this.slots.values()) {
             const busy = slot.busy;
@@ -294,15 +340,3 @@ export class WorkerPool {
         }
     }
 }
-function failure(item, message, startTime = Date.now()) {
-    return {
-        name: item.testName,
-        file: item.file,
-        browser: item.browser,
-        status: "failed",
-        type: "error",
-        message,
-        startTime,
-        duration: Date.now() - startTime,
-    };
-}

package/dist/runner/worker-protocol.d.ts

@@ -9,8 +9,14 @@ export interface WorkItem {
     /** The test's full name */
     testName: string;
     browser: SupportedBrowser;
+    /** The group this run of the test is for (`--group`): its `use` applies and the result names it */
+    group?: string;
     /** Items with the same key never run at the same time */
     serialKey?: string;
+    /** Items with the same unit run one after the other on the same worker, in the order they are queued */
+    unit?: string;
+    /** The unit is a serial block: a failure skips the rest, and `retries` run the whole block again (the pool does it, not the worker) */
+    serial?: boolean;
     /** Where this test's screenshots go, instead of the run's `outputDir` (a flow gives each node its own) */
     outputDir?: string;
     /** The values the run supplies for the inputs tests declare (`runTests({ inputs })`) */
@@ -21,6 +27,8 @@ export interface WorkerInit {
     projectDir: string;
     dataDir?: string;
     environment?: string;
+    /** The run's base URL when no config sets one (it came from a web server): used where the config has none */
+    baseURL?: string;
     timeout?: number;
     expectTimeout?: number;
     headless?: boolean;

package/dist/runner/worker-session.js

@@ -3,7 +3,8 @@ import logger from "../logger/index.js";
 import { withProject } from "./project.js";
 import TestReporter from "./reporter.js";
 import { AllHooks } from "./all-hooks.js";
-import { allHookFixtures, runOneTest } from "./run-test.js";
+import { allHookFixtures } from "./fixtures.js";
+import { runOneTest } from "./run-test.js";
 import { runWithRetries } from "./retry.js";
 import { loadSpecs } from "./select.js";
 /**
@@ -28,6 +29,8 @@ export function openSession(init) {
             }),
             ...(headless !== undefined && { headless }),
         }, async (run) => {
+            if (run.settings.baseURL === undefined && init.baseURL !== undefined)
+                run.settings.baseURL = init.baseURL;
             const loaded = await loadSpecs(init.files);
             // The `beforeAll` hooks this worker ran: their `afterAll` hooks run when it closes
             const allHooks = new AllHooks({
@@ -55,7 +58,7 @@ export function openSession(init) {
                         port,
                         signal,
                     };
-                    await runWithRetries(test, browser, init.retries ?? 0, signal, (sink, attempt) => runOneTest(test, browser, run, options, loaded.srcDir, sink, {
+                    await runWithRetries(test, browser, perTest.retries ?? init.retries ?? 0, signal, (sink, attempt) => runOneTest(test, browser, run, options, loaded.srcDir, sink, {
                         port,
                         profileDir: workerProfileDir(browser, workerIndex),
                         workerIndex,

package/dist/runner/worker.d.ts

@@ -9,6 +9,8 @@ export interface WorkerSession {
     runTest(test: RegisteredTestCase, browser: SupportedBrowser, signal: AbortSignal, onEvent: (event: RunEvent) => void, options?: {
         outputDir?: string;
         inputs?: Record<string, string>;
+        /** Instead of the run's `retries` */
+        retries?: number;
     }): Promise<void>;
     /** Ends the project's session */
     close(): Promise<void>;

package/dist/runner/worker.js

@@ -3,6 +3,7 @@ const failedResult = (item, worker, message) => ({
     name: item.testName,
     file: item.file,
     browser: item.browser,
+    ...(item.group !== undefined && { group: item.group }),
     worker,
     status: "failed",
     startTime: performance.now(),
@@ -29,7 +30,10 @@ export function handleMessages(channel, { open, exit }) {
         if (!session)
             throw new Error("The worker was sent a test before it was initialised");
         channel.send({ type: "test-start", id });
-        const test = session.tests.find(({ file, name }) => file === item.file && name === item.testName);
+        const found = session.tests.find(({ file, name }) => file === item.file && name === item.testName);
+        const test = found && item.group !== undefined
+            ? { ...found, group: item.group }
+            : found;
         if (!test) {
             const message = `Test "${item.testName}" was not found in ${item.file}`;
             channel.send({
@@ -57,6 +61,8 @@ export function handleMessages(channel, { open, exit }) {
             }, {
                 ...(item.outputDir !== undefined && { outputDir: item.outputDir }),
                 ...(item.inputs !== undefined && { inputs: item.inputs }),
+                // The pool runs a whole serial block again, not the test
+                ...(item.serial && { retries: 0 }),
             });
         }
         finally {

package/dist/script/element-handle.d.ts

@@ -1,7 +1,7 @@
 /**
  * A reference to a DOM node in the browser. The `sharedId` stays valid across scripts in the
  * same document; after navigation or removal the browser answers `no such node`.
- * Internal: user code keeps using `Locator`.
+ * What `evaluate` returns for a DOM node; user code keeps using `Locator` to act on elements.
  */
 export declare class ElementHandle {
     readonly sharedId: string;

package/dist/script/element-handle.js

@@ -1,7 +1,7 @@
 /**
  * A reference to a DOM node in the browser. The `sharedId` stays valid across scripts in the
  * same document; after navigation or removal the browser answers `no such node`.
- * Internal: user code keeps using `Locator`.
+ * What `evaluate` returns for a DOM node; user code keeps using `Locator` to act on elements.
  */
 export class ElementHandle {
     constructor(sharedId) {

package/dist/types/config.d.ts

@@ -1,5 +1,7 @@
 import type { StepDefinition } from "../flows/steps/define.js";
 import type { SupportedBrowser } from "./browser.js";
+import type { UseOptions } from "./use.js";
+import type { ResolvedConfig } from "../config/resolved-config.js";
 
 interface SamuraiTest {
   /** Path relative to srcDir, e.g. 'checkout/coupons.spec.ts' */
@@ -32,8 +34,67 @@ export interface SamuraiGroup {
    * nothing but such a group uses one worker.
    */
   workers?: number;
+  /**
+   * Options for this group's tests, over the config's `use` and under the `test.use()` of a spec: the same options
+   * as config `use` that make sense per group (`locale`, `isMobile`, `viewport`, ...). They apply when the group is
+   * selected (`--group`); a run without `--group` ignores them. The way to port a Playwright project.
+   */
+  use?: GroupUse;
+}
+
+/** The options of a group's `use`: those of `test.use()`, and the page's `viewport` */
+export interface GroupUse extends UseOptions {
+  /** Size of the page's viewport, instead of config `use.viewport`. `null` keeps the browser's own size */
+  viewport?: { width: number; height: number } | null;
+}
+
+/** One server the run starts before the first test (see `SamuraiTestConfig.webServer`). Give either `url` or `port`. */
+export interface WebServerConfig {
+  /** Shell command that starts the server. It leads its own process group, so the whole tree is stopped at the end */
+  command: string;
+  /** Where the server answers: ready once it responds with a status from 200 to 403 */
+  url?: string;
+  /** Where the server listens: ready once `localhost` accepts TCP connections on it */
+  port?: number;
+  /** Working directory of the command, relative to the project folder. @default the project folder */
+  cwd?: string;
+  /** Extra environment variables for the command, on top of this process's. Never printed in errors */
+  env?: Record<string, string>;
+  /** Time (ms) the server may take to be ready. @default 60000 */
+  timeout?: number;
+  /** When `url`/`port` already answers, use that server and start nothing (it is not stopped either). @default false */
+  reuseExistingServer?: boolean;
+  /** Send the command's stdout to this process's stdout, or drop it. @default "ignore" */
+  stdout?: "pipe" | "ignore";
+  /** Send the command's stderr to this process's stderr, or drop it. @default "pipe" */
+  stderr?: "pipe" | "ignore";
+  /** Do not fail on a certificate error while waiting for an `https` `url`. @default false */
+  ignoreHTTPSErrors?: boolean;
+  /**
+   * How the server is stopped: `signal` goes to the whole process group, and after `timeout` ms what is left is
+   * killed. @default { signal: "SIGTERM", timeout: 5000 }
+   */
+  gracefulShutdown?: { signal?: "SIGTERM" | "SIGINT"; timeout?: number };
 }
 
+/** What `globalSetup` and `globalTeardown` receive: the project's resolved config, with the run's environment and base URL */
+export type GlobalHookConfig = ResolvedConfig & {
+  /** The environment the run uses */
+  environment: string;
+};
+
+/** A run-level hook; `globalSetup` may return a function that runs as teardown too */
+export type GlobalSetupFunction = (
+  config: GlobalHookConfig,
+) =>
+  | void
+  | (() => void | Promise<void>)
+  | Promise<void | (() => void | Promise<void>)>;
+
+export type GlobalTeardownFunction = (
+  config: GlobalHookConfig,
+) => void | Promise<void>;
+
 export interface SamuraiEnvironment {
   /** Where page.goto('/…') resolves to in this environment, and where recordings start */
   baseURL?: string;
@@ -64,6 +125,14 @@ export interface SamuraiTestConfig {
    * @default half the CPUs
    */
   workers?: number | string;
+  /**
+   * Whether the tests of one file may run on different workers. Not set: they do, one test at a time goes to the
+   * next free worker (the same as `true`). `false` keeps the tests of a file together, in order, on one worker;
+   * `describe.configure({ mode: 'parallel' })` frees a part of it again. A serial block (`describe.serial`) is
+   * never split, whatever this says.
+   * @default not set
+   */
+  fullyParallel?: boolean;
   /**
    * How many times a failing test is run again, each time with a fresh browser. A test that passes on a retry is
    * reported as flaky and does not fail the run; a run that is aborted is never retried. `--retries` and
@@ -129,8 +198,8 @@ export interface SamuraiTestConfig {
    * @default "files"
    */
   filesDir?: string;
-  /** How pages are opened */
-  use?: {
+  /** How pages are opened, and what the browser pretends: the options of `test.use()` apply to every test */
+  use?: UseOptions & {
     /**
      * Size of the page's viewport, so screenshots and layout are the same on every machine.
      * `null` keeps the browser's own size
@@ -209,6 +278,21 @@ export interface SamuraiTestConfig {
      */
     maxBodySize?: number;
   };
+  /**
+   * Servers the run starts before the first test and stops after the last. Each waits until its `url` answers or
+   * its `port` accepts connections. When no `baseURL` is configured (project or environment), the first server's
+   * `url` origin (or `http://localhost:<port>`) becomes the base URL. Only a run (`samurai run`, `runTests`)
+   * starts them, not `list` or the recorder.
+   */
+  webServer?: WebServerConfig | WebServerConfig[];
+  /**
+   * Module (path relative to the project folder) whose default export `async (config) => void | teardown` runs once
+   * before the tests, after the web servers are up; or the function itself. Variables it sets in `process.env`
+   * reach the tests, workers included. A function it returns runs as teardown.
+   */
+  globalSetup?: string | GlobalSetupFunction;
+  /** Module (path relative to the project folder) whose default export runs once after the tests; or the function itself */
+  globalTeardown?: string | GlobalTeardownFunction;
   /** Keyed by name: 'dev', 'staging', 'production' */
   environments?: Record<string, SamuraiEnvironment>;
   /** Used when no environment is chosen with --env or SAMURAI_ENV */

package/dist/types/extend.d.ts

@@ -0,0 +1,30 @@
+import type { TestFixtures } from "./test.js";
+
+/** What a test sees when it is built with `test.extend()`: the built-in fixtures, a custom one of the same name replacing it */
+export type FixturesOf<X> = Omit<TestFixtures, keyof X> & X;
+
+/** `B` on top of `A`: a key in both takes the type from `B` */
+export type MergeFixtures<A, B> = Omit<A, keyof B> & B;
+
+/** The options of a fixture written as `[fixture, options]` */
+export interface FixtureOptions {
+  /** Set the fixture up for every test, whether or not the test asks for it */
+  auto?: boolean;
+  /** Fixtures live as long as one test; worker-scoped fixtures are not supported */
+  scope?: "test";
+}
+
+/** A value, or a function that sets the value up, calls `use(value)` and cleans up once `use` returns */
+export type FixtureValue<R, Args> =
+  | Exclude<R, (...args: never[]) => unknown>
+  | ((args: Args, use: (value: R) => Promise<void>) => Promise<void>);
+
+/** What `test.extend()` takes: each fixture alone, or as `[fixture, options]` */
+export type FixtureDefinitions<T, Existing> = {
+  [K in keyof T]:
+    | FixtureValue<T[K], FixturesOf<MergeFixtures<Existing, T>>>
+    | [
+        FixtureValue<T[K], FixturesOf<MergeFixtures<Existing, T>>>,
+        FixtureOptions,
+      ];
+};