@itmetsam/samurai-framework 0.21.5 → 0.22.1

package/dist/runner/fixture-outputs.js

@@ -0,0 +1,28 @@
+/**
+ * The declared outputs of a fixture that the report and the app may show: the non-secret keys of its value.
+ * Throws when a declared output is not a string, a number or a boolean (the value must be a flat record).
+ */
+export function outputsOf(def, value) {
+    const shown = (def.outputs ?? []).filter((output) => !output.secret);
+    if (shown.length === 0)
+        return undefined;
+    const record = typeof value === "object" && value !== null
+        ? value
+        : {};
+    const outputs = {};
+    for (const { name } of shown) {
+        const output = record[name];
+        if (typeof output !== "string" &&
+            typeof output !== "number" &&
+            typeof output !== "boolean")
+            throw new Error(`output "${name}" must be a string, number or boolean`);
+        outputs[name] = output;
+    }
+    return outputs;
+}
+/** JSON with the keys of every object in order, so equal options give equal text */
+export function stableJson(value) {
+    return JSON.stringify(value, (_key, item) => typeof item === "object" && item !== null && !Array.isArray(item)
+        ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b)))
+        : item);
+}

package/dist/runner/fixture-registry.d.ts

@@ -0,0 +1,35 @@
+import type { FixtureField, FixtureOutput } from "../types/extend.js";
+import { type FixtureLayers } from "./fixture-defs.js";
+/** Remembers the fixtures of an extended `test`, so `config.fixtures` can be read without running a test */
+export declare function rememberLayers(test: object, layers: FixtureLayers): void;
+/** The fixtures of an extended `test`; nothing for the plain one */
+export declare const layersOf: (test: object) => FixtureLayers | undefined;
+/** What the app and the CLI show of one fixture of a project, read without running a test */
+export interface FixtureCatalogEntry {
+    name: string;
+    /** The `label` option, else the name */
+    label: string;
+    icon?: string;
+    description?: string;
+    scope: "test" | "worker";
+    /** Set up for every test */
+    auto: boolean;
+    /** The name is a built-in fixture's: this one replaces it */
+    overridesBuiltIn: boolean;
+    /** The fixtures it destructures, built-in ones included */
+    dependencies: string[];
+    /** The `<name>Options` fixture: its form, and its values when `use` sets none */
+    options?: {
+        name: string;
+        fields: FixtureField[];
+        defaults: Record<string, unknown>;
+    };
+    /** The declared keys of its value, without the secret ones */
+    outputs?: FixtureOutput[];
+}
+/**
+ * The catalog of the fixtures in `config.fixtures`, read from the extended tests without running anything. Throws
+ * `config.fixtures: ...` for a list entry that is not an extended test, a name that does not match
+ * `/^[a-z][A-Za-z0-9]*$/` and a name defined in two places.
+ */
+export declare function fixtureCatalog(tests: readonly unknown[]): FixtureCatalogEntry[];

package/dist/runner/fixture-registry.js

@@ -0,0 +1,73 @@
+import { BUILT_IN_FIXTURES, } from "./fixture-defs.js";
+/** The fixtures each `test.extend()` result carries; the test function itself holds nothing a config could read */
+const layersByTest = new WeakMap();
+/** Remembers the fixtures of an extended `test`, so `config.fixtures` can be read without running a test */
+export function rememberLayers(test, layers) {
+    layersByTest.set(test, layers);
+}
+/** The fixtures of an extended `test`; nothing for the plain one */
+export const layersOf = (test) => layersByTest.get(test);
+const NAME = /^[a-z][A-Za-z0-9]*$/;
+const fail = (problem) => {
+    throw new Error(`config.fixtures: ${problem}`);
+};
+/** The last definition of every name in the listed tests, with the place of the first test that has it */
+function collect(tests) {
+    const found = new Map();
+    tests.forEach((test, index) => {
+        const layers = (typeof test === "function" ||
+            (typeof test === "object" && test !== null)) &&
+            layersByTest.has(test)
+            ? layersByTest.get(test)
+            : undefined;
+        if (layers === undefined &&
+            !(typeof test === "function" && "extend" in test))
+            throw new Error(`config.fixtures[${index}] is not a test (use the result of test.extend())`);
+        for (const [name, defs] of layers ?? []) {
+            const def = defs[defs.length - 1];
+            const earlier = found.get(name);
+            if (earlier && earlier.def !== def)
+                fail(`fixture "${name}" is defined twice (config.fixtures[${earlier.index}] and config.fixtures[${index}])`);
+            if (!earlier)
+                found.set(name, { def, index });
+        }
+    });
+    return new Map([...found].map(([name, { def }]) => [name, def]));
+}
+/**
+ * The catalog of the fixtures in `config.fixtures`, read from the extended tests without running anything. Throws
+ * `config.fixtures: ...` for a list entry that is not an extended test, a name that does not match
+ * `/^[a-z][A-Za-z0-9]*$/` and a name defined in two places.
+ */
+export function fixtureCatalog(tests) {
+    if (!Array.isArray(tests))
+        throw new Error("config.fixtures must be an array");
+    const defs = collect(tests);
+    for (const name of defs.keys())
+        if (!NAME.test(name))
+            fail(`fixture name "${name}" must match ${NAME} (letters and digits, starting with a lower-case letter)`);
+    return [...defs.values()]
+        .filter((def) => !def.option)
+        .map((def) => {
+        const option = defs.get(`${def.name}Options`);
+        const outputs = def.outputs?.filter((output) => !output.secret);
+        return {
+            name: def.name,
+            label: def.label ?? def.name,
+            ...(def.icon !== undefined && { icon: def.icon }),
+            ...(def.description !== undefined && { description: def.description }),
+            scope: def.scope,
+            auto: def.auto,
+            overridesBuiltIn: BUILT_IN_FIXTURES.has(def.name),
+            dependencies: def.deps,
+            ...(option?.option && {
+                options: {
+                    name: option.name,
+                    fields: option.fields ?? [],
+                    defaults: { ...option.value },
+                },
+            }),
+            ...(outputs && outputs.length > 0 && { outputs }),
+        };
+    });
+}

package/dist/runner/fixture-scope.d.ts

@@ -1,33 +1,63 @@
-import type { TestFixtures } from "../types/test.js";
+import type { TestFixtureEntry, TestFixtures } from "../types/test.js";
 import { type FixtureLayers } from "./fixture-defs.js";
-/** A custom fixture whose setup threw or never called `use` */
-export declare class FixtureSetupError extends Error {
-    constructor(name: string, cause: unknown);
+import { WorkerFixtures } from "./fixture-worker.js";
+/** What `use` and the run's environment say about the fixtures of one test */
+export interface ScopeRun {
+    /** Fixtures to set up with the `auto` ones */
+    attach?: readonly string[];
+    /** The values of the option fixtures from `use`, by option fixture name */
+    options?: Readonly<Record<string, Record<string, unknown>>>;
+    /** Fixtures the environment turns off, with the reason a test that needs one is skipped for */
+    disabled?: ReadonlyMap<string, string>;
+    /** The worker's fixtures, where worker-scoped ones live. Without it they live as long as this scope */
+    workers?: WorkerFixtures;
+    /** The run's environment: a worker fixture is shared only within one environment */
+    environment?: string;
 }
 /**
  * The custom fixtures of one test run. Fixtures are set up when a test or hook asks for them (by destructuring
  * them), their dependencies first, and torn down in the opposite order. The hooks and the body of a test share
- * one scope, so they see the same instances.
+ * one scope, so they see the same instances. Worker-scoped fixtures come from the worker's `WorkerFixtures`.
  */
 export declare class FixtureScope {
     private readonly layers;
     private readonly base;
+    private readonly run;
     /** The built-in fixtures, with each custom fixture added (replacing a built-in of that name) once it is set up */
     readonly fixtures: TestFixtures;
     private readonly values;
     private readonly pending;
+    /** What was set up for this test, last in the order it is torn down */
     private readonly started;
+    /** The key of each worker fixture set up here, which the fixtures that use it build theirs from */
+    private readonly keys;
+    private readonly entries;
+    private readonly entryOf;
+    private readonly workers;
+    private readonly ownsWorkers;
     private closed;
-    constructor(layers: FixtureLayers, base: TestFixtures);
-    /** Sets up the fixtures marked `auto` */
+    constructor(layers: FixtureLayers, base: TestFixtures, run?: ScopeRun);
+    /** Sets up the fixtures marked `auto`, then those `use` attaches */
     prepareAuto(): Promise<void>;
     /** Sets up the custom fixtures `fn` destructures from its first parameter */
     prepare(fn: (...args: never[]) => unknown): Promise<void>;
+    /** The fixtures this test set up, in the order they were set up, with how each went so far */
+    report(): TestFixtureEntry[];
     /**
-     * Finishes the fixtures that were set up, last first, each within `limitMs`. Returns the first error (a
-     * fixture's cleanup threw or ran out of time); the others still run.
+     * Finishes the fixtures that were set up, last first, each within its own `timeout` else `limitMs`. Returns the
+     * first error (a fixture's cleanup threw or ran out of time); the others still run. The worker fixtures of a
+     * scope that has no worker are finished too.
      */
     teardown(limitMs: number): Promise<Error | undefined>;
     private ensure;
     private setUp;
+    /** Runs the setup function of `def` (for this test, or through the worker) and records how it went */
+    private instantiate;
+    /** Starts the setup of `def` until it calls `use`, and checks the declared outputs of the value */
+    private make;
+    /**
+     * What makes one instance of a worker fixture different from another: the environment, the values of the
+     * option fixtures it reads, the instances of the worker fixtures it uses and the built-ins that differ
+     */
+    private workerKey;
 }

package/dist/runner/fixture-scope.js

@@ -1,65 +1,49 @@
+import { toTestError } from "./test-error.js";
 import { requestedBy } from "./fixture-args.js";
 import { BUILT_IN_FIXTURES, dependencyOf, } from "./fixture-defs.js";
-/** A custom fixture whose setup threw or never called `use` */
-export class FixtureSetupError extends Error {
-    constructor(name, cause) {
-        super(`fixture "${name}" setup failed: ${messageOf(cause)}`);
-        this.name = "FixtureSetupError";
-        if (cause instanceof Error && cause.stack)
-            this.stack = cause.stack;
-    }
-}
-const messageOf = (err) => err instanceof Error ? err.message : String(err);
-/** Runs `setup` until it calls `use`; resolves with the value and the means to finish it */
-function startSetup(def, args) {
-    return new Promise((resolve, reject) => {
-        let release = () => { };
-        const released = new Promise((r) => (release = r));
-        let used = false;
-        let running;
-        const done = () => running ?? Promise.resolve();
-        const use = async (value) => {
-            if (used)
-                throw new Error("use() was called twice");
-            used = true;
-            resolve({ value, release, done });
-            await released;
-        };
-        running = (async () => def.setup?.(args, use))();
-        running.then(() => {
-            if (!used)
-                reject(new Error("the setup never called use()"));
-        }, (err) => {
-            if (!used)
-                reject(err);
-        });
-        // A failure after use() is read by teardown; this keeps it from being reported as unhandled meanwhile
-        running.catch(() => { });
-    });
-}
+import { resolveOptions } from "./fixture-options.js";
+import { outputsOf, stableJson } from "./fixture-outputs.js";
+import { finishWithin, FixtureSetupError, startSetup, } from "./fixture-start.js";
+import { WorkerFixtures } from "./fixture-worker.js";
+import { SkipSignal } from "./skip.js";
+/** The built-in fixtures whose value tells worker fixtures apart (the rest are the same for every test of a worker) */
+const KEYED_BUILT_INS = ["browserName", "workerIndex", "baseURL"];
 /**
  * The custom fixtures of one test run. Fixtures are set up when a test or hook asks for them (by destructuring
  * them), their dependencies first, and torn down in the opposite order. The hooks and the body of a test share
- * one scope, so they see the same instances.
+ * one scope, so they see the same instances. Worker-scoped fixtures come from the worker's `WorkerFixtures`.
  */
 export class FixtureScope {
-    constructor(layers, base) {
+    constructor(layers, base, run = {}) {
         this.layers = layers;
         this.base = base;
+        this.run = run;
         this.values = new Map();
         this.pending = new Map();
+        /** What was set up for this test, last in the order it is torn down */
         this.started = [];
+        /** The key of each worker fixture set up here, which the fixtures that use it build theirs from */
+        this.keys = new Map();
+        this.entries = [];
+        this.entryOf = new Map();
         this.closed = false;
         const own = Object.getOwnPropertyDescriptors(base);
         for (const descriptor of Object.values(own))
             descriptor.configurable = true;
         this.fixtures = Object.defineProperties({}, own);
+        this.ownsWorkers = run.workers === undefined;
+        this.workers = run.workers ?? new WorkerFixtures(60_000);
     }
-    /** Sets up the fixtures marked `auto` */
+    /** Sets up the fixtures marked `auto`, then those `use` attaches */
     async prepareAuto() {
         for (const [name, defs] of this.layers)
             if (defs[defs.length - 1]?.auto)
                 await this.ensure(name, defs.length - 1);
+        for (const name of this.run.attach ?? []) {
+            const defs = this.layers.get(name);
+            if (defs)
+                await this.ensure(name, defs.length - 1);
+        }
     }
     /** Sets up the custom fixtures `fn` destructures from its first parameter */
     async prepare(fn) {
@@ -75,36 +59,38 @@ export class FixtureScope {
                 throw new Error(`extend(): unknown fixture "${name}"`);
         }
     }
+    /** The fixtures this test set up, in the order they were set up, with how each went so far */
+    report() {
+        return this.entries.map((entry) => structuredClone(entry));
+    }
     /**
-     * Finishes the fixtures that were set up, last first, each within `limitMs`. Returns the first error (a
-     * fixture's cleanup threw or ran out of time); the others still run.
+     * Finishes the fixtures that were set up, last first, each within its own `timeout` else `limitMs`. Returns the
+     * first error (a fixture's cleanup threw or ran out of time); the others still run. The worker fixtures of a
+     * scope that has no worker are finished too.
      */
     async teardown(limitMs) {
         this.closed = true;
         let first;
         for (let one = this.started.pop(); one; one = this.started.pop()) {
-            one.release();
-            let timer;
-            try {
-                await Promise.race([
-                    one.done(),
-                    new Promise((_, reject) => {
-                        timer = setTimeout(() => reject(new Error(`timed out after ${limitMs / 1000} seconds`)), limitMs);
-                    }),
-                ]);
-            }
-            catch (err) {
-                first ??= new Error(`fixture "${one.name}" teardown failed: ${messageOf(err)}`);
-                if (err instanceof Error && err.stack)
-                    first.stack = err.stack;
-            }
-            finally {
-                clearTimeout(timer);
-            }
+            const begin = performance.now();
+            const error = await finishWithin(one, limitMs);
+            const entry = this.entryOf.get(one);
+            if (entry)
+                entry.teardown = {
+                    status: error ? "failed" : "passed",
+                    durationMs: performance.now() - begin,
+                    ...(error && { error: toTestError(error) }),
+                };
+            first ??= error;
         }
+        if (this.ownsWorkers)
+            first ??= (await this.workers.closeAll())[0];
         return first;
     }
     ensure(name, level) {
+        const off = this.run.disabled?.get(name);
+        if (off !== undefined)
+            return Promise.reject(new SkipSignal("skip", off));
         const key = `${name}#${level}`;
         let promise = this.pending.get(key);
         if (!promise) {
@@ -129,15 +115,10 @@ export class FixtureScope {
                 }
             }
             let value = def.value;
-            if (def.setup) {
-                const started = await startSetup(def, args);
-                if (this.closed) {
-                    started.release();
-                    throw new Error("the test is already over");
-                }
-                value = started.value;
-                this.started.push({ name, ...started });
-            }
+            if (def.option)
+                value = resolveOptions(def, this.run.options?.[name], this.base);
+            if (def.setup)
+                value = await this.instantiate(def, level, args);
             this.values.set(`${name}#${level}`, value);
             if (level === (this.layers.get(name)?.length ?? 0) - 1)
                 Object.defineProperty(this.fixtures, name, {
@@ -148,9 +129,97 @@ export class FixtureScope {
                 });
         }
         catch (err) {
-            throw err instanceof FixtureSetupError
+            throw err instanceof FixtureSetupError || err instanceof SkipSignal
                 ? err
                 : new FixtureSetupError(name, err);
         }
     }
+    /** Runs the setup function of `def` (for this test, or through the worker) and records how it went */
+    async instantiate(def, level, args) {
+        const entry = {
+            name: def.name,
+            scope: def.scope,
+            setup: { status: "passed", durationMs: 0 },
+        };
+        const begin = performance.now();
+        try {
+            let instance;
+            if (def.scope === "worker") {
+                const key = this.workerKey(def, level);
+                this.keys.set(`${def.name}#${level}`, key);
+                const got = await this.workers.acquire(def, key, () => this.make(def, args));
+                if (got.reused)
+                    entry.setup.status = "skipped";
+                instance = got;
+            }
+            else {
+                instance = await this.make(def, args);
+                if (this.closed) {
+                    instance.started.release();
+                    throw new Error("the test is already over");
+                }
+                this.started.push(instance.started);
+                this.entryOf.set(instance.started, entry);
+            }
+            if (instance.outputs)
+                entry.outputs = instance.outputs;
+            return instance.value;
+        }
+        catch (err) {
+            entry.setup = {
+                status: "failed",
+                durationMs: 0,
+                error: toTestError(new FixtureSetupError(def.name, err)),
+            };
+            throw err;
+        }
+        finally {
+            if (entry.setup.status !== "skipped")
+                entry.setup.durationMs = performance.now() - begin;
+            this.entries.push(entry);
+        }
+    }
+    /** Starts the setup of `def` until it calls `use`, and checks the declared outputs of the value */
+    async make(def, args) {
+        const started = await startSetup(def, args, def.timeout);
+        const finish = {
+            name: def.name,
+            release: started.release,
+            done: started.done,
+            timeout: def.timeout,
+        };
+        try {
+            const outputs = outputsOf(def, started.value);
+            return { value: started.value, started: finish, outputs };
+        }
+        catch (err) {
+            started.release();
+            await started.done().catch(() => undefined);
+            throw err;
+        }
+    }
+    /**
+     * What makes one instance of a worker fixture different from another: the environment, the values of the
+     * option fixtures it reads, the instances of the worker fixtures it uses and the built-ins that differ
+     */
+    workerKey(def, level) {
+        const parts = [this.run.environment ?? ""];
+        for (const dep of def.deps) {
+            const target = dependencyOf(this.layers, def.name, level, dep);
+            if (target === "built-in" || target === undefined) {
+                if (KEYED_BUILT_INS.includes(dep))
+                    parts.push([dep, this.base[dep]]);
+                continue;
+            }
+            const used = this.layers.get(target.name)?.[target.level];
+            if (used?.option)
+                parts.push([
+                    dep,
+                    { ...used.value, ...this.run.options?.[target.name] },
+                ]);
+            else if (used?.scope === "worker")
+                parts.push([dep, this.keys.get(`${target.name}#${target.level}`)]);
+        }
+        return stableJson(parts);
+    }
 }

package/dist/runner/fixture-start.d.ts

@@ -0,0 +1,27 @@
+import type { FixtureDef } from "./fixture-defs.js";
+/** A custom fixture whose setup threw or never called `use` */
+export declare class FixtureSetupError extends Error {
+    constructor(name: string, cause: unknown);
+}
+/** A fixture that is set up: `release` lets its setup function finish, `done` is that function's end */
+export interface Started {
+    name: string;
+    release: () => void;
+    done: () => Promise<void>;
+    /** The fixture's own `timeout` for its teardown */
+    timeout?: number | undefined;
+}
+/**
+ * Runs `setup` until it calls `use`; resolves with the value and the means to finish it. A setup that takes longer
+ * than `limitMs` fails; if it calls `use` later anyway, it is let go at once.
+ */
+export declare function startSetup(def: FixtureDef, args: Record<string, unknown>, limitMs?: number): Promise<{
+    value: unknown;
+    release: () => void;
+    done: () => Promise<void>;
+}>;
+/**
+ * Lets a set-up fixture finish its cleanup, within the fixture's own `timeout` else `limitMs`. Returns the error
+ * (`fixture "x" teardown failed: ...`) when the cleanup threw or ran out of time.
+ */
+export declare function finishWithin(one: Started, limitMs: number): Promise<Error | undefined>;

package/dist/runner/fixture-start.js

@@ -0,0 +1,76 @@
+/** A custom fixture whose setup threw or never called `use` */
+export class FixtureSetupError extends Error {
+    constructor(name, cause) {
+        super(`fixture "${name}" setup failed: ${messageOf(cause)}`);
+        this.name = "FixtureSetupError";
+        if (cause instanceof Error && cause.stack)
+            this.stack = cause.stack;
+    }
+}
+const messageOf = (err) => err instanceof Error ? err.message : String(err);
+/**
+ * Runs `setup` until it calls `use`; resolves with the value and the means to finish it. A setup that takes longer
+ * than `limitMs` fails; if it calls `use` later anyway, it is let go at once.
+ */
+export function startSetup(def, args, limitMs) {
+    return new Promise((resolve, reject) => {
+        let release = () => { };
+        const released = new Promise((r) => (release = r));
+        let used = false;
+        let running;
+        const done = () => running ?? Promise.resolve();
+        const timer = limitMs === undefined
+            ? undefined
+            : setTimeout(() => {
+                release();
+                reject(new Error(`timed out after ${limitMs / 1000} seconds`));
+            }, limitMs);
+        const use = async (value) => {
+            if (used)
+                throw new Error("use() was called twice");
+            used = true;
+            clearTimeout(timer);
+            resolve({ value, release, done });
+            await released;
+        };
+        running = (async () => def.setup?.(args, use))();
+        running.then(() => {
+            clearTimeout(timer);
+            if (!used)
+                reject(new Error("the setup never called use()"));
+        }, (err) => {
+            clearTimeout(timer);
+            if (!used)
+                reject(err);
+        });
+        // A failure after use() is read by teardown; this keeps it from being reported as unhandled meanwhile
+        running.catch(() => { });
+    });
+}
+/**
+ * Lets a set-up fixture finish its cleanup, within the fixture's own `timeout` else `limitMs`. Returns the error
+ * (`fixture "x" teardown failed: ...`) when the cleanup threw or ran out of time.
+ */
+export async function finishWithin(one, limitMs) {
+    one.release();
+    const limit = one.timeout ?? limitMs;
+    let timer;
+    try {
+        await Promise.race([
+            one.done(),
+            new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new Error(`timed out after ${limit / 1000} seconds`)), limit);
+            }),
+        ]);
+        return undefined;
+    }
+    catch (err) {
+        const failure = new Error(`fixture "${one.name}" teardown failed: ${messageOf(err)}`);
+        if (err instanceof Error && err.stack)
+            failure.stack = err.stack;
+        return failure;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}

package/dist/runner/fixture-use.d.ts

@@ -0,0 +1,23 @@
+import type { SamuraiTestConfig } from "../types/config.js";
+import type { RegisteredTestCase } from "../types/test.js";
+import type { FixtureLayers } from "./fixture-defs.js";
+import type { FixtureCatalogEntry } from "./fixture-registry.js";
+/** What the fixture keys of the `use` layers come to for one test */
+export interface ResolvedFixtureUse {
+    /** The fixtures to set up before `beforeEach`, besides the `auto` ones; one the environment turns off is still listed when a later layer insists, and skips the test */
+    attach: string[];
+    /** Option fixture name to its values from `use`, over the fixture's own default */
+    options: Record<string, Record<string, unknown>>;
+    /** Fixtures the run's environment turns off, with the reason a test that needs one is skipped for */
+    disabled: Map<string, string>;
+}
+/**
+ * The fixtures a test gets from `use`: config, then the run's environment, then the group, then `test.use()`, a
+ * later layer winning key by key. The environment's `false` is the exception: it wins over a `true` anywhere, so a
+ * fixture an environment turns off cannot be switched back on for it.
+ */
+export declare function effectiveFixtureUse(test: RegisteredTestCase): Promise<ResolvedFixtureUse>;
+/** Refuses a config whose `use` (config, environments, groups) names a fixture that is not in `config.fixtures` */
+export declare function checkFixtureUseNames(config: SamuraiTestConfig, catalog: readonly FixtureCatalogEntry[]): void;
+/** Refuses a `test.use()` that names a fixture the test's own `test.extend()` does not define */
+export declare function checkTestFixtureUse(use: object, layers: FixtureLayers | undefined): void;