@itmetsam/samurai-framework 0.12.1 → 0.15.1

package/dist/runner/pooled-run.d.ts

@@ -27,6 +27,11 @@ export declare function planItems(selected: RegisteredTestCase[], browsers: Supp
  * and `configGroups` are the same list, resolved and as written.
  */
 export declare function serialKeyOf(groups: ResolvedGroup[], configGroups: SamuraiGroup[] | undefined, srcDir: string): (test: RegisteredTestCase) => string | undefined;
+/**
+ * How many items can run at the same time: each item alone counts one, and the items of one serial key together
+ * count one. More workers than this would only sit idle.
+ */
+export declare function concurrency(items: WorkItem[]): number;
 /**
  * How a run goes: through the pool, in this process, or in this process because the workers could not read the
  * config (it was given as an object; a worker reads `samurai.config.ts` from the project folder)
@@ -37,7 +42,8 @@ export declare function poolMode(input: {
     inlineConfig: boolean;
 }): "pool" | "in-process" | "inline-config";
 /**
- * Runs the planned items on the pool and reports them as they happen. An item the pool answers without events
+ * Runs the planned items on the pool and reports them as they happen. Tests that do not run (skip, fixme) are
+ * reported first, here, and never reach a worker. An item the pool answers without events
  * (a queued one cut short by an abort, or one whose worker died before it began) is reported here at the end.
  */
 export declare function runPooled(pool: PooledRunner, planned: PlannedTest[], items: WorkItem[], reporter: TestReporter, signal?: AbortSignal): Promise<void>;

package/dist/runner/pooled-run.js

@@ -1,19 +1,25 @@
 import { idOf } from "./select.js";
+import { staticSkip } from "./skip.js";
 /**
  * Lists the run: every selected test in the first browser, then every selected test in the next one. The
  * position in the list is the item's `index`. `serialKey` says which group makes a test run alone.
  */
 export function planItems(selected, browsers, serialKey) {
     const planned = browsers.flatMap((browser) => selected.map((test) => ({ test, browser })));
-    const items = planned.map(({ test, browser }, index) => {
+    // A test that does not run (skip, fixme) is in the plan but not in the queue; `index` is its place in the plan
+    const items = planned.flatMap(({ test, browser }, index) => {
+        if (staticSkip(test))
+            return [];
         const key = serialKey(test);
-        return {
-            index,
-            file: test.file,
-            testName: test.name,
-            browser,
-            ...(key !== undefined && { serialKey: key }),
-        };
+        return [
+            {
+                index,
+                file: test.file,
+                testName: test.name,
+                browser,
+                ...(key !== undefined && { serialKey: key }),
+            },
+        ];
     });
     return { items, planned };
 }
@@ -32,6 +38,21 @@ export function serialKeyOf(groups, configGroups, srcDir) {
     });
     return (test) => keys.get(idOf(test, srcDir));
 }
+/**
+ * How many items can run at the same time: each item alone counts one, and the items of one serial key together
+ * count one. More workers than this would only sit idle.
+ */
+export function concurrency(items) {
+    const keys = new Set();
+    let free = 0;
+    for (const item of items) {
+        if (item.serialKey === undefined)
+            free++;
+        else
+            keys.add(item.serialKey);
+    }
+    return free + keys.size;
+}
 /**
  * How a run goes: through the pool, in this process, or in this process because the workers could not read the
  * config (it was given as an object; a worker reads `samurai.config.ts` from the project folder)
@@ -42,19 +63,26 @@ export function poolMode(input) {
     return input.inlineConfig ? "inline-config" : "pool";
 }
 /**
- * Runs the planned items on the pool and reports them as they happen. An item the pool answers without events
+ * Runs the planned items on the pool and reports them as they happen. Tests that do not run (skip, fixme) are
+ * reported first, here, and never reach a worker. An item the pool answers without events
  * (a queued one cut short by an abort, or one whose worker died before it began) is reported here at the end.
  */
 export async function runPooled(pool, planned, items, reporter, signal) {
     reporter.setOrder(planned);
+    for (const { test, browser } of planned) {
+        const annotation = staticSkip(test);
+        if (annotation)
+            reporter.onTestSkipped(test, browser, { annotation });
+    }
     const started = new Set();
     const ended = new Set();
-    const begin = (index) => {
-        if (started.has(index))
+    const begin = (index, retry) => {
+        // A test run again starts again; the first start is reported once however it arrives
+        if (started.has(index) && !retry)
             return;
         started.add(index);
         const { test, browser } = planned[index];
-        reporter.onTestStart(test, browser);
+        reporter.onTestStart(test, browser, retry);
     };
     const end = (index, result) => {
         begin(index);
@@ -65,10 +93,11 @@ export async function runPooled(pool, planned, items, reporter, signal) {
     const results = await pool.run(items, {
         ...(signal && { signal }),
         onEvent: (event) => event.type === "test-start"
-            ? begin(event.item.index)
+            ? begin(event.item.index, event.retry)
             : end(event.item.index, event.result),
     });
-    results.forEach((result, index) => {
+    results.forEach((result, position) => {
+        const index = items[position].index;
         if (!ended.has(index))
             end(index, result);
     });

package/dist/runner/project.js

@@ -1,7 +1,9 @@
+import path from "node:path";
 import { removeGeneratedFiles } from "../browser/generated-files.js";
 import { loadConfig, projectDir, setActiveProject } from "../config/config.js";
 import { resetSecrets } from "../config/mask.js";
 import { setRunSettings } from "../config/run-settings.js";
+import { keepLogFiles } from "../logger/index.js";
 import { prepareRun } from "./prepare-run.js";
 let active = false;
 /**
@@ -14,6 +16,8 @@ export async function withProject(options, work) {
     active = true;
     const { projectDir: dir, config, dataDir, environment, timeout, expectTimeout, headless, } = options;
     const previousDir = projectDir();
+    // The log goes to the project's `logs/` folder, not to wherever this process was started
+    const releaseLogs = keepLogFiles(path.resolve(dir ?? previousDir));
     try {
         setActiveProject({ dir: dir ?? previousDir, config, dataDir });
         const loaded = config ?? (await loadConfig());
@@ -32,5 +36,6 @@ export async function withProject(options, work) {
         setRunSettings(undefined);
         setActiveProject(undefined);
         active = false;
+        releaseLogs();
     }
 }

package/dist/runner/registry.d.ts

@@ -1,7 +1,38 @@
-import type { RegisteredTestCase, TestCaseBase } from "../types/test.js";
-/** Groups the tests registered inside `fn` under `title`. `fn` runs immediately and must be synchronous. */
-export declare function describe(title: string, fn: () => void): void;
-export declare function test(title: string, fn: TestCaseBase["function"]): void;
+import type { AllHook, EachHook, RegisteredTestCase, TestCaseBase } from "../types/test.js";
+type Body = TestCaseBase["function"];
+/** `test(title, fn)`, `test.skip(title, fn)`, `test.skip(condition, reason)` and the other forms */
+export interface TestFunction {
+    (title: string, fn: Body): void;
+    /** Registers a test that is not run */
+    skip: SkipFunction;
+    /** Registers a test that is not run because it is known to be broken */
+    fixme: SkipFunction;
+    /** Runs only this test (and the other `only` ones) */
+    only(title: string, fn: Body): void;
+}
+/** With a title and a body it registers a skipped test; with a condition it skips the running test when the condition is true */
+interface SkipFunction {
+    (title: string, fn: Body): void;
+    (condition: boolean, reason?: string): void;
+}
+/** `describe(title, fn)` and the same with a modifier on every test inside */
+export interface DescribeFunction {
+    (title: string, fn: () => void): void;
+    skip(title: string, fn: () => void): void;
+    fixme(title: string, fn: () => void): void;
+    only(title: string, fn: () => void): void;
+}
+export declare const test: TestFunction;
+export declare const describe: DescribeFunction;
+/** Runs before each test of the `describe` block it is written in (of the file, at the top level) */
+export declare function beforeEach(hook: EachHook): void;
+/** Runs after each test of the `describe` block it is written in, also when the test failed */
+export declare function afterEach(hook: EachHook): void;
+/** Runs once per worker, before the first test of its `describe` block that the worker runs. It gets no page. */
+export declare function beforeAll(hook: AllHook): void;
+/** Runs once per worker, when the worker is done, for every `describe` block it ran a `beforeAll` for */
+export declare function afterAll(hook: AllHook): void;
 export declare function registeredTests(): readonly RegisteredTestCase[];
 /** Forgets every registered test: a run registers its specs afresh */
 export declare function clearRegistry(): void;
+export {};

package/dist/runner/registry.js

@@ -1,11 +1,11 @@
 import { fileURLToPath } from "node:url";
 import logger from "../logger/index.js";
+import { SkipSignal } from "./skip.js";
+import { addHook, currentSuite, enterSuite, leaveSuite, modifierOf, resetSuites, } from "./suite.js";
 const registered = [];
-/** Titles of the describe blocks being registered, outermost first */
-const titleStack = [];
-/** Groups the tests registered inside `fn` under `title`. `fn` runs immediately and must be synchronous. */
-export function describe(title, fn) {
-    titleStack.push(title);
+/** Runs `fn` as the `describe` block `title`, under `modifier` when there is one. `fn` runs immediately and must be synchronous. */
+function openDescribe(file, title, fn, modifier) {
+    enterSuite(file, title, modifier);
     try {
         const result = fn();
         if (result instanceof Promise) {
@@ -14,24 +14,79 @@ export function describe(title, fn) {
         }
     }
     finally {
-        titleStack.pop();
+        leaveSuite();
     }
 }
-export function test(title, fn) {
-    const titlePath = [...titleStack, title];
+function add(file, title, fn, modifier) {
+    const suite = currentSuite(file);
+    const titlePath = [...titlesOf(suite), title];
     const name = titlePath.join(" > ");
-    registered.push({ name, titlePath, function: fn, file: callerFile() });
+    const effective = modifierOf(modifier, suite);
+    registered.push({
+        name,
+        titlePath,
+        function: fn,
+        file,
+        suite,
+        ...(effective && { modifier: effective }),
+    });
     logger.debug("Registered test %s", name);
 }
+/** The `describe` titles from the file down to `suite` */
+function titlesOf(suite) {
+    const titles = [];
+    for (let s = suite; s; s = s.parent)
+        if (s.title !== undefined)
+            titles.unshift(s.title);
+    return titles;
+}
+/** `test.skip` and `test.fixme`: a title and a body register a test with the modifier; a condition skips the running test */
+function skipping(annotation) {
+    return ((first, second) => {
+        if (typeof first === "string") {
+            // Frames: Error, callerFile, this function, the spec
+            add(callerFile(), first, second, annotation);
+            return;
+        }
+        if (first)
+            throw new SkipSignal(annotation, second);
+    });
+}
+export const test = Object.assign((title, fn) => add(callerFile(), title, fn), {
+    skip: skipping("skip"),
+    fixme: skipping("fixme"),
+    only: (title, fn) => add(callerFile(), title, fn, "only"),
+});
+export const describe = Object.assign((title, fn) => openDescribe(callerFile(), title, fn), {
+    skip: (title, fn) => openDescribe(callerFile(), title, fn, "skip"),
+    fixme: (title, fn) => openDescribe(callerFile(), title, fn, "fixme"),
+    only: (title, fn) => openDescribe(callerFile(), title, fn, "only"),
+});
+/** Runs before each test of the `describe` block it is written in (of the file, at the top level) */
+export function beforeEach(hook) {
+    addHook(currentSuite(callerFile()), "beforeEach", hook);
+}
+/** Runs after each test of the `describe` block it is written in, also when the test failed */
+export function afterEach(hook) {
+    addHook(currentSuite(callerFile()), "afterEach", hook);
+}
+/** Runs once per worker, before the first test of its `describe` block that the worker runs. It gets no page. */
+export function beforeAll(hook) {
+    addHook(currentSuite(callerFile()), "beforeAll", hook);
+}
+/** Runs once per worker, when the worker is done, for every `describe` block it ran a `beforeAll` for */
+export function afterAll(hook) {
+    addHook(currentSuite(callerFile()), "afterAll", hook);
+}
 export function registeredTests() {
     return registered;
 }
 /** Forgets every registered test: a run registers its specs afresh */
 export function clearRegistry() {
     registered.length = 0;
-    titleStack.length = 0;
+    resetSuites();
 }
-/** File of the code that called test(): stack lines are Error, callerFile, test, caller */
+/** File of the code that called the entry point: stack lines are Error, callerFile, the entry point, the caller */
 function callerFile() {
     const caller = new Error().stack?.split("\n")[3] ?? "";
     const match = caller.match(/\((.+?):\d+:\d+\)$/) ?? caller.match(/at (.+?):\d+:\d+$/);

package/dist/runner/reporter.d.ts

@@ -1,4 +1,4 @@
-import type { RegisteredTestCase, TestError, TestLogEntry, TestResult, TestSummary } from "../types/test.js";
+import type { RegisteredTestCase, TestError, TestLogEntry, TestResult, TestRetries, TestSummary } from "../types/test.js";
 import type { SupportedBrowser } from "../types/browser.js";
 /** What a run reports while it happens */
 export type RunEvent = {
@@ -14,6 +14,8 @@ export type RunEvent = {
     name: string;
     file: string;
     browser: SupportedBrowser;
+    /** The attempt, counting from 1, when the test is run again after a failure; absent for the first */
+    retry?: number;
 } | {
     type: "test-end";
     name: string;
@@ -36,11 +38,18 @@ export interface ReporterOptions {
 }
 /** What `runOneTest` reports a test to: the reporter of a run, or a worker that relays it */
 export interface TestSink {
-    onTestStart(test: RegisteredTestCase, browser: SupportedBrowser): void;
+    onTestStart(test: RegisteredTestCase, browser: SupportedBrowser, retry?: number): void;
     onTestEnd(test: RegisteredTestCase, browser: SupportedBrowser, error?: TestError, extras?: TestEndExtras): void;
+    /** A test that does not run: reported as started and ended at once */
+    onTestSkipped(test: RegisteredTestCase, browser: SupportedBrowser, skip: TestSkip): void;
+}
+/** Why a test was skipped */
+export interface TestSkip {
+    annotation: "skip" | "fixme";
+    reason?: string | undefined;
 }
 /** What a test's report entry lists besides its status */
-export interface TestEndExtras {
+export interface TestEndExtras extends TestRetries {
     logs?: TestLogEntry[];
     logsDropped?: number;
     screenshots?: {
@@ -66,7 +75,8 @@ export default class TestReporter implements TestSink {
     }[]): void;
     onStart(total: number, workers?: number): void;
     onEnd(): Promise<TestSummary>;
-    onTestStart(test: RegisteredTestCase, browser: SupportedBrowser): void;
+    onTestStart(test: RegisteredTestCase, browser: SupportedBrowser, retry?: number): void;
+    onTestSkipped(test: RegisteredTestCase, browser: SupportedBrowser, { annotation, reason }: TestSkip): void;
     /**
      * Ends a test that ran in a worker: the worker built the result (masked, with its own duration), this keeps the
      * start time from this process's clock, as the worker's clock has another origin

package/dist/runner/reporter.js

@@ -62,7 +62,7 @@ export default class TestReporter {
             environment: this.environment,
             browsers: this.browsers,
             tests,
-            status: tests.every((r) => r.status === "success")
+            status: tests.every((r) => r.status === "success" || r.status === "skipped")
                 ? "success"
                 : "failed",
         };
@@ -75,7 +75,7 @@ export default class TestReporter {
         this.emit({ type: "run-end", summary: this.summary });
         return this.summary;
     }
-    onTestStart(test, browser) {
+    onTestStart(test, browser, retry) {
         const key = keyOf(test, browser);
         const t = performance.mark(`${key}-start`);
         this.testResults.set(key, {
@@ -90,6 +90,28 @@ export default class TestReporter {
             name: test.name,
             file: test.file,
             browser,
+            ...(retry ? { retry } : {}),
+        });
+    }
+    onTestSkipped(test, browser, { annotation, reason }) {
+        const key = keyOf(test, browser);
+        // A test skipped from inside already started; one that never ran starts and ends at once
+        if (this.testResults.get(key)?.status !== "started")
+            this.onTestStart(test, browser);
+        const result = maskDeep({
+            ...this.testResults.get(key),
+            status: "skipped",
+            duration: 0,
+            annotation,
+            ...(reason !== undefined && { reason }),
+        });
+        this.testResults.set(key, result);
+        this.emit({
+            type: "test-end",
+            name: test.name,
+            file: test.file,
+            browser,
+            result,
         });
     }
     /**
@@ -124,6 +146,11 @@ export default class TestReporter {
             ...(extras?.logs && { logs: extras.logs }),
             ...(extras?.logsDropped && { logsDropped: extras.logsDropped }),
             ...(extras?.screenshots?.length && { screenshots: extras.screenshots }),
+            ...(extras?.retry && { retry: extras.retry }),
+            ...(extras?.flaky && { flaky: true }),
+            ...(extras?.previousAttempts?.length && {
+                previousAttempts: extras.previousAttempts,
+            }),
         });
         this.testResults.set(key, result);
         this.emit({

package/dist/runner/retries-option.d.ts

@@ -0,0 +1,10 @@
+export interface RetriesInput {
+    /** The command line's `--retries` (or the `retries` run option) */
+    flag?: number | string | undefined;
+    /** `SAMURAI_RETRIES` */
+    env?: string | undefined;
+    /** The config's `retries` */
+    config?: number | string | undefined;
+}
+/** How many times a failing test is run again: the flag, else `SAMURAI_RETRIES`, else the config's `retries`, else 0 */
+export declare function resolveRetries({ flag, env, config }: RetriesInput): number;

package/dist/runner/retries-option.js

@@ -0,0 +1,12 @@
+import { parseRetries } from "../config/retries.js";
+/** How many times a failing test is run again: the flag, else `SAMURAI_RETRIES`, else the config's `retries`, else 0 */
+export function resolveRetries({ flag, env, config }) {
+    const fromEnv = env?.trim() ? env : undefined;
+    if (flag !== undefined)
+        return parseRetries(flag, "--retries");
+    if (fromEnv !== undefined)
+        return parseRetries(fromEnv, "SAMURAI_RETRIES");
+    if (config !== undefined)
+        return parseRetries(config, "retries");
+    return 0;
+}

package/dist/runner/retry.d.ts

@@ -0,0 +1,13 @@
+import type { SupportedBrowser } from "../types/browser.js";
+import type { RegisteredTestCase } from "../types/test.js";
+import type { TestSink } from "./reporter.js";
+/** One attempt at a test: it reports to `sink` and `attempt` counts from 0 */
+export type Attempt = (sink: TestSink, attempt: number) => Promise<void>;
+/**
+ * Runs a test, and again after a failure with a fresh browser, up to `retries` more times. The test starts once
+ * per attempt (`retry` counts from 1), and ends once, with the last attempt's result: `retry` says which attempt
+ * that was, `previousAttempts` holds the errors before it, and `flaky` marks a pass after a failure.
+ *
+ * An aborted run is never run again. A test that skips itself ends there and then.
+ */
+export declare function runWithRetries(test: RegisteredTestCase, browser: SupportedBrowser, retries: number, signal: AbortSignal | undefined, attempt: Attempt, sink: TestSink): Promise<void>;

package/dist/runner/retry.js

@@ -0,0 +1,43 @@
+/**
+ * Runs a test, and again after a failure with a fresh browser, up to `retries` more times. The test starts once
+ * per attempt (`retry` counts from 1), and ends once, with the last attempt's result: `retry` says which attempt
+ * that was, `previousAttempts` holds the errors before it, and `flaky` marks a pass after a failure.
+ *
+ * An aborted run is never run again. A test that skips itself ends there and then.
+ */
+export async function runWithRetries(test, browser, retries, signal, attempt, sink) {
+    if (retries <= 0)
+        return attempt(sink, 0);
+    const previous = [];
+    for (let n = 0;; n++) {
+        let held;
+        const attemptSink = {
+            onTestStart: (t, b) => sink.onTestStart(t, b, n > 0 ? n : undefined),
+            onTestEnd: (_t, _b, error, extras) => {
+                held = { error, extras };
+            },
+            onTestSkipped: (t, b, skip) => sink.onTestSkipped(t, b, skip),
+        };
+        await attempt(attemptSink, n);
+        // Skipped: already reported, and nothing is run again
+        if (!held)
+            return;
+        const { error, extras } = held;
+        if (error &&
+            n < retries &&
+            !signal?.aborted &&
+            error.message !== "Run aborted") {
+            previous.push(error);
+            continue;
+        }
+        sink.onTestEnd(test, browser, error, {
+            ...extras,
+            ...(n > 0 && {
+                retry: n,
+                previousAttempts: [...previous],
+                ...(!error && { flaky: true }),
+            }),
+        });
+        return;
+    }
+}

package/dist/runner/run-test.d.ts

@@ -1,6 +1,7 @@
 import type { SupportedBrowser } from "../types/browser.js";
-import type { RegisteredTestCase } from "../types/test.js";
+import type { AllHookFixtures, RegisteredTestCase } from "../types/test.js";
 import type { RunnerOptions } from "./options.js";
+import type { AllHooks } from "./all-hooks.js";
 import type { PreparedRun } from "./prepare-run.js";
 import type { TestSink } from "./reporter.js";
 /** What a worker gives its tests that the main process does not: its own port and profile, and its number */
@@ -11,6 +12,12 @@ export interface WorkerResources {
     profileDir?: string;
     /** The `workerIndex` fixture. @default 0 */
     workerIndex?: number;
+    /** Which attempt this is, counting from 0: screenshots of a retry go to a `retry-<n>` folder of their own. @default 0 */
+    attempt?: number;
+    /** The `beforeAll` / `afterAll` hooks of this worker: kept across tests, closed when the worker is done */
+    allHooks?: AllHooks;
 }
+/** What `beforeAll` and `afterAll` hooks get: no page, only what a run knows */
+export declare function allHookFixtures(run: PreparedRun, browserName: SupportedBrowser, workerIndex?: number): AllHookFixtures;
 /** Runs one test in one browser and reports it: launch, run the body, judge the page's logs, close the browser */
 export declare function runOneTest(test: RegisteredTestCase, browserName: SupportedBrowser, run: PreparedRun, options: RunnerOptions, srcDir: string, reporter: TestSink, resources?: WorkerResources): Promise<void>;

package/dist/runner/run-test.js

@@ -9,6 +9,8 @@ import logger from "../logger/index.js";
 import { ConnectionClosedError } from "../transport/bidi-connection.js";
 import { judgePageLogs } from "./page-logs-report.js";
 import { DEFAULT_PORT } from "./ports.js";
+import { runWithEachHooks } from "./hooks.js";
+import { isSkipSignal, staticSkip } from "./skip.js";
 import { testDirOf } from "./test-dir.js";
 import { toTestError } from "./test-error.js";
 /**
@@ -30,6 +32,16 @@ class TestOutcome {
     get isEnded() {
         return this.ended;
     }
+    /** The test skipped itself (`test.skip(condition)`): reported as skipped, not as a failure */
+    skip(signal) {
+        if (this.ended)
+            return;
+        this.ended = true;
+        this.reporter.onTestSkipped(this.test, this.browser, {
+            annotation: signal.annotation,
+            reason: signal.reason,
+        });
+    }
     finish(error, logs) {
         if (this.ended)
             return;
@@ -40,8 +52,24 @@ class TestOutcome {
         });
     }
 }
+/** What `beforeAll` and `afterAll` hooks get: no page, only what a run knows */
+export function allHookFixtures(run, browserName, workerIndex = 0) {
+    const { settings, secrets } = run;
+    return {
+        browserName,
+        workerIndex,
+        env: createEnvFixture(settings.environment, settings.variables),
+        secrets: createSecretsFixture(settings.environment, secrets),
+    };
+}
 /** Runs one test in one browser and reports it: launch, run the body, judge the page's logs, close the browser */
 export async function runOneTest(test, browserName, run, options, srcDir, reporter, resources = {}) {
+    // A test marked skip or fixme never runs, whoever asked for it (a flow's test node, a worker)
+    const marked = staticSkip(test);
+    if (marked) {
+        reporter.onTestSkipped(test, browserName, { annotation: marked });
+        return;
+    }
     logger.verbose("Starting test: %s in %s", test.name, browserName);
     reporter.onTestStart(test, browserName);
     const outcome = new TestOutcome(reporter, test, browserName);
@@ -81,6 +109,8 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
     /** Launches the browser, runs the test body and reports how it went */
     const runBody = async () => {
         try {
+            // The `beforeAll` hooks come first, before a browser exists
+            await resources.allHooks?.ensure(test, browserName);
             const launched = await Browser.launch(browserName, {
                 port: resources.port ?? options.port ?? DEFAULT_PORT,
                 ...(resources.profileDir && { profileDir: resources.profileDir }),
@@ -101,7 +131,7 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
                 browserGone();
             });
             page = launched.page;
-            outcome.evidence = new Evidence(resultDir, `${browserName}/${testDirOf(test, srcDir)}`);
+            outcome.evidence = new Evidence(resultDir, `${browserName}/${testDirOf(test, srcDir)}${resources.attempt ? `/retry-${resources.attempt}` : ""}`);
             launched.page.setEvidence(outcome.evidence);
             const fixtures = {
                 page: launched.page,
@@ -111,7 +141,7 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
                 env: createEnvFixture(settings.environment, settings.variables),
                 secrets: createSecretsFixture(settings.environment, secrets),
             };
-            await test.function(fixtures);
+            await runWithEachHooks(test.suite, fixtures, timeoutController.signal, () => test.function(fixtures));
             // Already reported as timed out
             if (timeoutController.signal.aborted)
                 return;
@@ -131,6 +161,8 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
         catch (err) {
             if (timeoutController.signal.aborted)
                 return;
+            if (isSkipSignal(err))
+                return outcome.skip(err);
             outcome.finish(outcome.browserClosed || err instanceof ConnectionClosedError
                 ? {
                     message: "The browser was closed while the test was running",

package/dist/runner/run.d.ts

@@ -1,4 +1,4 @@
-import type { TestSummary } from "../types/test.js";
+import type { Modifier, TestSummary } from "../types/test.js";
 import { type ProjectOptions } from "./project.js";
 import type { ResolvedGroup } from "./groups.js";
 import type { RunnerOptions } from "./options.js";
@@ -19,6 +19,8 @@ export interface FoundTest {
     /** Full name: `describe` titles and the test title joined with " > " */
     name: string;
     file: string;
+    /** `skip` or `fixme` (not run), or `only` (focused); absent for a plain test. A modifier on a `describe` counts. */
+    modifier?: Modifier;
 }
 /** Imports the project's specs without running anything and lists the tests the options select */
 export declare function listTests(options?: RunTestsOptions): Promise<FoundTest[]>;
@@ -26,8 +28,9 @@ export declare function listTests(options?: RunTestsOptions): Promise<FoundTest[
 export declare function listGroups(options?: RunTestsOptions): Promise<ResolvedGroup[]>;
 export { BROWSER_NAMES, checkBrowserList } from "../config/browsers.js";
 export { parseWorkers } from "../config/workers.js";
+export { parseRetries } from "../config/retries.js";
 export type { RunEvent } from "./reporter.js";
-export type { TestError, TestResult, TestSummary } from "../types/test.js";
+export type { Modifier, TestError, TestResult, TestSummary, } from "../types/test.js";
 export type { ResolvedGroup } from "./groups.js";
 export type { RunnerOptions } from "./options.js";
 export type { ProjectOptions } from "./project.js";

package/dist/runner/run.js

@@ -35,7 +35,11 @@ export async function runTests(options = {}) {
 }
 /** Imports the project's specs without running anything and lists the tests the options select */
 export async function listTests(options = {}) {
-    return inProject(options, async (runner) => (await runner.register()).map(({ name, file }) => ({ name, file })));
+    return inProject(options, async (runner) => (await runner.register()).map(({ name, file, modifier }) => ({
+        name,
+        file,
+        ...(modifier && { modifier }),
+    })));
 }
 /** Imports the project's specs without running anything and lists the config's groups with the tests each holds */
 export async function listGroups(options = {}) {
@@ -43,4 +47,5 @@ export async function listGroups(options = {}) {
 }
 export { BROWSER_NAMES, checkBrowserList } from "../config/browsers.js";
 export { parseWorkers } from "../config/workers.js";
+export { parseRetries } from "../config/retries.js";
 export { DEFAULT_PORT } from "./ports.js";

package/dist/runner/skip.d.ts

@@ -0,0 +1,10 @@
+import type { RegisteredTestCase } from "../types/test.js";
+/** Thrown by `test.skip(condition, reason)` and `test.fixme(condition, reason)` inside a test: the test ends as skipped */
+export declare class SkipSignal extends Error {
+    readonly annotation: "skip" | "fixme";
+    readonly reason?: string | undefined;
+    constructor(annotation: "skip" | "fixme", reason?: string | undefined);
+}
+export declare function isSkipSignal(error: unknown): error is SkipSignal;
+/** Why a test does not run at all: it is marked `skip` or `fixme`, itself or by a `describe` around it */
+export declare function staticSkip(test: Pick<RegisteredTestCase, "modifier">): "skip" | "fixme" | undefined;