@itmetsam/samurai-framework 0.15.0 → 0.15.1

package/dist/cli/args.d.ts

@@ -12,6 +12,7 @@ export declare function parseCommandLine(argv: string[]): {
         headless?: boolean | undefined;
         port?: string | undefined;
         workers?: string | undefined;
+        retries?: string | undefined;
         grep?: string | undefined;
         group?: string | undefined;
         browser?: string[] | undefined;
@@ -40,13 +41,15 @@ export interface RunnerFlags {
     port?: number;
     /** `--workers`: a whole number or a percentage like `50%`, as typed */
     workers?: string;
+    /** `--retries`: a whole number from 0, as typed */
+    retries?: string;
     grep?: string;
     group?: string;
     files?: string[];
     /** `--browser`, repeated for more than one; replaces the config's `browsers` for this run */
     browsers?: SupportedBrowser[];
 }
-/** Reads `--headless`, `--port <n>`, `--workers <n|n%>`, `--grep <text>`, `--group <name>`, `--file <spec>` and `--browser <name>` (both repeatable) from `argv`, or from values that were parsed already */
+/** Reads `--headless`, `--port <n>`, `--workers <n|n%>`, `--retries <n>`, `--grep <text>`, `--group <name>`, `--file <spec>` and `--browser <name>` (both repeatable) from `argv`, or from values that were parsed already */
 export declare function parseRunnerFlags(source: string[] | CommandValues): RunnerFlags;
 /** Reads `--env`, `--timeout` and `--expect-timeout` from `argv` (or values parsed already); the environment falls back to SAMURAI_ENV */
 export declare function parseRunOverrides(source: string[] | CommandValues, env?: Partial<NodeJS.ProcessEnv>): EnvAndTimeouts;

package/dist/cli/args.js

@@ -1,5 +1,5 @@
 import { parseArgs } from "node:util";
-import { checkBrowserList, parseWorkers, } from "../runner/run.js";
+import { checkBrowserList, parseRetries, parseWorkers, } from "../runner/run.js";
 const FLAGS = {
     env: { type: "string" },
     timeout: { type: "string" },
@@ -7,6 +7,7 @@ const FLAGS = {
     headless: { type: "boolean" },
     port: { type: "string" },
     workers: { type: "string" },
+    retries: { type: "string" },
     grep: { type: "string" },
     group: { type: "string" },
     browser: { type: "string", multiple: true },
@@ -34,7 +35,7 @@ export function parseCommandLine(argv) {
     });
 }
 const valuesOf = (source) => Array.isArray(source) ? parseCommandLine(source).values : source;
-/** Reads `--headless`, `--port <n>`, `--workers <n|n%>`, `--grep <text>`, `--group <name>`, `--file <spec>` and `--browser <name>` (both repeatable) from `argv`, or from values that were parsed already */
+/** Reads `--headless`, `--port <n>`, `--workers <n|n%>`, `--retries <n>`, `--grep <text>`, `--group <name>`, `--file <spec>` and `--browser <name>` (both repeatable) from `argv`, or from values that were parsed already */
 export function parseRunnerFlags(source) {
     const values = valuesOf(source);
     const flags = {};
@@ -51,6 +52,10 @@ export function parseRunnerFlags(source) {
         parseWorkers(values.workers, "--workers");
         flags.workers = values.workers;
     }
+    if (values.retries !== undefined) {
+        parseRetries(values.retries, "--retries");
+        flags.retries = values.retries;
+    }
     if (values.grep !== undefined)
         flags.grep = values.grep;
     if (values.group !== undefined)

package/dist/cli/console-reporter.js

@@ -1,5 +1,13 @@
 import { painter } from "./paint.js";
 const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`;
+/** What a result says about its retries: `passed on retry 2`, `failed after 2 retries`, or nothing */
+function retried(result) {
+    if (!("retry" in result) || !result.retry)
+        return undefined;
+    if (result.status === "success")
+        return `passed on retry ${result.retry}`;
+    return `failed after ${result.retry} ${result.retry === 1 ? "retry" : "retries"}`;
+}
 /** Prints a run for a person: one line per test as it ends, then every failure in full and a count */
 export class ConsoleReporter {
     constructor({ write, color }) {
@@ -30,7 +38,7 @@ export class ConsoleReporter {
     }
     line(name, result, browser) {
         const took = "duration" in result
-            ? ` ${this.paint(90, `(${seconds(result.duration)})`)}`
+            ? ` ${this.paint(90, `(${[seconds(result.duration), retried(result)].filter(Boolean).join(", ")})`)}`
             : "";
         if (result.status === "skipped")
             return this.paint(90, `- ${name}${this.tag(browser)} (${result.annotation === "fixme" ? "fixme" : "skipped"}${result.reason ? `: ${result.reason}` : ""})`);
@@ -54,9 +62,10 @@ export class ConsoleReporter {
                 this.write(`    ${this.paint(90, test.file)}\n`);
         }
         const passed = summary.tests.length - failed.length - skipped.length;
+        const flaky = summary.tests.filter((test) => "flaky" in test && test.flaky).length;
         const parts = [
             failed.length > 0 && this.paint(31, `${failed.length} failed`),
-            this.paint(32, `${passed} passed`),
+            this.paint(32, `${passed} passed${flaky > 0 ? ` (${flaky} flaky)` : ""}`),
             skipped.length > 0 && this.paint(90, `${skipped.length} skipped`),
         ].filter(Boolean);
         this.write(`\n${parts.join(", ")} ${this.paint(90, `(${seconds(summary.duration)})`)}\n`);

package/dist/cli/flow.js

@@ -25,6 +25,7 @@ Options for run:
   --headless, --no-headless   Run the browser without a window, or with one
   --port <n>                  Browser debugging port (default ${DEFAULT_PORT})
   --workers <n|n%>            Worker processes the tests share (or SAMURAI_WORKERS; default half the CPUs)
+  --retries <n>               Run a failing test again up to n times, each with a fresh browser (or SAMURAI_RETRIES; default 0)
 
 Each run writes result/flows/<id>.json.
 Exit code: 0 when every node that should run passed, 1 when a node failed or the run was cancelled,
@@ -228,6 +229,10 @@ export async function flowCommand(argv, io) {
                     ...(io.env.SAMURAI_WORKERS && {
                         workersEnv: io.env.SAMURAI_WORKERS,
                     }),
+                    ...(flags.retries !== undefined && { retries: flags.retries }),
+                    ...(io.env.SAMURAI_RETRIES && {
+                        retriesEnv: io.env.SAMURAI_RETRIES,
+                    }),
                 },
             });
         }

package/dist/cli/main.js

@@ -48,6 +48,7 @@ export async function main(argv, io) {
             ...(flags.headless !== undefined && { headless: flags.headless }),
             ...(flags.port !== undefined && { port: flags.port }),
             ...(flags.workers !== undefined && { workers: flags.workers }),
+            ...(flags.retries !== undefined && { retries: flags.retries }),
             ...(flags.grep !== undefined && { grep: flags.grep }),
             ...(flags.group !== undefined && { group: flags.group }),
             ...(flags.files && { files: flags.files }),

package/dist/cli/usage.d.ts

@@ -1 +1 @@
-export declare const USAGE = "samurai: browser tests over WebDriver BiDi\n\nUsage:\n  samurai run [options]       Run the tests (the default command)\n  samurai list [options]      List the tests without running them\n  samurai record <spec>       Open a browser and record what you do into a spec file\n  samurai init [folder]       Scaffold a project: config, an example test, .env.example\n  samurai flow <command>      List, check and run the flows in ./flows (see below)\n  samurai browsers [command]  List, install and remove the browsers SAMURAI launches\n\nOptions for run and list:\n  --env <name>                Environment to run against (or SAMURAI_ENV)\n  --file <spec>               Only this spec file; repeatable\n  --grep <text>               Only tests whose full name contains the text\n  --group <name>              Only the tests of this group of the config's groups\n  --browser <name>            Run in this browser (firefox or chrome) instead of the config's\n                              browsers; repeat it for more than one\n  --json                      Machine-readable output: JSON lines for run, one array for list\n\nOptions for run:\n  --timeout <ms>              Time one test may take\n  --expect-timeout <ms>       Time actions and assertions retry\n  --headless, --no-headless   Run the browser without a window, or with one.\n                              Default: the config's use.headless, else headless when CI is set\n  --port <n>                  Browser debugging port (default 9223)\n  --workers <n|n%>            Tests that run at the same time, each in a worker process: a number\n                              from 1, or a share of the CPUs like 50%. Default: the config's\n                              workers (or SAMURAI_WORKERS), else half the CPUs\n\nOptions for record (also --env, --port, --json, --headless, --no-headless):\n  --browser <name>            Record in this browser (default: the first of the config's browsers)\n  --new <title>               Add a test with this title (creates the spec file if needed)\n  --test <name|index>         Record into this test; not needed when the file has one\n  --at <n>                    Insert from step n (default: after the last step)\n  --url <url>                 Page to start on (default: the environment's baseURL)\n  Stop with Ctrl+C or by closing the browser window.\n\n  -h, --help                  Show this help\n  -v, --version               Show the version\n\nExit code: 0 when every test passed, 1 when a test failed, 2 for a usage or configuration error.\n\nsamurai flow:\n  list                        The flows in ./flows\n  check [<flow>...]           Check flows without running them\n  run <flow>... | --all       Run flows; each writes result/flows/<id>.json\n  Options for run: --env, --headless, --timeout, --expect-timeout, --port, --json, --allow-unsupported\n  Run `samurai flow` alone for the details.\n\nsamurai browsers:\n  list                        The pinned browsers and every build in the cache (the default)\n  install [firefox|chrome|all]\n                              Download the pinned builds (default: all)\n  uninstall <name> [--build <id>] | --unused\n\nRun it in the project folder, the one holding samurai.config.ts.\n";
+export declare const USAGE = "samurai: browser tests over WebDriver BiDi\n\nUsage:\n  samurai run [options]       Run the tests (the default command)\n  samurai list [options]      List the tests without running them\n  samurai record <spec>       Open a browser and record what you do into a spec file\n  samurai init [folder]       Scaffold a project: config, an example test, .env.example\n  samurai flow <command>      List, check and run the flows in ./flows (see below)\n  samurai browsers [command]  List, install and remove the browsers SAMURAI launches\n\nOptions for run and list:\n  --env <name>                Environment to run against (or SAMURAI_ENV)\n  --file <spec>               Only this spec file; repeatable\n  --grep <text>               Only tests whose full name contains the text\n  --group <name>              Only the tests of this group of the config's groups\n  --browser <name>            Run in this browser (firefox or chrome) instead of the config's\n                              browsers; repeat it for more than one\n  --json                      Machine-readable output: JSON lines for run, one array for list\n\nOptions for run:\n  --timeout <ms>              Time one test may take\n  --expect-timeout <ms>       Time actions and assertions retry\n  --headless, --no-headless   Run the browser without a window, or with one.\n                              Default: the config's use.headless, else headless when CI is set\n  --port <n>                  Browser debugging port (default 9223)\n  --workers <n|n%>            Tests that run at the same time, each in a worker process: a number\n                              from 1, or a share of the CPUs like 50%. Default: the config's\n                              workers (or SAMURAI_WORKERS), else half the CPUs\n  --retries <n>               Run a failing test again up to n times, each with a fresh browser\n                              (or SAMURAI_RETRIES; default 0)\n\nOptions for record (also --env, --port, --json, --headless, --no-headless):\n  --browser <name>            Record in this browser (default: the first of the config's browsers)\n  --new <title>               Add a test with this title (creates the spec file if needed)\n  --test <name|index>         Record into this test; not needed when the file has one\n  --at <n>                    Insert from step n (default: after the last step)\n  --url <url>                 Page to start on (default: the environment's baseURL)\n  Stop with Ctrl+C or by closing the browser window.\n\n  -h, --help                  Show this help\n  -v, --version               Show the version\n\nExit code: 0 when every test passed, 1 when a test failed, 2 for a usage or configuration error.\n\nsamurai flow:\n  list                        The flows in ./flows\n  check [<flow>...]           Check flows without running them\n  run <flow>... | --all       Run flows; each writes result/flows/<id>.json\n  Options for run: --env, --headless, --timeout, --expect-timeout, --port, --json, --allow-unsupported\n  Run `samurai flow` alone for the details.\n\nsamurai browsers:\n  list                        The pinned browsers and every build in the cache (the default)\n  install [firefox|chrome|all]\n                              Download the pinned builds (default: all)\n  uninstall <name> [--build <id>] | --unused\n\nRun it in the project folder, the one holding samurai.config.ts.\n";

package/dist/cli/usage.js

@@ -27,6 +27,8 @@ Options for run:
   --workers <n|n%>            Tests that run at the same time, each in a worker process: a number
                               from 1, or a share of the CPUs like 50%. Default: the config's
                               workers (or SAMURAI_WORKERS), else half the CPUs
+  --retries <n>               Run a failing test again up to n times, each with a fresh browser
+                              (or SAMURAI_RETRIES; default 0)
 
 Options for record (also --env, --port, --json, --headless, --no-headless):
   --browser <name>            Record in this browser (default: the first of the config's browsers)

package/dist/config/config.js

@@ -3,6 +3,7 @@ import logger from "../logger/index.js";
 import { pathToFileURL } from "url";
 import { validateBrowserConfig } from "./browsers.js";
 import { validateForbidOnlyConfig } from "./forbid-only.js";
+import { validateRetriesConfig } from "./retries.js";
 import { validateWorkersConfig } from "./workers.js";
 let activeProject;
 /** Makes `project` the one config, secrets, reports and downloads are resolved for; `undefined` ends it */
@@ -29,6 +30,7 @@ export async function loadConfig() {
         validateBrowserConfig(activeProject.config);
         validateWorkersConfig(activeProject.config);
         validateForbidOnlyConfig(activeProject.config);
+        validateRetriesConfig(activeProject.config);
         return activeProject.config;
     }
     const configUrl = path.join(projectDir(), "samurai.config.ts");
@@ -40,6 +42,7 @@ export async function loadConfig() {
     validateBrowserConfig(config.default);
     validateWorkersConfig(config.default);
     validateForbidOnlyConfig(config.default);
+    validateRetriesConfig(config.default);
     return config.default;
 }
 /** A project's config without keeping the project active (`withProject` is one at a time): the given one, else its samurai.config.ts */

package/dist/config/retries.d.ts

@@ -0,0 +1,8 @@
+import type { SamuraiTestConfig } from "../types/config.js";
+/**
+ * Reads a `retries` value: a whole number from 0, as a number or as text. `where` names the source (`--retries`,
+ * `SAMURAI_RETRIES`, a config key) in the error.
+ */
+export declare function parseRetries(value: unknown, where: string): number;
+/** Checks the `retries` key of a config. Throws an Error that names the key */
+export declare function validateRetriesConfig(config: SamuraiTestConfig): void;

package/dist/config/retries.js

@@ -0,0 +1,19 @@
+/**
+ * Reads a `retries` value: a whole number from 0, as a number or as text. `where` names the source (`--retries`,
+ * `SAMURAI_RETRIES`, a config key) in the error.
+ */
+export function parseRetries(value, where) {
+    if (typeof value === "number" && Number.isInteger(value) && value >= 0)
+        return value;
+    if (typeof value === "string" && /^\s*\d+\s*$/.test(value))
+        return Number(value.trim());
+    throw new Error(`${where} must be a whole number from 0, got "${String(value)}"`);
+}
+/** Checks the `retries` key of a config. Throws an Error that names the key */
+export function validateRetriesConfig(config) {
+    if (config.retries === undefined)
+        return;
+    if (typeof config.retries !== "number")
+        throw new Error(`retries must be a whole number from 0, got "${String(config.retries)}"`);
+    parseRetries(config.retries, "retries");
+}

package/dist/flows/run/node-pool.d.ts

@@ -0,0 +1,9 @@
+import type { RunEvent } from "../../runner/reporter.js";
+import type { WorkerPool } from "../../runner/worker-pool.js";
+import type { WorkItem } from "../../runner/worker-protocol.js";
+import type { SupportedBrowser } from "../../types/browser.js";
+import type { NodeTest } from "./events.js";
+/** The work items of a node: each test in each browser, the browsers one after the other */
+export declare function itemsOf(tests: NodeTest[], browsers: readonly SupportedBrowser[]): WorkItem[];
+/** Runs the tests of a node on the pool and reports them as a run would */
+export declare function runOnPool(pool: Pick<WorkerPool, "run">, items: WorkItem[], onEvent: (event: RunEvent) => void, signal: AbortSignal): Promise<"passed" | "failed" | "cancelled">;

package/dist/flows/run/node-pool.js

@@ -0,0 +1,27 @@
+/** The work items of a node: each test in each browser, the browsers one after the other */
+export function itemsOf(tests, browsers) {
+    return browsers
+        .flatMap((browser) => tests.map((t) => ({ file: t.file, testName: t.name, browser })))
+        .map((item, index) => ({ ...item, index }));
+}
+/** Runs the tests of a node on the pool and reports them as a run would */
+export async function runOnPool(pool, items, onEvent, signal) {
+    const results = await pool.run(items, {
+        signal,
+        onEvent: (event) => onEvent({
+            type: event.type,
+            name: event.item.testName,
+            file: event.item.file,
+            browser: event.item.browser,
+            ...(event.type === "test-start" &&
+                event.retry && { retry: event.retry }),
+            ...(event.type === "test-end" && { result: event.result }),
+        }),
+    });
+    if (signal.aborted)
+        return "cancelled";
+    // A skipped test is not a failure
+    return results.every((r) => r.status === "success" || r.status === "skipped")
+        ? "passed"
+        : "failed";
+}

package/dist/flows/run/project-options.d.ts

@@ -0,0 +1,44 @@
+import type { ProjectOptions } from "../../runner/project.js";
+import type { PoolOptions, WorkerPool } from "../../runner/worker-pool.js";
+import type { runTests } from "../../runner/run.js";
+import type { FlowFile, FlowProblem } from "../core/index.js";
+import type { FlowEvent } from "./events.js";
+/** The flow has errors; nothing was run */
+export declare class FlowCheckError extends Error {
+    readonly problems: FlowProblem[];
+    constructor(problems: FlowProblem[]);
+}
+export interface RunFlowInProjectOptions extends ProjectOptions {
+    /** The flow to run... */
+    flow?: FlowFile;
+    /** ...or the id of one in `<project>/flows` */
+    flowId?: string;
+    /** Beats the flow's `envDefault`, the config's `defaultEnvironment` and the first environment */
+    environment?: string;
+    signal?: AbortSignal;
+    onEvent?: (event: FlowEvent) => void;
+    /** `"fail"` (default): a flow with a node that can't run is refused. `"skip"`: those nodes are skipped. */
+    unsupported?: "fail" | "skip";
+    port?: number;
+    /**
+     * How many worker processes the tests of the whole run share: a number, or a percentage like `"50%"`. Unset
+     * takes `SAMURAI_WORKERS` (`workersEnv`), then the config's `workers`, then half the CPUs. 1 runs each Test
+     * and Group node through `runTests` in this process; more starts one pool that every node, and every branch
+     * of a Parallel node, takes its workers from
+     */
+    workers?: number | string;
+    /** The value of `SAMURAI_WORKERS`, when set */
+    workersEnv?: string;
+    /** How many times a failing test is run again: a number, or numeric text like the command line's `--retries`. Unset takes `SAMURAI_RETRIES` (`retriesEnv`), then the config's `retries`, then 0 */
+    retries?: number | string;
+    /** The value of `SAMURAI_RETRIES`, when set */
+    retriesEnv?: string;
+    /** @internal Replaces the worker pool (tests) */
+    createPool?: (options: PoolOptions) => Pick<WorkerPool, "run" | "dispose">;
+    /** @internal Replaces the runner of each Test and Group node (tests) */
+    testRunner?: typeof runTests;
+    /** @internal */
+    sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
+}
+/** What `runFlowInProject` takes when the flow itself is given separately: everything but `flow` and `flowId` */
+export type FlowRunOptions = Omit<RunFlowInProjectOptions, "flow" | "flowId">;

package/dist/flows/run/project-options.js

@@ -0,0 +1,8 @@
+/** The flow has errors; nothing was run */
+export class FlowCheckError extends Error {
+    constructor(problems) {
+        super(`This flow can't run:\n${problems.map((p) => `  - ${p.message}`).join("\n")}`);
+        this.problems = problems;
+        this.name = "FlowCheckError";
+    }
+}

package/dist/flows/run/project.d.ts

@@ -1,46 +1,11 @@
 import { type FlowFile, type FlowProblem } from "../core/index.js";
 import { type ResolvedGroup } from "../../runner/groups.js";
-import { type ProjectOptions } from "../../runner/project.js";
-import { runTests } from "../../runner/run.js";
-import { WorkerPool, type PoolOptions } from "../../runner/worker-pool.js";
 import type { SamuraiTestConfig } from "../../types/config.js";
-import type { FlowEvent, FlowSummary, NodeTest } from "./events.js";
+import type { FlowSummary, NodeTest } from "./events.js";
+import { FlowCheckError, type FlowRunOptions, type RunFlowInProjectOptions } from "./project-options.js";
 import type { SupportedBrowser } from "../../types/browser.js";
-/** The flow has errors; nothing was run */
-export declare class FlowCheckError extends Error {
-    readonly problems: FlowProblem[];
-    constructor(problems: FlowProblem[]);
-}
-export interface RunFlowInProjectOptions extends ProjectOptions {
-    /** The flow to run... */
-    flow?: FlowFile;
-    /** ...or the id of one in `<project>/flows` */
-    flowId?: string;
-    /** Beats the flow's `envDefault`, the config's `defaultEnvironment` and the first environment */
-    environment?: string;
-    signal?: AbortSignal;
-    onEvent?: (event: FlowEvent) => void;
-    /** `"fail"` (default): a flow with a node that can't run is refused. `"skip"`: those nodes are skipped. */
-    unsupported?: "fail" | "skip";
-    port?: number;
-    /**
-     * How many worker processes the tests of the whole run share: a number, or a percentage like `"50%"`. Unset
-     * takes `SAMURAI_WORKERS` (`workersEnv`), then the config's `workers`, then half the CPUs. 1 runs each Test
-     * and Group node through `runTests` in this process; more starts one pool that every node, and every branch
-     * of a Parallel node, takes its workers from
-     */
-    workers?: number | string;
-    /** The value of `SAMURAI_WORKERS`, when set */
-    workersEnv?: string;
-    /** @internal Replaces the worker pool (tests) */
-    createPool?: (options: PoolOptions) => Pick<WorkerPool, "run" | "dispose">;
-    /** @internal Replaces the runner of each Test and Group node (tests) */
-    testRunner?: typeof runTests;
-    /** @internal */
-    sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
-}
-/** What `runFlowInProject` takes when the flow itself is given separately: everything but `flow` and `flowId` */
-export type FlowRunOptions = Omit<RunFlowInProjectOptions, "flow" | "flowId">;
+export { FlowCheckError };
+export type { FlowRunOptions, RunFlowInProjectOptions };
 /** The environment: the option, else the flow's `envDefault` (when the project has it), else the config's default, else the first, else "default" */
 export declare function chooseEnvironment(config: SamuraiTestConfig, flow: FlowFile, option?: string): string;
 /** A test of the project as flows see it */
@@ -71,4 +36,3 @@ export declare function checkFlowInProject(options: Pick<RunFlowInProjectOptions
  * One at a time per process.
  */
 export declare function runFlowInProject(options: RunFlowInProjectOptions): Promise<FlowSummary>;
-export {};

package/dist/flows/run/project.js

@@ -3,6 +3,7 @@ import { projectConfig } from "../../config/config.js";
 import { checkFlow, withKeys, } from "../core/index.js";
 import { DEFAULT_SRC_DIR, realPath, resolveGroups, testId, toPosix, } from "../../runner/groups.js";
 import logger, { keepLogFiles } from "../../logger/index.js";
+import { resolveRetries } from "../../runner/retries-option.js";
 import { resolveWorkers } from "../../runner/workers-option.js";
 import { withProject } from "../../runner/project.js";
 import { browsersFor } from "../../runner/browsers.js";
@@ -12,17 +13,12 @@ import TestRunner from "../../runner/test-runner.js";
 import { WorkerPool } from "../../runner/worker-pool.js";
 import { processWorkerFactory } from "../../runner/worker-process.js";
 import { readFlow } from "./files.js";
+import { itemsOf, runOnPool } from "./node-pool.js";
+import { FlowCheckError, } from "./project-options.js";
 import { projectSteps } from "../steps/set.js";
 import { HANDLERS } from "./handlers/index.js";
 import { runFlow } from "./run.js";
-/** The flow has errors; nothing was run */
-export class FlowCheckError extends Error {
-    constructor(problems) {
-        super(`This flow can't run:\n${problems.map((p) => `  - ${p.message}`).join("\n")}`);
-        this.problems = problems;
-        this.name = "FlowCheckError";
-    }
-}
+export { FlowCheckError };
 /** Whether the project's config defines an environment of this name */
 function hasEnvironment(config, name) {
     return Object.prototype.hasOwnProperty.call(config.environments ?? {}, name);
@@ -36,31 +32,6 @@ export function chooseEnvironment(config, flow, option) {
         return flow.envDefault;
     return config.defaultEnvironment ?? names[0] ?? "default";
 }
-/** The work items of a node: each test in each browser, the browsers one after the other */
-function itemsOf(tests, browsers) {
-    return browsers
-        .flatMap((browser) => tests.map((t) => ({ file: t.file, testName: t.name, browser })))
-        .map((item, index) => ({ ...item, index }));
-}
-/** Runs the tests of a node on the pool and reports them as a run would */
-async function runOnPool(pool, items, onEvent, signal) {
-    const results = await pool.run(items, {
-        signal,
-        onEvent: (event) => onEvent({
-            type: event.type,
-            name: event.item.testName,
-            file: event.item.file,
-            browser: event.item.browser,
-            ...(event.type === "test-end" && { result: event.result }),
-        }),
-    });
-    if (signal.aborted)
-        return "cancelled";
-    // A skipped test is not a failure
-    return results.every((r) => r.status === "success" || r.status === "skipped")
-        ? "passed"
-        : "failed";
-}
 /** The tests, and the browsers, of each Test and Group node of a flow; a Test node uses the project's browsers */
 export function nodeTestsOf(flow, tests, groups) {
     const byId = new Map(tests.map((t) => [t.id, t]));
@@ -182,6 +153,12 @@ async function walkFlowInProject(options) {
     const files = [
         ...new Set(Object.values(nodeTests).flatMap((list) => list.map((t) => t.file))),
     ];
+    // A failing test is run again as often as the project says, in a node as in a plain run
+    const retries = resolveRetries({
+        flag: options.retries,
+        env: options.retriesEnv,
+        config: config.retries,
+    });
     // One pool for the whole run: the nodes, and the branches of a Parallel node, share its workers
     const pool = workers > 1
         ? (options.createPool ?? ((o) => new WorkerPool(o)))({
@@ -200,6 +177,7 @@ async function walkFlowInProject(options) {
                 }),
                 files,
                 basePort: options.port ?? DEFAULT_PORT,
+                ...(retries > 0 && { retries }),
             },
         })
         : undefined;
@@ -230,6 +208,7 @@ async function walkFlowInProject(options) {
                         expectTimeout: options.expectTimeout,
                     }),
                     ...(options.port !== undefined && { port: options.port }),
+                    ...(retries > 0 && { retries }),
                     ...(browsers && { browsers }),
                     files: [...new Set(tests.map((t) => t.file))],
                     testNames: tests.map((t) => t.name),

package/dist/recorder/candidates.d.ts

@@ -1,6 +1,7 @@
 /**
  * Declarations that turn an element into locator candidates (test id, role and name, label, text, css and
- * an absolute xpath), plus `inToolbar(event)`, which is true for events from the recorder toolbar. A string
+ * an absolute xpath), `uniqueOf(el, candidates)`, the candidates that match only that element right now, and
+ * `inToolbar(event)`, which is true for events from the recorder toolbar. A string
  * of plain declarations, interpolated into the capture script (page realm) and the toolbar script
  * (sandbox realm), so no transpiler helper can end up in it.
  */

package/dist/recorder/candidates.js

@@ -1,6 +1,7 @@
 /**
  * Declarations that turn an element into locator candidates (test id, role and name, label, text, css and
- * an absolute xpath), plus `inToolbar(event)`, which is true for events from the recorder toolbar. A string
+ * an absolute xpath), `uniqueOf(el, candidates)`, the candidates that match only that element right now, and
+ * `inToolbar(event)`, which is true for events from the recorder toolbar. A string
  * of plain declarations, interpolated into the capture script (page realm) and the toolbar script
  * (sandbox realm), so no transpiler helper can end up in it.
  */
@@ -100,6 +101,48 @@ const candidatesOf = (el) => {
   return found;
 };
 
+// Not searched by text, as the framework's text locator skips them
+const UNTEXTED = ["SCRIPT", "STYLE", "NOSCRIPT", "TEMPLATE", "HEAD", "TITLE", "META", "LINK"];
+
+// The elements a candidate matches in this document, found the way the framework's locators find them
+const matchesOf = (call) => {
+  const all = () => Array.prototype.slice.call(document.querySelectorAll("*"));
+  switch (call.method) {
+    case "getByTestId":
+      return all().filter((node) => node.getAttribute("data-testid") === call.testId);
+    case "getByRole":
+      return all().filter((node) => roleOf(node) === call.role && nameOf(node, call.role) === call.name);
+    case "getByLabel":
+      return all().filter((node) => labelTexts(node).indexOf(call.text) >= 0);
+    case "getByText": {
+      const matches = all().filter((node) => UNTEXTED.indexOf(node.tagName) < 0 && textOf(node) === call.text);
+      // Only the innermost count: a wrapper around the matching element is not a match
+      return matches.filter((node) => !matches.some((other) => other !== node && node.contains(other)));
+    }
+    case "getByCss":
+      try {
+        return Array.prototype.slice.call(document.querySelectorAll(call.css));
+      } catch (error) {
+        return [];
+      }
+    case "locator": {
+      const found = document.evaluate(call.xpath, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+      const nodes = [];
+      for (let i = 0; i < found.snapshotLength; i++) nodes.push(found.snapshotItem(i));
+      return nodes;
+    }
+    default:
+      return [];
+  }
+};
+
+// The candidates that match only el right now: what the recorder falls back to once the page has removed el
+const uniqueOf = (el, candidates) =>
+  candidates.filter((call) => {
+    const found = matchesOf(call);
+    return found.length === 1 && found[0] === el;
+  });
+
 const isTextEntry = (el) => {
   const tag = el.tagName.toLowerCase();
   if (tag === "textarea") return true;

package/dist/recorder/capture.d.ts

@@ -5,17 +5,19 @@
  * listeners; those of the earlier run go quiet, as their run is no longer the current one.
  * Reports what the tester does in the top-level document:
  *
- * - `{ type: "click", target, candidates, textEntry }`: a click, retargeted to the nearest interactive
+ * - `{ type: "click", target, candidates, unique, textEntry }`: a click, retargeted to the nearest interactive
  *   element (a label click reports its control; the click the browser forwards to the control is dropped).
  * - `{ type: "input", target, candidates, value, secret, secretName }`: text typed into a field. Password
  *   values never leave the page: `secret` is set and `value` is left out.
- * - `{ type: "press", target, candidates, pressed }`: Enter, Tab or Escape (Enter in a button, link or textarea is a
+ * - `{ type: "press", target, candidates, unique, pressed }`: Enter, Tab or Escape (Enter in a button, link or textarea is a
  *   click or typing, which are reported as such).
  * - `{ type: "assert", target, candidates, text, value }`: an Alt+click, which does not reach the page.
  *   While Alt is held, the element under the pointer is outlined.
  *
  * `candidates` are locator calls for the element, most stable first (test id, role and name, label,
  * text, css, and an absolute xpath that always exists). The recorder checks each one against the page.
+ * `unique` lists the candidates that matched only the element when it was clicked or pressed, before the
+ * page's own handlers ran: a button the click removes (Sign in, Delete) is gone by the time the recorder checks.
  * Runs in the page's own realm; a string, so no transpiler helper can end up in it.
  */
 export declare const CAPTURE_SOURCE: string;

package/dist/recorder/capture.js

@@ -12,17 +12,19 @@ const RUN_KEY = "samurai.recorder.run";
  * listeners; those of the earlier run go quiet, as their run is no longer the current one.
  * Reports what the tester does in the top-level document:
  *
- * - `{ type: "click", target, candidates, textEntry }`: a click, retargeted to the nearest interactive
+ * - `{ type: "click", target, candidates, unique, textEntry }`: a click, retargeted to the nearest interactive
  *   element (a label click reports its control; the click the browser forwards to the control is dropped).
  * - `{ type: "input", target, candidates, value, secret, secretName }`: text typed into a field. Password
  *   values never leave the page: `secret` is set and `value` is left out.
- * - `{ type: "press", target, candidates, pressed }`: Enter, Tab or Escape (Enter in a button, link or textarea is a
+ * - `{ type: "press", target, candidates, unique, pressed }`: Enter, Tab or Escape (Enter in a button, link or textarea is a
  *   click or typing, which are reported as such).
  * - `{ type: "assert", target, candidates, text, value }`: an Alt+click, which does not reach the page.
  *   While Alt is held, the element under the pointer is outlined.
  *
  * `candidates` are locator calls for the element, most stable first (test id, role and name, label,
  * text, css, and an absolute xpath that always exists). The recorder checks each one against the page.
+ * `unique` lists the candidates that matched only the element when it was clicked or pressed, before the
+ * page's own handlers ran: a button the click removes (Sign in, Delete) is gone by the time the recorder checks.
  * Runs in the page's own realm; a string, so no transpiler helper can end up in it.
  */
 export const CAPTURE_SOURCE = String.raw `(send) => {
@@ -39,6 +41,7 @@ export const CAPTURE_SOURCE = String.raw `(send) => {
   const MUTED = Symbol.for("${MUTED_KEY}");
   const off = () => window[RUN] !== run || window[OFF] === true || window[MUTED] === true;
   let lastClick = { el: null, at: 0 };
+  let lastEnter = { form: null, at: 0 };
   let outlined = null;
   let outlinedStyle = "";
   const unhighlight = () => {
@@ -69,8 +72,11 @@ export const CAPTURE_SOURCE = String.raw `(send) => {
     const now = Date.now();
     // The browser forwards a label click to its control as a second click
     if (lastClick.el === el && now - lastClick.at < 150) return;
+    // An Enter in a field submits its form with a click on the submit button: the Enter is the step
+    if (event.detail === 0 && el.form && el.form === lastEnter.form && now - lastEnter.at < 150) return;
     lastClick = { el, at: now };
-    report({ type: "click", target: el, candidates: candidatesOf(el), textEntry: isTextEntry(el) });
+    const candidates = candidatesOf(el);
+    report({ type: "click", target: el, candidates, unique: uniqueOf(el, candidates), textEntry: isTextEntry(el) });
   }, true);
 
   document.addEventListener("input", (event) => {
@@ -98,7 +104,9 @@ export const CAPTURE_SOURCE = String.raw `(send) => {
     if (RECORDED_KEYS.indexOf(event.key) < 0 || !(event.target instanceof Element)) return;
     const el = event.target;
     if (event.key === "Enter" && (el.closest(ACTIVATES) || el.tagName === "TEXTAREA" || el.isContentEditable)) return;
-    report({ type: "press", target: el, candidates: candidatesOf(el), pressed: event.key });
+    const candidates = candidatesOf(el);
+    report({ type: "press", target: el, candidates, unique: uniqueOf(el, candidates), pressed: event.key });
+    if (event.key === "Enter") lastEnter = { form: el.form || null, at: Date.now() };
   }, true);
 
   document.addEventListener("mousemove", (event) => {

package/dist/recorder/page-messages.d.ts

@@ -23,6 +23,8 @@ export interface PageMessage {
     type: "click" | "input" | "press" | "assert";
     target: ElementHandle;
     candidates: LocatorCall[];
+    /** Of a click or press: the candidates that matched only the target when it happened, checked in the page */
+    unique?: LocatorCall[];
     textEntry?: boolean;
     /** The key of a `press` message */
     pressed?: string;

package/dist/recorder/ranking.d.ts

@@ -7,3 +7,5 @@ import type { LocatorCall, LocatorSpec } from "../steps/model.js";
  * become its fallbacks. Returns undefined when no candidate qualifies.
  */
 export declare function rankLocator(page: Page, target: ElementHandle, candidates: readonly LocatorCall[], maxFallbacks?: number): Promise<LocatorSpec | undefined>;
+/** The first of `verified` as the primary, up to `maxFallbacks` more as its fallbacks; undefined when empty */
+export declare function specOf(verified: readonly LocatorCall[], maxFallbacks?: number): LocatorSpec | undefined;

package/dist/recorder/ranking.js

@@ -17,6 +17,13 @@ export async function rankLocator(page, target, candidates, maxFallbacks = 2) {
         if (found.length === 1 && found[0].sharedId === target.sharedId)
             verified.push(candidate);
     }
+    return specOf(verified, maxFallbacks);
+}
+/** The first of `verified` as the primary, up to `maxFallbacks` more as its fallbacks; undefined when empty */
+export function specOf(verified, maxFallbacks = 2) {
     const [primary, ...fallbacks] = verified;
-    return (primary && { chain: [primary], fallbacks: fallbacks.map((call) => [call]) });
+    return (primary && {
+        chain: [primary],
+        fallbacks: fallbacks.slice(0, maxFallbacks).map((call) => [call]),
+    });
 }

package/dist/recorder/recorder.js

@@ -9,7 +9,7 @@ import { FilesRecorder, missingFileText } from "./files-recorder.js";
 import { Normaliser } from "./normalise.js";
 import { observationFor, } from "./page-messages.js";
 import { listSandboxRealms } from "./realms.js";
-import { rankLocator } from "./ranking.js";
+import { rankLocator, specOf } from "./ranking.js";
 import { TOOLBAR_HOOK, ToolbarBridge } from "./toolbar-bridge.js";
 import { TOOLBAR_SANDBOX } from "./toolbar/protocol.js";
 import { TOOLBAR_SOURCE } from "./toolbar/source.js";
@@ -216,7 +216,10 @@ export class Recorder {
         const { sharedId } = message.target;
         if (this.known.has(sharedId))
             return this.known.get(sharedId);
-        const spec = await rankLocator(this.page, message.target, message.candidates);
+        // The page may have removed the element by now (a Sign in that leaves the form): then what matched
+        // only it when it was used is the best there is
+        const spec = (await rankLocator(this.page, message.target, message.candidates)) ??
+            specOf(message.unique ?? []);
         this.known.set(sharedId, spec);
         return spec;
     }

package/dist/runner/options.d.ts

@@ -18,6 +18,12 @@ export interface RunnerOptions {
      * @default the config's `workers`, else half the CPUs
      */
     workers?: number | string;
+    /**
+     * How many times a failing test is run again, each time with a fresh browser (the command line's `--retries`).
+     * Beats `SAMURAI_RETRIES` and the config's `retries`.
+     * @default the config's `retries`, else 0
+     */
+    retries?: number | string;
     /** Run the browser without a window. @default false */
     headless?: boolean;
     /** Remote debugging port of the browser. @default DEFAULT_PORT (9223) */

package/dist/runner/pooled-run.js

@@ -76,12 +76,13 @@ export async function runPooled(pool, planned, items, reporter, signal) {
     }
     const started = new Set();
     const ended = new Set();
-    const begin = (index) => {
-        if (started.has(index))
+    const begin = (index, retry) => {
+        // A test run again starts again; the first start is reported once however it arrives
+        if (started.has(index) && !retry)
             return;
         started.add(index);
         const { test, browser } = planned[index];
-        reporter.onTestStart(test, browser);
+        reporter.onTestStart(test, browser, retry);
     };
     const end = (index, result) => {
         begin(index);
@@ -92,7 +93,7 @@ export async function runPooled(pool, planned, items, reporter, signal) {
     const results = await pool.run(items, {
         ...(signal && { signal }),
         onEvent: (event) => event.type === "test-start"
-            ? begin(event.item.index)
+            ? begin(event.item.index, event.retry)
             : end(event.item.index, event.result),
     });
     results.forEach((result, position) => {

package/dist/runner/reporter.d.ts

@@ -1,4 +1,4 @@
-import type { RegisteredTestCase, TestError, TestLogEntry, TestResult, TestSummary } from "../types/test.js";
+import type { RegisteredTestCase, TestError, TestLogEntry, TestResult, TestRetries, TestSummary } from "../types/test.js";
 import type { SupportedBrowser } from "../types/browser.js";
 /** What a run reports while it happens */
 export type RunEvent = {
@@ -14,6 +14,8 @@ export type RunEvent = {
     name: string;
     file: string;
     browser: SupportedBrowser;
+    /** The attempt, counting from 1, when the test is run again after a failure; absent for the first */
+    retry?: number;
 } | {
     type: "test-end";
     name: string;
@@ -36,7 +38,7 @@ export interface ReporterOptions {
 }
 /** What `runOneTest` reports a test to: the reporter of a run, or a worker that relays it */
 export interface TestSink {
-    onTestStart(test: RegisteredTestCase, browser: SupportedBrowser): void;
+    onTestStart(test: RegisteredTestCase, browser: SupportedBrowser, retry?: number): void;
     onTestEnd(test: RegisteredTestCase, browser: SupportedBrowser, error?: TestError, extras?: TestEndExtras): void;
     /** A test that does not run: reported as started and ended at once */
     onTestSkipped(test: RegisteredTestCase, browser: SupportedBrowser, skip: TestSkip): void;
@@ -47,7 +49,7 @@ export interface TestSkip {
     reason?: string | undefined;
 }
 /** What a test's report entry lists besides its status */
-export interface TestEndExtras {
+export interface TestEndExtras extends TestRetries {
     logs?: TestLogEntry[];
     logsDropped?: number;
     screenshots?: {
@@ -73,7 +75,7 @@ export default class TestReporter implements TestSink {
     }[]): void;
     onStart(total: number, workers?: number): void;
     onEnd(): Promise<TestSummary>;
-    onTestStart(test: RegisteredTestCase, browser: SupportedBrowser): void;
+    onTestStart(test: RegisteredTestCase, browser: SupportedBrowser, retry?: number): void;
     onTestSkipped(test: RegisteredTestCase, browser: SupportedBrowser, { annotation, reason }: TestSkip): void;
     /**
      * Ends a test that ran in a worker: the worker built the result (masked, with its own duration), this keeps the

package/dist/runner/reporter.js

@@ -75,7 +75,7 @@ export default class TestReporter {
         this.emit({ type: "run-end", summary: this.summary });
         return this.summary;
     }
-    onTestStart(test, browser) {
+    onTestStart(test, browser, retry) {
         const key = keyOf(test, browser);
         const t = performance.mark(`${key}-start`);
         this.testResults.set(key, {
@@ -90,6 +90,7 @@ export default class TestReporter {
             name: test.name,
             file: test.file,
             browser,
+            ...(retry ? { retry } : {}),
         });
     }
     onTestSkipped(test, browser, { annotation, reason }) {
@@ -145,6 +146,11 @@ export default class TestReporter {
             ...(extras?.logs && { logs: extras.logs }),
             ...(extras?.logsDropped && { logsDropped: extras.logsDropped }),
             ...(extras?.screenshots?.length && { screenshots: extras.screenshots }),
+            ...(extras?.retry && { retry: extras.retry }),
+            ...(extras?.flaky && { flaky: true }),
+            ...(extras?.previousAttempts?.length && {
+                previousAttempts: extras.previousAttempts,
+            }),
         });
         this.testResults.set(key, result);
         this.emit({

package/dist/runner/retries-option.d.ts

@@ -0,0 +1,10 @@
+export interface RetriesInput {
+    /** The command line's `--retries` (or the `retries` run option) */
+    flag?: number | string | undefined;
+    /** `SAMURAI_RETRIES` */
+    env?: string | undefined;
+    /** The config's `retries` */
+    config?: number | string | undefined;
+}
+/** How many times a failing test is run again: the flag, else `SAMURAI_RETRIES`, else the config's `retries`, else 0 */
+export declare function resolveRetries({ flag, env, config }: RetriesInput): number;

package/dist/runner/retries-option.js

@@ -0,0 +1,12 @@
+import { parseRetries } from "../config/retries.js";
+/** How many times a failing test is run again: the flag, else `SAMURAI_RETRIES`, else the config's `retries`, else 0 */
+export function resolveRetries({ flag, env, config }) {
+    const fromEnv = env?.trim() ? env : undefined;
+    if (flag !== undefined)
+        return parseRetries(flag, "--retries");
+    if (fromEnv !== undefined)
+        return parseRetries(fromEnv, "SAMURAI_RETRIES");
+    if (config !== undefined)
+        return parseRetries(config, "retries");
+    return 0;
+}

package/dist/runner/retry.d.ts

@@ -0,0 +1,13 @@
+import type { SupportedBrowser } from "../types/browser.js";
+import type { RegisteredTestCase } from "../types/test.js";
+import type { TestSink } from "./reporter.js";
+/** One attempt at a test: it reports to `sink` and `attempt` counts from 0 */
+export type Attempt = (sink: TestSink, attempt: number) => Promise<void>;
+/**
+ * Runs a test, and again after a failure with a fresh browser, up to `retries` more times. The test starts once
+ * per attempt (`retry` counts from 1), and ends once, with the last attempt's result: `retry` says which attempt
+ * that was, `previousAttempts` holds the errors before it, and `flaky` marks a pass after a failure.
+ *
+ * An aborted run is never run again. A test that skips itself ends there and then.
+ */
+export declare function runWithRetries(test: RegisteredTestCase, browser: SupportedBrowser, retries: number, signal: AbortSignal | undefined, attempt: Attempt, sink: TestSink): Promise<void>;

package/dist/runner/retry.js

@@ -0,0 +1,43 @@
+/**
+ * Runs a test, and again after a failure with a fresh browser, up to `retries` more times. The test starts once
+ * per attempt (`retry` counts from 1), and ends once, with the last attempt's result: `retry` says which attempt
+ * that was, `previousAttempts` holds the errors before it, and `flaky` marks a pass after a failure.
+ *
+ * An aborted run is never run again. A test that skips itself ends there and then.
+ */
+export async function runWithRetries(test, browser, retries, signal, attempt, sink) {
+    if (retries <= 0)
+        return attempt(sink, 0);
+    const previous = [];
+    for (let n = 0;; n++) {
+        let held;
+        const attemptSink = {
+            onTestStart: (t, b) => sink.onTestStart(t, b, n > 0 ? n : undefined),
+            onTestEnd: (_t, _b, error, extras) => {
+                held = { error, extras };
+            },
+            onTestSkipped: (t, b, skip) => sink.onTestSkipped(t, b, skip),
+        };
+        await attempt(attemptSink, n);
+        // Skipped: already reported, and nothing is run again
+        if (!held)
+            return;
+        const { error, extras } = held;
+        if (error &&
+            n < retries &&
+            !signal?.aborted &&
+            error.message !== "Run aborted") {
+            previous.push(error);
+            continue;
+        }
+        sink.onTestEnd(test, browser, error, {
+            ...extras,
+            ...(n > 0 && {
+                retry: n,
+                previousAttempts: [...previous],
+                ...(!error && { flaky: true }),
+            }),
+        });
+        return;
+    }
+}

package/dist/runner/run-test.d.ts

@@ -12,6 +12,8 @@ export interface WorkerResources {
     profileDir?: string;
     /** The `workerIndex` fixture. @default 0 */
     workerIndex?: number;
+    /** Which attempt this is, counting from 0: screenshots of a retry go to a `retry-<n>` folder of their own. @default 0 */
+    attempt?: number;
     /** The `beforeAll` / `afterAll` hooks of this worker: kept across tests, closed when the worker is done */
     allHooks?: AllHooks;
 }

package/dist/runner/run-test.js

@@ -10,7 +10,7 @@ import { ConnectionClosedError } from "../transport/bidi-connection.js";
 import { judgePageLogs } from "./page-logs-report.js";
 import { DEFAULT_PORT } from "./ports.js";
 import { runWithEachHooks } from "./hooks.js";
-import { isSkipSignal } from "./skip.js";
+import { isSkipSignal, staticSkip } from "./skip.js";
 import { testDirOf } from "./test-dir.js";
 import { toTestError } from "./test-error.js";
 /**
@@ -64,6 +64,12 @@ export function allHookFixtures(run, browserName, workerIndex = 0) {
 }
 /** Runs one test in one browser and reports it: launch, run the body, judge the page's logs, close the browser */
 export async function runOneTest(test, browserName, run, options, srcDir, reporter, resources = {}) {
+    // A test marked skip or fixme never runs, whoever asked for it (a flow's test node, a worker)
+    const marked = staticSkip(test);
+    if (marked) {
+        reporter.onTestSkipped(test, browserName, { annotation: marked });
+        return;
+    }
     logger.verbose("Starting test: %s in %s", test.name, browserName);
     reporter.onTestStart(test, browserName);
     const outcome = new TestOutcome(reporter, test, browserName);
@@ -125,7 +131,7 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
                 browserGone();
             });
             page = launched.page;
-            outcome.evidence = new Evidence(resultDir, `${browserName}/${testDirOf(test, srcDir)}`);
+            outcome.evidence = new Evidence(resultDir, `${browserName}/${testDirOf(test, srcDir)}${resources.attempt ? `/retry-${resources.attempt}` : ""}`);
             launched.page.setEvidence(outcome.evidence);
             const fixtures = {
                 page: launched.page,

package/dist/runner/run.d.ts

@@ -28,6 +28,7 @@ export declare function listTests(options?: RunTestsOptions): Promise<FoundTest[
 export declare function listGroups(options?: RunTestsOptions): Promise<ResolvedGroup[]>;
 export { BROWSER_NAMES, checkBrowserList } from "../config/browsers.js";
 export { parseWorkers } from "../config/workers.js";
+export { parseRetries } from "../config/retries.js";
 export type { RunEvent } from "./reporter.js";
 export type { Modifier, TestError, TestResult, TestSummary, } from "../types/test.js";
 export type { ResolvedGroup } from "./groups.js";

package/dist/runner/run.js

@@ -47,4 +47,5 @@ export async function listGroups(options = {}) {
 }
 export { BROWSER_NAMES, checkBrowserList } from "../config/browsers.js";
 export { parseWorkers } from "../config/workers.js";
+export { parseRetries } from "../config/retries.js";
 export { DEFAULT_PORT } from "./ports.js";

package/dist/runner/test-runner.js

@@ -14,6 +14,8 @@ import { concurrency, planItems, poolMode, runPooled, serialKeyOf, } from "./poo
 import { allHookFixtures, runOneTest } from "./run-test.js";
 import { staticSkip } from "./skip.js";
 import { browsersOfRun, loadSpecs, selectTests } from "./select.js";
+import { resolveRetries } from "./retries-option.js";
+import { runWithRetries } from "./retry.js";
 import { resolveWorkers } from "./workers-option.js";
 import { WorkerPool } from "./worker-pool.js";
 import { processWorkerFactory } from "./worker-process.js";
@@ -75,6 +77,12 @@ export default class TestRunner {
             env: process.env.SAMURAI_WORKERS,
             config: config.workers,
         });
+        // A bad `retries` is an error here too, before any browser starts
+        const retries = resolveRetries({
+            flag: this.options.retries,
+            env: process.env.SAMURAI_RETRIES,
+            config: config.retries,
+        });
         const plan = planItems(selected, browsers, serialKeyOf(loaded.groups, config.groups, loaded.srcDir));
         // A pool larger than what can overlap would only hold idle workers: a group that runs one at a time needs one
         const lanes = concurrency(plan.items);
@@ -97,7 +105,7 @@ export default class TestRunner {
         if (mode === "pool") {
             const workers = Math.min(requested, lanes);
             reporter.onStart(plan.planned.length, workers);
-            const pool = this.createPool(workers);
+            const pool = this.createPool(workers, retries);
             try {
                 await runPooled(pool, plan.planned, plan.items, reporter, signal);
             }
@@ -128,7 +136,7 @@ export default class TestRunner {
                         });
                         continue;
                     }
-                    await runOneTest(test, browser, this.run, this.options, loaded.srcDir, reporter, { allHooks });
+                    await runWithRetries(test, browser, retries, signal, (sink, attempt) => runOneTest(test, browser, this.run, this.options, loaded.srcDir, sink, { allHooks, attempt }), reporter);
                 }
             }
         }
@@ -145,7 +153,7 @@ export default class TestRunner {
             fromConfig !== undefined);
     }
     /** The pool of a run: up to `workers` worker processes that read the same project and settings as this run */
-    createPool(workers) {
+    createPool(workers, retries) {
         const { settings } = this.run;
         const { reportPath, headless, port } = this.options;
         const options = {
@@ -163,6 +171,7 @@ export default class TestRunner {
                 ...(typeof reportPath === "string" && { reportPath }),
                 files: this.testFiles,
                 basePort: port ?? DEFAULT_PORT,
+                ...(retries > 0 && { retries }),
             },
         };
         return (this.project.createPool ?? ((o) => new WorkerPool(o)))(options);

package/dist/runner/worker-pool.d.ts

@@ -4,6 +4,7 @@ export type PoolEvent = {
     type: "test-start";
     item: WorkItem;
     worker: number;
+    retry?: number;
 } | {
     type: "test-end";
     item: WorkItem;

package/dist/runner/worker-pool.js

@@ -187,6 +187,7 @@ export class WorkerPool {
                     type: "test-start",
                     item: busy.entry.item,
                     worker: slot.index,
+                    ...(message.retry && { retry: message.retry }),
                 });
             case "test-end":
                 if (busy?.id !== message.id)

package/dist/runner/worker-protocol.d.ts

@@ -20,6 +20,8 @@ export interface WorkerInit {
     timeout?: number;
     expectTimeout?: number;
     headless?: boolean;
+    /** How many times a failing test is run again in this worker. @default 0 */
+    retries?: number;
     /** Where the run's report goes when it is not the default; screenshots are kept beside it */
     reportPath?: string;
     /** Spec files to import, absolute */
@@ -48,6 +50,7 @@ export type FromWorker = {
 } | {
     type: "test-start";
     id: number;
+    retry?: number;
 } | {
     type: "test-end";
     id: number;

package/dist/runner/worker-session.js

@@ -4,6 +4,7 @@ import { withProject } from "./project.js";
 import TestReporter from "./reporter.js";
 import { AllHooks } from "./all-hooks.js";
 import { allHookFixtures, runOneTest } from "./run-test.js";
+import { runWithRetries } from "./retry.js";
 import { loadSpecs } from "./select.js";
 /**
  * Opens the project for a worker and keeps it open until `close`: `withProject` holds the config, environment
@@ -42,19 +43,21 @@ export function openSession(init) {
                         output: false,
                         onEvent,
                     });
-                    await runOneTest(test, browser, run, {
+                    const options = {
                         ...(headless !== undefined && { headless }),
                         ...(init.reportPath !== undefined && {
                             reportPath: init.reportPath,
                         }),
                         port,
                         signal,
-                    }, loaded.srcDir, reporter, {
+                    };
+                    await runWithRetries(test, browser, init.retries ?? 0, signal, (sink, attempt) => runOneTest(test, browser, run, options, loaded.srcDir, sink, {
                         port,
                         profileDir: workerProfileDir(browser, workerIndex),
                         workerIndex,
                         allHooks,
-                    });
+                        attempt,
+                    }), reporter);
                 },
                 async close() {
                     for (const error of await allHooks.closeAll())

package/dist/runner/worker.js

@@ -43,6 +43,9 @@ export function handleMessages(channel, { open, exit }) {
         current = new AbortController();
         try {
             await session.runTest(test, item.browser, current.signal, (event) => {
+                // The first start was sent above; a test run again starts again, saying which attempt
+                if (event.type === "test-start" && event.retry)
+                    channel.send({ type: "test-start", id, retry: event.retry });
                 if (event.type !== "test-end")
                     return;
                 ended = true;

package/dist/steps/model.d.ts

@@ -1,3 +1,4 @@
+import type { Modifier } from "../types/test.js";
 /** One call of a locator expression, as written in a spec: `page.getByRole("button", { name: "Save" })` */
 export type LocatorCall = {
     method: "locator";
@@ -155,4 +156,6 @@ export interface ParsedTest {
         at: number;
         shape: "list" | "empty" | "none";
     };
+    /** `skip` or `fixme` (not run), or `only` (focused), as the runner applies it: a modifier on a `describe` counts */
+    modifier?: Modifier;
 }

package/dist/steps/parse.js

@@ -44,32 +44,40 @@ function callbackBody(node) {
         ? node.body
         : undefined;
 }
-/** `describe.skip`, `describe.fixme` and `describe.only`: tests written inside keep their place in the title */
-function isDescribeWithModifier(expression) {
-    return (ts.isPropertyAccessExpression(expression) &&
+/** `test(...)` or `describe(...)` with its modifier: `test.skip(...)` gives `{ name: "test", modifier: "skip" }` */
+function calleeOf(expression) {
+    if (ts.isIdentifier(expression))
+        return { name: expression.text };
+    if (ts.isPropertyAccessExpression(expression) &&
         ts.isIdentifier(expression.expression) &&
-        expression.expression.text === "describe" &&
-        ["skip", "fixme", "only"].includes(expression.name.text));
+        ["skip", "fixme", "only"].includes(expression.name.text))
+        return {
+            name: expression.expression.text,
+            modifier: expression.name.text,
+        };
+    return undefined;
+}
+/** What the runner applies to a test (see `modifierOf` in the runner): the innermost `skip` or `fixme`, else `only` */
+function effectiveModifier(own, describes) {
+    const chain = [own, ...[...describes].reverse()];
+    return (chain.find((m) => m === "skip" || m === "fixme") ??
+        (chain.includes("only") ? "only" : undefined));
 }
 /** The tests in a spec file, with their steps and where each statement sits */
 export function parseSpec(source) {
     const sourceFile = ts.createSourceFile("spec.ts", source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
     const tests = [];
-    const visit = (node, titles) => {
-        if (ts.isCallExpression(node) && isDescribeWithModifier(node.expression)) {
-            // `describe.skip("Old", () => ...)`: a describe block for the names of the tests inside it
-            const title = stringOf(node.arguments[0]);
-            const body = callbackBody(node.arguments[1]);
-            if (title !== undefined && body) {
-                ts.forEachChild(body, (child) => visit(child, [...titles, title]));
-                return;
-            }
-        }
-        if (ts.isCallExpression(node) && ts.isIdentifier(node.expression)) {
+    const visit = (node, titles, describes) => {
+        const callee = ts.isCallExpression(node)
+            ? calleeOf(node.expression)
+            : undefined;
+        if (ts.isCallExpression(node) && callee) {
             const title = stringOf(node.arguments[0]);
             const body = callbackBody(node.arguments[1]);
-            if (title !== undefined && body && node.expression.text === "test") {
+            if (title !== undefined && body && callee.name === "test") {
+                // `test.skip(condition, reason)` has no title and no body, so it stays a step of the test it is in
                 const titlePath = [...titles, title];
+                const modifier = effectiveModifier(callee.modifier, describes);
                 tests.push({
                     titlePath,
                     name: titlePath.join(" > "),
@@ -77,16 +85,17 @@ export function parseSpec(source) {
                     bodyStart: body.getStart(sourceFile) + 1,
                     bodyEnd: body.end - 1,
                     fixtures: callbackFixtures(node.arguments[1], sourceFile),
+                    ...(modifier && { modifier }),
                 });
                 return;
             }
-            if (title !== undefined && body && node.expression.text === "describe") {
-                ts.forEachChild(body, (child) => visit(child, [...titles, title]));
+            if (title !== undefined && body && callee.name === "describe") {
+                ts.forEachChild(body, (child) => visit(child, [...titles, title], [...describes, callee.modifier]));
                 return;
             }
         }
-        ts.forEachChild(node, (child) => visit(child, titles));
+        ts.forEachChild(node, (child) => visit(child, titles, describes));
     };
-    visit(sourceFile, []);
+    visit(sourceFile, [], []);
     return tests;
 }

package/dist/types/config.d.ts

@@ -64,6 +64,13 @@ export interface SamuraiTestConfig {
    * @default half the CPUs
    */
   workers?: number | string;
+  /**
+   * How many times a failing test is run again, each time with a fresh browser. A test that passes on a retry is
+   * reported as flaky and does not fail the run; a run that is aborted is never retried. `--retries` and
+   * `SAMURAI_RETRIES` win over it.
+   * @default 0
+   */
+  retries?: number;
   /**
    * Refuse a run when a spec contains `test.only` or `describe.only`, so a forgotten focus cannot quietly shrink a
    * suite. The run stops before any test starts and lists the tests.

package/dist/types/test.d.ts

@@ -70,7 +70,7 @@ export interface PartialTestResult extends TestResultBase {
 export type TestResult =
   TestResultSuccess | TestResultFailed | TestResultSkipped | TestResultBase;
 
-interface TestResultSuccess extends TestResultIdentity {
+interface TestResultSuccess extends TestResultIdentity, TestRetries {
   status: "success";
   duration: number;
   logs?: TestLogEntry[];
@@ -81,7 +81,7 @@ interface TestResultSuccess extends TestResultIdentity {
 }
 
 /** The error sits flat on the entry, as in the report: `message`, `type`, `stack`, and for assertions `expected` and `actual` */
-interface TestResultFailed extends TestResultIdentity, TestError {
+interface TestResultFailed extends TestResultIdentity, TestError, TestRetries {
   status: "failed";
   duration: number;
   logs?: TestLogEntry[];
@@ -90,6 +90,16 @@ interface TestResultFailed extends TestResultIdentity, TestError {
   screenshots?: { name: string; path: string }[];
 }
 
+/** What a result says about the retries of its test; absent when it passed or failed on the first attempt */
+export interface TestRetries {
+  /** The attempt this result is from, counting from 1: `retry: 1` is the first run again */
+  retry?: number;
+  /** The test passed after at least one failed attempt */
+  flaky?: true;
+  /** The errors of the failed attempts before this one, in order */
+  previousAttempts?: TestError[];
+}
+
 /** A test that was not run: marked `skip` or `fixme`, or skipped from inside with `test.skip(condition)` */
 interface TestResultSkipped extends TestResultIdentity {
   status: "skipped";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itmetsam/samurai-framework",
-  "version": "0.15.0",
+  "version": "0.15.1",
   "description": "Browser test framework over WebDriver BiDi: auto-waiting locators, environments and secrets, a recorder, and a command line.",
   "main": "dist/api.js",
   "exports": {
@@ -53,6 +53,7 @@
   "scripts": {
     "dev": "tsx src/index.ts",
     "build": "rm -rf dist && tsc -p tsconfig.build.json && node scripts/copy-types.mjs",
+    "prepack": "npm run build",
     "test": "SAMURAI_LOG_LEVEL=${SAMURAI_LOG_LEVEL:-warn} tsx --import ./scripts/test-tmp.mjs --test --test-timeout=10000 \"src/**/*.test.ts\"",
     "test:browser": "SAMURAI_LOG_LEVEL=${SAMURAI_LOG_LEVEL:-warn} tsx --import ./scripts/test-tmp.mjs --test --test-concurrency=1 --test-timeout=60000 \"src/**/*.browser-test.ts\"",
     "test:browser:chrome": "SAMURAI_TEST_BROWSER=chrome npm run test:browser",