@itmetsam/samurai-framework 0.12.1 → 0.15.1

package/dist/runner/skip.js

@@ -0,0 +1,18 @@
+/** Thrown by `test.skip(condition, reason)` and `test.fixme(condition, reason)` inside a test: the test ends as skipped */
+export class SkipSignal extends Error {
+    constructor(annotation, reason) {
+        super(reason ?? "skipped");
+        this.annotation = annotation;
+        this.reason = reason;
+        this.name = "SkipSignal";
+    }
+}
+export function isSkipSignal(error) {
+    return error instanceof SkipSignal;
+}
+/** Why a test does not run at all: it is marked `skip` or `fixme`, itself or by a `describe` around it */
+export function staticSkip(test) {
+    return test.modifier === "skip" || test.modifier === "fixme"
+        ? test.modifier
+        : undefined;
+}

package/dist/runner/suite.d.ts

@@ -0,0 +1,24 @@
+import type { EachHook, Modifier, Suite } from "../types/test.js";
+/** The four hook lists a suite can hold */
+export type HookKind = "beforeEach" | "afterEach" | "beforeAll" | "afterAll";
+export declare function newSuite(title?: string, parent?: Suite, modifier?: Modifier): Suite;
+/** The suite a test or hook written now belongs to: the innermost open `describe`, else the file's own */
+export declare function currentSuite(file: string): Suite;
+/** Opens a `describe` block under the current suite; `leaveSuite` closes it */
+export declare function enterSuite(file: string, title: string, modifier?: Modifier): Suite;
+export declare function leaveSuite(): void;
+/** Forgets every suite: a run registers its specs afresh */
+export declare function resetSuites(): void;
+/**
+ * What applies to a test: the innermost `skip` or `fixme` (the test's own, then its suites', inside out), else
+ * `only` when the test or any suite around it is focused. A skip beats a focus.
+ */
+export declare function modifierOf(own: Modifier | undefined, suite: Suite): Modifier | undefined;
+/** The suites from the file's root down to `suite` */
+export declare function suiteChain(suite: Suite): Suite[];
+/** The `beforeEach` hooks of a test: outermost suite first, in the order they were written */
+export declare function beforeEachOf(suite: Suite): EachHook[];
+/** The `afterEach` hooks of a test: innermost suite first, in the order they were written */
+export declare function afterEachOf(suite: Suite): EachHook[];
+/** Adds a hook to the suite being registered */
+export declare function addHook<K extends HookKind>(suite: Suite, kind: K, hook: NonNullable<Suite[K]>[number]): void;

package/dist/runner/suite.js

@@ -0,0 +1,73 @@
+/** One root suite per spec file; a `describe` block is a child of the suite it is written in */
+const roots = new Map();
+/** The `describe` blocks being registered, outermost first */
+const open = [];
+export function newSuite(title, parent, modifier) {
+    return {
+        ...(title !== undefined && { title }),
+        ...(parent && { parent }),
+        ...(modifier && { modifier }),
+    };
+}
+/** The suite a test or hook written now belongs to: the innermost open `describe`, else the file's own */
+export function currentSuite(file) {
+    const inner = open[open.length - 1];
+    if (inner)
+        return inner;
+    let root = roots.get(file);
+    if (!root) {
+        root = newSuite();
+        roots.set(file, root);
+    }
+    return root;
+}
+/** Opens a `describe` block under the current suite; `leaveSuite` closes it */
+export function enterSuite(file, title, modifier) {
+    const suite = newSuite(title, currentSuite(file), modifier);
+    open.push(suite);
+    return suite;
+}
+export function leaveSuite() {
+    open.pop();
+}
+/** Forgets every suite: a run registers its specs afresh */
+export function resetSuites() {
+    roots.clear();
+    open.length = 0;
+}
+/**
+ * What applies to a test: the innermost `skip` or `fixme` (the test's own, then its suites', inside out), else
+ * `only` when the test or any suite around it is focused. A skip beats a focus.
+ */
+export function modifierOf(own, suite) {
+    let focused = own === "only";
+    let found = own === "only" ? undefined : own;
+    for (let s = suite; s && !found; s = s.parent) {
+        if (s.modifier === "only")
+            focused = true;
+        else
+            found = s.modifier;
+    }
+    return found ?? (focused ? "only" : undefined);
+}
+/** The suites from the file's root down to `suite` */
+export function suiteChain(suite) {
+    const chain = [];
+    for (let s = suite; s; s = s.parent)
+        chain.unshift(s);
+    return chain;
+}
+/** The `beforeEach` hooks of a test: outermost suite first, in the order they were written */
+export function beforeEachOf(suite) {
+    return suiteChain(suite).flatMap((s) => s.beforeEach ?? []);
+}
+/** The `afterEach` hooks of a test: innermost suite first, in the order they were written */
+export function afterEachOf(suite) {
+    return suiteChain(suite)
+        .reverse()
+        .flatMap((s) => s.afterEach ?? []);
+}
+/** Adds a hook to the suite being registered */
+export function addHook(suite, kind, hook) {
+    (suite[kind] ??= []).push(hook);
+}

package/dist/runner/test-error.js

@@ -1,6 +1,10 @@
 import { AssertionError, toReportValue } from "../assert/assertion-error.js";
+import { HookError } from "./hooks.js";
 /** Maps anything a test threw to the error stored in the report */
 export function toTestError(err) {
+    // A hook's failure keeps what it threw (an assertion stays one) under the hook's own message
+    if (err instanceof HookError)
+        return { ...toTestError(err.cause), message: err.message };
     if (err instanceof AssertionError) {
         return {
             message: err.message,

package/dist/runner/test-runner.js

@@ -3,13 +3,19 @@ import path from "path";
 import { ensureWarm } from "../browser/warm-up.js";
 import { projectBrowser } from "../browsers/resolve.js";
 import { loadConfig, projectDir, readConfig } from "../config/config.js";
+import { forbidsOnly } from "../config/forbid-only.js";
 import logger from "../logger/index.js";
+import { AllHooks } from "./all-hooks.js";
+import { assertNoOnly, narrowToOnly } from "./focus.js";
 import { DEFAULT_SRC_DIR, DEFAULT_TEST_MATCH } from "./groups.js";
 import { DEFAULT_PORT } from "./ports.js";
 import TestReporter from "./reporter.js";
-import { planItems, poolMode, runPooled, serialKeyOf, } from "./pooled-run.js";
-import { runOneTest } from "./run-test.js";
+import { concurrency, planItems, poolMode, runPooled, serialKeyOf, } from "./pooled-run.js";
+import { allHookFixtures, runOneTest } from "./run-test.js";
+import { staticSkip } from "./skip.js";
 import { browsersOfRun, loadSpecs, selectTests } from "./select.js";
+import { resolveRetries } from "./retries-option.js";
+import { runWithRetries } from "./retry.js";
 import { resolveWorkers } from "./workers-option.js";
 import { WorkerPool } from "./worker-pool.js";
 import { processWorkerFactory } from "./worker-process.js";
@@ -54,7 +60,10 @@ export default class TestRunner {
         const { reportPath, onEvent, signal } = this.options;
         // The specs are imported once: their top-level code runs once per run
         const loaded = await loadSpecs(this.testFiles);
-        const selected = selectTests(loaded, this.options);
+        const config = await loadConfig();
+        // A focused test the config refuses stops the run before anything starts; else only the focused ones run
+        assertNoOnly(loaded.all, forbidsOnly(config), loaded.srcDir);
+        const selected = narrowToOnly(selectTests(loaded, this.options));
         const browsers = await browsersOfRun(loaded.groups, this.options);
         const reporter = new TestReporter({
             environment: this.run.settings.environment,
@@ -63,32 +72,40 @@ export default class TestRunner {
             ...(onEvent && { onEvent }),
         });
         // How the run goes is settled first: a bad `workers` is an error before any browser starts
-        const config = await loadConfig();
         const requested = resolveWorkers({
             flag: this.options.workers,
             env: process.env.SAMURAI_WORKERS,
             config: config.workers,
         });
+        // A bad `retries` is an error here too, before any browser starts
+        const retries = resolveRetries({
+            flag: this.options.retries,
+            env: process.env.SAMURAI_RETRIES,
+            config: config.retries,
+        });
         const plan = planItems(selected, browsers, serialKeyOf(loaded.groups, config.groups, loaded.srcDir));
+        // A pool larger than what can overlap would only hold idle workers: a group that runs one at a time needs one
+        const lanes = concurrency(plan.items);
         const mode = poolMode({
             workers: requested,
-            items: plan.items.length,
+            items: lanes,
             inlineConfig: this.project.inlineConfig ?? false,
         });
         if (mode === "inline-config" && this.workersAsked(config.workers))
             logger.warn("workers needs samurai.config.ts in the project folder; running on one worker");
         // A missing browser is one clear error before the run, not a failure per test, and before any browser has run a
         // test; a fresh build's slow first start happens here too, outside any test's timeout
-        if (selected.length > 0) {
+        // Skipped tests need no browser, so a run of only those needs none installed
+        if (selected.some((test) => !staticSkip(test))) {
             for (const browser of browsers)
                 await projectBrowser(browser);
             for (const browser of browsers)
                 await ensureWarm(browser, { port: this.options.port ?? DEFAULT_PORT }, signal);
         }
         if (mode === "pool") {
-            const workers = Math.min(requested, plan.items.length);
-            reporter.onStart(plan.items.length, workers);
-            const pool = this.createPool(workers);
+            const workers = Math.min(requested, lanes);
+            reporter.onStart(plan.planned.length, workers);
+            const pool = this.createPool(workers, retries);
             try {
                 await runPooled(pool, plan.planned, plan.items, reporter, signal);
             }
@@ -98,19 +115,35 @@ export default class TestRunner {
             return reporter.onEnd();
         }
         reporter.onStart(selected.length * browsers.length);
-        for (const browser of browsers) {
-            for (const test of selected) {
-                if (signal?.aborted) {
-                    reporter.onTestStart(test, browser);
-                    reporter.onTestEnd(test, browser, {
-                        message: "Run aborted",
-                        type: "error",
-                    });
-                    continue;
+        // The `afterAll` hooks run when the last test is done, before the run ends
+        const allHooks = new AllHooks({
+            timeout: this.run.settings.timeout,
+            fixtures: (browser) => allHookFixtures(this.run, browser),
+        });
+        try {
+            for (const browser of browsers) {
+                for (const test of selected) {
+                    const annotation = staticSkip(test);
+                    if (annotation) {
+                        reporter.onTestSkipped(test, browser, { annotation });
+                        continue;
+                    }
+                    if (signal?.aborted) {
+                        reporter.onTestStart(test, browser);
+                        reporter.onTestEnd(test, browser, {
+                            message: "Run aborted",
+                            type: "error",
+                        });
+                        continue;
+                    }
+                    await runWithRetries(test, browser, retries, signal, (sink, attempt) => runOneTest(test, browser, this.run, this.options, loaded.srcDir, sink, { allHooks, attempt }), reporter);
                 }
-                await runOneTest(test, browser, this.run, this.options, loaded.srcDir, reporter);
             }
         }
+        finally {
+            for (const error of await allHooks.closeAll())
+                logger.error("%s", error.message);
+        }
         return reporter.onEnd();
     }
     /** Whether anything but the default says how many workers to use */
@@ -120,7 +153,7 @@ export default class TestRunner {
             fromConfig !== undefined);
     }
     /** The pool of a run: up to `workers` worker processes that read the same project and settings as this run */
-    createPool(workers) {
+    createPool(workers, retries) {
         const { settings } = this.run;
         const { reportPath, headless, port } = this.options;
         const options = {
@@ -138,6 +171,7 @@ export default class TestRunner {
                 ...(typeof reportPath === "string" && { reportPath }),
                 files: this.testFiles,
                 basePort: port ?? DEFAULT_PORT,
+                ...(retries > 0 && { retries }),
             },
         };
         return (this.project.createPool ?? ((o) => new WorkerPool(o)))(options);

package/dist/runner/worker-pool.d.ts

@@ -4,6 +4,7 @@ export type PoolEvent = {
     type: "test-start";
     item: WorkItem;
     worker: number;
+    retry?: number;
 } | {
     type: "test-end";
     item: WorkItem;
@@ -17,6 +18,8 @@ export interface PoolOptions {
     factory: WorkerFactory;
     /** The abort grace period in ms; 2000 by default */
     graceMs?: number;
+    /** How long `dispose` waits for idle workers to leave on their own, in ms: their `afterAll` hooks run first. 60000 by default */
+    shutdownMs?: number;
 }
 export interface RunHooks {
     onEvent?: (event: PoolEvent) => void;
@@ -40,8 +43,13 @@ export declare class WorkerPool {
     constructor(options: PoolOptions);
     /** Results in item order; may be called several times at once (flows): all calls share the cap */
     run(items: WorkItem[], hooks?: RunHooks): Promise<TestResult[]>;
-    /** Tells every worker to exit and kills it; safe to call twice */
+    /**
+     * Ends the pool; safe to call twice. A worker that has nothing to do is told to exit and given time to leave
+     * (it runs its `afterAll` hooks first); one that is busy is killed, as nobody waits for its test.
+     */
     dispose(): Promise<void>;
+    /** Asks the idle workers to exit and waits for them (up to `shutdownMs`), kills the busy ones and the stragglers */
+    private sendHome;
     /** Hands queued items to ready workers, then starts workers for what is left */
     private pump;
     private busyKeys;

package/dist/runner/worker-pool.js

@@ -1,5 +1,6 @@
 /** How long a busy worker may take to wind down after an abort before it is killed */
 const GRACE_MS = 2000;
+const SHUTDOWN_MS = 60000;
 /** Workers that die before they are ready this many times in a row fail the run, instead of being replaced for ever */
 const STARTUP_ATTEMPTS = 3;
 /**
@@ -49,14 +50,45 @@ export class WorkerPool {
             this.pump();
         });
     }
-    /** Tells every worker to exit and kills it; safe to call twice */
+    /**
+     * Ends the pool; safe to call twice. A worker that has nothing to do is told to exit and given time to leave
+     * (it runs its `afterAll` hooks first); one that is busy is killed, as nobody waits for its test.
+     */
     async dispose() {
         if (this.disposed)
             return;
         this.disposed = true;
         this.queue.length = 0;
-        this.stopWorkers(true);
+        const leaving = this.sendHome();
         this.rejectAll(new Error("WorkerPool.dispose(): the pool was disposed during a run"));
+        await leaving;
+    }
+    /** Asks the idle workers to exit and waits for them (up to `shutdownMs`), kills the busy ones and the stragglers */
+    async sendHome() {
+        for (const timer of this.timers)
+            clearTimeout(timer);
+        this.timers.clear();
+        const slots = [...this.slots.values()];
+        this.slots.clear();
+        const gone = [];
+        for (const { handle, busy } of slots) {
+            if (busy) {
+                handle.kill();
+                continue;
+            }
+            gone.push(new Promise((resolve) => handle.onExit(() => resolve())));
+            handle.send({ type: "exit" });
+        }
+        let timer;
+        await Promise.race([
+            Promise.all(gone),
+            new Promise((resolve) => {
+                timer = setTimeout(resolve, this.options.shutdownMs ?? SHUTDOWN_MS);
+            }),
+        ]);
+        clearTimeout(timer);
+        for (const { handle } of slots)
+            handle.kill();
     }
     /** Hands queued items to ready workers, then starts workers for what is left */
     pump() {
@@ -155,6 +187,7 @@ export class WorkerPool {
                     type: "test-start",
                     item: busy.entry.item,
                     worker: slot.index,
+                    ...(message.retry && { retry: message.retry }),
                 });
             case "test-end":
                 if (busy?.id !== message.id)

package/dist/runner/worker-protocol.d.ts

@@ -20,6 +20,8 @@ export interface WorkerInit {
     timeout?: number;
     expectTimeout?: number;
     headless?: boolean;
+    /** How many times a failing test is run again in this worker. @default 0 */
+    retries?: number;
     /** Where the run's report goes when it is not the default; screenshots are kept beside it */
     reportPath?: string;
     /** Spec files to import, absolute */
@@ -48,6 +50,7 @@ export type FromWorker = {
 } | {
     type: "test-start";
     id: number;
+    retry?: number;
 } | {
     type: "test-end";
     id: number;

package/dist/runner/worker-session.js

@@ -1,7 +1,10 @@
 import { workerProfileDir } from "../browser/launch-flags.js";
+import logger from "../logger/index.js";
 import { withProject } from "./project.js";
 import TestReporter from "./reporter.js";
-import { runOneTest } from "./run-test.js";
+import { AllHooks } from "./all-hooks.js";
+import { allHookFixtures, runOneTest } from "./run-test.js";
+import { runWithRetries } from "./retry.js";
 import { loadSpecs } from "./select.js";
 /**
  * Opens the project for a worker and keeps it open until `close`: `withProject` holds the config, environment
@@ -26,6 +29,11 @@ export function openSession(init) {
             ...(headless !== undefined && { headless }),
         }, async (run) => {
             const loaded = await loadSpecs(init.files);
+            // The `beforeAll` hooks this worker ran: their `afterAll` hooks run when it closes
+            const allHooks = new AllHooks({
+                timeout: run.settings.timeout,
+                fixtures: (browser) => allHookFixtures(run, browser, workerIndex),
+            });
             resolve({
                 tests: loaded.all,
                 async runTest(test, browser, signal, onEvent) {
@@ -35,20 +43,25 @@ export function openSession(init) {
                         output: false,
                         onEvent,
                     });
-                    await runOneTest(test, browser, run, {
+                    const options = {
                         ...(headless !== undefined && { headless }),
                         ...(init.reportPath !== undefined && {
                             reportPath: init.reportPath,
                         }),
                         port,
                         signal,
-                    }, loaded.srcDir, reporter, {
+                    };
+                    await runWithRetries(test, browser, init.retries ?? 0, signal, (sink, attempt) => runOneTest(test, browser, run, options, loaded.srcDir, sink, {
                         port,
                         profileDir: workerProfileDir(browser, workerIndex),
                         workerIndex,
-                    });
+                        allHooks,
+                        attempt,
+                    }), reporter);
                 },
                 async close() {
+                    for (const error of await allHooks.closeAll())
+                        logger.error("%s", error.message);
                     release();
                     await finished;
                 },

package/dist/runner/worker.js

@@ -43,6 +43,9 @@ export function handleMessages(channel, { open, exit }) {
         current = new AbortController();
         try {
             await session.runTest(test, item.browser, current.signal, (event) => {
+                // The first start was sent above; a test run again starts again, saying which attempt
+                if (event.type === "test-start" && event.retry)
+                    channel.send({ type: "test-start", id, retry: event.retry });
                 if (event.type !== "test-end")
                     return;
                 ended = true;

package/dist/steps/model.d.ts

@@ -1,3 +1,4 @@
+import type { Modifier } from "../types/test.js";
 /** One call of a locator expression, as written in a spec: `page.getByRole("button", { name: "Save" })` */
 export type LocatorCall = {
     method: "locator";
@@ -155,4 +156,6 @@ export interface ParsedTest {
         at: number;
         shape: "list" | "empty" | "none";
     };
+    /** `skip` or `fixme` (not run), or `only` (focused), as the runner applies it: a modifier on a `describe` counts */
+    modifier?: Modifier;
 }

package/dist/steps/parse.js

@@ -44,16 +44,40 @@ function callbackBody(node) {
         ? node.body
         : undefined;
 }
+/** `test(...)` or `describe(...)` with its modifier: `test.skip(...)` gives `{ name: "test", modifier: "skip" }` */
+function calleeOf(expression) {
+    if (ts.isIdentifier(expression))
+        return { name: expression.text };
+    if (ts.isPropertyAccessExpression(expression) &&
+        ts.isIdentifier(expression.expression) &&
+        ["skip", "fixme", "only"].includes(expression.name.text))
+        return {
+            name: expression.expression.text,
+            modifier: expression.name.text,
+        };
+    return undefined;
+}
+/** What the runner applies to a test (see `modifierOf` in the runner): the innermost `skip` or `fixme`, else `only` */
+function effectiveModifier(own, describes) {
+    const chain = [own, ...[...describes].reverse()];
+    return (chain.find((m) => m === "skip" || m === "fixme") ??
+        (chain.includes("only") ? "only" : undefined));
+}
 /** The tests in a spec file, with their steps and where each statement sits */
 export function parseSpec(source) {
     const sourceFile = ts.createSourceFile("spec.ts", source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
     const tests = [];
-    const visit = (node, titles) => {
-        if (ts.isCallExpression(node) && ts.isIdentifier(node.expression)) {
+    const visit = (node, titles, describes) => {
+        const callee = ts.isCallExpression(node)
+            ? calleeOf(node.expression)
+            : undefined;
+        if (ts.isCallExpression(node) && callee) {
             const title = stringOf(node.arguments[0]);
             const body = callbackBody(node.arguments[1]);
-            if (title !== undefined && body && node.expression.text === "test") {
+            if (title !== undefined && body && callee.name === "test") {
+                // `test.skip(condition, reason)` has no title and no body, so it stays a step of the test it is in
                 const titlePath = [...titles, title];
+                const modifier = effectiveModifier(callee.modifier, describes);
                 tests.push({
                     titlePath,
                     name: titlePath.join(" > "),
@@ -61,16 +85,17 @@ export function parseSpec(source) {
                     bodyStart: body.getStart(sourceFile) + 1,
                     bodyEnd: body.end - 1,
                     fixtures: callbackFixtures(node.arguments[1], sourceFile),
+                    ...(modifier && { modifier }),
                 });
                 return;
             }
-            if (title !== undefined && body && node.expression.text === "describe") {
-                ts.forEachChild(body, (child) => visit(child, [...titles, title]));
+            if (title !== undefined && body && callee.name === "describe") {
+                ts.forEachChild(body, (child) => visit(child, [...titles, title], [...describes, callee.modifier]));
                 return;
             }
         }
-        ts.forEachChild(node, (child) => visit(child, titles));
+        ts.forEachChild(node, (child) => visit(child, titles, describes));
     };
-    visit(sourceFile, []);
+    visit(sourceFile, [], []);
     return tests;
 }

package/dist/types/config.d.ts

@@ -28,7 +28,8 @@ export interface SamuraiGroup {
   browsers?: SupportedBrowser[];
   /**
    * How many of this group's tests may run at the same time. Only `1` has an effect: the group's tests run one
-   * after the other, even when the run uses more workers (for tests that share an account or data).
+   * after the other, even when the run uses more workers (for tests that share an account or data). A run of
+   * nothing but such a group uses one worker.
    */
   workers?: number;
 }
@@ -63,6 +64,19 @@ export interface SamuraiTestConfig {
    * @default half the CPUs
    */
   workers?: number | string;
+  /**
+   * How many times a failing test is run again, each time with a fresh browser. A test that passes on a retry is
+   * reported as flaky and does not fail the run; a run that is aborted is never retried. `--retries` and
+   * `SAMURAI_RETRIES` win over it.
+   * @default 0
+   */
+  retries?: number;
+  /**
+   * Refuse a run when a spec contains `test.only` or `describe.only`, so a forgotten focus cannot quietly shrink a
+   * suite. The run stops before any test starts and lists the tests.
+   * @default true when the `CI` environment variable is set, else false
+   */
+  forbidOnly?: boolean;
   /**
    * Browser binaries to launch instead of the builds SAMURAI downloads, relative to the project.
    * `SAMURAI_FIREFOX_PATH` / `SAMURAI_CHROME_PATH` win over these.

package/dist/types/test.d.ts

@@ -18,11 +18,42 @@ export interface TestFixtures {
 
 export type TestCase = TestCaseBase;
 
+/** `skip`: not run. `fixme`: not run, known to be broken. `only`: the only test that runs when any is focused */
+export type Modifier = "skip" | "fixme" | "only";
+
+/** Runs before or after each test of a suite, in the test's browser, with the test's fixtures */
+export type EachHook = (fixtures: TestFixtures) => Promise<void> | void;
+
+/** What `beforeAll` and `afterAll` receive: no page, as they run once for many tests */
+export type AllHookFixtures = Pick<
+  TestFixtures,
+  "env" | "secrets" | "browserName" | "workerIndex"
+>;
+
+/** Runs once per worker before the first test of a suite, or after the last one the worker ran */
+export type AllHook = (fixtures: AllHookFixtures) => Promise<void> | void;
+
+/** A spec file (the root) or a `describe` block: what a modifier or a hook applies to */
+export interface Suite {
+  /** The `describe` title; the root of a file has none */
+  title?: string;
+  parent?: Suite;
+  modifier?: Modifier;
+  beforeEach?: EachHook[];
+  afterEach?: EachHook[];
+  beforeAll?: AllHook[];
+  afterAll?: AllHook[];
+}
+
 export interface RegisteredTestCase extends TestCaseBase {
   /** Path of the spec file that registered the test */
   file: string;
   /** describe titles, then the test title */
   titlePath: string[];
+  /** The `describe` block (or the file) the test is written in */
+  suite: Suite;
+  /** What applies to the test: its own modifier, or the one of a suite around it */
+  modifier?: Modifier;
 }
 
 export interface TestCaseBase {
@@ -36,9 +67,10 @@ export interface PartialTestResult extends TestResultBase {
   file: string;
 }
 
-export type TestResult = TestResultSuccess | TestResultFailed | TestResultBase;
+export type TestResult =
+  TestResultSuccess | TestResultFailed | TestResultSkipped | TestResultBase;
 
-interface TestResultSuccess extends TestResultIdentity {
+interface TestResultSuccess extends TestResultIdentity, TestRetries {
   status: "success";
   duration: number;
   logs?: TestLogEntry[];
@@ -49,7 +81,7 @@ interface TestResultSuccess extends TestResultIdentity {
 }
 
 /** The error sits flat on the entry, as in the report: `message`, `type`, `stack`, and for assertions `expected` and `actual` */
-interface TestResultFailed extends TestResultIdentity, TestError {
+interface TestResultFailed extends TestResultIdentity, TestError, TestRetries {
   status: "failed";
   duration: number;
   logs?: TestLogEntry[];
@@ -58,6 +90,25 @@ interface TestResultFailed extends TestResultIdentity, TestError {
   screenshots?: { name: string; path: string }[];
 }
 
+/** What a result says about the retries of its test; absent when it passed or failed on the first attempt */
+export interface TestRetries {
+  /** The attempt this result is from, counting from 1: `retry: 1` is the first run again */
+  retry?: number;
+  /** The test passed after at least one failed attempt */
+  flaky?: true;
+  /** The errors of the failed attempts before this one, in order */
+  previousAttempts?: TestError[];
+}
+
+/** A test that was not run: marked `skip` or `fixme`, or skipped from inside with `test.skip(condition)` */
+interface TestResultSkipped extends TestResultIdentity {
+  status: "skipped";
+  duration: 0;
+  /** `"fixme"` for a test known to be broken, `"skip"` for the rest */
+  annotation: "skip" | "fixme";
+  reason?: string;
+}
+
 /** One line of the browser's log, as stored in the report */
 export interface TestLogEntry {
   level: "debug" | "info" | "warn" | "error";
@@ -100,6 +151,7 @@ export interface TestError {
 export type TestSummary = TestSummarySuccess | TestSummaryFailed;
 
 interface TestSummarySuccess extends TestSummaryBase {
+  /** No test failed (skipped tests do not count as failed) */
   status: "success";
 }