@itmetsam/samurai-framework 0.15.0 → 0.15.2

package/dist/runner/worker-protocol.d.ts

@@ -20,6 +20,8 @@ export interface WorkerInit {
     timeout?: number;
     expectTimeout?: number;
     headless?: boolean;
+    /** How many times a failing test is run again in this worker. @default 0 */
+    retries?: number;
     /** Where the run's report goes when it is not the default; screenshots are kept beside it */
     reportPath?: string;
     /** Spec files to import, absolute */
@@ -48,6 +50,7 @@ export type FromWorker = {
 } | {
     type: "test-start";
     id: number;
+    retry?: number;
 } | {
     type: "test-end";
     id: number;

package/dist/runner/worker-session.js

@@ -4,6 +4,7 @@ import { withProject } from "./project.js";
 import TestReporter from "./reporter.js";
 import { AllHooks } from "./all-hooks.js";
 import { allHookFixtures, runOneTest } from "./run-test.js";
+import { runWithRetries } from "./retry.js";
 import { loadSpecs } from "./select.js";
 /**
  * Opens the project for a worker and keeps it open until `close`: `withProject` holds the config, environment
@@ -42,19 +43,21 @@ export function openSession(init) {
                         output: false,
                         onEvent,
                     });
-                    await runOneTest(test, browser, run, {
+                    const options = {
                         ...(headless !== undefined && { headless }),
                         ...(init.reportPath !== undefined && {
                             reportPath: init.reportPath,
                         }),
                         port,
                         signal,
-                    }, loaded.srcDir, reporter, {
+                    };
+                    await runWithRetries(test, browser, init.retries ?? 0, signal, (sink, attempt) => runOneTest(test, browser, run, options, loaded.srcDir, sink, {
                         port,
                         profileDir: workerProfileDir(browser, workerIndex),
                         workerIndex,
                         allHooks,
-                    });
+                        attempt,
+                    }), reporter);
                 },
                 async close() {
                     for (const error of await allHooks.closeAll())

package/dist/runner/worker.js

@@ -43,6 +43,9 @@ export function handleMessages(channel, { open, exit }) {
         current = new AbortController();
         try {
             await session.runTest(test, item.browser, current.signal, (event) => {
+                // The first start was sent above; a test run again starts again, saying which attempt
+                if (event.type === "test-start" && event.retry)
+                    channel.send({ type: "test-start", id, retry: event.retry });
                 if (event.type !== "test-end")
                     return;
                 ended = true;

package/dist/steps/model.d.ts

@@ -1,3 +1,4 @@
+import type { Modifier } from "../types/test.js";
 /** One call of a locator expression, as written in a spec: `page.getByRole("button", { name: "Save" })` */
 export type LocatorCall = {
     method: "locator";
@@ -155,4 +156,8 @@ export interface ParsedTest {
         at: number;
         shape: "list" | "empty" | "none";
     };
+    /** `skip` or `fixme` (not run), or `only` (focused), as the runner applies it: a modifier on a `describe` counts */
+    modifier?: Modifier;
+    /** The `{ reason }` written with the `skip` or `fixme` that applies, when it is a string literal */
+    reason?: string;
 }

package/dist/steps/parse.js

@@ -1,5 +1,5 @@
 import ts from "typescript";
-import { stringOf } from "./parse/literals.js";
+import { readObject, stringOf } from "./parse/literals.js";
 import { readStep } from "./parse/step.js";
 function parseStatement(source, statement, sourceFile) {
     const start = statement.getStart(sourceFile);
@@ -44,31 +44,65 @@ function callbackBody(node) {
         ? node.body
         : undefined;
 }
-/** `describe.skip`, `describe.fixme` and `describe.only`: tests written inside keep their place in the title */
-function isDescribeWithModifier(expression) {
-    return (ts.isPropertyAccessExpression(expression) &&
+/** `test(...)` or `describe(...)` with its modifier: `test.skip(...)` gives `{ name: "test", modifier: "skip" }` */
+function calleeOf(expression) {
+    if (ts.isIdentifier(expression))
+        return { name: expression.text };
+    if (ts.isPropertyAccessExpression(expression) &&
         ts.isIdentifier(expression.expression) &&
-        expression.expression.text === "describe" &&
-        ["skip", "fixme", "only"].includes(expression.name.text));
+        ["skip", "fixme", "only"].includes(expression.name.text))
+        return {
+            name: expression.expression.text,
+            modifier: expression.name.text,
+        };
+    return undefined;
+}
+/**
+ * The callback of `test(title, fn)` and `describe(title, fn)`, and for a skip or fixme also of
+ * `(title, { reason }, fn)` with the reason when it is a literal
+ */
+function callbackOf(node, modifier) {
+    const [, second, third] = node.arguments;
+    if ((modifier === "skip" || modifier === "fixme") &&
+        second &&
+        ts.isObjectLiteralExpression(second)) {
+        const reason = readObject(second, ["reason"])?.reason;
+        return {
+            callback: third,
+            ...(typeof reason === "string" && { reason }),
+        };
+    }
+    return { callback: second };
+}
+/** What the runner applies to a test (see `modifierOf` in the runner): the innermost `skip` or `fixme` with its reason, else `only` */
+function effectiveMark(own, describes) {
+    const chain = [own, ...[...describes].reverse()];
+    const skip = chain.find((m) => m.modifier === "skip" || m.modifier === "fixme");
+    if (skip)
+        return {
+            modifier: skip.modifier,
+            ...(skip.reason !== undefined && { reason: skip.reason }),
+        };
+    return chain.some((m) => m.modifier === "only") ? { modifier: "only" } : {};
 }
 /** The tests in a spec file, with their steps and where each statement sits */
 export function parseSpec(source) {
     const sourceFile = ts.createSourceFile("spec.ts", source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
     const tests = [];
-    const visit = (node, titles) => {
-        if (ts.isCallExpression(node) && isDescribeWithModifier(node.expression)) {
-            // `describe.skip("Old", () => ...)`: a describe block for the names of the tests inside it
-            const title = stringOf(node.arguments[0]);
-            const body = callbackBody(node.arguments[1]);
-            if (title !== undefined && body) {
-                ts.forEachChild(body, (child) => visit(child, [...titles, title]));
-                return;
-            }
-        }
-        if (ts.isCallExpression(node) && ts.isIdentifier(node.expression)) {
+    const visit = (node, titles, describes) => {
+        const callee = ts.isCallExpression(node)
+            ? calleeOf(node.expression)
+            : undefined;
+        if (ts.isCallExpression(node) && callee) {
             const title = stringOf(node.arguments[0]);
-            const body = callbackBody(node.arguments[1]);
-            if (title !== undefined && body && node.expression.text === "test") {
+            const { callback, reason } = callbackOf(node, callee.modifier);
+            const body = callbackBody(callback);
+            const mark = {
+                ...(callee.modifier && { modifier: callee.modifier }),
+                ...(reason !== undefined && { reason }),
+            };
+            if (title !== undefined && body && callee.name === "test") {
+                // `test.skip(condition, reason)` has no title and no body, so it stays a step of the test it is in
                 const titlePath = [...titles, title];
                 tests.push({
                     titlePath,
@@ -76,17 +110,18 @@ export function parseSpec(source) {
                     steps: body.statements.map((statement) => parseStatement(source, statement, sourceFile)),
                     bodyStart: body.getStart(sourceFile) + 1,
                     bodyEnd: body.end - 1,
-                    fixtures: callbackFixtures(node.arguments[1], sourceFile),
+                    fixtures: callbackFixtures(callback, sourceFile),
+                    ...effectiveMark(mark, describes),
                 });
                 return;
             }
-            if (title !== undefined && body && node.expression.text === "describe") {
-                ts.forEachChild(body, (child) => visit(child, [...titles, title]));
+            if (title !== undefined && body && callee.name === "describe") {
+                ts.forEachChild(body, (child) => visit(child, [...titles, title], [...describes, mark]));
                 return;
             }
         }
-        ts.forEachChild(node, (child) => visit(child, titles));
+        ts.forEachChild(node, (child) => visit(child, titles, describes));
     };
-    visit(sourceFile, []);
+    visit(sourceFile, [], []);
     return tests;
 }

package/dist/types/config.d.ts

@@ -64,6 +64,13 @@ export interface SamuraiTestConfig {
    * @default half the CPUs
    */
   workers?: number | string;
+  /**
+   * How many times a failing test is run again, each time with a fresh browser. A test that passes on a retry is
+   * reported as flaky and does not fail the run; a run that is aborted is never retried. `--retries` and
+   * `SAMURAI_RETRIES` win over it.
+   * @default 0
+   */
+  retries?: number;
   /**
    * Refuse a run when a spec contains `test.only` or `describe.only`, so a forgotten focus cannot quietly shrink a
    * suite. The run stops before any test starts and lists the tests.

package/dist/types/test.d.ts

@@ -21,6 +21,12 @@ export type TestCase = TestCaseBase;
 /** `skip`: not run. `fixme`: not run, known to be broken. `only`: the only test that runs when any is focused */
 export type Modifier = "skip" | "fixme" | "only";
 
+/** The optional second argument of `test.skip`, `test.fixme`, `describe.skip` and `describe.fixme` */
+export interface TestDetails {
+  /** Why the tests are not run; shown in the report and by `samurai list` */
+  reason?: string;
+}
+
 /** Runs before or after each test of a suite, in the test's browser, with the test's fixtures */
 export type EachHook = (fixtures: TestFixtures) => Promise<void> | void;
 
@@ -39,6 +45,8 @@ export interface Suite {
   title?: string;
   parent?: Suite;
   modifier?: Modifier;
+  /** The `{ reason }` of a `describe.skip` or `describe.fixme` */
+  reason?: string;
   beforeEach?: EachHook[];
   afterEach?: EachHook[];
   beforeAll?: AllHook[];
@@ -54,6 +62,8 @@ export interface RegisteredTestCase extends TestCaseBase {
   suite: Suite;
   /** What applies to the test: its own modifier, or the one of a suite around it */
   modifier?: Modifier;
+  /** The reason given with the `skip` or `fixme` that applies */
+  reason?: string;
 }
 
 export interface TestCaseBase {
@@ -70,7 +80,7 @@ export interface PartialTestResult extends TestResultBase {
 export type TestResult =
   TestResultSuccess | TestResultFailed | TestResultSkipped | TestResultBase;
 
-interface TestResultSuccess extends TestResultIdentity {
+interface TestResultSuccess extends TestResultIdentity, TestRetries {
   status: "success";
   duration: number;
   logs?: TestLogEntry[];
@@ -81,7 +91,7 @@ interface TestResultSuccess extends TestResultIdentity {
 }
 
 /** The error sits flat on the entry, as in the report: `message`, `type`, `stack`, and for assertions `expected` and `actual` */
-interface TestResultFailed extends TestResultIdentity, TestError {
+interface TestResultFailed extends TestResultIdentity, TestError, TestRetries {
   status: "failed";
   duration: number;
   logs?: TestLogEntry[];
@@ -90,6 +100,16 @@ interface TestResultFailed extends TestResultIdentity, TestError {
   screenshots?: { name: string; path: string }[];
 }
 
+/** What a result says about the retries of its test; absent when it passed or failed on the first attempt */
+export interface TestRetries {
+  /** The attempt this result is from, counting from 1: `retry: 1` is the first run again */
+  retry?: number;
+  /** The test passed after at least one failed attempt */
+  flaky?: true;
+  /** The errors of the failed attempts before this one, in order */
+  previousAttempts?: TestError[];
+}
+
 /** A test that was not run: marked `skip` or `fixme`, or skipped from inside with `test.skip(condition)` */
 interface TestResultSkipped extends TestResultIdentity {
   status: "skipped";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itmetsam/samurai-framework",
-  "version": "0.15.0",
+  "version": "0.15.2",
   "description": "Browser test framework over WebDriver BiDi: auto-waiting locators, environments and secrets, a recorder, and a command line.",
   "main": "dist/api.js",
   "exports": {
@@ -53,6 +53,7 @@
   "scripts": {
     "dev": "tsx src/index.ts",
     "build": "rm -rf dist && tsc -p tsconfig.build.json && node scripts/copy-types.mjs",
+    "prepack": "npm run build",
     "test": "SAMURAI_LOG_LEVEL=${SAMURAI_LOG_LEVEL:-warn} tsx --import ./scripts/test-tmp.mjs --test --test-timeout=10000 \"src/**/*.test.ts\"",
     "test:browser": "SAMURAI_LOG_LEVEL=${SAMURAI_LOG_LEVEL:-warn} tsx --import ./scripts/test-tmp.mjs --test --test-concurrency=1 --test-timeout=60000 \"src/**/*.browser-test.ts\"",
     "test:browser:chrome": "SAMURAI_TEST_BROWSER=chrome npm run test:browser",