@itmetsam/samurai-framework 0.12.0 → 0.15.0

package/dist/runner/test-runner.js

@@ -3,12 +3,16 @@ import path from "path";
 import { ensureWarm } from "../browser/warm-up.js";
 import { projectBrowser } from "../browsers/resolve.js";
 import { loadConfig, projectDir, readConfig } from "../config/config.js";
+import { forbidsOnly } from "../config/forbid-only.js";
 import logger from "../logger/index.js";
+import { AllHooks } from "./all-hooks.js";
+import { assertNoOnly, narrowToOnly } from "./focus.js";
 import { DEFAULT_SRC_DIR, DEFAULT_TEST_MATCH } from "./groups.js";
 import { DEFAULT_PORT } from "./ports.js";
 import TestReporter from "./reporter.js";
-import { planItems, poolMode, runPooled, serialKeyOf, } from "./pooled-run.js";
-import { runOneTest } from "./run-test.js";
+import { concurrency, planItems, poolMode, runPooled, serialKeyOf, } from "./pooled-run.js";
+import { allHookFixtures, runOneTest } from "./run-test.js";
+import { staticSkip } from "./skip.js";
 import { browsersOfRun, loadSpecs, selectTests } from "./select.js";
 import { resolveWorkers } from "./workers-option.js";
 import { WorkerPool } from "./worker-pool.js";
@@ -54,7 +58,10 @@ export default class TestRunner {
         const { reportPath, onEvent, signal } = this.options;
         // The specs are imported once: their top-level code runs once per run
         const loaded = await loadSpecs(this.testFiles);
-        const selected = selectTests(loaded, this.options);
+        const config = await loadConfig();
+        // A focused test the config refuses stops the run before anything starts; else only the focused ones run
+        assertNoOnly(loaded.all, forbidsOnly(config), loaded.srcDir);
+        const selected = narrowToOnly(selectTests(loaded, this.options));
         const browsers = await browsersOfRun(loaded.groups, this.options);
         const reporter = new TestReporter({
             environment: this.run.settings.environment,
@@ -63,31 +70,33 @@ export default class TestRunner {
             ...(onEvent && { onEvent }),
         });
         // How the run goes is settled first: a bad `workers` is an error before any browser starts
-        const config = await loadConfig();
         const requested = resolveWorkers({
             flag: this.options.workers,
             env: process.env.SAMURAI_WORKERS,
             config: config.workers,
         });
         const plan = planItems(selected, browsers, serialKeyOf(loaded.groups, config.groups, loaded.srcDir));
+        // A pool larger than what can overlap would only hold idle workers: a group that runs one at a time needs one
+        const lanes = concurrency(plan.items);
         const mode = poolMode({
             workers: requested,
-            items: plan.items.length,
+            items: lanes,
             inlineConfig: this.project.inlineConfig ?? false,
         });
         if (mode === "inline-config" && this.workersAsked(config.workers))
             logger.warn("workers needs samurai.config.ts in the project folder; running on one worker");
         // A missing browser is one clear error before the run, not a failure per test, and before any browser has run a
         // test; a fresh build's slow first start happens here too, outside any test's timeout
-        if (selected.length > 0) {
+        // Skipped tests need no browser, so a run of only those needs none installed
+        if (selected.some((test) => !staticSkip(test))) {
             for (const browser of browsers)
                 await projectBrowser(browser);
             for (const browser of browsers)
                 await ensureWarm(browser, { port: this.options.port ?? DEFAULT_PORT }, signal);
         }
         if (mode === "pool") {
-            const workers = Math.min(requested, plan.items.length);
-            reporter.onStart(plan.items.length, workers);
+            const workers = Math.min(requested, lanes);
+            reporter.onStart(plan.planned.length, workers);
             const pool = this.createPool(workers);
             try {
                 await runPooled(pool, plan.planned, plan.items, reporter, signal);
@@ -98,19 +107,35 @@ export default class TestRunner {
             return reporter.onEnd();
         }
         reporter.onStart(selected.length * browsers.length);
-        for (const browser of browsers) {
-            for (const test of selected) {
-                if (signal?.aborted) {
-                    reporter.onTestStart(test, browser);
-                    reporter.onTestEnd(test, browser, {
-                        message: "Run aborted",
-                        type: "error",
-                    });
-                    continue;
+        // The `afterAll` hooks run when the last test is done, before the run ends
+        const allHooks = new AllHooks({
+            timeout: this.run.settings.timeout,
+            fixtures: (browser) => allHookFixtures(this.run, browser),
+        });
+        try {
+            for (const browser of browsers) {
+                for (const test of selected) {
+                    const annotation = staticSkip(test);
+                    if (annotation) {
+                        reporter.onTestSkipped(test, browser, { annotation });
+                        continue;
+                    }
+                    if (signal?.aborted) {
+                        reporter.onTestStart(test, browser);
+                        reporter.onTestEnd(test, browser, {
+                            message: "Run aborted",
+                            type: "error",
+                        });
+                        continue;
+                    }
+                    await runOneTest(test, browser, this.run, this.options, loaded.srcDir, reporter, { allHooks });
                 }
-                await runOneTest(test, browser, this.run, this.options, loaded.srcDir, reporter);
             }
         }
+        finally {
+            for (const error of await allHooks.closeAll())
+                logger.error("%s", error.message);
+        }
         return reporter.onEnd();
     }
     /** Whether anything but the default says how many workers to use */

package/dist/runner/worker-pool.d.ts

@@ -17,6 +17,8 @@ export interface PoolOptions {
     factory: WorkerFactory;
     /** The abort grace period in ms; 2000 by default */
     graceMs?: number;
+    /** How long `dispose` waits for idle workers to leave on their own, in ms: their `afterAll` hooks run first. 60000 by default */
+    shutdownMs?: number;
 }
 export interface RunHooks {
     onEvent?: (event: PoolEvent) => void;
@@ -40,8 +42,13 @@ export declare class WorkerPool {
     constructor(options: PoolOptions);
     /** Results in item order; may be called several times at once (flows): all calls share the cap */
     run(items: WorkItem[], hooks?: RunHooks): Promise<TestResult[]>;
-    /** Tells every worker to exit and kills it; safe to call twice */
+    /**
+     * Ends the pool; safe to call twice. A worker that has nothing to do is told to exit and given time to leave
+     * (it runs its `afterAll` hooks first); one that is busy is killed, as nobody waits for its test.
+     */
     dispose(): Promise<void>;
+    /** Asks the idle workers to exit and waits for them (up to `shutdownMs`), kills the busy ones and the stragglers */
+    private sendHome;
     /** Hands queued items to ready workers, then starts workers for what is left */
     private pump;
     private busyKeys;

package/dist/runner/worker-pool.js

@@ -1,5 +1,6 @@
 /** How long a busy worker may take to wind down after an abort before it is killed */
 const GRACE_MS = 2000;
+const SHUTDOWN_MS = 60000;
 /** Workers that die before they are ready this many times in a row fail the run, instead of being replaced for ever */
 const STARTUP_ATTEMPTS = 3;
 /**
@@ -49,14 +50,45 @@ export class WorkerPool {
             this.pump();
         });
     }
-    /** Tells every worker to exit and kills it; safe to call twice */
+    /**
+     * Ends the pool; safe to call twice. A worker that has nothing to do is told to exit and given time to leave
+     * (it runs its `afterAll` hooks first); one that is busy is killed, as nobody waits for its test.
+     */
     async dispose() {
         if (this.disposed)
             return;
         this.disposed = true;
         this.queue.length = 0;
-        this.stopWorkers(true);
+        const leaving = this.sendHome();
         this.rejectAll(new Error("WorkerPool.dispose(): the pool was disposed during a run"));
+        await leaving;
+    }
+    /** Asks the idle workers to exit and waits for them (up to `shutdownMs`), kills the busy ones and the stragglers */
+    async sendHome() {
+        for (const timer of this.timers)
+            clearTimeout(timer);
+        this.timers.clear();
+        const slots = [...this.slots.values()];
+        this.slots.clear();
+        const gone = [];
+        for (const { handle, busy } of slots) {
+            if (busy) {
+                handle.kill();
+                continue;
+            }
+            gone.push(new Promise((resolve) => handle.onExit(() => resolve())));
+            handle.send({ type: "exit" });
+        }
+        let timer;
+        await Promise.race([
+            Promise.all(gone),
+            new Promise((resolve) => {
+                timer = setTimeout(resolve, this.options.shutdownMs ?? SHUTDOWN_MS);
+            }),
+        ]);
+        clearTimeout(timer);
+        for (const { handle } of slots)
+            handle.kill();
     }
     /** Hands queued items to ready workers, then starts workers for what is left */
     pump() {

package/dist/runner/worker-session.js

@@ -1,7 +1,9 @@
 import { workerProfileDir } from "../browser/launch-flags.js";
+import logger from "../logger/index.js";
 import { withProject } from "./project.js";
 import TestReporter from "./reporter.js";
-import { runOneTest } from "./run-test.js";
+import { AllHooks } from "./all-hooks.js";
+import { allHookFixtures, runOneTest } from "./run-test.js";
 import { loadSpecs } from "./select.js";
 /**
  * Opens the project for a worker and keeps it open until `close`: `withProject` holds the config, environment
@@ -26,6 +28,11 @@ export function openSession(init) {
             ...(headless !== undefined && { headless }),
         }, async (run) => {
             const loaded = await loadSpecs(init.files);
+            // The `beforeAll` hooks this worker ran: their `afterAll` hooks run when it closes
+            const allHooks = new AllHooks({
+                timeout: run.settings.timeout,
+                fixtures: (browser) => allHookFixtures(run, browser, workerIndex),
+            });
             resolve({
                 tests: loaded.all,
                 async runTest(test, browser, signal, onEvent) {
@@ -46,9 +53,12 @@ export function openSession(init) {
                         port,
                         profileDir: workerProfileDir(browser, workerIndex),
                         workerIndex,
+                        allHooks,
                     });
                 },
                 async close() {
+                    for (const error of await allHooks.closeAll())
+                        logger.error("%s", error.message);
                     release();
                     await finished;
                 },

package/dist/steps/parse.js

@@ -44,11 +44,27 @@ function callbackBody(node) {
         ? node.body
         : undefined;
 }
+/** `describe.skip`, `describe.fixme` and `describe.only`: tests written inside keep their place in the title */
+function isDescribeWithModifier(expression) {
+    return (ts.isPropertyAccessExpression(expression) &&
+        ts.isIdentifier(expression.expression) &&
+        expression.expression.text === "describe" &&
+        ["skip", "fixme", "only"].includes(expression.name.text));
+}
 /** The tests in a spec file, with their steps and where each statement sits */
 export function parseSpec(source) {
     const sourceFile = ts.createSourceFile("spec.ts", source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
     const tests = [];
     const visit = (node, titles) => {
+        if (ts.isCallExpression(node) && isDescribeWithModifier(node.expression)) {
+            // `describe.skip("Old", () => ...)`: a describe block for the names of the tests inside it
+            const title = stringOf(node.arguments[0]);
+            const body = callbackBody(node.arguments[1]);
+            if (title !== undefined && body) {
+                ts.forEachChild(body, (child) => visit(child, [...titles, title]));
+                return;
+            }
+        }
         if (ts.isCallExpression(node) && ts.isIdentifier(node.expression)) {
             const title = stringOf(node.arguments[0]);
             const body = callbackBody(node.arguments[1]);

package/dist/types/config.d.ts

@@ -28,7 +28,8 @@ export interface SamuraiGroup {
   browsers?: SupportedBrowser[];
   /**
    * How many of this group's tests may run at the same time. Only `1` has an effect: the group's tests run one
-   * after the other, even when the run uses more workers (for tests that share an account or data).
+   * after the other, even when the run uses more workers (for tests that share an account or data). A run of
+   * nothing but such a group uses one worker.
    */
   workers?: number;
 }
@@ -63,6 +64,12 @@ export interface SamuraiTestConfig {
    * @default half the CPUs
    */
   workers?: number | string;
+  /**
+   * Refuse a run when a spec contains `test.only` or `describe.only`, so a forgotten focus cannot quietly shrink a
+   * suite. The run stops before any test starts and lists the tests.
+   * @default true when the `CI` environment variable is set, else false
+   */
+  forbidOnly?: boolean;
   /**
    * Browser binaries to launch instead of the builds SAMURAI downloads, relative to the project.
    * `SAMURAI_FIREFOX_PATH` / `SAMURAI_CHROME_PATH` win over these.

package/dist/types/test.d.ts

@@ -18,11 +18,42 @@ export interface TestFixtures {
 
 export type TestCase = TestCaseBase;
 
+/** `skip`: not run. `fixme`: not run, known to be broken. `only`: the only test that runs when any is focused */
+export type Modifier = "skip" | "fixme" | "only";
+
+/** Runs before or after each test of a suite, in the test's browser, with the test's fixtures */
+export type EachHook = (fixtures: TestFixtures) => Promise<void> | void;
+
+/** What `beforeAll` and `afterAll` receive: no page, as they run once for many tests */
+export type AllHookFixtures = Pick<
+  TestFixtures,
+  "env" | "secrets" | "browserName" | "workerIndex"
+>;
+
+/** Runs once per worker before the first test of a suite, or after the last one the worker ran */
+export type AllHook = (fixtures: AllHookFixtures) => Promise<void> | void;
+
+/** A spec file (the root) or a `describe` block: what a modifier or a hook applies to */
+export interface Suite {
+  /** The `describe` title; the root of a file has none */
+  title?: string;
+  parent?: Suite;
+  modifier?: Modifier;
+  beforeEach?: EachHook[];
+  afterEach?: EachHook[];
+  beforeAll?: AllHook[];
+  afterAll?: AllHook[];
+}
+
 export interface RegisteredTestCase extends TestCaseBase {
   /** Path of the spec file that registered the test */
   file: string;
   /** describe titles, then the test title */
   titlePath: string[];
+  /** The `describe` block (or the file) the test is written in */
+  suite: Suite;
+  /** What applies to the test: its own modifier, or the one of a suite around it */
+  modifier?: Modifier;
 }
 
 export interface TestCaseBase {
@@ -36,7 +67,8 @@ export interface PartialTestResult extends TestResultBase {
   file: string;
 }
 
-export type TestResult = TestResultSuccess | TestResultFailed | TestResultBase;
+export type TestResult =
+  TestResultSuccess | TestResultFailed | TestResultSkipped | TestResultBase;
 
 interface TestResultSuccess extends TestResultIdentity {
   status: "success";
@@ -58,6 +90,15 @@ interface TestResultFailed extends TestResultIdentity, TestError {
   screenshots?: { name: string; path: string }[];
 }
 
+/** A test that was not run: marked `skip` or `fixme`, or skipped from inside with `test.skip(condition)` */
+interface TestResultSkipped extends TestResultIdentity {
+  status: "skipped";
+  duration: 0;
+  /** `"fixme"` for a test known to be broken, `"skip"` for the rest */
+  annotation: "skip" | "fixme";
+  reason?: string;
+}
+
 /** One line of the browser's log, as stored in the report */
 export interface TestLogEntry {
   level: "debug" | "info" | "warn" | "error";
@@ -100,6 +141,7 @@ export interface TestError {
 export type TestSummary = TestSummarySuccess | TestSummaryFailed;
 
 interface TestSummarySuccess extends TestSummaryBase {
+  /** No test failed (skipped tests do not count as failed) */
   status: "success";
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itmetsam/samurai-framework",
-  "version": "0.12.0",
+  "version": "0.15.0",
   "description": "Browser test framework over WebDriver BiDi: auto-waiting locators, environments and secrets, a recorder, and a command line.",
   "main": "dist/api.js",
   "exports": {
@@ -53,10 +53,10 @@
   "scripts": {
     "dev": "tsx src/index.ts",
     "build": "rm -rf dist && tsc -p tsconfig.build.json && node scripts/copy-types.mjs",
-    "test": "tsx --test --test-timeout=10000 \"src/**/*.test.ts\"",
-    "test:browser": "tsx --test --test-concurrency=1 --test-timeout=60000 \"src/**/*.browser-test.ts\"",
+    "test": "SAMURAI_LOG_LEVEL=${SAMURAI_LOG_LEVEL:-warn} tsx --import ./scripts/test-tmp.mjs --test --test-timeout=10000 \"src/**/*.test.ts\"",
+    "test:browser": "SAMURAI_LOG_LEVEL=${SAMURAI_LOG_LEVEL:-warn} tsx --import ./scripts/test-tmp.mjs --test --test-concurrency=1 --test-timeout=60000 \"src/**/*.browser-test.ts\"",
     "test:browser:chrome": "SAMURAI_TEST_BROWSER=chrome npm run test:browser",
-    "test:cli": "tsx --test --test-concurrency=1 \"src/**/*.cli-test.ts\""
+    "test:cli": "tsx --import ./scripts/test-tmp.mjs --test --test-concurrency=1 \"src/**/*.cli-test.ts\""
   },
   "keywords": [
     "testing",