@itmetsam/samurai-framework 0.8.0 → 0.10.0

package/dist/runner/test-runner.js

@@ -4,6 +4,8 @@ import path from "path";
 import { pathToFileURL } from "url";
 import { Browser } from "../browser/browser.js";
 import { ensureWarm } from "../browser/warm-up.js";
+import { projectBrowser } from "../browsers/resolve.js";
+import { browsersFor } from "./browsers.js";
 import { loadConfig, projectDir, readConfig } from "../config/config.js";
 import logger from "../logger/index.js";
 import { assertModuleProject } from "./module-type.js";
@@ -108,34 +110,59 @@ export default class TestRunner {
         }
         return all.filter((test) => inGroup(test) && isSelected(test.name, this.options));
     }
-    /** Runs the selected tests and returns the summary that was also written to the report */
+    /**
+     * The browsers the run uses: the options' (`--browser`), else the selected group's, else the project's, else
+     * Firefox. A group is only looked up when there is something to look up
+     */
+    async browsersOfRun() {
+        const { browsers: flag, group } = this.options;
+        const config = await loadConfig();
+        let ofGroup;
+        if (group !== undefined && !(flag && flag.length > 0)) {
+            const { groups } = await this.resolveAll(await this.registerAll());
+            ofGroup = findGroup(groups, group).browsers;
+        }
+        return browsersFor({ flag, group: ofGroup, project: config.browsers });
+    }
+    /** Runs the selected tests in every browser of the run, one browser after the other, and returns the summary that was also written to the report */
     async start() {
         const { reportPath, onEvent, signal } = this.options;
+        const selected = await this.register();
+        const browsers = await this.browsersOfRun();
         const reporter = new TestReporter({
             environment: this.run.settings.environment,
+            browsers,
             ...(reportPath !== undefined && { output: reportPath }),
             ...(onEvent && { onEvent }),
         });
-        const selected = await this.register();
         this.srcDir = realDirs((await loadConfig()).srcDir).srcDir;
-        // A missing browser is one clear error before the run, not a failure per test; a fresh build's slow
-        // first start happens here, outside any test's timeout
-        if (selected.length > 0)
-            await ensureWarm((await readConfig("browser")) ?? "firefox", { port: this.options.port ?? 9223 }, signal);
-        reporter.onStart(selected.length);
-        for (const test of selected) {
-            if (signal?.aborted) {
-                reporter.onTestStart(test);
-                reporter.onTestEnd(test, { message: "Run aborted", type: "error" });
-                continue;
+        // A missing browser is one clear error before the run, not a failure per test, and before any browser has run a
+        // test; a fresh build's slow first start happens here too, outside any test's timeout
+        if (selected.length > 0) {
+            for (const browser of browsers)
+                await projectBrowser(browser);
+            for (const browser of browsers)
+                await ensureWarm(browser, { port: this.options.port ?? 9223 }, signal);
+        }
+        reporter.onStart(selected.length * browsers.length);
+        for (const browser of browsers) {
+            for (const test of selected) {
+                if (signal?.aborted) {
+                    reporter.onTestStart(test, browser);
+                    reporter.onTestEnd(test, browser, {
+                        message: "Run aborted",
+                        type: "error",
+                    });
+                    continue;
+                }
+                await this.executeTestCase(test, browser, reporter);
             }
-            await this.executeTestCase(test, reporter);
         }
         return reporter.onEnd();
     }
-    async executeTestCase(test, reporter) {
-        logger.verbose("Starting test: %s", test.name);
-        reporter.onTestStart(test);
+    async executeTestCase(test, selectedBrowser, reporter) {
+        logger.verbose("Starting test: %s in %s", test.name, selectedBrowser);
+        reporter.onTestStart(test, selectedBrowser);
         // A test ends once, whichever of finishing, failing, timing out, an abort or a closed browser comes first
         let ended = false;
         let evidence;
@@ -143,7 +170,7 @@ export default class TestRunner {
             if (ended)
                 return;
             ended = true;
-            reporter.onTestEnd(test, error, {
+            reporter.onTestEnd(test, selectedBrowser, error, {
                 ...logs,
                 screenshots: evidence?.screenshots,
             });
@@ -151,7 +178,6 @@ export default class TestRunner {
         let browserClosed = false;
         /** Set once the runner itself closes the browser */
         let closing = false;
-        const selectedBrowser = (await readConfig("browser")) ?? "firefox";
         const { settings, secrets } = this.run;
         const timeout = settings.timeout;
         const { reportPath } = this.options;
@@ -205,11 +231,12 @@ export default class TestRunner {
                         resolve();
                     });
                     page = launched.page;
-                    evidence = new Evidence(resultDir, testDirOf(test, this.srcDir));
+                    evidence = new Evidence(resultDir, `${selectedBrowser}/${testDirOf(test, this.srcDir)}`);
                     launched.page.setEvidence(evidence);
                     const fixtures = {
                         page: launched.page,
                         browser: launched.browser,
+                        browserName: selectedBrowser,
                         env: createEnvFixture(settings.environment, settings.variables),
                         secrets: createSecretsFixture(settings.environment, secrets),
                     };

package/dist/script/preload.js

@@ -16,11 +16,24 @@ export async function addPreload(connector, options) {
             arguments: options.arguments.map((arg) => toLocalValue(arg)),
         }),
     });
+    let stopRerun;
+    if (needsRerun(connector, options)) {
+        try {
+            stopRerun = await rerunInNewRealms(connector, options);
+        }
+        catch (err) {
+            await connector
+                .send("script.removePreloadScript", { script })
+                .catch(() => undefined);
+            throw err;
+        }
+    }
     let removed = false;
     const dispose = async () => {
         if (removed)
             return;
         removed = true;
+        await stopRerun?.();
         try {
             await connector.send("script.removePreloadScript", { script });
         }
@@ -42,6 +55,58 @@ export async function addPreload(connector, options) {
     }
     return { id: script, dispose };
 }
+/**
+ * Chrome drops the messages of a channel that a preload script in a sandbox was given, while a channel handed to a
+ * call into the same sandbox works. So there the script is also run, with its channels, in every new sandbox realm.
+ */
+function needsRerun(connector, options) {
+    return (connector.browserName === "chrome" &&
+        options.sandbox !== undefined &&
+        (options.arguments?.length ?? 0) > 0);
+}
+async function rerunInNewRealms(connector, options) {
+    const { contexts, sandbox, source } = options;
+    const inScope = async (context) => {
+        if (!contexts)
+            return true;
+        if (contexts.includes(context))
+            return true;
+        const trees = await Promise.all(contexts.map((root) => connector.send("browsingContext.getTree", { root })));
+        return trees.some((tree) => tree.contexts.flatMap(flatten).includes(context));
+    };
+    const onRealm = (realm) => {
+        if (realm.type !== "window" || realm.sandbox !== sandbox || !realm.context)
+            return;
+        const context = realm.context;
+        void (async () => {
+            try {
+                if (!(await inScope(context)))
+                    return;
+                await callFunction(connector, context, source, options.arguments ?? [], {
+                    awaitPromise: false,
+                    ...(sandbox !== undefined && { sandbox }),
+                });
+            }
+            catch (err) {
+                // The document went away again, or the script threw: the registration covers what it can
+                logger.debug("Could not run a preload script in a new realm", { err });
+            }
+        })();
+    };
+    connector.onEvent("script.realmCreated", onRealm);
+    let subscription;
+    try {
+        subscription = await connector.subscribe(["script.realmCreated"]);
+    }
+    catch (err) {
+        connector.offEvent("script.realmCreated", onRealm);
+        throw err;
+    }
+    return async () => {
+        connector.offEvent("script.realmCreated", onRealm);
+        await subscription.unsubscribe().catch(() => undefined);
+    };
+}
 /** Runs `source` once in every loaded context (iframes included) under `contexts` */
 export async function runInLoadedContexts(connector, { source, contexts, sandbox, onRunError = "throw", arguments: args = [], }) {
     const trees = contexts

package/dist/transport/bidi-connection.d.ts

@@ -25,12 +25,18 @@ export declare class BiDiConnector {
     private currentId;
     /** Set once the socket closed or failed: nothing sent after that gets an answer */
     private closed;
+    /** Which browser is on the other end, when the launcher says; code that must treat browsers apart asks this */
+    browserName?: "firefox" | "chrome";
     private webSocket;
     private eventEmitter;
     private commandTimeout;
     private subscriptions;
     constructor(ws: WebSocket, options?: ConnectorOptions);
     static connect(url: string, options?: ConnectorOptions): Promise<BiDiConnector>;
+    /** Calls `listener` once when the socket closes or fails (not when `kill()` ends it) */
+    onClosed(listener: () => void): void;
+    private closeListeners;
+    private notifyClosed;
     private onWebsocketClose;
     private onWebsocketError;
     private get isClosed();
@@ -59,3 +65,16 @@ export declare class BiDiConnector {
     private messageListener;
     kill(): void;
 }
+/**
+ * The message as the debug log gets it: a `script.message` on a private channel is logged by channel and
+ * action only, so a value the tester types or a secret added from the recorder toolbar never reaches the log
+ * (an added secret is only registered for masking after it is saved).
+ */
+export declare function forLog(message: {
+    type?: string;
+    method?: string;
+    params?: {
+        channel?: unknown;
+        data?: unknown;
+    };
+}): unknown;

package/dist/transport/bidi-connection.js

@@ -22,12 +22,15 @@ export class BiDiConnector {
         this.currentId = 0;
         /** Set once the socket closed or failed: nothing sent after that gets an answer */
         this.closed = false;
+        this.closeListeners = new Set();
         this.onWebsocketClose = () => {
             this.closed = true;
+            this.notifyClosed();
             this.rejectAll((pending) => new ConnectionClosedError(`Websocket closed${pendingSuffix(pending)}`));
         };
         this.onWebsocketError = (ev) => {
             this.closed = true;
+            this.notifyClosed();
             this.rejectAll((pending) => new ConnectionClosedError(`Websocket error${pendingSuffix(pending)}`, {
                 cause: ev,
             }));
@@ -84,6 +87,16 @@ export class BiDiConnector {
             });
         });
     }
+    /** Calls `listener` once when the socket closes or fails (not when `kill()` ends it) */
+    onClosed(listener) {
+        this.closeListeners.add(listener);
+    }
+    notifyClosed() {
+        const listeners = [...this.closeListeners];
+        this.closeListeners.clear();
+        for (const listener of listeners)
+            listener();
+    }
     get isClosed() {
         // A socket that is closing or closed (readyState 2 or 3) has already lost the browser
         const state = this.webSocket.readyState;
@@ -203,7 +216,27 @@ const PRIVATE_CHANNEL = /^samurai-(recorder|toolbar)-/;
  * action only, so a value the tester types or a secret added from the recorder toolbar never reaches the log
  * (an added secret is only registered for masking after it is saved).
  */
-function forLog(message) {
+export function forLog(message) {
+    return shortened(redacted(message));
+}
+/** A string longer than this is cut in the log: a file's bytes travel in 1 MB chunks, 50 MB makes 70 MB of debug log */
+const LOG_STRING_LIMIT = 1000;
+/** A copy of `value` with every long string cut to its start and its length; short values come back as they are */
+function shortened(value, depth = 0) {
+    if (typeof value === "string")
+        return value.length > LOG_STRING_LIMIT
+            ? `${value.slice(0, 200)}… (${value.length} characters)`
+            : value;
+    if (value === null || typeof value !== "object" || depth > 30)
+        return value;
+    if (Array.isArray(value))
+        return value.map((item) => shortened(item, depth + 1));
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [
+        key,
+        shortened(item, depth + 1),
+    ]));
+}
+function redacted(message) {
     const channel = message.params?.channel;
     if (message.type !== "event" ||
         message.method !== "script.message" ||

package/dist/types/browser.d.ts

@@ -1 +1 @@
-export type SupportedBrowser = "firefox";
+export type SupportedBrowser = "firefox" | "chrome";

package/dist/types/config.d.ts

@@ -24,6 +24,8 @@ export interface SamuraiGroup {
    * Overrides 'src' and 'testMatch'  options.
    */
   tests?: SamuraiTest[];
+  /** The browsers this group's tests run in, instead of the project's `browsers` */
+  browsers?: SupportedBrowser[];
 }
 
 export interface SamuraiEnvironment {
@@ -45,10 +47,10 @@ export interface SamuraiTestConfig {
    */
   srcDir?: string;
   /**
-   * The browser to use for all tests. Only "firefox" can launch today
-   * @default "firefox"
+   * The browsers the tests run in, one after the other: every selected test runs once in each. A group can name its own.
+   * @default ["firefox"]
    */
-  browser?: SupportedBrowser;
+  browsers?: SupportedBrowser[];
   /**
    * Browser binaries to launch instead of the builds SAMURAI downloads, relative to the project.
    * `SAMURAI_FIREFOX_PATH` / `SAMURAI_CHROME_PATH` win over these.

package/dist/types/test.d.ts

@@ -1,10 +1,13 @@
 import type { Browser } from "../browser/browser.ts";
 import type Page from "../browser/page.ts";
+import type { SupportedBrowser } from "./browser.ts";
 
 /** What a test function receives */
 export interface TestFixtures {
   page: Page;
   browser: Browser;
+  /** The browser this run of the test is in */
+  browserName: SupportedBrowser;
   /** The chosen environment's variables; reading an unknown name throws */
   env: Readonly<Record<string, string | number | boolean>>;
   /** Secrets by name (SAMURAI_SECRET_<NAME>); reading an unset name throws */
@@ -66,12 +69,15 @@ export interface TestLogEntry {
 interface TestResultIdentity {
   name: string;
   file: string;
+  /** The browser the test ran in */
+  browser: SupportedBrowser;
   startTime: number;
 }
 
 interface TestResultBase {
   name: string;
   file: string;
+  browser: SupportedBrowser;
   startTime: number;
   status: "started";
 }
@@ -102,4 +108,6 @@ interface TestSummaryBase {
   tests: TestResult[];
   /** The environment the run used */
   environment: string;
+  /** The browsers the run used, in the order they ran */
+  browsers: SupportedBrowser[];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itmetsam/samurai-framework",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "description": "Browser test framework over WebDriver BiDi: auto-waiting locators, environments and secrets, a recorder, and a command line.",
   "main": "dist/api.js",
   "exports": {
@@ -54,7 +54,8 @@
     "dev": "tsx src/index.ts",
     "build": "rm -rf dist && tsc -p tsconfig.build.json && node scripts/copy-types.mjs",
     "test": "tsx --test --test-timeout=10000 \"src/**/*.test.ts\"",
-    "test:browser": "tsx --test --test-concurrency=1 \"src/**/*.browser-test.ts\"",
+    "test:browser": "tsx --test --test-concurrency=1 --test-timeout=60000 \"src/**/*.browser-test.ts\"",
+    "test:browser:chrome": "SAMURAI_TEST_BROWSER=chrome npm run test:browser",
     "test:cli": "tsx --test --test-concurrency=1 \"src/**/*.cli-test.ts\""
   },
   "keywords": [