@itmetsam/samurai-framework 0.1.0 → 0.2.0

package/README.md

@@ -0,0 +1,129 @@
+# Samurai
+
+A browser test framework written in TypeScript. It drives a real browser over the [WebDriver BiDi](https://www.w3.org/TR/webdriver-bidi/) protocol, with no WebDriver server and no Playwright or Puppeteer underneath.
+
+```ts
+import { expect, test } from "../api.js";
+
+test("Visitor can sign in", async ({ page, env, secrets }) => {
+  await page.goto("/login");
+  await page.getByLabel("Email").fill(env.testUser as string);
+  await page.getByLabel("Password").fill(secrets.TEST_PASSWORD as string);
+  await page.getByRole("button", { name: "Sign in" }).click();
+  await expect(page.getByTestId("welcome")).toHaveText("Welcome back");
+});
+```
+
+- **Auto-waiting.** Actions wait until the element is attached, visible, stable, enabled and not covered. Assertions retry until they pass or time out.
+- **Semantic locators.** By role, label, text, test id, CSS or XPath, with ordered fallbacks.
+- **Environments and secrets.** One test file runs against `dev`, `staging` or `production`; secrets come from the environment or a `.env.<name>` file and are masked in logs and reports.
+- **Network control.** Mock, block or modify requests; wait for requests, responses and network idle; read response bodies.
+- **Recorder building blocks.** Specs are parsed into steps and edited back with minimal diffs, and an engine records what a person does in the browser. The UI that ties them together is not built yet (see the [roadmap](docs/roadmap.md)).
+
+> **Status:** early. Firefox only, run from a checkout of this repository (not published to npm yet). See [what is still open](docs/roadmap.md).
+
+## Getting started
+
+### 1. Requirements
+
+- [Node.js](https://nodejs.org) 26 or newer and [Bun](https://bun.sh) 1.3 or newer (CI uses these versions)
+- Firefox installed on the machine. It is found in the usual install locations on Linux, macOS and Windows.
+
+### 2. Install
+
+```sh
+git clone https://github.com/degrootsam/samurai-framework.git
+cd samurai-framework
+bun install
+```
+
+### 3. Write a test
+
+Tests live in the folder `srcDir` points to (`./src/tests` in `samurai.config.ts`) and end in `.spec.ts`. Create `src/tests/example.spec.ts`:
+
+```ts
+import { expect, test } from "../api.js";
+
+test("example.com has its heading", async ({ page }) => {
+  await page.goto("https://example.com");
+  await expect(
+    page.getByRole("heading", { name: "Example Domain" }),
+  ).toBeVisible();
+  await page.getByText("More information", { match: "partial" }).click();
+  await page.waitForLoadState();
+  expect(await page.url()).toContain("iana.org");
+});
+```
+
+### 4. Configure
+
+`samurai.config.ts` in the project root:
+
+```ts
+import { defineConfig } from "./src/config/config.js";
+
+export default defineConfig({
+  srcDir: "./src/tests",
+  browser: "firefox",
+  timeout: 30000,
+  environments: {
+    staging: { baseURL: "https://staging.example.com" },
+    production: { baseURL: "https://www.example.com" },
+  },
+  defaultEnvironment: "staging",
+});
+```
+
+With `baseURL` set, `page.goto("/login")` opens `https://staging.example.com/login`.
+
+### 5. Run
+
+```sh
+bun run dev                    # every *.spec.ts under srcDir, in the default environment
+bun run dev --env production   # another environment (or SAMURAI_ENV=production)
+bun run dev --timeout 60000    # per-test timeout in ms; also --expect-timeout
+```
+
+Once the package is installed in a project, `samurai run` does the same with a readable summary and a proper exit code, and `samurai init` scaffolds a new project (see [Command line](docs/cli.md)).
+
+A browser window opens for each test and closes when the test ends. The result is written to `result/report.json` (status, duration and error per test); nothing summarises it on the console yet. The process exits with code 0 when every test passed and 1 otherwise (add `--headless` to hide the window). Framework logs go to the console and `logs/`.
+
+> `src/tests/index.spec.ts` is an example that submits a real contact form. Delete it or replace it before running everything.
+
+### 6. Secrets (optional)
+
+```sh
+cp .env.example .env.staging
+# SAMURAI_SECRET_TEST_PASSWORD=...
+```
+
+Read it in a test as `secrets.TEST_PASSWORD`. Real environment variables win over the file, and `.env.*` files are git-ignored. See [Environments and secrets](docs/environments-and-secrets.md).
+
+## Documentation
+
+| Guide                                                        | What it covers                                                                                        |
+| ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- |
+| [Writing tests](docs/writing-tests.md)                       | `test`, `describe`, fixtures, how a test passes or fails                                              |
+| [Locators](docs/locators.md)                                 | Finding elements, chaining, fallbacks, actions and their auto-waiting                                 |
+| [Assertions](docs/assertions.md)                             | `expect` for elements and for plain values                                                            |
+| [Pages and browsers](docs/pages-and-browsers.md)             | Navigation, waits, screenshots, PDF, viewport, emulation, dialogs, downloads, contexts, cookies       |
+| [Network](docs/network.md)                                   | Waiting for traffic, mocking, blocking, response bodies                                               |
+| [Configuration](docs/configuration.md)                       | Every option in `samurai.config.ts`, CLI flags and precedence                                         |
+| [Environments and secrets](docs/environments-and-secrets.md) | Per-environment settings, variables, secrets and masking                                              |
+| [Command line](docs/cli.md)                                  | `samurai run`, `list`, `record` and `init`, options, output, exit codes, `--json`                     |
+| [Running tests and reports](docs/running-and-reports.md)     | What a run does, `report.json`, page logs, timeouts                                                   |
+| [Steps and the recorder](docs/recorder.md)                   | The step codec and the recording engine                                                               |
+| [Embedding the framework](docs/embedding.md)                 | Using the package from another app (such as the Electron UI): build, entry points, `runTests`, events |
+| [Contributing](docs/contributing.md)                         | Project layout, running the framework's own tests, conventions                                        |
+| [Roadmap](docs/roadmap.md)                                   | What is built and what is still open                                                                  |
+
+Design notes for every feature live in [`docs/superpowers/specs`](docs/superpowers/specs).
+
+## Scripts
+
+| Command                | Does                                            |
+| ---------------------- | ----------------------------------------------- |
+| `bun run dev`          | Runs the test suite (`tsx src/index.ts`)        |
+| `bun run test`         | Unit tests of the framework (no browser needed) |
+| `bun run test:browser` | Browser tests of the framework (needs Firefox)  |
+| `bun run build`        | Typechecks and compiles to `dist/`              |

package/bin/samurai.mjs

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+// Quiet the framework's console logging unless the caller asked for more; the log files keep everything
+process.env.SAMURAI_LOG_LEVEL ??= "warn";
+
+// Specs are TypeScript: let this process import them. tsx is an optional peer dependency of the package
+try {
+  const { register } = await import("tsx/esm/api");
+  register();
+} catch (err) {
+  if (err?.code !== "ERR_MODULE_NOT_FOUND") throw err;
+  process.stderr.write(
+    "samurai loads TypeScript specs with tsx, which is not installed. Install it: npm install --save-dev tsx\n",
+  );
+  process.exit(2);
+}
+
+const { main } = await import("../dist/cli/main.js");
+// Ctrl+C ends a recording cleanly (the browser is closed, the file is complete); a second one stops at once
+const interrupt = new AbortController();
+process.on("SIGINT", () => {
+  if (interrupt.signal.aborted) process.exit(130);
+  interrupt.abort();
+});
+const code = await main(process.argv.slice(2), {
+  out: (text) => process.stdout.write(text),
+  err: (text) => process.stderr.write(text),
+  color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR,
+  env: process.env,
+  signal: interrupt.signal,
+});
+process.exit(code);

package/dist/api.js

@@ -0,0 +1,3 @@
+export { test, describe } from "./runner/registry.js";
+export { expect } from "./assert/expect.js";
+export { defineConfig } from "./config/config.js";

package/dist/assert/assertion-error.d.ts

@@ -0,0 +1,23 @@
+/** Formats a value for assertion messages */
+export declare function format(value: unknown): string;
+/**
+ * Converts a value into something JSON can store in the report: regexps, bigints,
+ * symbols, functions and anything else JSON.stringify rejects (e.g. circular objects)
+ * become their `format()` string; other values are kept as-is.
+ */
+export declare function toReportValue(value: unknown): unknown;
+export interface AssertionErrorDetails {
+    /** Matcher name, prefixed with `not.` when negated */
+    matcher: string;
+    expected: unknown;
+    actual: unknown;
+    /** XPath of the asserted locator, only for locator assertions */
+    locator?: string | undefined;
+}
+export declare class AssertionError extends Error {
+    readonly matcher: string;
+    readonly expected: unknown;
+    readonly actual: unknown;
+    readonly locator: string | undefined;
+    constructor({ matcher, expected, actual, locator }: AssertionErrorDetails);
+}

package/dist/assert/assertion-error.js

@@ -0,0 +1,45 @@
+import { inspect } from "node:util";
+/** Formats a value for assertion messages */
+export function format(value) {
+    if (typeof value === "string")
+        return JSON.stringify(value);
+    if (value instanceof RegExp)
+        return value.toString();
+    return inspect(value, { depth: 2 });
+}
+/**
+ * Converts a value into something JSON can store in the report: regexps, bigints,
+ * symbols, functions and anything else JSON.stringify rejects (e.g. circular objects)
+ * become their `format()` string; other values are kept as-is.
+ */
+export function toReportValue(value) {
+    if (value instanceof RegExp ||
+        typeof value === "bigint" ||
+        typeof value === "symbol" ||
+        typeof value === "function") {
+        return format(value);
+    }
+    try {
+        JSON.stringify(value);
+        return value;
+    }
+    catch {
+        return format(value);
+    }
+}
+export class AssertionError extends Error {
+    constructor({ matcher, expected, actual, locator }) {
+        const subject = locator === undefined ? "value" : "locator";
+        const lines = [`expect(${subject}).${matcher}`];
+        if (locator !== undefined) {
+            lines.push(`  locator: ${locator}`);
+        }
+        lines.push(`  expected: ${format(expected)}`, `  received: ${format(actual)}`);
+        super(lines.join("\n"));
+        this.name = "AssertionError";
+        this.matcher = matcher;
+        this.expected = expected;
+        this.actual = actual;
+        this.locator = locator;
+    }
+}

package/dist/assert/expect.d.ts

@@ -0,0 +1,55 @@
+import Locator from "../locator/locator.js";
+/**
+ * Returns the matcher names of locator assertions that were never awaited and are
+ * still running or already failed, and stops tracking them. The runner calls this
+ * after a test to detect a missing `await`.
+ */
+export declare function takePendingAssertions(): string[];
+export interface LocatorAssertionOptions {
+    /** Time (ms) to keep retrying. Defaults to config `expect.timeout`, then 5000 */
+    timeout?: number;
+}
+export declare class ValueAssertions<T> {
+    private readonly actual;
+    private readonly negated;
+    constructor(actual: T, negated?: boolean);
+    /** Inverts the next matcher */
+    get not(): ValueAssertions<T>;
+    private assert;
+    private compare;
+    /** Strict identity (`Object.is`) */
+    toBe(expected: unknown): void;
+    /** Deep structural equality */
+    toEqual(expected: unknown): void;
+    toBeTruthy(): void;
+    toBeFalsy(): void;
+    /** Array includes `item`, or string contains substring `item` */
+    toContain(item: unknown): void;
+    /** String contains `pattern` (string) or matches it (RegExp) */
+    toMatch(pattern: string | RegExp): void;
+    toBeGreaterThan(expected: number | bigint): void;
+    toBeGreaterThanOrEqual(expected: number | bigint): void;
+    toBeLessThan(expected: number | bigint): void;
+    toBeLessThanOrEqual(expected: number | bigint): void;
+}
+export declare class LocatorAssertions {
+    private readonly locator;
+    private readonly negated;
+    constructor(locator: Locator, negated?: boolean);
+    /** Inverts the next matcher: it retries until the condition is false */
+    get not(): LocatorAssertions;
+    /** Re-reads the element until `check` passes or the timeout expires */
+    private assertEventually;
+    toBeVisible(options?: LocatorAssertionOptions): Promise<void>;
+    /** Trimmed text content equals `expected` (string) or matches it (RegExp) */
+    toHaveText(expected: string | RegExp, options?: LocatorAssertionOptions): Promise<void>;
+    /** Text content contains `expected` (string) or matches it (RegExp) */
+    toContainText(expected: string | RegExp, options?: LocatorAssertionOptions): Promise<void>;
+    toHaveValue(expected: string | RegExp, options?: LocatorAssertionOptions): Promise<void>;
+    toHaveAttribute(name: string, expected: string | RegExp, options?: LocatorAssertionOptions): Promise<void>;
+    toHaveCount(expected: number, options?: LocatorAssertionOptions): Promise<void>;
+}
+/** Auto-retrying assertions on a page element */
+export declare function expect(locator: Locator): LocatorAssertions;
+/** Synchronous assertions on a plain value */
+export declare function expect<T>(value: T): ValueAssertions<T>;

package/dist/assert/expect.js

@@ -0,0 +1,188 @@
+import { isDeepStrictEqual } from "node:util";
+import { AssertionError } from "./assertion-error.js";
+import Locator from "../locator/locator.js";
+import { resolveTimeout, waitUntil, WaitTimeoutError } from "../wait/wait-until.js";
+/**
+ * Locator assertions nobody has awaited yet that are still running or have failed,
+ * mapped to their matcher name
+ */
+const pendingAssertions = new Map();
+/**
+ * Returns the matcher names of locator assertions that were never awaited and are
+ * still running or already failed, and stops tracking them. The runner calls this
+ * after a test to detect a missing `await`.
+ */
+export function takePendingAssertions() {
+    const names = [...pendingAssertions.values()];
+    pendingAssertions.clear();
+    return names;
+}
+/**
+ * Promise that stops being pending as soon as anyone subscribes to it. `await`,
+ * `.catch`, `.finally`, `Promise.all` and `assert.rejects` all go through `then`
+ * (`await` calls it because the constructor is not the native Promise).
+ */
+class TrackedAssertion extends Promise {
+    /** Promises derived through `then` are plain promises */
+    static get [Symbol.species]() {
+        return Promise;
+    }
+    then(onfulfilled, onrejected) {
+        pendingAssertions.delete(this);
+        return super.then(onfulfilled, onrejected);
+    }
+}
+function track(matcher, assertion) {
+    const tracked = new TrackedAssertion((resolve, reject) => {
+        assertion.then(resolve, reject);
+    });
+    pendingAssertions.set(tracked, matcher);
+    // The native then does not count as awaiting: a pass stops tracking, a failure stays
+    // tracked. Handling the rejection keeps an un-awaited failure from being unhandled.
+    Promise.prototype.then.call(tracked, () => pendingAssertions.delete(tracked), () => { });
+    return tracked;
+}
+function matchesText(actual, expected, mode) {
+    // search() ignores lastIndex, so global regexps behave the same on every call
+    if (expected instanceof RegExp)
+        return actual.search(expected) !== -1;
+    return mode === "exact" ? actual === expected : actual.includes(expected);
+}
+function isNumeric(value) {
+    return typeof value === "number" || typeof value === "bigint";
+}
+export class ValueAssertions {
+    constructor(actual, negated = false) {
+        this.actual = actual;
+        this.negated = negated;
+    }
+    /** Inverts the next matcher */
+    get not() {
+        return new ValueAssertions(this.actual, !this.negated);
+    }
+    assert(pass, matcher, expected) {
+        if (pass !== this.negated)
+            return;
+        throw new AssertionError({
+            matcher: this.negated ? `not.${matcher}` : matcher,
+            expected,
+            actual: this.actual,
+        });
+    }
+    compare(matcher, expected, check) {
+        const actual = this.actual;
+        if (!isNumeric(actual) || !isNumeric(expected)) {
+            throw new TypeError(`${matcher} expects numbers, received ${typeof actual} and ${typeof expected}`);
+        }
+        this.assert(check(actual, expected), matcher, expected);
+    }
+    /** Strict identity (`Object.is`) */
+    toBe(expected) {
+        this.assert(Object.is(this.actual, expected), "toBe", expected);
+    }
+    /** Deep structural equality */
+    toEqual(expected) {
+        this.assert(isDeepStrictEqual(this.actual, expected), "toEqual", expected);
+    }
+    toBeTruthy() {
+        this.assert(Boolean(this.actual), "toBeTruthy", "truthy");
+    }
+    toBeFalsy() {
+        this.assert(!this.actual, "toBeFalsy", "falsy");
+    }
+    /** Array includes `item`, or string contains substring `item` */
+    toContain(item) {
+        const actual = this.actual;
+        if (typeof actual === "string") {
+            if (typeof item !== "string") {
+                throw new TypeError(`toContain on a string expects a string, received ${typeof item}`);
+            }
+            this.assert(actual.includes(item), "toContain", item);
+            return;
+        }
+        if (Array.isArray(actual)) {
+            this.assert(actual.includes(item), "toContain", item);
+            return;
+        }
+        throw new TypeError(`toContain expects an array or string, received ${typeof actual}`);
+    }
+    /** String contains `pattern` (string) or matches it (RegExp) */
+    toMatch(pattern) {
+        const actual = this.actual;
+        if (typeof actual !== "string") {
+            throw new TypeError(`toMatch expects a string, received ${typeof actual}`);
+        }
+        this.assert(matchesText(actual, pattern, "contains"), "toMatch", pattern);
+    }
+    toBeGreaterThan(expected) {
+        this.compare("toBeGreaterThan", expected, (a, b) => a > b);
+    }
+    toBeGreaterThanOrEqual(expected) {
+        this.compare("toBeGreaterThanOrEqual", expected, (a, b) => a >= b);
+    }
+    toBeLessThan(expected) {
+        this.compare("toBeLessThan", expected, (a, b) => a < b);
+    }
+    toBeLessThanOrEqual(expected) {
+        this.compare("toBeLessThanOrEqual", expected, (a, b) => a <= b);
+    }
+}
+export class LocatorAssertions {
+    constructor(locator, negated = false) {
+        this.locator = locator;
+        this.negated = negated;
+    }
+    /** Inverts the next matcher: it retries until the condition is false */
+    get not() {
+        return new LocatorAssertions(this.locator, !this.negated);
+    }
+    /** Re-reads the element until `check` passes or the timeout expires */
+    assertEventually(matcher, expected, read, check, options) {
+        const name = this.negated ? `not.${matcher}` : matcher;
+        return track(name, (async () => {
+            const timeout = await resolveTimeout(options?.timeout);
+            try {
+                await waitUntil(read, (actual) => check(actual) !== this.negated, { timeout });
+            }
+            catch (err) {
+                if (!(err instanceof WaitTimeoutError))
+                    throw err;
+                // Reads never resolve to undefined, so an undefined `last` means no read completed
+                if (err.last === undefined) {
+                    throw new Error(`expect(locator).${name}: could not read ${this.locator.selector} within ${timeout}ms`);
+                }
+                throw new AssertionError({
+                    matcher: name,
+                    expected,
+                    actual: err.last,
+                    locator: this.locator.selector,
+                });
+            }
+        })());
+    }
+    toBeVisible(options) {
+        return this.assertEventually("toBeVisible", true, () => this.locator.isVisible(), (visible) => visible, options);
+    }
+    /** Trimmed text content equals `expected` (string) or matches it (RegExp) */
+    toHaveText(expected, options) {
+        return this.assertEventually("toHaveText", expected, () => this.locator.textContent(), (text) => text !== null && matchesText(text, expected, "exact"), options);
+    }
+    /** Text content contains `expected` (string) or matches it (RegExp) */
+    toContainText(expected, options) {
+        return this.assertEventually("toContainText", expected, () => this.locator.textContent(), (text) => text !== null && matchesText(text, expected, "contains"), options);
+    }
+    toHaveValue(expected, options) {
+        return this.assertEventually("toHaveValue", expected, () => this.locator.inputValue(), (value) => value !== null && matchesText(value, expected, "exact"), options);
+    }
+    toHaveAttribute(name, expected, options) {
+        return this.assertEventually("toHaveAttribute", expected, () => this.locator.getAttribute(name), (value) => value !== null && matchesText(value, expected, "exact"), options);
+    }
+    toHaveCount(expected, options) {
+        return this.assertEventually("toHaveCount", expected, () => this.locator.count(), (count) => count === expected, options);
+    }
+}
+export function expect(value) {
+    return value instanceof Locator
+        ? new LocatorAssertions(value)
+        : new ValueAssertions(value);
+}

package/dist/browser/browser-context.d.ts

@@ -0,0 +1,91 @@
+import type { BiDiConnector } from "../transport/bidi-connection.js";
+import type { SameSite } from "../types/bidi-modules/network.js";
+import { type EmulationOptions } from "./emulation.js";
+import type Page from "./page.js";
+export type { SameSite } from "../types/bidi-modules/network.js";
+/** A cookie as `context.cookies()` returns it */
+export interface Cookie {
+    name: string;
+    value: string;
+    domain: string;
+    path: string;
+    httpOnly: boolean;
+    secure: boolean;
+    sameSite: SameSite;
+    /** Seconds since the epoch; absent for a session cookie */
+    expiry?: number;
+}
+/** A cookie to add: `domain`, or a `url` to take domain, path and `secure` from */
+export interface CookieInput {
+    name: string;
+    value: string;
+    domain?: string;
+    /** @default "/", or the path of `url` */
+    path?: string;
+    url?: string;
+    httpOnly?: boolean;
+    secure?: boolean;
+    sameSite?: SameSite;
+    /** Seconds since the epoch */
+    expiry?: number;
+}
+export interface CookieFilter {
+    name?: string;
+    domain?: string;
+    path?: string;
+}
+/** What a context needs from its browser */
+export interface ContextHost {
+    connector: BiDiConnector;
+    newPage(options: {
+        type: "tab";
+        userContext?: string;
+    }): Promise<Page>;
+    /** Called when a context has been closed */
+    forget(context: BrowserContext): void;
+}
+/**
+ * An isolated browsing session inside one browser: its own cookies, storage and cache. Pages opened in it
+ * share them; pages of other contexts see none of it.
+ */
+export declare class BrowserContext {
+    private host;
+    /** The BiDi user context id; "default" for the browser's own */
+    readonly id: string;
+    readonly isDefault: boolean;
+    private opened;
+    private isClosed;
+    constructor(host: ContextHost, 
+    /** The BiDi user context id; "default" for the browser's own */
+    id: string, isDefault: boolean);
+    /** True once `close()` has run */
+    get closed(): boolean;
+    /** Opens a tab in this context */
+    newPage(): Promise<Page>;
+    /** The pages opened in this context that are still open */
+    pages(): Page[];
+    /** @internal Called when a page is opened in this context */
+    register(page: Page): void;
+    /** Adds cookies to this context's storage. Every cookie is checked before any is stored */
+    addCookies(cookies: CookieInput[]): Promise<void>;
+    /** The cookies of this context, optionally only those matching `filter` */
+    cookies(filter?: CookieFilter): Promise<Cookie[]>;
+    /** Deletes all cookies of this context, or those matching `filter` */
+    clearCookies(filter?: CookieFilter): Promise<void>;
+    /**
+     * Pretends things about every page of this context, including pages opened later: locale, timezone, user agent,
+     * geolocation, offline, orientation, screen size. A page's own `emulate()` wins over the context's.
+     * @example
+     *  await context.emulate({ locale: "nl-NL", timezone: "Europe/Amsterdam", offline: false });
+     */
+    emulate(options: EmulationOptions): Promise<void>;
+    /**
+     * Grants, denies or resets a browser permission ("geolocation", "notifications", "camera", …) for `origin`
+     * in this context. Geolocation emulation only answers once the permission is granted.
+     */
+    setPermission(name: string, state: "granted" | "denied" | "prompt", origin: string): Promise<void>;
+    /** Removes the context; the browser closes its pages. The default context cannot be closed */
+    close(): Promise<void>;
+    private partition;
+    private assertOpen;
+}