browxai 0.7.0

package/dist/page/recording.js

@@ -0,0 +1,172 @@
+// Calibration-walk → flow-file scaffold — .
+//
+// Records action-tool calls during a session and emits a draft `flow-file.yaml`
+// the agent can hand to site-docs (or any consumer with a similar YAML shape).
+//
+// The recording captures: the action's type/target, the resolved selectorHint
+// (so flow-files transcribe mechanically), the URL, and an optional human-readable
+// note. Annotations are agent-supplied via `record_annotate({ copy, arrow })`.
+//
+// Format: minimal site-docs-flavoured YAML (the canonical shape lives in the
+// site-docs repo; consumers can post-process if they need a different dialect).
+export class Recorder {
+    steps = [];
+    name = null;
+    autoCounter = 0;
+    start(flowName) {
+        if (this.name) {
+            // Replace silently — calibration restarts are common.
+        }
+        this.name = flowName;
+        this.steps = [];
+        this.autoCounter = 0;
+        return { ok: true, name: flowName };
+    }
+    active() {
+        return this.name !== null;
+    }
+    /** Read-only access to the recorded trace + the flow name without ending
+     *  the recording. Used by trace-export tools (e.g. the Playwright-script
+     *  exporter) that need to lower the steps to a runnable artefact while the
+     *  recording is still in progress. Returns null when no recording is
+     *  active. */
+    inspect() {
+        if (!this.name)
+            return null;
+        return { name: this.name, steps: this.steps.slice() };
+    }
+    /** Record an action that just happened. The caller (server.ts action handlers)
+     *  passes the descriptor + the URL it ended at + whatever selectorHint was
+     *  used to resolve the target. Best-effort: if no recording is active, this
+     *  is a no-op. */
+    record(descriptor, url, hint) {
+        if (!this.name)
+            return;
+        const id = this.suggestId(descriptor);
+        this.steps.push({
+            id,
+            action: descriptor,
+            url,
+            selectorHint: hint?.selectorHint,
+            stability: hint?.stability,
+            ts: Date.now(),
+        });
+    }
+    /** Attach an annotation to the most-recent step (or by id). */
+    annotate(args) {
+        if (!this.name)
+            return { ok: false, error: "no active recording" };
+        if (this.steps.length === 0)
+            return { ok: false, error: "no steps recorded yet" };
+        const step = args.stepId
+            ? this.steps.find((s) => s.id === args.stepId)
+            : this.steps[this.steps.length - 1];
+        if (!step)
+            return { ok: false, error: `no step with id "${args.stepId}"` };
+        step.annotation = { copy: args.copy, arrow: args.arrow, target: args.target };
+        return { ok: true };
+    }
+    /** End the recording. Returns the YAML draft + the step count. */
+    end() {
+        if (!this.name)
+            throw new Error("end_recording: no active recording");
+        const name = this.name;
+        const yaml = this.toYaml();
+        const stepCount = this.steps.length;
+        this.name = null;
+        this.steps = [];
+        return { name, yaml, stepCount };
+    }
+    suggestId(d) {
+        const base = d.type === "navigate"
+            ? "open"
+            : d.type === "click"
+                ? "click"
+                : d.type === "fill"
+                    ? "fill"
+                    : d.type === "press"
+                        ? "press"
+                        : d.type === "hover"
+                            ? "hover"
+                            : d.type === "select"
+                                ? "select"
+                                : d.type === "waitFor"
+                                    ? "wait"
+                                    : d.type;
+        return `${base}-${++this.autoCounter}`;
+    }
+    /** Render the recording as YAML. Minimal site-docs-flavoured shape; consumers
+     *  can post-process. We don't pull in a YAML library to keep deps lean — the
+     *  output is small + predictable. */
+    toYaml() {
+        const lines = [];
+        lines.push(`name: ${this.name}`);
+        lines.push(`# Drafted via browxai recording — review locator stability + add prerequisites/assertions before committing.`);
+        // Locators block — pulled from steps that have a selectorHint.
+        const locatorEntries = this.collectLocators();
+        if (locatorEntries.length > 0) {
+            lines.push("locators:");
+            for (const { name, hint, stability } of locatorEntries) {
+                const stabilityTag = stability && stability !== "high" ? `   # stability: ${stability} — review` : "";
+                lines.push(`  ${name}: ${quote(hint)}${stabilityTag}`);
+            }
+        }
+        lines.push("steps:");
+        for (const step of this.steps) {
+            lines.push(`  - id: ${step.id}`);
+            lines.push(`    action: ${step.action.type}`);
+            const valueOrUrl = step.action.url ?? step.action.value;
+            if (valueOrUrl !== undefined)
+                lines.push(`    value: ${quote(valueOrUrl)}`);
+            if (step.selectorHint && step.action.type !== "navigate") {
+                const locName = this.locatorNameFor(step);
+                lines.push(`    target: $${locName}`);
+            }
+            if (step.annotation) {
+                lines.push(`    annotation:`);
+                lines.push(`      copy: ${quote(step.annotation.copy)}`);
+                if (step.annotation.arrow)
+                    lines.push(`      arrow: ${step.annotation.arrow}`);
+                if (step.annotation.target)
+                    lines.push(`      target: ${step.annotation.target}`);
+            }
+        }
+        return lines.join("\n") + "\n";
+    }
+    collectLocators() {
+        const seen = new Map();
+        for (const step of this.steps) {
+            if (!step.selectorHint || step.action.type === "navigate")
+                continue;
+            const name = this.locatorNameFor(step);
+            if (!seen.has(name))
+                seen.set(name, { name, hint: step.selectorHint, stability: step.stability });
+        }
+        return [...seen.values()];
+    }
+    locatorNameFor(step) {
+        // Derive a stable locator name from the selectorHint: extract the testId/
+        // role+name if obvious, else mash the step id.
+        const hint = step.selectorHint ?? "";
+        // Tier-1 attribute selector: require `data-*` prefix so we don't match
+        // e.g. `[name="Submit"]` inside a role=… hint.
+        const testId = hint.match(/\[(data-[a-z-]+)="([^"]+)"\]/);
+        if (testId)
+            return slugify(testId[2]);
+        const roleName = hint.match(/role=([a-z]+)\[name="([^"]+)"\]/i);
+        if (roleName)
+            return slugify(`${roleName[1]}_${roleName[2]}`);
+        return slugify(step.id);
+    }
+}
+function slugify(s) {
+    return (s
+        .replace(/[^a-z0-9]+/gi, "_")
+        .replace(/^_+|_+$/g, "")
+        .toLowerCase() || "step");
+}
+function quote(s) {
+    // Minimal YAML scalar quoting — wrap in double quotes if contains spaces /
+    // special chars; otherwise leave bare. Always quote for predictability.
+    return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+}

package/dist/page/refs.d.ts

@@ -0,0 +1,92 @@
+import type { Frame } from "playwright-core";
+export interface KeyInputs {
+    role: string;
+    name?: string;
+    /** A structural path through the a11y tree, e.g. "WebArea/main/list/listitem[2]/button". */
+    path: string;
+    /** Optional test-id-ish attribute value to disambiguate identical roles. */
+    testId?: string;
+    /** stable frame ID this node lives in. Namespaces the key so two
+     *  iframes with the same internal markup don't collide on the same ref.
+     *  Absent / empty means the main frame, preserving the pre-v0.5.0 hash. */
+    frameId?: string;
+}
+export declare function elementKey(inputs: KeyInputs): string;
+/** What we remember about a ref so an action can rebuild a Playwright Locator for it. */
+export interface RefLocatorInputs {
+    role: string;
+    name?: string;
+    testId?: string;
+    /** Attribute *name* that yielded `testId` (e.g. "data-testid", "data-type").
+     *  Locator-resolution uses it to build `[<attr>=...]` so non-standard test attributes
+     *  Just Work. */
+    testIdAttr?: string;
+    /** Ref provenance. Drives locator routing: a11y refs use role/name locators
+     *  (auto-wait + strict semantics); dom refs use the structural CSS path that
+     *  built the ref (refs whose role is a bare tag like `td`/`div`/`generic`
+     *  produce ambiguous role-locators that don't actually find anything). `both`
+     *  means the same element was discovered by both passes — a11y-tier locators
+     *  win, with cssPath available as fallback. */
+    source?: "a11y" | "dom" | "both";
+    /** Structural CSS path captured at DOM-walk time, e.g.
+     *  `body > div:nth-child(2) > table > tbody > tr:nth-child(4) > td:nth-child(3)`.
+     *  Used when role/name locators would be ambiguous. Only populated for refs
+     *  whose `source` includes `dom`. */
+    cssPath?: string;
+    /** stable frame ID the ref was minted in. Absent / `f0` means the
+     *  main frame (existing behaviour). Anything else: a child iframe — locator
+     *  resolution routes through `frame.locator(...)` instead of `page.locator(...)`
+     *  so subsequent actions land inside the correct frame. Same-origin and
+     *  cross-origin frames work transparently through Playwright's frame API. */
+    frameId?: string;
+}
+export declare class RefRegistry {
+    private refByKey;
+    private keyByRef;
+    private locatorByRef;
+    /** persistent named refs. Maps an agent-chosen mnemonic (e.g.
+     *  "play_btn") to the underlying ref. Refs themselves are stable across
+     *  snapshots (see elementKey()) so the name effectively pins an element
+     *  identity for the whole session. */
+    private refByName;
+    /** Frame handle owning each child-frame ref. Main-frame refs
+     *  omit the entry — the page-level locator resolution handles them
+     *  unchanged. When a child-frame ref is acted on, `locatorFor` uses
+     *  this Frame handle (instead of `page.locator(...)`) so the action
+     *  lands inside the right OOPIF / same-origin iframe. */
+    private frameByRef;
+    private counter;
+    /** Resolve (or mint) the ref for a node's stable key. */
+    forKey(key: string, locator?: RefLocatorInputs): string;
+    has(ref: string): boolean;
+    hasKey(key: string): boolean;
+    keyOf(ref: string): string | undefined;
+    locatorOf(ref: string): RefLocatorInputs | undefined;
+    updateLocator(ref: string, locator: RefLocatorInputs): void;
+    /**
+     * Merge new locator inputs into an existing ref's record. Existing richness
+     * wins (a11y-discovered `name` survives); missing fields fill in from the
+     * partial; `source` combines (`a11y` + `dom` → `both`). When the ref has
+     * no prior locator inputs, the partial is installed wholesale provided
+     * `role` is present.
+     */
+    augmentLocator(ref: string, partial: Partial<RefLocatorInputs>): void;
+    /** Bind a Playwright Frame handle to a ref. Call when minting a ref in a
+     *  child-frame snapshot/find so action-time `locatorFor` can route through
+     *  the frame instead of the page. Main-frame refs don't need to call this;
+     *  absence of a binding means "resolve through the page". */
+    bindFrame(ref: string, frame: Frame): void;
+    /** Resolve a ref to its bound Frame, or undefined for main-frame refs. */
+    frameOf(ref: string): Frame | undefined;
+    /** Bind a mnemonic name to a ref. Overwrites any prior binding for that name. */
+    nameRef(name: string, ref: string): void;
+    /** Resolve a name back to a ref. */
+    refByNameLookup(name: string): string | undefined;
+    /** List all current name → ref bindings. */
+    listNames(): Array<{
+        name: string;
+        ref: string;
+    }>;
+    /** Useful for tests / introspection only. */
+    size(): number;
+}

package/dist/page/refs.js

@@ -0,0 +1,134 @@
+// Stable `eN` ref scheme — the cross-snapshot coherence constraint.
+//
+// A ref is assigned by a *stable element key* (a hash of role + accessible name +
+// structural path + testid-if-any), NOT by enumeration order. When a node persists
+// across snapshots it keeps its eN; new nodes get fresh ones. This is what makes
+// `tree_diff` line-stable and what lets refs from a `find()` candidate survive
+// the next `snapshot()`.
+import { createHash } from "node:crypto";
+export function elementKey(inputs) {
+    const raw = [
+        inputs.role,
+        inputs.name ?? "",
+        inputs.path,
+        inputs.testId ?? "",
+        inputs.frameId ?? "",
+    ].join("");
+    // Short hash — collision-resistant within a single page session.
+    return createHash("sha256").update(raw).digest("hex").slice(0, 16);
+}
+export class RefRegistry {
+    refByKey = new Map();
+    keyByRef = new Map();
+    locatorByRef = new Map();
+    /** persistent named refs. Maps an agent-chosen mnemonic (e.g.
+     *  "play_btn") to the underlying ref. Refs themselves are stable across
+     *  snapshots (see elementKey()) so the name effectively pins an element
+     *  identity for the whole session. */
+    refByName = new Map();
+    /** Frame handle owning each child-frame ref. Main-frame refs
+     *  omit the entry — the page-level locator resolution handles them
+     *  unchanged. When a child-frame ref is acted on, `locatorFor` uses
+     *  this Frame handle (instead of `page.locator(...)`) so the action
+     *  lands inside the right OOPIF / same-origin iframe. */
+    frameByRef = new Map();
+    counter = 0;
+    /** Resolve (or mint) the ref for a node's stable key. */
+    forKey(key, locator) {
+        let ref = this.refByKey.get(key);
+        if (!ref) {
+            ref = `e${++this.counter}`;
+            this.refByKey.set(key, ref);
+            this.keyByRef.set(ref, key);
+        }
+        if (locator)
+            this.locatorByRef.set(ref, locator);
+        return ref;
+    }
+    has(ref) {
+        return this.keyByRef.has(ref);
+    }
+    hasKey(key) {
+        return this.refByKey.has(key);
+    }
+    keyOf(ref) {
+        return this.keyByRef.get(ref);
+    }
+    locatorOf(ref) {
+        return this.locatorByRef.get(ref);
+    }
+    updateLocator(ref, locator) {
+        if (this.keyByRef.has(ref))
+            this.locatorByRef.set(ref, locator);
+    }
+    /**
+     * Merge new locator inputs into an existing ref's record. Existing richness
+     * wins (a11y-discovered `name` survives); missing fields fill in from the
+     * partial; `source` combines (`a11y` + `dom` → `both`). When the ref has
+     * no prior locator inputs, the partial is installed wholesale provided
+     * `role` is present.
+     */
+    augmentLocator(ref, partial) {
+        if (!this.keyByRef.has(ref))
+            return;
+        const existing = this.locatorByRef.get(ref);
+        if (!existing) {
+            if (partial.role !== undefined)
+                this.locatorByRef.set(ref, partial);
+            return;
+        }
+        const merged = {
+            role: existing.role,
+            name: existing.name ?? partial.name,
+            testId: existing.testId ?? partial.testId,
+            testIdAttr: existing.testIdAttr ?? partial.testIdAttr,
+            cssPath: existing.cssPath ?? partial.cssPath,
+            source: combineSource(existing.source, partial.source),
+            frameId: existing.frameId ?? partial.frameId,
+        };
+        this.locatorByRef.set(ref, merged);
+    }
+    // --- frame binding ---
+    /** Bind a Playwright Frame handle to a ref. Call when minting a ref in a
+     *  child-frame snapshot/find so action-time `locatorFor` can route through
+     *  the frame instead of the page. Main-frame refs don't need to call this;
+     *  absence of a binding means "resolve through the page". */
+    bindFrame(ref, frame) {
+        if (!this.keyByRef.has(ref))
+            return;
+        this.frameByRef.set(ref, frame);
+    }
+    /** Resolve a ref to its bound Frame, or undefined for main-frame refs. */
+    frameOf(ref) {
+        return this.frameByRef.get(ref);
+    }
+    // --- named refs ---
+    /** Bind a mnemonic name to a ref. Overwrites any prior binding for that name. */
+    nameRef(name, ref) {
+        if (!this.keyByRef.has(ref))
+            throw new Error(`name_ref: ref "${ref}" not in registry`);
+        this.refByName.set(name, ref);
+    }
+    /** Resolve a name back to a ref. */
+    refByNameLookup(name) {
+        return this.refByName.get(name);
+    }
+    /** List all current name → ref bindings. */
+    listNames() {
+        return [...this.refByName.entries()].map(([name, ref]) => ({ name, ref }));
+    }
+    /** Useful for tests / introspection only. */
+    size() {
+        return this.refByKey.size;
+    }
+}
+function combineSource(a, b) {
+    if (!a)
+        return b;
+    if (!b)
+        return a;
+    if (a === b)
+        return a;
+    // Any mix of a11y + dom (or already-both) ⇒ both.
+    return "both";
+}

package/dist/page/regions.d.ts

@@ -0,0 +1,23 @@
+export interface Box {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface NamedRegion {
+    name: string;
+    box: Box;
+    /** box centre — the point to hand to a coords-based action. */
+    center: {
+        x: number;
+        y: number;
+    };
+}
+/** One per SessionEntry. In-memory; nothing persisted. */
+export declare class RegionRegistry {
+    private regions;
+    set(name: string, box: Box): NamedRegion;
+    get(name: string): NamedRegion | undefined;
+    list(): NamedRegion[];
+    private view;
+}

package/dist/page/regions.js

@@ -0,0 +1,32 @@
+// Named visual regions — capability `human`.
+//
+// In virtualised timelines / canvas / unlabelled positioned divs the target
+// is "the third row segment at x=…", not an element ref. `name_region` binds
+// a box to a mnemonic so a sub-agent can re-select, copy, and re-check the
+// same media segment without re-deriving coordinates each time (coordinate
+// drift). `region` resolves the name back to its box + centre point, which
+// the caller passes to `click({coords})` etc.
+/** One per SessionEntry. In-memory; nothing persisted. */
+export class RegionRegistry {
+    regions = new Map();
+    set(name, box) {
+        this.regions.set(name, box);
+        return this.view(name);
+    }
+    get(name) {
+        return this.view(name);
+    }
+    list() {
+        return [...this.regions.keys()].map((n) => this.view(n));
+    }
+    view(name) {
+        const box = this.regions.get(name);
+        if (!box)
+            return undefined;
+        return {
+            name,
+            box,
+            center: { x: box.x + box.width / 2, y: box.y + box.height / 2 },
+        };
+    }
+}

package/dist/page/routes.d.ts

@@ -0,0 +1,40 @@
+import type { Page } from "playwright-core";
+export interface RouteResponse {
+    status?: number;
+    body?: string;
+    contentType?: string;
+    delayMs?: number;
+}
+export interface RouteSpec extends RouteResponse {
+    urlPattern: string;
+    method?: string;
+}
+export interface RouteQueueSpec {
+    urlPattern: string;
+    method?: string;
+    responses: RouteResponse[];
+}
+/** Per-session registry of active route interceptions. Routes are also
+ *  discarded when the session's browser context closes; this exists for
+ *  `unroute` (mid-session clearing) and audit. */
+export declare class RouteRegistry {
+    private routes;
+    /** Single canned response for every matching request. */
+    add(page: Page, spec: RouteSpec): Promise<{
+        key: string;
+    }>;
+    /** Successive matches consume successive responses; once the list is
+     *  exhausted, matches fall through to the real network. */
+    addQueue(page: Page, spec: RouteQueueSpec): Promise<{
+        key: string;
+        queued: number;
+    }>;
+    private removeByKey;
+    /** Remove one route (by urlPattern[+method]) or, when no pattern is given,
+     *  every route this session registered. Returns the removed keys. */
+    remove(page: Page, sel: {
+        urlPattern?: string;
+        method?: string;
+    }): Promise<string[]>;
+    list(): string[];
+}

package/dist/page/routes.js

@@ -0,0 +1,87 @@
+// Scoped network route mocking — capability `action`.
+//
+// Race-condition QA needs to make backend responses arrive in a controlled
+// (often out-of-request) order, or substitute a canned response. Rather than
+// app-specific `eval_js` monkey-patching, this drives Playwright's own
+// `page.route` interception, per-session and torn down with the session.
+//
+// `route` fulfils every match with one canned response (optional delay).
+// `route_queue` fulfils successive matches from a list — each entry with its
+// own delay, so e.g. response #2 (delay 0) can land before response #1
+// (delay 400) — the exact "responses out of request order" failure class.
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const MAX_DELAY_MS = 60_000;
+function keyOf(urlPattern, method) {
+    return `${(method ?? "*").toUpperCase()} ${urlPattern}`;
+}
+async function fulfil(route, r) {
+    if (r.delayMs && r.delayMs > 0)
+        await sleep(Math.min(r.delayMs, MAX_DELAY_MS));
+    await route.fulfill({
+        status: r.status ?? 200,
+        contentType: r.contentType ?? "application/json",
+        body: r.body ?? "",
+    });
+}
+/** Per-session registry of active route interceptions. Routes are also
+ *  discarded when the session's browser context closes; this exists for
+ *  `unroute` (mid-session clearing) and audit. */
+export class RouteRegistry {
+    routes = new Map();
+    /** Single canned response for every matching request. */
+    async add(page, spec) {
+        const key = keyOf(spec.urlPattern, spec.method);
+        await this.removeByKey(page, key); // replace any prior route on the same key
+        const handler = async (route) => {
+            if (spec.method && route.request().method().toUpperCase() !== spec.method.toUpperCase()) {
+                return route.fallback();
+            }
+            await fulfil(route, spec);
+        };
+        await page.route(spec.urlPattern, handler);
+        this.routes.set(key, { url: spec.urlPattern, handler });
+        return { key };
+    }
+    /** Successive matches consume successive responses; once the list is
+     *  exhausted, matches fall through to the real network. */
+    async addQueue(page, spec) {
+        const key = keyOf(spec.urlPattern, spec.method);
+        await this.removeByKey(page, key);
+        let i = 0;
+        const handler = async (route) => {
+            if (spec.method && route.request().method().toUpperCase() !== spec.method.toUpperCase()) {
+                return route.fallback();
+            }
+            const r = spec.responses[i++];
+            if (!r)
+                return route.fallback();
+            await fulfil(route, r);
+        };
+        await page.route(spec.urlPattern, handler);
+        this.routes.set(key, { url: spec.urlPattern, handler });
+        return { key, queued: spec.responses.length };
+    }
+    async removeByKey(page, key) {
+        const e = this.routes.get(key);
+        if (!e)
+            return false;
+        await page.unroute(e.url, e.handler).catch(() => undefined);
+        this.routes.delete(key);
+        return true;
+    }
+    /** Remove one route (by urlPattern[+method]) or, when no pattern is given,
+     *  every route this session registered. Returns the removed keys. */
+    async remove(page, sel) {
+        if (!sel.urlPattern) {
+            const all = [...this.routes.keys()];
+            for (const k of all)
+                await this.removeByKey(page, k);
+            return all;
+        }
+        const key = keyOf(sel.urlPattern, sel.method);
+        return (await this.removeByKey(page, key)) ? [key] : [];
+    }
+    list() {
+        return [...this.routes.keys()];
+    }
+}

package/dist/page/safari-actions.d.ts

@@ -0,0 +1,12 @@
+import type { SafariSessionHandle } from "../engine/index.js";
+import type { RefRegistry } from "./refs.js";
+import type { ActionResult, DispatchedAction } from "./actionresult.js";
+import type { ActionTarget } from "./locator.js";
+/** A clean refusal for an action family outside Safari's curated subset (hover,
+ *  select, scroll, …). Keeps the gating in the adapter rather than the handler,
+ *  so the tool surface stays engine-blind. */
+export declare function safariUnsupportedAction(type: DispatchedAction["type"]): ActionResult;
+export declare function safariNavigate(handle: SafariSessionHandle, url: string): Promise<ActionResult>;
+export declare function safariClick(handle: SafariSessionHandle, refs: RefRegistry, target: ActionTarget): Promise<ActionResult>;
+export declare function safariFill(handle: SafariSessionHandle, refs: RefRegistry, target: ActionTarget, value: string): Promise<ActionResult>;
+export declare function safariPress(handle: SafariSessionHandle, refs: RefRegistry, target: ActionTarget, key: string): Promise<ActionResult>;