browxai 0.7.0

package/dist/page/dom-walk.js

@@ -0,0 +1,267 @@
+// DOM-walk fallback —  .
+//
+// The 2026-05-13 target-app adoption found that `Accessibility.getFullAXTree` returns
+// root-only on heavy SPAs whose markup is mostly `div`s with `data-testid`/`data-type`
+// (legacy-React / Reflux). With nothing in the a11y tree, find() degrades to tier-5
+// `low` and the curated surface is no better than grep. The fix is to walk the DOM
+// directly for interactive / data-attribute-bearing elements when the a11y signal is
+// thin, and combine the two sources into the snapshot tree.
+//
+// Implementation: run a single `Runtime.evaluate` in page context that returns a
+// JSON array of { role, name, testId, testIdAttr, tag, id, structuralPath } for
+// every visible element matching the interactive predicate set OR carrying any of
+// the configured test attributes. Convert to A11yNode (leaf nodes — DOM walk
+// doesn't produce children; the a11y tree is the structural source).
+import { elementKey } from "./refs.js";
+const DEFAULT_TEST_ATTRS = ["data-testid", "data-test", "data-cy", "data-qa"];
+const DEFAULT_MAX = 500;
+/**
+ * Run the in-page DOM-walk and return the discovered entries.
+ *
+ * Notes:
+ *  - The script runs in page context — keep it ECMAScript-only (no TS).
+ *  - Stringified function so we can pass it through `Runtime.evaluate` with
+ *    `returnByValue: true`.
+ *  - Visibility is checked at walk time (`getBoundingClientRect` non-zero + computed
+ *    `visibility !== hidden` + `display !== none`). This intentionally skips offscreen
+ *    elements (they wouldn't be clickable from the agent's POV right now).
+ */
+export async function runDomWalk(cdp, opts = {}) {
+    const testAttrs = opts.testAttributes ?? DEFAULT_TEST_ATTRS;
+    const max = opts.maxEntries ?? DEFAULT_MAX;
+    // Back-compat: `pierce: undefined` preserves pre-v0.5.0 behaviour
+    // (top-document walk only). `pierce: "open"` / `"closed"` opts into the
+    // shadow-aware walk that recurses through every open shadow root we can
+    // see from the page side. Closed shadow roots are platform-inaccessible
+    // here; the CDP path in src/page/shadow.ts covers them and adds the
+    // result via the merge layer.
+    const walkOpen = opts.pierce === "open" || opts.pierce === "closed";
+    const expr = `(${PAGE_SCRIPT})(${JSON.stringify(testAttrs)}, ${max}, ${walkOpen})`;
+    try {
+        const { result } = (await cdp.send("Runtime.evaluate", {
+            expression: expr,
+            returnByValue: true,
+            awaitPromise: false,
+        }));
+        return result.value ?? [];
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Frame-scoped DOM walk. Same `PAGE_SCRIPT`, but evaluated inside
+ * a Playwright `Frame` via `frame.evaluate(...)` instead of the top-level
+ * CDP `Runtime.evaluate`. Works transparently for both same-origin and
+ * cross-origin (OOPIF) child frames — Playwright's frame API spans both.
+ *
+ * Honours `pierce` the same way the top-level walk does: `"open"` / `"closed"`
+ * recurses into reachable open shadow roots inside the frame; `false` /
+ * undefined sticks to the frame's top document. Closed-shadow CDP harvesting
+ * is not run for child frames (the CDP path is rooted at the top target).
+ */
+export async function runDomWalkOnFrame(frame, opts = {}) {
+    const testAttrs = opts.testAttributes ?? DEFAULT_TEST_ATTRS;
+    const max = opts.maxEntries ?? DEFAULT_MAX;
+    const walkOpen = opts.pierce === "open" || opts.pierce === "closed";
+    try {
+        const raw = await frame.evaluate(({ script, attrs, cap, openShadow, }) => {
+            // The page-side `Function` constructor produces an untyped callable;
+            // annotate its precise call signature (the PAGE_SCRIPT IIFE returns
+            // `DomWalkEntry[]`) so the invocation result is typed, not `any`.
+            // eslint-disable-next-line @typescript-eslint/no-implied-eval
+            const run = new Function("attrs", "cap", "openShadow", `return (${script})(attrs, cap, openShadow)`);
+            return run(attrs, cap, openShadow);
+        }, { script: PAGE_SCRIPT, attrs: testAttrs, cap: max, openShadow: walkOpen });
+        return raw ?? [];
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Transport-agnostic DOM walk. Runs the SAME `PAGE_SCRIPT` via an injected
+ * `exec` that takes a function BODY + an args array and returns the value —
+ * exactly the WebDriver-Classic `execute/sync` shape (`{script, args}`). This is
+ * the seam the Safari snapshot substrate uses: Safari has neither
+ * CDP nor a Playwright Frame, but `safaridriver`'s `execute/sync` runs the script
+ * identically (the returned `DomWalkEntry` shape matches
+ * `frame.evaluate` byte-for-byte).
+ * `PAGE_SCRIPT` stays encapsulated here; callers pass only the transport.
+ */
+export async function runDomWalkViaExecute(exec, opts = {}) {
+    const testAttrs = opts.testAttributes ?? DEFAULT_TEST_ATTRS;
+    const max = opts.maxEntries ?? DEFAULT_MAX;
+    const walkOpen = opts.pierce === "open" || opts.pierce === "closed";
+    try {
+        const raw = await exec(`return (${PAGE_SCRIPT})(arguments[0], arguments[1], arguments[2])`, [
+            testAttrs,
+            max,
+            walkOpen,
+        ]);
+        return raw ?? [];
+    }
+    catch {
+        return [];
+    }
+}
+/** Build the page-side script. Returned as a stringified IIFE.
+ *  `walkOpenShadow` — when true, the DOM-walk additionally recurses into every
+ *  `Element.shadowRoot` (open mode) and runs the same predicate-match on its
+ *  descendants. Closed shadow roots are platform-protected and unreachable from
+ *  the page side; the CDP path in `src/page/shadow.ts` covers them. */
+const PAGE_SCRIPT = `function(testAttrs, max, walkOpenShadow) {
+  var ATTR_INTERACTIVE_SEL = '[role],button,a[href],input,select,textarea,[onclick],[tabindex],[contenteditable="true"]';
+  var attrSel = testAttrs.map(function(a){ return '['+a+']'; }).join(',');
+  var sel = ATTR_INTERACTIVE_SEL + (attrSel ? ',' + attrSel : '');
+  // Collect matches from the top document AND, when walkOpenShadow is set,
+  // from every open shadow root we can reach. querySelectorAll does NOT
+  // pierce shadow boundaries by web-platform design, so we walk shadow
+  // roots explicitly.
+  var els = Array.prototype.slice.call(document.querySelectorAll(sel));
+  if (walkOpenShadow) {
+    var shadowHosts = [];
+    function collectHosts(root) {
+      var all = root.querySelectorAll('*');
+      for (var i = 0; i < all.length; i++) {
+        var sr = all[i].shadowRoot;
+        if (sr) shadowHosts.push(sr);
+      }
+    }
+    collectHosts(document);
+    var seen = 0;
+    while (shadowHosts.length > seen) {
+      var sr2 = shadowHosts[seen++];
+      var matches = sr2.querySelectorAll(sel);
+      for (var j = 0; j < matches.length; j++) els.push(matches[j]);
+      collectHosts(sr2);
+      // Hard bound — a pathological page could embed thousands of nested
+      // shadow roots; cap the walk to keep snapshot latency predictable.
+      if (seen > 500) break;
+    }
+  }
+
+  function isVisible(el) {
+    if (!el.isConnected) return false;
+    var r = el.getBoundingClientRect();
+    if (r.width === 0 && r.height === 0) return false;
+    var cs = getComputedStyle(el);
+    return cs.visibility !== 'hidden' && cs.display !== 'none';
+  }
+  function structuralPath(el) {
+    var parts = [];
+    var n = el;
+    while (n && n.nodeType === 1 && n.tagName !== 'HTML') {
+      var tag = n.tagName.toLowerCase();
+      var role = n.getAttribute('role') || '';
+      var parent = n.parentElement;
+      var idx = parent ? Array.prototype.indexOf.call(parent.children, n) : 0;
+      parts.unshift(tag + (role ? '@' + role : '') + '[' + idx + ']');
+      n = parent;
+    }
+    return parts.join('/');
+  }
+  function cssPath(el) {
+    var parts = [];
+    var n = el;
+    while (n && n.nodeType === 1 && n.tagName !== 'HTML') {
+      var tag = n.tagName.toLowerCase();
+      var parent = n.parentElement;
+      if (!parent) { parts.unshift(tag); break; }
+      var idx = Array.prototype.indexOf.call(parent.children, n) + 1;
+      parts.unshift(tag + ':nth-child(' + idx + ')');
+      n = parent;
+    }
+    return parts.join(' > ');
+  }
+  function nameFor(el) {
+    var aria = el.getAttribute('aria-label');
+    if (aria) return aria.trim().slice(0, 120);
+    var lid = el.getAttribute('aria-labelledby');
+    if (lid) {
+      var parts = lid.split(/\\s+/).map(function(id){ var n = document.getElementById(id); return n ? (n.textContent || '') : ''; });
+      var s = parts.join(' ').trim();
+      if (s) return s.slice(0, 120);
+    }
+    if (el.tagName === 'INPUT') {
+      if (el.placeholder) return el.placeholder.trim().slice(0, 120);
+      if (el.value && el.type !== 'password') return String(el.value).trim().slice(0, 120);
+    }
+    var text = (el.textContent || '').replace(/\\s+/g, ' ').trim();
+    if (text && text.length <= 120) return text;
+    // title attribute as last-resort label source. Icon-only buttons
+    // commonly carry their visible label here when neither aria-label nor
+    // textContent (the icon's empty span) is set.
+    var title = el.getAttribute('title');
+    if (title) return title.trim().slice(0, 120);
+    return '';
+  }
+  function testIdFor(el) {
+    for (var i = 0; i < testAttrs.length; i++) {
+      var a = testAttrs[i];
+      var v = el.getAttribute(a);
+      if (v) return { attr: a, value: v };
+    }
+    return null;
+  }
+  var out = [];
+  for (var i = 0; i < els.length && out.length < max; i++) {
+    var el = els[i];
+    if (!isVisible(el)) continue;
+    var tag = el.tagName.toLowerCase();
+    var role = el.getAttribute('role') || tag;
+    var tid = testIdFor(el);
+    out.push({
+      role: role,
+      name: nameFor(el),
+      testId: tid ? tid.value : '',
+      testIdAttr: tid ? tid.attr : '',
+      tag: tag,
+      id: el.id || '',
+      structuralPath: structuralPath(el),
+      cssPath: cssPath(el)
+    });
+  }
+  return out;
+}`;
+export function mergeDomWalkIntoTree(root, entries, refs, opts = {}) {
+    let added = 0;
+    let combined = 0;
+    const { frameId, frame } = opts;
+    for (const e of entries) {
+        const name = e.name || undefined;
+        const testId = e.testId || undefined;
+        const testIdAttr = e.testIdAttr || undefined;
+        const key = elementKey({ role: e.role, name, path: e.structuralPath, testId, frameId });
+        const wasNew = !refs.hasKey(key);
+        const ref = refs.forKey(key);
+        refs.augmentLocator(ref, {
+            role: e.role,
+            name,
+            testId,
+            testIdAttr,
+            cssPath: e.cssPath,
+            source: wasNew ? "dom" : "both",
+            ...(frameId ? { frameId } : {}),
+        });
+        if (frame)
+            refs.bindFrame(ref, frame);
+        const node = {
+            ref,
+            role: e.role,
+            name,
+            testId,
+            testIdAttr,
+            tag: e.tag,
+            id: e.id || undefined,
+            source: wasNew ? "dom" : "both",
+            children: [],
+        };
+        root.children.push(node);
+        if (wasNew)
+            added++;
+        else
+            combined++;
+    }
+    return { added, combined };
+}

package/dist/page/dom_diff.d.ts

@@ -0,0 +1,48 @@
+import type { Page } from "playwright-core";
+export interface DomNode {
+    tag: string;
+    testId?: string;
+    classes: string;
+    style: string;
+    attrs: Record<string, string>;
+}
+/** keyed by a structural index path from the scope root, e.g. "0/2/1". */
+export type DomMap = Record<string, DomNode>;
+export declare function captureDomMap(page: Page, scopeSelector?: string): Promise<DomMap | null>;
+export interface DomChange {
+    path: string;
+    tag: string;
+    testId?: string;
+    classDelta?: {
+        added: string[];
+        removed: string[];
+    };
+    styleDelta?: {
+        before: string;
+        after: string;
+    };
+    attrDelta?: Record<string, {
+        before?: string;
+        after?: string;
+    }>;
+}
+export interface DomDiff {
+    changed: DomChange[];
+    added: Array<{
+        path: string;
+        tag: string;
+        testId?: string;
+    }>;
+    removed: Array<{
+        path: string;
+        tag: string;
+        testId?: string;
+    }>;
+    counts: {
+        changed: number;
+        added: number;
+        removed: number;
+    };
+    note?: string;
+}
+export declare function diffDomMaps(before: DomMap | null, after: DomMap | null): DomDiff;

package/dist/page/dom_diff.js

@@ -0,0 +1,121 @@
+// Scoped DOM diff around an action.
+//
+// For selection-heavy UIs (timeline editors, kanban, custom toggles) the
+// state change an agent must verify — "which clip became selected" — is
+// expressed only as class / `aria-*` / `data-*` / inline-style changes, not
+// visible text or accessibility-tree changes that snapshot/find would catch.
+// `act_and_diff` captures a structural DOM map before and after one action
+// and reports exactly which elements changed which of those fields.
+const MAX_NODES = 3000;
+// Fixed in-page snapshot — no agent JS. Walks the scope subtree, recording
+// each element's class/style and only its aria-*/data-* attributes (the
+// fields selection state hides in), keyed by structural index path.
+//
+// `page.evaluate(string)` treats the string as an *expression* (a
+// `function(arg){…}` string is never called, args ignored) — so this is an
+// arg-less IIFE with the scope selector interpolated as a JSON string literal.
+function buildSnapScript(scopeSelector) {
+    return `(() => {
+  var scopeSel = ${JSON.stringify(scopeSelector)};
+  var root = scopeSel ? document.querySelector(scopeSel) : document.body;
+  if (!root) return null;
+  var out = {};
+  var count = 0;
+  function walk(el, path) {
+    if (count >= ${MAX_NODES}) return;
+    count++;
+    var attrs = {};
+    for (var i = 0; i < el.attributes.length; i++) {
+      var a = el.attributes[i];
+      if (a.name.indexOf('aria-') === 0 || a.name.indexOf('data-') === 0) attrs[a.name] = a.value;
+    }
+    out[path] = {
+      tag: el.tagName.toLowerCase(),
+      testId: el.getAttribute('data-testid') || undefined,
+      classes: el.getAttribute('class') || '',
+      style: el.getAttribute('style') || '',
+      attrs: attrs,
+    };
+    var k = 0;
+    for (var c = 0; c < el.children.length; c++) walk(el.children[c], path + '/' + (k++));
+  }
+  walk(root, '0');
+  return out;
+})()`;
+}
+export async function captureDomMap(page, scopeSelector) {
+    return await page.evaluate(buildSnapScript(scopeSelector ?? null));
+}
+function tokens(s) {
+    return new Set(s.split(/\s+/).filter(Boolean));
+}
+/** Pure structural diff of two DOM maps. Exported for unit tests. */
+/** Compute the per-attribute delta between two attribute maps, or null when
+ *  nothing changed. */
+function attrDeltaOf(b, a) {
+    const delta = {};
+    for (const k of new Set([...Object.keys(b), ...Object.keys(a)])) {
+        if (b[k] !== a[k])
+            delta[k] = { before: b[k], after: a[k] };
+    }
+    return Object.keys(delta).length ? delta : null;
+}
+/** Diff one node (present before AND after) into a `DomChange`, or null when it
+ *  is unchanged across class / style / attribute dimensions. */
+function diffNode(path, b, a) {
+    const change = { path, tag: a.tag, ...(a.testId ? { testId: a.testId } : {}) };
+    let dirty = false;
+    if (b.classes !== a.classes) {
+        const bt = tokens(b.classes);
+        const at = tokens(a.classes);
+        change.classDelta = {
+            added: [...at].filter((t) => !bt.has(t)),
+            removed: [...bt].filter((t) => !at.has(t)),
+        };
+        dirty = true;
+    }
+    if (b.style !== a.style) {
+        change.styleDelta = { before: b.style, after: a.style };
+        dirty = true;
+    }
+    const attrDelta = attrDeltaOf(b.attrs, a.attrs);
+    if (attrDelta) {
+        change.attrDelta = attrDelta;
+        dirty = true;
+    }
+    return dirty ? change : null;
+}
+export function diffDomMaps(before, after) {
+    const changed = [];
+    const added = [];
+    const removed = [];
+    if (!before || !after) {
+        return {
+            changed,
+            added,
+            removed,
+            counts: { changed: 0, added: 0, removed: 0 },
+            note: "scope did not resolve before and/or after the action — pass a `scope` selector that exists across the transition",
+        };
+    }
+    for (const [path, b] of Object.entries(before)) {
+        const a = after[path];
+        if (!a) {
+            removed.push({ path, tag: b.tag, ...(b.testId ? { testId: b.testId } : {}) });
+            continue;
+        }
+        const change = diffNode(path, b, a);
+        if (change)
+            changed.push(change);
+    }
+    for (const [path, a] of Object.entries(after)) {
+        if (!before[path])
+            added.push({ path, tag: a.tag, ...(a.testId ? { testId: a.testId } : {}) });
+    }
+    return {
+        changed,
+        added,
+        removed,
+        counts: { changed: changed.length, added: added.length, removed: removed.length },
+    };
+}

package/dist/page/downloads.d.ts

@@ -0,0 +1,80 @@
+import type { BrowserContext } from "playwright-core";
+/** What lands on `ActionResult.downloads[]` and `download_get`. Workspace-rooted
+ *  paths only; never absolute outside `$BROWX_WORKSPACE`. */
+export interface CapturedDownload {
+    /** session-local monotonic id (`d1`, `d2`, …) — pass back to `download_get`. */
+    id: string;
+    /** filename the page suggested (sanitised; raw value kept too). */
+    suggestedFilename: string;
+    /** raw page-supplied filename pre-sanitisation, only present when sanitisation
+     *  changed it. Useful when an agent wants to know the original (e.g. routing
+     *  decisions) but the on-disk name diverged. */
+    rawSuggestedFilename?: string;
+    /** best-effort MIME type. Playwright doesn't expose the HTTP `Content-Type`
+     *  on `Download`, so this is filename-extension-inferred. May be undefined. */
+    mimeType?: string;
+    /** size of the persisted file in bytes. 0 if the file vanished before we
+     *  could stat it (best-effort). */
+    sizeBytes: number;
+    /** absolute on-disk path, ALWAYS rooted under `$BROWX_WORKSPACE/.downloads/`. */
+    path: string;
+    /** epoch-ms when the download fired. */
+    capturedAt: number;
+}
+/** Per-session download registry. One instance per SessionEntry. */
+export declare class DownloadsRegistry {
+    /** Per-session storage dir: `$BROWX_WORKSPACE/.downloads/<sessionId>/`. */
+    readonly storageDir: string;
+    /** Toggled by the `downloads_capture` MCP tool. */
+    captureOn: boolean;
+    /** Active captures, keyed by id. Bounded to MAX_ENTRIES — oldest evicted. */
+    private entries;
+    private nextId;
+    /** Captures that have fired and are not yet sliced into an ActionResult. */
+    private pendingSince;
+    /** Max captures kept in memory + on disk per session before LRU-evicting
+     *  the oldest. Prevents an unbounded download loop from filling the disk. */
+    private static MAX_ENTRIES;
+    constructor(
+    /** Per-session storage dir: `$BROWX_WORKSPACE/.downloads/<sessionId>/`. */
+    storageDir: string);
+    /** List all captured downloads for this session (most-recent first). */
+    list(): CapturedDownload[];
+    /** Look up a capture by id; undefined if not present. */
+    get(id: string): CapturedDownload | undefined;
+    /** Slice captures that fired after `tsMs` (action-window slice). Returns a
+     *  snapshot of the captures so the action-window can include them on the
+     *  ActionResult without exposing the live registry. */
+    since(tsMs: number): CapturedDownload[];
+    /** Record a capture. Caller has already persisted the file at `path`. */
+    record(record: Omit<CapturedDownload, "id">): CapturedDownload;
+}
+/** Sanitise a page-supplied filename for safe on-disk use. Rules (mirrors the
+ *  workspace-escape posture in `upload.ts`):
+ *    - strip path separators (`/`, `\`) and NUL/control bytes — collapses any
+ *      traversal attempt to a flat filename.
+ *    - reject leading dots so we never write to a hidden `.foo` file.
+ *    - cap at 200 chars (leaves room for the `<id>-` prefix on filesystems
+ *      with 255-byte name limits).
+ *    - empty / all-stripped → fall back to `"download"`.
+ *  Exported for unit tests. */
+export declare function sanitiseFilename(raw: string): string;
+/** Best-effort MIME type from a filename extension. Tiny built-in table; this
+ *  is metadata only (we never reject on it), so an unknown extension just
+ *  yields `undefined`. */
+export declare function mimeTypeFromName(name: string): string | undefined;
+/** Attach the Playwright `download` listener to a context. The listener fires
+ *  for every download on every page in the context (now or later). When
+ *  capture is OFF the artifact is silently deleted; when ON it's persisted and
+ *  the registry records it. Errors during capture never propagate — they
+ *  surface as warnings on the session's log only. */
+export declare function attachDownloadCapture(context: BrowserContext, registry: DownloadsRegistry): void;
+/** Read a captured download's bytes. Returns base64. Throws if the id is
+ *  unknown or the file vanished. */
+export declare function readCapturedBytes(reg: DownloadsRegistry, id: string): {
+    base64: string;
+    bytes: number;
+    path: string;
+    mimeType?: string;
+    suggestedFilename: string;
+};