browxai 0.7.0

package/dist/page/set-of-marks.js

@@ -0,0 +1,245 @@
+// set-of-marks — composed PNG with numbered bounding boxes painted over
+// caller-supplied candidates. Pure compose over existing primitives (find /
+// snapshot / refs / bbox); no new browser interaction beyond a transient
+// in-page overlay.
+//
+// Numbering shares the `name_ref` / `eN` namespace deliberately. When
+// `label:"ref"`, each box renders the existing `eN` ref; when `label:"index"`
+// (the default), the box renders the 1..N array position AND the result also
+// carries an `{ index → ref }` mapping so the LLM can address the same
+// element either way. We do not invent a parallel ID space.
+//
+// Image-library choice: an in-page overlay drawn via DOM + an absolute-positioned
+// container, then `page.screenshot()`. browxai has no Node-side image library
+// (sharp / canvas / jimp) in `dependencies`, and adding one would pull native
+// bindings + ~MBs of install weight for a single drawing primitive. The
+// in-page overlay is dependency-free, runs in the same coordinate space as
+// `find().evidence.bbox` (CSS pixels, viewport-relative — exactly the rect
+// `visibleRect` returns), and is removed before we return so it never leaks
+// state into a follow-up read.
+import { buildSelectorHint } from "./find.js";
+import { visibleRect, locatorBoundingBox } from "./bbox.js";
+import { walk } from "./a11y.js";
+/** Type guard: does this candidate already carry a bbox (i.e. is it the
+ *  full `FindCandidate`-shaped object)? */
+function hasBbox(c) {
+    return Object.prototype.hasOwnProperty.call(c, "bbox");
+}
+/** Build the ref→node lookup by walking the composed tree once, so each bare-ref
+ *  candidate's `backendDOMNodeId` resolves exactly as `find()` would. Returns
+ *  null + pushes a warning on compose failure. */
+async function buildRefLookup(deps, warnings) {
+    try {
+        const { tree } = await deps.substrate.compose(deps.refs, deps.testAttributes);
+        if (!tree)
+            return null;
+        const m = new Map();
+        for (const { node } of walk(tree)) {
+            if (!m.has(node.ref))
+                m.set(node.ref, node);
+        }
+        return m;
+    }
+    catch (err) {
+        warnings.push(`set-of-marks: bbox lookup for bare ref candidates failed (${err instanceof Error ? err.message : String(err)}); ` +
+            `they will be reported without bboxes. Pass the full find() candidate to avoid the lookup.`);
+        return null;
+    }
+}
+/** Build a mark entry from a node (carrying its optional role/name/testId). */
+function markEntryFrom(index, node, bbox) {
+    return {
+        index,
+        ref: node.ref,
+        ...(node.role !== undefined ? { role: node.role } : {}),
+        ...(node.name !== undefined ? { name: node.name } : {}),
+        ...(node.testId !== undefined ? { testId: node.testId } : {}),
+        bbox,
+        painted: bbox !== null,
+    };
+}
+/** Resolve one bare-ref candidate's bbox the same way `find()` does (CDP
+ *  visibleRect → portable locatorBoundingBox fallback, capped at 1s). */
+async function resolveBareRef(deps, index, ref, lookupByRef, warnings) {
+    const looked = lookupByRef?.get(ref);
+    if (!looked) {
+        warnings.push(`set-of-marks: ref "${ref}" was not surfaced by the current snapshot walk — ` +
+            `no bbox to paint. Pass the full find() candidate (with bbox) or call snapshot/find first.`);
+        return { index, ref, bbox: null, painted: false };
+    }
+    let bbox = deps.cdp !== undefined && looked.backendDOMNodeId !== undefined
+        ? await visibleRect(deps.cdp, looked.backendDOMNodeId)
+        : null;
+    if (bbox === null) {
+        const { hint } = buildSelectorHint(looked);
+        bbox = await locatorBoundingBox(deps.page, hint, { timeoutMs: 1000 });
+    }
+    return markEntryFrom(index, looked, bbox);
+}
+export async function resolveCandidates(deps, candidates) {
+    const warnings = [];
+    const entries = [];
+    // Fast-path: when every candidate already carries a bbox (e.g. piped straight
+    // from a prior find()), skip the tree walk entirely.
+    const lookupByRef = candidates.every(hasBbox) ? null : await buildRefLookup(deps, warnings);
+    for (let i = 0; i < candidates.length; i++) {
+        const c = candidates[i];
+        if (hasBbox(c)) {
+            entries.push(markEntryFrom(i + 1, c, c.bbox));
+        }
+        else {
+            entries.push(await resolveBareRef(deps, i + 1, c.ref, lookupByRef, warnings));
+        }
+    }
+    return { entries, warnings };
+}
+/** Pure label-builder. Exported for unit-test direct coverage. */
+export function labelFor(entry, mode) {
+    switch (mode) {
+        case "ref":
+            return entry.ref;
+        case "role":
+            return entry.role ?? entry.ref;
+        case "index":
+        default:
+            return String(entry.index);
+    }
+}
+/** Build the in-page overlay-painter script. Inlines the painted boxes
+ *  (already resolved server-side) — page-side does no DOM walking, only
+ *  paints absolute-positioned <div>s + a label badge per box, returning the
+ *  element id of the overlay container so we can remove it after. */
+function buildOverlayScript(paintedBoxes) {
+    const data = JSON.stringify(paintedBoxes);
+    return `(() => {
+    var BOXES = ${data};
+    var OVERLAY_ID = 'browxai-set-of-marks-' + Math.random().toString(36).slice(2);
+    var root = document.createElement('div');
+    root.id = OVERLAY_ID;
+    root.setAttribute('aria-hidden', 'true');
+    root.style.cssText = [
+      'position:fixed',
+      'left:0','top:0','right:0','bottom:0',
+      'pointer-events:none',
+      'z-index:2147483647',
+      'margin:0','padding:0',
+      'background:transparent',
+    ].join(';');
+    BOXES.forEach(function (b) {
+      var box = document.createElement('div');
+      box.style.cssText = [
+        'position:absolute',
+        'left:' + b.x + 'px','top:' + b.y + 'px',
+        'width:' + b.width + 'px','height:' + b.height + 'px',
+        'box-sizing:border-box',
+        'border:2px solid #ff0066',
+        'background:rgba(255,0,102,0.08)',
+        'border-radius:2px',
+      ].join(';');
+      var badge = document.createElement('div');
+      badge.textContent = b.label;
+      // Default badge corner: top-left of the box, pulled outward so it
+      // doesn't occlude the box's contents. When the box hugs an edge of the
+      // viewport, flip the badge to the box's inside corner so it stays in
+      // frame for the screenshot.
+      var anchorTop = b.y >= 22;
+      var anchorLeft = b.x >= 24;
+      var tx = anchorLeft ? '-100%' : '100%';
+      var ty = anchorTop ? '-100%' : '100%';
+      var styles = [
+        'position:absolute',
+        anchorTop ? 'top:-2px' : 'bottom:-2px',
+        anchorLeft ? 'left:-2px' : 'right:-2px',
+        'transform:translate(' + tx + ',' + ty + ')',
+        'min-width:18px','height:18px','line-height:18px',
+        'padding:0 5px','box-sizing:border-box',
+        'background:#ff0066','color:#fff',
+        'font:bold 12px/18px ui-sans-serif,system-ui,-apple-system,sans-serif',
+        'text-align:center','border-radius:9px',
+        'box-shadow:0 1px 2px rgba(0,0,0,0.35)',
+        'white-space:nowrap',
+      ];
+      badge.style.cssText = styles.join(';');
+      box.appendChild(badge);
+      root.appendChild(box);
+    });
+    document.documentElement.appendChild(root);
+    return OVERLAY_ID;
+  })()`;
+}
+/** Remove the overlay element previously installed by buildOverlayScript. */
+function buildRemoveScript(overlayId) {
+    return `(() => {
+    var el = document.getElementById(${JSON.stringify(overlayId)});
+    if (el && el.parentNode) el.parentNode.removeChild(el);
+    return true;
+  })()`;
+}
+/**
+ * Compose a single PNG screenshot of the current viewport with numbered
+ * bounding boxes painted over the supplied candidates. The numbering scheme
+ * shares the existing `eN` ref namespace (index↔ref mapping returned).
+ *
+ * `bbox` is honoured exactly as `find()` reported it — visible-rect with
+ * ancestor-overflow + viewport intersection applied (see `src/page/bbox.ts`).
+ * A candidate with `bbox: null` is **not** painted; it's listed in `marks`
+ * with `painted: false` and a `warnings` entry, so the caller knows the
+ * mapping still resolves but the box wasn't visible.
+ *
+ * The overlay is installed for the duration of the screenshot only and
+ * removed before this function returns. Failures during removal are
+ * best-effort (we still return the image + the warning).
+ */
+export async function screenshotMarks(page, substrate, refs, opts, 
+/** CDP handle for the visible-rect bbox fast path — chromium only. */
+cdp) {
+    const testAttributes = opts.testAttributes ?? [];
+    const label = opts.label ?? "index";
+    const { entries, warnings } = await resolveCandidates({ page, substrate, refs, testAttributes, cdp }, opts.candidates);
+    // Only paint entries that have a bbox to paint.
+    const paintedBoxes = entries
+        .filter((e) => e.bbox !== null)
+        .map((e) => ({
+        x: e.bbox.x,
+        y: e.bbox.y,
+        width: e.bbox.width,
+        height: e.bbox.height,
+        label: labelFor(e, label),
+    }));
+    let overlayId = null;
+    let imageBase64 = "";
+    try {
+        if (paintedBoxes.length > 0) {
+            overlayId = await page.evaluate(buildOverlayScript(paintedBoxes));
+        }
+        const buf = await page.screenshot({ type: "png", fullPage: false });
+        imageBase64 = Buffer.from(buf).toString("base64");
+    }
+    finally {
+        if (overlayId) {
+            try {
+                await page.evaluate(buildRemoveScript(overlayId));
+            }
+            catch {
+                warnings.push(`set-of-marks: overlay removal failed; a stray <div id="${overlayId}"> may persist until the next navigation.`);
+            }
+        }
+    }
+    // Tally un-paintable candidates as warnings (one terse line, not one per
+    // entry — the per-entry `painted:false` already encodes it).
+    const skipped = entries.filter((e) => !e.painted).length;
+    if (skipped > 0) {
+        warnings.push(`set-of-marks: ${skipped} of ${entries.length} candidate(s) had no bbox (clipped / off-screen / unresolved) and were not painted on the image. ` +
+            `Their entries remain in \`marks\` with \`painted:false\` so the index↔ref mapping is still complete.`);
+    }
+    const mapping = {};
+    for (const e of entries)
+        mapping[String(e.index)] = e.ref;
+    return {
+        imageBase64,
+        mimeType: "image/png",
+        marks: entries,
+        mapping,
+        warnings,
+    };
+}

package/dist/page/shadow.d.ts

@@ -0,0 +1,136 @@
+import type { CDPSession } from "playwright-core";
+/** One shadow-tree-entry surfaced by `shadow_trees`. */
+export interface ShadowTreeEntry {
+    /** The host element's `ref` (from a prior `snapshot` / `find`) when the
+     *  registry knows it; otherwise the CDP `backendNodeId` rendered as
+     *  `"backend:<n>"` so an agent can correlate against `inspect()` output. */
+    hostRef: string;
+    /** Host element tag (lowercased). */
+    hostTag: string;
+    /** Shadow root mode as CDP reports it. */
+    mode: "open" | "closed";
+    /** Direct children of the shadow root (depth-1 summary — keeps the output
+     *  token-bounded). Use `dom_export({includeShadow:true})` for the full
+     *  recursive dump. */
+    children: ShadowChildSummary[];
+    /** Total descendant element count under the shadow root, including nested
+     *  shadow subtrees we descended into. Cheap heuristic for "is this a
+     *  thin wrapper or a whole sub-app?" */
+    descendantCount: number;
+}
+export interface ShadowChildSummary {
+    tag: string;
+    /** Short text label — first non-empty `textContent`-equivalent slice
+     *  (capped at 80 chars). Null when the element carries no direct text. */
+    text?: string;
+    /** Element child count of *this* shadow child (one level deep — same
+     *  shape as `Element.childElementCount`). Lets the caller decide whether
+     *  to drill in via `dom_export`. */
+    childCount: number;
+}
+export interface ShadowTreesOptions {
+    /** Backend node id of the host to limit the walk to. When omitted, walks
+     *  the entire document and returns every shadow root it finds. */
+    rootBackendNodeId?: number;
+    /** Hard cap on returned hosts. Default 200 — enough for typical
+     *  shadow-heavy pages, bounded enough to keep tokens predictable. */
+    maxHosts?: number;
+}
+export interface ShadowTreesResult {
+    trees: ShadowTreeEntry[];
+    /** `true` when CDP's `pierce:true` view of the document came back with
+     *  at least one closed-mode root (proves the closed-shadow path is live
+     *  on this browser/page). `false` is informational — the page may simply
+     *  not contain a closed root, or CDP refused the call. */
+    closedShadowAvailable: boolean;
+    /** Non-fatal warnings — e.g. CDP fell through to open-only, the host
+     *  ref didn't resolve, we hit the `maxHosts` cap. */
+    warnings: string[];
+}
+/** CDP `DOM.Node` subset we care about. Fields are optional — different
+ *  Chromium versions populate different ones. */
+interface CdpDomNode {
+    nodeId?: number;
+    backendNodeId?: number;
+    nodeType?: number;
+    nodeName?: string;
+    localName?: string;
+    attributes?: string[];
+    children?: CdpDomNode[];
+    shadowRoots?: CdpDomNode[];
+    shadowRootType?: "open" | "closed" | "user-agent";
+    contentDocument?: CdpDomNode;
+    textValue?: string;
+    nodeValue?: string;
+}
+/**
+ * Read the full pierced DOM. Returns the root node and a "closed shadow root
+ * exists somewhere" flag. Falls back to an open-only document when CDP refuses
+ * the `pierce` parameter (older Chromium, attached BYOB endpoints whose CDP
+ * vintage differs from the launcher's). Caller is responsible for treating a
+ * `null` result as "shadow walk unavailable" — no exception is thrown so the
+ * outer tool can still emit a partial result.
+ */
+export declare function fetchPiercedDocument(cdp: CDPSession): Promise<{
+    root: CdpDomNode | null;
+    closedAvailable: boolean;
+    warning?: string;
+}>;
+/**
+ * Walk a CDP-pierced document and collect shadow-root entries. When
+ * `rootBackendNodeId` is set, only the subtree rooted at that backend id
+ * contributes. When omitted, the entire document is walked.
+ */
+export declare function collectShadowTrees(root: CdpDomNode, opts: ShadowTreesOptions): {
+    entries: ShadowTreeEntry[];
+    cappedAt?: number;
+};
+/** Page-side counterpart for `shadow_trees` when only open shadow piercing
+ *  is needed. Cheaper than the CDP `pierce:true` path (no full-document
+ *  serialisation) and entirely Chromium-version-agnostic. The `hostRef`
+ *  field is filled in by the caller (the registry is server-side). */
+export declare function runOpenShadowWalk(cdp: CDPSession, rootSelector: string | undefined, max: number): Promise<Array<Omit<ShadowTreeEntry, "hostRef">>>;
+/**
+ * A pseudo-DOM-walk entry for an interactive element discovered inside a
+ * CLOSED shadow root via the CDP pierce path. Same shape as the open-side
+ * `DomWalkEntry` so it can flow through `mergeDomWalkIntoTree` without a
+ * second path. Locator fields (`cssPath`) are best-effort — the host's
+ * `data-browx-host` data-id-style attribute would let us address closed-
+ * shadow elements deterministically, but the live page rarely carries
+ * such markers. We surface `tag` + (when present) any test attribute we
+ * recognise; agents can act on these via `find()` on the test-attr value
+ * (`[data-testid="…"]:visible` works through Playwright's normal
+ * locator engine because Playwright auto-pierces open shadow only —
+ * closed-shadow elements remain platform-protected and CANNOT be acted on
+ * by the action tools, only inspected). The result envelope warns when
+ * a closed-shadow candidate is returned.
+ */
+export interface ClosedShadowDomEntry {
+    role: string;
+    name: string;
+    testId: string;
+    testIdAttr: string;
+    tag: string;
+    id: string;
+    structuralPath: string;
+    cssPath: string;
+    /** Always `true` for these entries — drives the snapshot warning. */
+    closedShadow: true;
+}
+/**
+ * Harvest interactive / test-attr-bearing elements from CLOSED shadow roots
+ * only. Open-shadow elements are skipped (the page-side `runDomWalk` already
+ * covers them when `pierce: "open"` or `"closed"`). Returns an empty list +
+ * no warning when CDP's `pierce` path is unavailable.
+ *
+ * The walker reads the CDP-pierced DOM tree and identifies closed-mode
+ * shadow roots; for each, it harvests element-shaped descendants and
+ * extracts the same fields the page-side walker would. Doesn't fetch
+ * computed style / `getBoundingClientRect` — closed shadow can't be
+ * resolved to a Playwright locator anyway, so a bbox would be a lie.
+ */
+export declare function harvestClosedShadowElements(cdp: CDPSession, testAttrs: string[], maxEntries: number): Promise<{
+    entries: ClosedShadowDomEntry[];
+    warning?: string;
+}>;
+export {};