@volter-ai-dev/supercode-browser-playwright 0.1.1 → 0.2.0

package/README.md

@@ -58,7 +58,8 @@ Both carry the same envelope, `{protocol, id, token, call}` in and
 
 - **TCP** is the wire the harness speaks (`crates/harness/src/browser.rs`): one
   newline-terminated JSON request per connection, one JSON response, then the
-  provider closes. The provider listens on `127.0.0.1` and writes an owner-only
+  provider closes. A provider waiting for the person may first write
+  `{protocol, id, pending}` lines; each extends the harness's wait to 120 s. The provider listens on `127.0.0.1` and writes an owner-only
   discovery record (file `0600`, directory `0700`) to
   `<providers dir>/browser/playwright.cdp.<port>.json`, scoped to the
   canonical workspace, and removes it on stop.
@@ -129,15 +130,37 @@ Two entry points use no Node API:
   `@volter/almostcdp/playwright`, and answers the operations against an
   AlmostCDP surface with no Node process.
 
-A host's `actionGuard(action)` is asked before each locator action (`click`,
-`fill`, `press`, `hover`, `focus`, `check`, `uncheck`, `select`, `drag`) with
-the action's `Locator` and value, and before each `browser.script` with
-`{action: "script", source, args}`; throwing a `BrowserActionRefusal` answers
-the call with its code, such as `APPROVAL_REQUIRED`. A script's own locator
-calls are the real `page`'s and are not asked again, so a host that guards
-locator actions guards scripts as a whole.
+A host's `actionGuard(action)` is asked before each element action and before
+each `browser.script` (`{action: "script", source, args}`); throwing a
+`BrowserActionRefusal` answers the call with its code, such as
+`APPROVAL_REQUIRED`. A script's own locator calls are the real `page`'s and are
+not asked again, so a host that guards element actions guards scripts as a
+whole.
+
+An element action (`click`, `fill`, `press`, `hover`, `focus`, `check`,
+`uncheck`, `select`, `drag`, and the coordinate actions `mousedown`, `mouseup`,
+`wheel`) carries a `target`, described from the browser side:
+
+- A locator is resolved once, waiting up to the action budget (5 s), so an
+  element that renders late is guarded, not skipped. The guard sees that
+  element and the action then runs on the same element handle.
+- A coordinate action (`browser.mouse` click/down/up/move, `browser.drag` from
+  or to x/y, `browser.wheel` at the mouse's position) is guarded on the element
+  at that point.
+- The elements are found in an isolated world the page's scripts cannot reach
+  (the element's exact box, open shadow roots included; for a point,
+  `elementFromPoint`), then described over CDP from each backend node:
+  `DOM.describeNode` for tag and attributes, `Accessibility.getPartialAXTree`
+  for role, name and accessibility ancestors. A page cannot change what the
+  guard reads by overriding DOM functions. Over AlmostCDP the CDP endpoint
+  itself runs in the page, so there the description is only as trustworthy as
+  the page's world.
+- A target that cannot be resolved or described (inside an embedded frame or a
+  closed shadow root, without a box, or a CDP failure) refuses the action with
+  `UNSUPPORTED`; the guard is never skipped.
 
 A host's `snapshotExclude` is a CSS selector for its own elements in the page
 (an overlay, a launcher): `browser.snapshot` leaves out the nodes of every
-element it matches, found by their aria refs, and a node left empty by that.
-The page is not changed, so assistive technology still reads those elements.
+element it matches, found by their aria refs, and a node left empty by that,
+and a coordinate action that lands on one is refused with `UNSUPPORTED`. The
+page is not changed, so assistive technology still reads those elements.

package/dist/executor.d.ts

@@ -4,7 +4,8 @@
  * over CDP. No Node API is used, so the executor also runs in a browser realm
  * with a browser build of playwright-core (`@volter/almostcdp/playwright`).
  */
-import { type Locator, type Page } from "playwright-core";
+import { type Page } from "playwright-core";
+import { type GuardedTarget } from "./guard.js";
 import { type BrowserOperationErrorCode, type BrowserOperationResult } from "./protocol.js";
 /** The implementation this provider reports; the pinned playwright-core. */
 export declare const PLAYWRIGHT_IMPLEMENTATION = "playwright-core 1.63.0";
@@ -23,24 +24,32 @@ export interface PlaywrightOperationExecutorOptions {
     endpoint: string;
     /**
      * The host's policy for one action, asked before it runs: each locator
-     * action, and each `browser.script` before its source is compiled. Throwing
-     * a `BrowserActionRefusal` refuses the operation with that code.
+     * action, each coordinate action (`browser.mouse`, `browser.drag`,
+     * `browser.wheel`), and each `browser.script` before its source is
+     * compiled. An element action's target is resolved once, described from the
+     * browser side (see `GuardedTarget`), and the action then runs on that same
+     * element. A target that cannot be resolved or described refuses the
+     * operation without asking. Throwing a `BrowserActionRefusal` refuses the
+     * operation with that code.
      */
     actionGuard?: (action: PlaywrightAction) => void | Promise<void>;
     /**
-     * A CSS selector (Playwright's, which pierces open shadow roots) for the
-     * host's own elements: `browser.snapshot` leaves out every matching
-     * element's nodes, and a node left with nothing by that. The page itself is
-     * not changed, so what assistive technology reads stays the same.
+     * A CSS selector for the host's own elements: `browser.snapshot` leaves out
+     * every matching element's nodes (Playwright's selector, which pierces open
+     * shadow roots), and a coordinate action that lands on one is refused. The
+     * page itself is not changed, so what assistive technology reads stays the
+     * same.
      */
     snapshotExclude?: string;
 }
 /** One action an operation is about to perform. */
-export type PlaywrightAction = PlaywrightLocatorAction | PlaywrightScriptAction;
-/** A locator action. */
-export interface PlaywrightLocatorAction {
-    action: "click" | "fill" | "press" | "hover" | "focus" | "check" | "uncheck" | "select" | "drag";
-    locator: Locator;
+export type PlaywrightAction = PlaywrightElementAction | PlaywrightScriptAction;
+export type { GuardedNode, GuardedTarget } from "./guard.js";
+/** An action that lands on an element: by locator, or at a viewport point. */
+export interface PlaywrightElementAction {
+    action: "click" | "fill" | "press" | "hover" | "focus" | "check" | "uncheck" | "select" | "drag" | "mousedown" | "mouseup" | "wheel";
+    /** What the action lands on, described from the browser side. */
+    target: GuardedTarget;
     value?: unknown;
 }
 export interface PlaywrightScriptAction {
@@ -61,6 +70,9 @@ export declare class PlaywrightOperationExecutor {
     private readonly page;
     private readonly options;
     private readonly ready;
+    private readonly targets;
+    /** Where the mouse is: `page.mouse` keeps it, but does not report it. */
+    private mouse;
     constructor(page: Page, options: PlaywrightOperationExecutorOptions);
     execute(raw: unknown): Promise<BrowserOperationResult>;
     private locator;
@@ -76,8 +88,17 @@ export declare class PlaywrightOperationExecutor {
     private inspect;
     /** Run a locator action; a failure on a locator that matches nothing is NOT_FOUND. */
     private act;
-    /** The host's policy for an action; an element that is not there is the action's own failure. */
-    private guard;
+    /**
+     * Resolves the locator's element once (waiting up to the action budget),
+     * asks the host's policy about it, and returns the handle the action must
+     * use, so the element guarded is the element acted on. Without a policy the
+     * action runs on the locator. Resolution or description failing refuses.
+     */
+    private guarded;
+    /** Asks the host's policy about the element at a viewport point. */
+    private guardPoint;
+    /** A browser-side description, or the refusal that replaces the action. */
+    private describe;
     /** The aria refs of the elements `snapshotExclude` matches, and of their contents. */
     private excludedRefs;
     private revision;

package/dist/executor.mjs

@@ -1,5 +1,143 @@
 // src/executor.ts
 import { errors } from "playwright-core";
+
+// src/guard.ts
+var UnidentifiedTarget = class extends Error {
+};
+var HostElementTarget = class extends Error {
+};
+var WORLD = "supercode-guard";
+var GROUP = "supercode-guard";
+var MAX_NODES = 8;
+var MAX_ANCESTORS = 6;
+function findInWorld(query) {
+  const onHost = (element) => {
+    if (!query.host) return false;
+    for (let node = element; node; ) {
+      if (node.matches(query.host)) return true;
+      const parent = node.parentElement;
+      node = parent ?? (node.getRootNode().host ?? null);
+    }
+    return false;
+  };
+  if (query.kind === "point") {
+    let element = document.elementFromPoint(query.x, query.y);
+    if (!element) return "Nothing is at that point.";
+    if (onHost(element)) return "host";
+    while (element.shadowRoot) {
+      const inner = element.shadowRoot.elementFromPoint(query.x, query.y);
+      if (!inner || inner === element) break;
+      element = inner;
+      if (onHost(element)) return "host";
+    }
+    if (/^(IFRAME|FRAME|OBJECT|EMBED)$/.test(element.tagName))
+      return "The point is inside an embedded frame, which the guard cannot inspect.";
+    return [element];
+  }
+  const { box } = query;
+  const found = [];
+  const near = (a, b) => Math.abs(a - b) <= 1;
+  const visit = (root) => {
+    for (const element of Array.from(root.querySelectorAll("*"))) {
+      if (found.length >= query.max) return;
+      const rect = element.getBoundingClientRect();
+      if (near(rect.x, box.x) && near(rect.y, box.y) && near(rect.width, box.width) && near(rect.height, box.height))
+        found.push(element);
+      if (element.shadowRoot) visit(element.shadowRoot);
+    }
+  };
+  visit(document);
+  return found.length ? found : "The element could not be found in the page's top document (it may be inside an embedded frame or a closed shadow root).";
+}
+var BrowserSideTargets = class {
+  constructor(page, host) {
+    this.page = page;
+    this.host = host;
+  }
+  session = null;
+  /** The element a resolved handle is, scrolled into view. */
+  async ofHandle(handle, timeout) {
+    await handle.scrollIntoViewIfNeeded({ timeout });
+    const box = await handle.boundingBox();
+    if (!box || box.width === 0 || box.height === 0)
+      throw new UnidentifiedTarget("The element has no box on the page.");
+    return await this.resolve({ kind: "box", box, host: this.host, max: MAX_NODES });
+  }
+  /** The element at a viewport point. */
+  async atPoint(x, y) {
+    return { ...await this.resolve({ kind: "point", x, y, host: this.host }), point: { x, y } };
+  }
+  cdp() {
+    this.session ??= this.page.context().newCDPSession(this.page).catch((error) => {
+      this.session = null;
+      throw error;
+    });
+    return this.session;
+  }
+  async resolve(query) {
+    const cdp = await this.cdp();
+    const tree = await cdp.send("Page.getFrameTree");
+    const world = await cdp.send("Page.createIsolatedWorld", { frameId: tree.frameTree.frame.id, worldName: WORLD });
+    try {
+      const evaluated = await cdp.send("Runtime.evaluate", {
+        expression: `(${findInWorld.toString()})(${JSON.stringify(query)})`,
+        contextId: world.executionContextId,
+        objectGroup: GROUP,
+        returnByValue: false
+      });
+      if (evaluated.exceptionDetails)
+        throw new UnidentifiedTarget(`The target could not be resolved: ${evaluated.exceptionDetails.text ?? "exception"}.`);
+      if (evaluated.result.type === "string") {
+        if (evaluated.result.value === "host") throw new HostElementTarget("The point is on the host's own interface, which agents cannot operate.");
+        throw new UnidentifiedTarget(String(evaluated.result.value));
+      }
+      if (!evaluated.result.objectId) throw new UnidentifiedTarget("The target could not be resolved.");
+      const properties = await cdp.send("Runtime.getProperties", { objectId: evaluated.result.objectId, ownProperties: true });
+      const objects = properties.result.filter((property) => /^\d+$/.test(property.name) && property.value?.objectId).map((property) => property.value.objectId);
+      if (!objects.length) throw new UnidentifiedTarget("The target could not be resolved.");
+      const nodes = [];
+      const ancestors = [];
+      const seen = /* @__PURE__ */ new Set();
+      for (const objectId of objects) {
+        const described = await cdp.send("DOM.describeNode", { objectId });
+        const attributes = {};
+        const list = described.node.attributes ?? [];
+        for (let index = 0; index + 1 < list.length; index += 2) attributes[list[index].toLowerCase()] = list[index + 1];
+        const ax = await this.accessibility(cdp, described.node.backendNodeId);
+        nodes.push({
+          backendNodeId: described.node.backendNodeId,
+          tag: described.node.nodeName.toLowerCase(),
+          attributes,
+          role: ax.self?.role ?? "",
+          name: ax.self?.name ?? ""
+        });
+        seen.add(described.node.backendNodeId);
+        for (const ancestor of ax.ancestors) {
+          if (ancestors.length >= MAX_ANCESTORS || seen.has(ancestor.backendNodeId)) continue;
+          seen.add(ancestor.backendNodeId);
+          ancestors.push(ancestor);
+        }
+      }
+      return { url: this.page.url(), nodes, ancestors };
+    } finally {
+      await cdp.send("Runtime.releaseObjectGroup", { objectGroup: GROUP }).catch(() => void 0);
+    }
+  }
+  async accessibility(cdp, backendNodeId) {
+    const tree = await cdp.send("Accessibility.getPartialAXTree", { backendNodeId, fetchRelatives: true });
+    const byId = new Map(tree.nodes.map((node) => [node.nodeId, node]));
+    const text = (value) => typeof value === "string" ? value.replace(/\s+/g, " ").trim() : "";
+    const self = tree.nodes.find((node) => node.backendDOMNodeId === backendNodeId) ?? null;
+    const ancestors = [];
+    for (let node = self?.parentId ? byId.get(self.parentId) : void 0; node && ancestors.length < MAX_ANCESTORS; node = node.parentId ? byId.get(node.parentId) : void 0) {
+      if (node.ignored || typeof node.backendDOMNodeId !== "number") continue;
+      ancestors.push({ backendNodeId: node.backendDOMNodeId, tag: "", attributes: {}, role: text(node.role?.value), name: text(node.name?.value) });
+    }
+    return { self: self ? { role: text(self.role?.value), name: text(self.name?.value) } : null, ancestors };
+  }
+};
+
+// src/executor.ts
 import {
   BROWSER_OPERATION_NAMES,
   BrowserActionRefusal,
@@ -225,9 +363,13 @@ var PlaywrightOperationExecutor = class {
   page;
   options;
   ready;
+  targets;
+  /** Where the mouse is: `page.mouse` keeps it, but does not report it. */
+  mouse = { x: 0, y: 0 };
   constructor(page, options) {
     this.page = page;
     this.options = options;
+    this.targets = new BrowserSideTargets(page, options.snapshotExclude ?? null);
     this.ready = (async () => {
       await page.addInitScript(installRevisionCounter, REVISION_KEY);
       await page.evaluate(installRevisionCounter, REVISION_KEY).catch(() => void 0);
@@ -301,10 +443,55 @@ var PlaywrightOperationExecutor = class {
       throw error;
     }
   }
-  /** The host's policy for an action; an element that is not there is the action's own failure. */
-  async guard(action) {
-    if (!this.options.actionGuard || await action.locator.count().catch(() => 0) === 0) return;
-    await this.options.actionGuard(action);
+  /**
+   * Resolves the locator's element once (waiting up to the action budget),
+   * asks the host's policy about it, and returns the handle the action must
+   * use, so the element guarded is the element acted on. Without a policy the
+   * action runs on the locator. Resolution or description failing refuses.
+   */
+  async guarded(locator, action, value) {
+    const guard = this.options.actionGuard;
+    if (!guard) return null;
+    let handle;
+    try {
+      handle = await locator.elementHandle({ timeout: ACTION_TIMEOUT_MS });
+    } catch (error) {
+      if (error instanceof errors.TimeoutError)
+        throw new BrowserActionRefusal("NOT_FOUND", `${action} found no element for the locator in ${ACTION_TIMEOUT_MS}ms.`);
+      throw error;
+    }
+    if (!handle) throw new BrowserActionRefusal("NOT_FOUND", `${action} found no element for the locator.`);
+    try {
+      await guard({
+        action,
+        target: await this.describe(() => this.targets.ofHandle(handle, ACTION_TIMEOUT_MS)),
+        ...value !== void 0 ? { value } : {}
+      });
+    } catch (error) {
+      await handle.dispose().catch(() => void 0);
+      throw error;
+    }
+    return handle;
+  }
+  /** Asks the host's policy about the element at a viewport point. */
+  async guardPoint(action, x, y) {
+    const guard = this.options.actionGuard;
+    if (!guard) return;
+    await guard({ action, target: await this.describe(() => this.targets.atPoint(x, y)) });
+  }
+  /** A browser-side description, or the refusal that replaces the action. */
+  async describe(run) {
+    try {
+      return await run();
+    } catch (error) {
+      if (error instanceof HostElementTarget) throw new BrowserActionRefusal("UNSUPPORTED", error.message);
+      if (error instanceof UnidentifiedTarget)
+        throw new BrowserActionRefusal("UNSUPPORTED", `${error.message} The action was refused because its target could not be checked.`);
+      throw new BrowserActionRefusal(
+        "UNSUPPORTED",
+        `The action was refused because its target could not be checked: ${errorMessage(error)}`
+      );
+    }
   }
   /** The aria refs of the elements `snapshotExclude` matches, and of their contents. */
   async excludedRefs() {
@@ -380,35 +567,48 @@ var PlaywrightOperationExecutor = class {
     if (call.operation === "browser.script") return this.script(call);
     if (call.operation === "browser.mouse") {
       const action = input.action;
-      if (action === "move") await page.mouse.move(input.x, input.y);
-      else if (action === "click") await page.mouse.click(input.x, input.y);
+      const at = typeof input.x === "number" && typeof input.y === "number" ? { x: input.x, y: input.y } : this.mouse;
+      if (action === "move") await this.guardPoint("hover", at.x, at.y);
+      else await this.guardPoint(action === "click" ? "click" : action === "down" ? "mousedown" : "mouseup", at.x, at.y);
+      if (action === "move") await page.mouse.move(at.x, at.y);
+      else if (action === "click") await page.mouse.click(at.x, at.y);
       else if (action === "down") await page.mouse.down();
       else await page.mouse.up();
+      if (action === "move" || action === "click") this.mouse = { x: at.x, y: at.y };
       return this.success(call.operation, { action, ...typeof input.x === "number" ? { x: input.x, y: input.y } : {} });
     }
     if (call.operation === "browser.wheel") {
       const deltaX = typeof input.deltaX === "number" ? input.deltaX : 0;
       const deltaY = typeof input.deltaY === "number" ? input.deltaY : 0;
+      await this.guardPoint("wheel", this.mouse.x, this.mouse.y);
       await page.mouse.wheel(deltaX, deltaY);
       return this.success(call.operation, { deltaX, deltaY });
     }
     if (call.operation === "browser.drag") {
       const resolve = async (endpoint) => {
-        if (endpoint.locator === void 0) return { x: endpoint.x, y: endpoint.y };
+        if (endpoint.locator === void 0) {
+          const point = { x: endpoint.x, y: endpoint.y };
+          await this.guardPoint("drag", point.x, point.y);
+          return point;
+        }
         await this.refreshRefs(endpoint.locator);
         const locator2 = this.locator(endpoint.locator).first();
-        const box = await locator2.count() === 0 ? null : await locator2.boundingBox({ timeout: ACTION_TIMEOUT_MS });
-        if (!box) throw new BrowserActionRefusal("NOT_FOUND", "The drag locator matched no element.");
-        return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+        const handle2 = await this.guarded(locator2, "drag");
+        try {
+          const box = handle2 ? await handle2.boundingBox() : await locator2.count() === 0 ? null : await locator2.boundingBox({ timeout: ACTION_TIMEOUT_MS });
+          if (!box) throw new BrowserActionRefusal("NOT_FOUND", "The drag locator matched no element.");
+          return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+        } finally {
+          await handle2?.dispose().catch(() => void 0);
+        }
       };
-      for (const end of [input.from, input.to])
-        if (end.locator !== void 0) await this.guard({ action: "drag", locator: this.locator(end.locator).first() });
       const from = await resolve(input.from);
       const to = await resolve(input.to);
       await page.mouse.move(from.x, from.y);
       await page.mouse.down();
       await page.mouse.move(to.x, to.y, { steps: typeof input.steps === "number" ? input.steps : 8 });
       await page.mouse.up();
+      this.mouse = to;
       return this.success(call.operation, { from, to });
     }
     if (call.operation === "browser.back") {
@@ -432,28 +632,38 @@ var PlaywrightOperationExecutor = class {
       return this.success(call.operation, box);
     }
     if (call.operation === "browser.press" && !locator) {
-      await this.guard({ action: "press", locator: page.locator("*:focus").first(), value: input.key });
-      await page.keyboard.press(input.key);
+      const focused = page.locator("*:focus").first();
+      const handle2 = await this.guarded(await focused.count() ? focused : page.locator("body"), "press", input.key);
+      try {
+        await page.keyboard.press(input.key);
+      } finally {
+        await handle2?.dispose().catch(() => void 0);
+      }
       return this.success(call.operation, { key: input.key });
     }
     const target = locator;
-    const guarded = call.operation === "browser.select" ? "select" : call.operation.slice("browser.".length);
-    await this.guard({
-      action: guarded,
-      locator: target.first(),
-      ...call.operation === "browser.fill" ? { value: input.value } : call.operation === "browser.press" ? { value: input.key } : call.operation === "browser.select" ? { value: input.values } : {}
-    });
+    const guardedAction = call.operation === "browser.select" ? "select" : call.operation.slice("browser.".length);
+    const handle = await this.guarded(
+      target.first(),
+      guardedAction,
+      call.operation === "browser.fill" ? input.value : call.operation === "browser.press" ? input.key : call.operation === "browser.select" ? input.values : void 0
+    );
+    const element = handle ?? { ...bind(target), focus: () => target.focus({ timeout }) };
     const before = await this.inspect(target).catch(() => null);
-    if (call.operation === "browser.click") await this.act(target, "click", () => target.click({ timeout }));
-    else if (call.operation === "browser.fill") await this.act(target, "fill", () => target.fill(input.value, { timeout }));
-    else if (call.operation === "browser.press") await this.act(target, "press", () => target.press(input.key, { timeout }));
-    else if (call.operation === "browser.hover") await this.act(target, "hover", () => target.hover({ timeout }));
-    else if (call.operation === "browser.focus") await this.act(target, "focus", () => target.focus({ timeout }));
-    else if (call.operation === "browser.check") await this.act(target, "check", () => target.check({ timeout }));
-    else if (call.operation === "browser.uncheck") await this.act(target, "uncheck", () => target.uncheck({ timeout }));
-    else if (call.operation === "browser.select") {
-      const values = await this.act(target, "selectOption", () => target.selectOption(input.values, { timeout }));
-      return this.success(call.operation, { values });
+    try {
+      if (call.operation === "browser.click") await this.act(target, "click", () => element.click({ timeout }));
+      else if (call.operation === "browser.fill") await this.act(target, "fill", () => element.fill(input.value, { timeout }));
+      else if (call.operation === "browser.press") await this.act(target, "press", () => element.press(input.key, { timeout }));
+      else if (call.operation === "browser.hover") await this.act(target, "hover", () => element.hover({ timeout }));
+      else if (call.operation === "browser.focus") await this.act(target, "focus", () => element.focus());
+      else if (call.operation === "browser.check") await this.act(target, "check", () => element.check({ timeout }));
+      else if (call.operation === "browser.uncheck") await this.act(target, "uncheck", () => element.uncheck({ timeout }));
+      else if (call.operation === "browser.select") {
+        const values = await this.act(target, "selectOption", () => element.selectOption(input.values, { timeout }));
+        return this.success(call.operation, { values });
+      }
+    } finally {
+      await handle?.dispose().catch(() => void 0);
     }
     const after = await this.inspect(target).catch(() => null);
     const inspection = after ?? before;
@@ -521,6 +731,17 @@ function operationName(raw) {
   const candidate = raw && typeof raw === "object" && !Array.isArray(raw) ? raw.operation : void 0;
   return typeof candidate === "string" && BROWSER_OPERATION_NAMES.includes(candidate) ? candidate : "browser.status";
 }
+function bind(locator) {
+  return {
+    click: locator.click.bind(locator),
+    fill: locator.fill.bind(locator),
+    press: locator.press.bind(locator),
+    hover: locator.hover.bind(locator),
+    check: locator.check.bind(locator),
+    uncheck: locator.uncheck.bind(locator),
+    selectOption: locator.selectOption.bind(locator)
+  };
+}
 export {
   ACTION_TIMEOUT_MS,
   PLAYWRIGHT_FEATURES,