@volter-ai-dev/supercode-browser-playwright 0.1.0 → 0.2.0

package/dist/index.mjs

@@ -1,6 +1,142 @@
 // src/executor.ts
 import { errors } from "playwright-core";
 
+// src/guard.ts
+var UnidentifiedTarget = class extends Error {
+};
+var HostElementTarget = class extends Error {
+};
+var WORLD = "supercode-guard";
+var GROUP = "supercode-guard";
+var MAX_NODES = 8;
+var MAX_ANCESTORS = 6;
+function findInWorld(query) {
+  const onHost = (element) => {
+    if (!query.host) return false;
+    for (let node = element; node; ) {
+      if (node.matches(query.host)) return true;
+      const parent = node.parentElement;
+      node = parent ?? (node.getRootNode().host ?? null);
+    }
+    return false;
+  };
+  if (query.kind === "point") {
+    let element = document.elementFromPoint(query.x, query.y);
+    if (!element) return "Nothing is at that point.";
+    if (onHost(element)) return "host";
+    while (element.shadowRoot) {
+      const inner = element.shadowRoot.elementFromPoint(query.x, query.y);
+      if (!inner || inner === element) break;
+      element = inner;
+      if (onHost(element)) return "host";
+    }
+    if (/^(IFRAME|FRAME|OBJECT|EMBED)$/.test(element.tagName))
+      return "The point is inside an embedded frame, which the guard cannot inspect.";
+    return [element];
+  }
+  const { box } = query;
+  const found = [];
+  const near = (a, b) => Math.abs(a - b) <= 1;
+  const visit = (root) => {
+    for (const element of Array.from(root.querySelectorAll("*"))) {
+      if (found.length >= query.max) return;
+      const rect = element.getBoundingClientRect();
+      if (near(rect.x, box.x) && near(rect.y, box.y) && near(rect.width, box.width) && near(rect.height, box.height))
+        found.push(element);
+      if (element.shadowRoot) visit(element.shadowRoot);
+    }
+  };
+  visit(document);
+  return found.length ? found : "The element could not be found in the page's top document (it may be inside an embedded frame or a closed shadow root).";
+}
+var BrowserSideTargets = class {
+  constructor(page, host) {
+    this.page = page;
+    this.host = host;
+  }
+  session = null;
+  /** The element a resolved handle is, scrolled into view. */
+  async ofHandle(handle, timeout) {
+    await handle.scrollIntoViewIfNeeded({ timeout });
+    const box = await handle.boundingBox();
+    if (!box || box.width === 0 || box.height === 0)
+      throw new UnidentifiedTarget("The element has no box on the page.");
+    return await this.resolve({ kind: "box", box, host: this.host, max: MAX_NODES });
+  }
+  /** The element at a viewport point. */
+  async atPoint(x, y) {
+    return { ...await this.resolve({ kind: "point", x, y, host: this.host }), point: { x, y } };
+  }
+  cdp() {
+    this.session ??= this.page.context().newCDPSession(this.page).catch((error) => {
+      this.session = null;
+      throw error;
+    });
+    return this.session;
+  }
+  async resolve(query) {
+    const cdp = await this.cdp();
+    const tree = await cdp.send("Page.getFrameTree");
+    const world = await cdp.send("Page.createIsolatedWorld", { frameId: tree.frameTree.frame.id, worldName: WORLD });
+    try {
+      const evaluated = await cdp.send("Runtime.evaluate", {
+        expression: `(${findInWorld.toString()})(${JSON.stringify(query)})`,
+        contextId: world.executionContextId,
+        objectGroup: GROUP,
+        returnByValue: false
+      });
+      if (evaluated.exceptionDetails)
+        throw new UnidentifiedTarget(`The target could not be resolved: ${evaluated.exceptionDetails.text ?? "exception"}.`);
+      if (evaluated.result.type === "string") {
+        if (evaluated.result.value === "host") throw new HostElementTarget("The point is on the host's own interface, which agents cannot operate.");
+        throw new UnidentifiedTarget(String(evaluated.result.value));
+      }
+      if (!evaluated.result.objectId) throw new UnidentifiedTarget("The target could not be resolved.");
+      const properties = await cdp.send("Runtime.getProperties", { objectId: evaluated.result.objectId, ownProperties: true });
+      const objects = properties.result.filter((property) => /^\d+$/.test(property.name) && property.value?.objectId).map((property) => property.value.objectId);
+      if (!objects.length) throw new UnidentifiedTarget("The target could not be resolved.");
+      const nodes = [];
+      const ancestors = [];
+      const seen = /* @__PURE__ */ new Set();
+      for (const objectId of objects) {
+        const described = await cdp.send("DOM.describeNode", { objectId });
+        const attributes = {};
+        const list = described.node.attributes ?? [];
+        for (let index = 0; index + 1 < list.length; index += 2) attributes[list[index].toLowerCase()] = list[index + 1];
+        const ax = await this.accessibility(cdp, described.node.backendNodeId);
+        nodes.push({
+          backendNodeId: described.node.backendNodeId,
+          tag: described.node.nodeName.toLowerCase(),
+          attributes,
+          role: ax.self?.role ?? "",
+          name: ax.self?.name ?? ""
+        });
+        seen.add(described.node.backendNodeId);
+        for (const ancestor of ax.ancestors) {
+          if (ancestors.length >= MAX_ANCESTORS || seen.has(ancestor.backendNodeId)) continue;
+          seen.add(ancestor.backendNodeId);
+          ancestors.push(ancestor);
+        }
+      }
+      return { url: this.page.url(), nodes, ancestors };
+    } finally {
+      await cdp.send("Runtime.releaseObjectGroup", { objectGroup: GROUP }).catch(() => void 0);
+    }
+  }
+  async accessibility(cdp, backendNodeId) {
+    const tree = await cdp.send("Accessibility.getPartialAXTree", { backendNodeId, fetchRelatives: true });
+    const byId = new Map(tree.nodes.map((node) => [node.nodeId, node]));
+    const text = (value) => typeof value === "string" ? value.replace(/\s+/g, " ").trim() : "";
+    const self = tree.nodes.find((node) => node.backendDOMNodeId === backendNodeId) ?? null;
+    const ancestors = [];
+    for (let node = self?.parentId ? byId.get(self.parentId) : void 0; node && ancestors.length < MAX_ANCESTORS; node = node.parentId ? byId.get(node.parentId) : void 0) {
+      if (node.ignored || typeof node.backendDOMNodeId !== "number") continue;
+      ancestors.push({ backendNodeId: node.backendDOMNodeId, tag: "", attributes: {}, role: text(node.role?.value), name: text(node.name?.value) });
+    }
+    return { self: self ? { role: text(self.role?.value), name: text(self.name?.value) } : null, ancestors };
+  }
+};
+
 // src/protocol.ts
 var SUPERCODE_BROWSER_PROVIDER_PROTOCOL = "supercode/browser-provider-v1";
 var SUPERCODE_BROWSER_OPERATION_PROTOCOL = "supercode/browser-operation-v1";
@@ -253,6 +389,49 @@ var PLAYWRIGHT_FEATURES = Object.freeze([
   "page.screenshot.png",
   "page.evaluate"
 ]);
+var SNAPSHOT_EXCLUDE_LIMIT = 16;
+function collectRefs(nodes, into) {
+  if (Array.isArray(nodes)) {
+    for (const node2 of nodes) collectRefs(node2, into);
+    return;
+  }
+  if (!nodes || typeof nodes !== "object") return;
+  const node = nodes;
+  if (typeof node.ref === "string") into.add(node.ref);
+  collectRefs(node.children, into);
+}
+function snapshotWithoutRefs(text, refs) {
+  if (refs.size === 0) return text;
+  const root = { text: "", indent: -1, children: [], removed: false };
+  const stack = [root];
+  const lines = [];
+  for (const raw of text.split("\n")) {
+    const indent = raw.length - raw.trimStart().length;
+    const line = { text: raw, indent, children: [], removed: false };
+    while (stack.length > 1 && stack[stack.length - 1].indent >= indent) stack.pop();
+    stack[stack.length - 1].children.push(line);
+    stack.push(line);
+    lines.push(line);
+  }
+  const refOf = (line) => /\[ref=([^\]\s]+)\]/.exec(line.text)?.[1];
+  const remove = (line) => {
+    line.removed = true;
+    for (const child of line.children) remove(child);
+  };
+  const visit = (line) => {
+    const ref = refOf(line);
+    if (ref !== void 0 && refs.has(ref)) {
+      remove(line);
+      return;
+    }
+    const before = line.children.length;
+    for (const child of line.children) visit(child);
+    if (line !== root && ref === void 0 && before > 0 && line.children.every((child) => child.removed))
+      line.removed = true;
+  };
+  visit(root);
+  return lines.filter((line) => !line.removed).map((line) => line.text).join("\n");
+}
 var REVISION_KEY = "__supercodeBrowserRevision";
 function installRevisionCounter(key) {
   const scope = globalThis;
@@ -386,9 +565,13 @@ var PlaywrightOperationExecutor = class {
   page;
   options;
   ready;
+  targets;
+  /** Where the mouse is: `page.mouse` keeps it, but does not report it. */
+  mouse = { x: 0, y: 0 };
   constructor(page, options) {
     this.page = page;
     this.options = options;
+    this.targets = new BrowserSideTargets(page, options.snapshotExclude ?? null);
     this.ready = (async () => {
       await page.addInitScript(installRevisionCounter, REVISION_KEY);
       await page.evaluate(installRevisionCounter, REVISION_KEY).catch(() => void 0);
@@ -462,10 +645,68 @@ var PlaywrightOperationExecutor = class {
       throw error;
     }
   }
-  /** The host's policy for an action; an element that is not there is the action's own failure. */
-  async guard(action) {
-    if (!this.options.actionGuard || await action.locator.count().catch(() => 0) === 0) return;
-    await this.options.actionGuard(action);
+  /**
+   * Resolves the locator's element once (waiting up to the action budget),
+   * asks the host's policy about it, and returns the handle the action must
+   * use, so the element guarded is the element acted on. Without a policy the
+   * action runs on the locator. Resolution or description failing refuses.
+   */
+  async guarded(locator, action, value) {
+    const guard = this.options.actionGuard;
+    if (!guard) return null;
+    let handle;
+    try {
+      handle = await locator.elementHandle({ timeout: ACTION_TIMEOUT_MS });
+    } catch (error) {
+      if (error instanceof errors.TimeoutError)
+        throw new BrowserActionRefusal("NOT_FOUND", `${action} found no element for the locator in ${ACTION_TIMEOUT_MS}ms.`);
+      throw error;
+    }
+    if (!handle) throw new BrowserActionRefusal("NOT_FOUND", `${action} found no element for the locator.`);
+    try {
+      await guard({
+        action,
+        target: await this.describe(() => this.targets.ofHandle(handle, ACTION_TIMEOUT_MS)),
+        ...value !== void 0 ? { value } : {}
+      });
+    } catch (error) {
+      await handle.dispose().catch(() => void 0);
+      throw error;
+    }
+    return handle;
+  }
+  /** Asks the host's policy about the element at a viewport point. */
+  async guardPoint(action, x, y) {
+    const guard = this.options.actionGuard;
+    if (!guard) return;
+    await guard({ action, target: await this.describe(() => this.targets.atPoint(x, y)) });
+  }
+  /** A browser-side description, or the refusal that replaces the action. */
+  async describe(run) {
+    try {
+      return await run();
+    } catch (error) {
+      if (error instanceof HostElementTarget) throw new BrowserActionRefusal("UNSUPPORTED", error.message);
+      if (error instanceof UnidentifiedTarget)
+        throw new BrowserActionRefusal("UNSUPPORTED", `${error.message} The action was refused because its target could not be checked.`);
+      throw new BrowserActionRefusal(
+        "UNSUPPORTED",
+        `The action was refused because its target could not be checked: ${errorMessage(error)}`
+      );
+    }
+  }
+  /** The aria refs of the elements `snapshotExclude` matches, and of their contents. */
+  async excludedRefs() {
+    const refs = /* @__PURE__ */ new Set();
+    const selector = this.options.snapshotExclude;
+    if (!selector) return refs;
+    const matches = this.page.locator(selector);
+    const count = Math.min(await matches.count().catch(() => 0), SNAPSHOT_EXCLUDE_LIMIT);
+    for (let index = 0; index < count; index += 1) {
+      const nodes = await matches.nth(index).ariaSnapshotJSON({ mode: "ai", timeout: ACTION_TIMEOUT_MS }).catch(() => null);
+      collectRefs(nodes, refs);
+    }
+    return refs;
   }
   async revision() {
     const value = await this.page.evaluate((key) => globalThis[key], REVISION_KEY).catch(() => void 0);
@@ -491,7 +732,8 @@ var PlaywrightOperationExecutor = class {
     if (call.operation === "browser.snapshot") {
       await this.refreshRefs(input.locator);
       const scoped = input.locator ? this.locator(input.locator).first() : null;
-      const text = scoped ? await this.act(scoped, "ariaSnapshot", () => scoped.ariaSnapshot({ mode: "ai", timeout: ACTION_TIMEOUT_MS })) : await page.ariaSnapshot({ mode: "ai", timeout: ACTION_TIMEOUT_MS });
+      const excluded = await this.excludedRefs();
+      const text = snapshotWithoutRefs(scoped ? await this.act(scoped, "ariaSnapshot", () => scoped.ariaSnapshot({ mode: "ai", timeout: ACTION_TIMEOUT_MS })) : await page.ariaSnapshot({ mode: "ai", timeout: ACTION_TIMEOUT_MS }), excluded);
       const bounded = text.slice(0, 64e3);
       return this.success(call.operation, { text: bounded, truncated: bounded.length < text.length });
     }
@@ -527,35 +769,48 @@ var PlaywrightOperationExecutor = class {
     if (call.operation === "browser.script") return this.script(call);
     if (call.operation === "browser.mouse") {
       const action = input.action;
-      if (action === "move") await page.mouse.move(input.x, input.y);
-      else if (action === "click") await page.mouse.click(input.x, input.y);
+      const at = typeof input.x === "number" && typeof input.y === "number" ? { x: input.x, y: input.y } : this.mouse;
+      if (action === "move") await this.guardPoint("hover", at.x, at.y);
+      else await this.guardPoint(action === "click" ? "click" : action === "down" ? "mousedown" : "mouseup", at.x, at.y);
+      if (action === "move") await page.mouse.move(at.x, at.y);
+      else if (action === "click") await page.mouse.click(at.x, at.y);
       else if (action === "down") await page.mouse.down();
       else await page.mouse.up();
+      if (action === "move" || action === "click") this.mouse = { x: at.x, y: at.y };
       return this.success(call.operation, { action, ...typeof input.x === "number" ? { x: input.x, y: input.y } : {} });
     }
     if (call.operation === "browser.wheel") {
       const deltaX = typeof input.deltaX === "number" ? input.deltaX : 0;
       const deltaY = typeof input.deltaY === "number" ? input.deltaY : 0;
+      await this.guardPoint("wheel", this.mouse.x, this.mouse.y);
       await page.mouse.wheel(deltaX, deltaY);
       return this.success(call.operation, { deltaX, deltaY });
     }
     if (call.operation === "browser.drag") {
       const resolve = async (endpoint) => {
-        if (endpoint.locator === void 0) return { x: endpoint.x, y: endpoint.y };
+        if (endpoint.locator === void 0) {
+          const point = { x: endpoint.x, y: endpoint.y };
+          await this.guardPoint("drag", point.x, point.y);
+          return point;
+        }
         await this.refreshRefs(endpoint.locator);
         const locator2 = this.locator(endpoint.locator).first();
-        const box = await locator2.count() === 0 ? null : await locator2.boundingBox({ timeout: ACTION_TIMEOUT_MS });
-        if (!box) throw new BrowserActionRefusal("NOT_FOUND", "The drag locator matched no element.");
-        return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+        const handle2 = await this.guarded(locator2, "drag");
+        try {
+          const box = handle2 ? await handle2.boundingBox() : await locator2.count() === 0 ? null : await locator2.boundingBox({ timeout: ACTION_TIMEOUT_MS });
+          if (!box) throw new BrowserActionRefusal("NOT_FOUND", "The drag locator matched no element.");
+          return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+        } finally {
+          await handle2?.dispose().catch(() => void 0);
+        }
       };
-      for (const end of [input.from, input.to])
-        if (end.locator !== void 0) await this.guard({ action: "drag", locator: this.locator(end.locator).first() });
       const from = await resolve(input.from);
       const to = await resolve(input.to);
       await page.mouse.move(from.x, from.y);
       await page.mouse.down();
       await page.mouse.move(to.x, to.y, { steps: typeof input.steps === "number" ? input.steps : 8 });
       await page.mouse.up();
+      this.mouse = to;
       return this.success(call.operation, { from, to });
     }
     if (call.operation === "browser.back") {
@@ -579,28 +834,38 @@ var PlaywrightOperationExecutor = class {
       return this.success(call.operation, box);
     }
     if (call.operation === "browser.press" && !locator) {
-      await this.guard({ action: "press", locator: page.locator("*:focus").first(), value: input.key });
-      await page.keyboard.press(input.key);
+      const focused = page.locator("*:focus").first();
+      const handle2 = await this.guarded(await focused.count() ? focused : page.locator("body"), "press", input.key);
+      try {
+        await page.keyboard.press(input.key);
+      } finally {
+        await handle2?.dispose().catch(() => void 0);
+      }
       return this.success(call.operation, { key: input.key });
     }
     const target = locator;
-    const guarded = call.operation === "browser.select" ? "select" : call.operation.slice("browser.".length);
-    await this.guard({
-      action: guarded,
-      locator: target.first(),
-      ...call.operation === "browser.fill" ? { value: input.value } : call.operation === "browser.press" ? { value: input.key } : call.operation === "browser.select" ? { value: input.values } : {}
-    });
+    const guardedAction = call.operation === "browser.select" ? "select" : call.operation.slice("browser.".length);
+    const handle = await this.guarded(
+      target.first(),
+      guardedAction,
+      call.operation === "browser.fill" ? input.value : call.operation === "browser.press" ? input.key : call.operation === "browser.select" ? input.values : void 0
+    );
+    const element = handle ?? { ...bind(target), focus: () => target.focus({ timeout }) };
     const before = await this.inspect(target).catch(() => null);
-    if (call.operation === "browser.click") await this.act(target, "click", () => target.click({ timeout }));
-    else if (call.operation === "browser.fill") await this.act(target, "fill", () => target.fill(input.value, { timeout }));
-    else if (call.operation === "browser.press") await this.act(target, "press", () => target.press(input.key, { timeout }));
-    else if (call.operation === "browser.hover") await this.act(target, "hover", () => target.hover({ timeout }));
-    else if (call.operation === "browser.focus") await this.act(target, "focus", () => target.focus({ timeout }));
-    else if (call.operation === "browser.check") await this.act(target, "check", () => target.check({ timeout }));
-    else if (call.operation === "browser.uncheck") await this.act(target, "uncheck", () => target.uncheck({ timeout }));
-    else if (call.operation === "browser.select") {
-      const values = await this.act(target, "selectOption", () => target.selectOption(input.values, { timeout }));
-      return this.success(call.operation, { values });
+    try {
+      if (call.operation === "browser.click") await this.act(target, "click", () => element.click({ timeout }));
+      else if (call.operation === "browser.fill") await this.act(target, "fill", () => element.fill(input.value, { timeout }));
+      else if (call.operation === "browser.press") await this.act(target, "press", () => element.press(input.key, { timeout }));
+      else if (call.operation === "browser.hover") await this.act(target, "hover", () => element.hover({ timeout }));
+      else if (call.operation === "browser.focus") await this.act(target, "focus", () => element.focus());
+      else if (call.operation === "browser.check") await this.act(target, "check", () => element.check({ timeout }));
+      else if (call.operation === "browser.uncheck") await this.act(target, "uncheck", () => element.uncheck({ timeout }));
+      else if (call.operation === "browser.select") {
+        const values = await this.act(target, "selectOption", () => element.selectOption(input.values, { timeout }));
+        return this.success(call.operation, { values });
+      }
+    } finally {
+      await handle?.dispose().catch(() => void 0);
     }
     const after = await this.inspect(target).catch(() => null);
     const inspection = after ?? before;
@@ -615,6 +880,7 @@ var PlaywrightOperationExecutor = class {
     const timeout = Math.min(requested, SCRIPT_TIMEOUT_CLAMP_MS);
     const AsyncFunction = Object.getPrototypeOf(async function() {
     }).constructor;
+    if (this.options.actionGuard) await this.options.actionGuard({ action: "script", source, args });
     let run;
     try {
       run = new AsyncFunction("page", "args", source);
@@ -667,6 +933,17 @@ function operationName(raw) {
   const candidate = raw && typeof raw === "object" && !Array.isArray(raw) ? raw.operation : void 0;
   return typeof candidate === "string" && BROWSER_OPERATION_NAMES.includes(candidate) ? candidate : "browser.status";
 }
+function bind(locator) {
+  return {
+    click: locator.click.bind(locator),
+    fill: locator.fill.bind(locator),
+    press: locator.press.bind(locator),
+    hover: locator.hover.bind(locator),
+    check: locator.check.bind(locator),
+    uncheck: locator.uncheck.bind(locator),
+    selectOption: locator.selectOption.bind(locator)
+  };
+}
 
 // src/provider.ts
 import { timingSafeEqual } from "node:crypto";
@@ -1011,6 +1288,7 @@ export {
   parseBrowserOperationCall,
   parseBrowserOperationResult,
   serveHttp,
-  serveTcp
+  serveTcp,
+  snapshotWithoutRefs
 };
 //# sourceMappingURL=index.mjs.map