@ricsam/r5d-browser 0.0.70 → 0.0.72

package/README.md

@@ -22,6 +22,8 @@ r5d-browser start --chrome-path "/Applications/Google Chrome.app/Contents/MacOS/
 
 The profile is stored under `~/.r5d/browser/chrome-profile`. The previous Chrome for Testing profile at `~/.r5d/browser/profile` is preserved but is not migrated automatically, so sign in again the first time you use the new profile. Browser binaries previously installed under `~/.r5d/browser/browsers` are also left untouched, but new versions of `r5d-browser` do not use them. Completed downloads are retained under `~/.r5d/browser/downloads`; agents can list them and copy a selected file into the active session's artifacts. That artifact is then synchronized to every connected worker. Browser screenshots are session artifacts in r5d.dev and are not retained by the browser process.
 
+Agents read pages through `browser_read_page`, an accessibility outline with stable element refs (`[ref=e12]`) on interactable elements and `[box=x,y,w,h]` viewport coordinates on everything visible. Refs feed the one-call interaction tools: `browser_mouse_click` clicks a ref (scrolled into view, with actionability checks) or an x/y point, `browser_form_input` sets values on text inputs, selects, checkboxes, and contenteditable elements with trusted events that controlled (React-style) components honor, and `browser_scroll` scrolls an element into view or dispatches trusted wheel input. Same-origin iframe content appears inline with frame-prefixed refs such as `f1e3`; cross-origin frame content may be omitted. Refs come from the most recent snapshot of a tab, so agents re-read the page after navigations or large re-renders. `browser_wait_for` waits for a selector, visible text, a URL glob, or a load state instead of sleeping, and `browser_keyboard_type` has a `keys` mode that presses each character as a real key event (up to 500 characters per call) for type-ahead widgets — characters without key codes, such as emoji, still fall back to text insertion. Screenshots capture the viewport by default; `fullPage: true` captures the scrollable page up to 4000 CSS px of height.
+
 Agents running JavaScript in a page can save captured text straight into the active session's artifacts by calling `r5dCreateArtifact(filename, content)`, which takes two strings and must be called before the agent's code returns. Each captured file is written to the session artifacts in r5d.dev, synchronized to every connected worker, and reported back as a `$R5D_ARTIFACTS_DIR` path; a single call may create at most 32 files totalling 4 MiB. The function is passed into the agent's code rather than assigned to `window`, so pages can neither call it nor use it to detect automation.
 
 Agents can also create loopback-only TCP port forwards from this Mac to the worker bound to a chat. For example, a forward can map Chrome's `localhost:4323` to `127.0.0.1:3232` on that worker. The relay preserves raw TCP traffic, including HTTP, HTTPS, WebSockets, HMR, and SSE; it never exposes a public listener or permits an arbitrary target host.

package/dist/cjs/browser-runtime.cjs

@@ -37,6 +37,38 @@ const MAX_DOWNLOAD_CHUNK_BYTES = 1024 * 1024;
 const MAX_RUN_JS_RESULT_BYTES = 1024 * 1024;
 const MAX_RUN_JS_ARTIFACT_BYTES = 4 * 1024 * 1024;
 const MAX_RUN_JS_ARTIFACT_COUNT = 32;
+const REF_ACTION_TIMEOUT_MS = 5e3;
+const NAVIGATION_TIMEOUT_MS = 2e4;
+const READ_PAGE_TIMEOUT_MS = 1e4;
+const MAX_FULL_PAGE_HEIGHT = 4e3;
+const MAX_FULL_PAGE_WIDTH = 4096;
+const MAX_TYPED_KEYS_CHARS = 500;
+const MAX_TYPED_KEYS_TOTAL_MS = 2e4;
+function clampInt(value, min, max, fallback) {
+  const num = value === void 0 ? fallback : Number(value);
+  if (!Number.isFinite(num)) return fallback;
+  return Math.min(max, Math.max(min, Math.round(num)));
+}
+function requireRef(value) {
+  if (typeof value !== "string" || !/^(f\d+)?e\d+$/.test(value)) {
+    throw new Error('ref must look like "e12" or "f1e3", exactly as printed by browser_read_page ([ref=e12]).');
+  }
+  return value;
+}
+function stripCallLog(message) {
+  const withoutCallLog = message.split(/\n\s*Call log:/)[0] ?? message;
+  return withoutCallLog.split("\n").filter((line) => !/^\s+at\s/.test(line)).join("\n").trim();
+}
+function isDestroyedContext(message) {
+  return /Execution context was destroyed|because of a navigation|Cannot find context with specified id|Target closed/i.test(message);
+}
+function refActionError(error, ref, verb) {
+  const message = stripCallLog(error instanceof Error ? error.message : String(error));
+  return new Error(
+    `${message}
+Ref ${ref} comes from the most recent browser_read_page snapshot of this tab. If the page navigated or re-rendered since, take a new snapshot and retry. If another element covers the target, dismiss the overlay before the ${verb}.`
+  );
+}
 class BrowserRuntime {
   constructor(context, downloadsPath) {
     this.context = context;
@@ -171,6 +203,18 @@ class BrowserRuntime {
     if (!page || page.isClosed()) throw new Error(`Browser tab ${tabId} is no longer open.`);
     return page;
   }
+  refLocator(page, ref) {
+    return page.locator(`aria-ref=${ref}`);
+  }
+  async gotoSettled(page, url) {
+    try {
+      await page.goto(url, { waitUntil: "domcontentloaded", timeout: NAVIGATION_TIMEOUT_MS });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      if (!/ERR_ABORTED|interrupted by another navigation/i.test(message)) throw new Error(stripCallLog(message));
+      await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => void 0);
+    }
+  }
   async tabInfo(page) {
     const tabId = this.track(page);
     let windowId = "unknown";
@@ -216,7 +260,7 @@ class BrowserRuntime {
         page2 = await this.context.newPage();
       }
       const url = typeof input.url === "string" ? input.url : "about:blank";
-      if (url !== "about:blank") await page2.goto(url, { waitUntil: "domcontentloaded" });
+      if (url !== "about:blank") await this.gotoSettled(page2, url);
       return { tab: await this.tabInfo(page2) };
     }
     const page = this.page(input.tabId);
@@ -226,10 +270,57 @@ class BrowserRuntime {
         await page.close();
         return { closed: true, tabId };
       case "navigate":
-        await page.goto(String(input.url), { waitUntil: "domcontentloaded" });
+        await this.gotoSettled(page, String(input.url));
         return { tab: await this.tabInfo(page) };
+      case "read_page": {
+        const maxChars = clampInt(input.maxChars, 1e3, 2e5, 5e4);
+        const depth = input.depth === void 0 ? void 0 : clampInt(input.depth, 1, 50, 50);
+        const options = { mode: "ai", boxes: true, timeout: READ_PAGE_TIMEOUT_MS, ...depth === void 0 ? {} : { depth } };
+        let snapshot;
+        try {
+          snapshot = await page.ariaSnapshot(options);
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          if (isDestroyedContext(message)) {
+            await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => void 0);
+            snapshot = await page.ariaSnapshot(options);
+          } else if (/Timeout/i.test(message)) {
+            throw new Error(
+              `read_page timed out after ${READ_PAGE_TIMEOUT_MS}ms building the outline. The page is very large or busy; retry with a depth limit (e.g. depth: 10).`
+            );
+          } else {
+            throw new Error(stripCallLog(message));
+          }
+        }
+        let text = snapshot;
+        let truncated = false;
+        if (text.length > maxChars) {
+          const cut = text.lastIndexOf("\n", maxChars);
+          text = text.slice(0, cut > 0 ? cut : maxChars);
+          truncated = true;
+        }
+        return {
+          tabId,
+          url: page.url(),
+          title: await page.title().catch(() => ""),
+          text,
+          truncated,
+          ...truncated ? { note: `Truncated at ${maxChars} of ${snapshot.length} chars. Raise maxChars, pass depth for an overview, or scroll and re-read.` } : {}
+        };
+      }
       case "screenshot": {
-        const bytes = await page.screenshot({ type: "png" });
+        if (input.fullPage === true) {
+          const metrics = await page.evaluate(() => ({
+            docWidth: Math.max(document.documentElement.scrollWidth, window.innerWidth),
+            docHeight: Math.max(document.documentElement.scrollHeight, window.innerHeight)
+          }));
+          const width = Math.min(metrics.docWidth, MAX_FULL_PAGE_WIDTH);
+          const height = Math.min(metrics.docHeight, MAX_FULL_PAGE_HEIGHT);
+          const truncated = metrics.docHeight > MAX_FULL_PAGE_HEIGHT || metrics.docWidth > MAX_FULL_PAGE_WIDTH;
+          const bytes2 = truncated ? await page.screenshot({ type: "png", scale: "css", timeout: 15e3, clip: { x: 0, y: 0, width, height } }) : await page.screenshot({ type: "png", scale: "css", timeout: 15e3, fullPage: true });
+          return { base64: bytes2.toString("base64"), width, height, fullPage: true, pageHeight: metrics.docHeight, truncated };
+        }
+        const bytes = await page.screenshot({ type: "png", scale: "css" });
         const viewport = page.viewportSize();
         const cursor = this.cursor.get(tabId);
         return {
@@ -249,12 +340,181 @@ class BrowserRuntime {
         return { tabId, x, y };
       }
       case "mouse_click": {
-        const cursor = this.cursor.get(tabId);
-        if (!cursor) throw new Error("Move the mouse in this tab before clicking.");
         const button = input.button === "middle" || input.button === "right" ? input.button : "left";
-        await page.mouse.click(cursor.x, cursor.y, { button, clickCount: input.clickCount === 2 ? 2 : 1 });
+        const clickCount = input.clickCount === 2 ? 2 : 1;
+        if (input.ref !== void 0) {
+          if (input.x !== void 0 || input.y !== void 0) throw new Error("mouse_click accepts ref or x/y, not both.");
+          const ref = requireRef(input.ref);
+          const locator = this.refLocator(page, ref);
+          try {
+            await locator.click({ button, clickCount, timeout: REF_ACTION_TIMEOUT_MS });
+          } catch (error) {
+            throw refActionError(error, ref, "click");
+          }
+          const box = await locator.boundingBox().catch(() => null);
+          if (box) this.cursor.set(tabId, { x: box.x + box.width / 2, y: box.y + box.height / 2 });
+          return {
+            tabId,
+            ref,
+            button,
+            clickCount,
+            ...box ? { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) } : {}
+          };
+        }
+        if (input.x !== void 0 || input.y !== void 0) {
+          const x = Number(input.x);
+          const y = Number(input.y);
+          if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error("mouse_click by coordinates requires numeric x and y.");
+          await page.mouse.move(x, y);
+          this.cursor.set(tabId, { x, y });
+          await page.mouse.click(x, y, { button, clickCount });
+          return { tabId, x, y, button, clickCount };
+        }
+        const cursor = this.cursor.get(tabId);
+        if (!cursor) throw new Error("Pass ref or x/y, or move the mouse in this tab before clicking.");
+        await page.mouse.click(cursor.x, cursor.y, { button, clickCount });
         return { tabId, ...cursor, button };
       }
+      case "scroll": {
+        if (input.ref !== void 0) {
+          if (input.direction !== void 0) throw new Error("scroll accepts ref or direction, not both.");
+          const ref = requireRef(input.ref);
+          const locator = this.refLocator(page, ref);
+          try {
+            await locator.scrollIntoViewIfNeeded({ timeout: REF_ACTION_TIMEOUT_MS });
+          } catch (error) {
+            throw refActionError(error, ref, "scroll");
+          }
+          const box = await locator.boundingBox().catch(() => null);
+          if (box) this.cursor.set(tabId, { x: box.x + box.width / 2, y: box.y + box.height / 2 });
+          return { tabId, ref, ...box ? { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) } : {} };
+        }
+        const direction = input.direction;
+        if (direction !== "up" && direction !== "down" && direction !== "left" && direction !== "right") {
+          throw new Error('scroll requires ref, or direction "up", "down", "left", or "right".');
+        }
+        const amount = clampInt(input.amount, 1, 2e4, 600);
+        let pointer;
+        if (input.x !== void 0 || input.y !== void 0) {
+          const x = Number(input.x);
+          const y = Number(input.y);
+          if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error("scroll requires numeric x and y when provided.");
+          pointer = { x, y };
+        } else {
+          pointer = this.cursor.get(tabId) ?? await page.evaluate(() => ({ x: Math.round(window.innerWidth / 2), y: Math.round(window.innerHeight / 2) }));
+        }
+        await page.mouse.move(pointer.x, pointer.y);
+        this.cursor.set(tabId, pointer);
+        const deltaX = direction === "left" ? -amount : direction === "right" ? amount : 0;
+        const deltaY = direction === "up" ? -amount : direction === "down" ? amount : 0;
+        await page.mouse.wheel(deltaX, deltaY);
+        return { tabId, direction, amount, ...pointer };
+      }
+      case "wait_for": {
+        const timeout = clampInt(input.timeoutMs, 500, 25e3, 1e4);
+        const provided = ["selector", "text", "url", "loadState"].filter((key) => input[key] !== void 0);
+        const condition = provided[0];
+        if (provided.length !== 1 || condition === void 0) {
+          throw new Error("wait_for requires exactly one of selector, text, url, or loadState.");
+        }
+        const state = input.state === "hidden" ? "hidden" : "visible";
+        const started = Date.now();
+        try {
+          if (typeof input.selector === "string") {
+            await page.locator(input.selector).first().waitFor({ state, timeout });
+          } else if (typeof input.text === "string") {
+            await page.getByText(input.text).first().waitFor({ state, timeout });
+          } else if (typeof input.url === "string") {
+            await page.waitForURL(input.url, { timeout, waitUntil: "domcontentloaded" });
+          } else if (input.loadState === "load" || input.loadState === "domcontentloaded" || input.loadState === "networkidle") {
+            await page.waitForLoadState(input.loadState, { timeout });
+          } else {
+            throw new Error('wait_for loadState must be "load", "domcontentloaded", or "networkidle".');
+          }
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          if (!/Timeout|exceeded/i.test(message)) throw new Error(stripCallLog(message));
+          throw new Error(
+            `wait_for gave up after ${timeout}ms waiting for ${condition} ${JSON.stringify(input[condition])}${condition === "selector" || condition === "text" ? ` to become ${state}` : ""}. Current URL: ${page.url()}. Take a browser_read_page snapshot to see the actual page state.`
+          );
+        }
+        return { tabId, matched: true, elapsedMs: Date.now() - started, url: page.url(), title: await page.title().catch(() => "") };
+      }
+      case "form_input": {
+        const ref = requireRef(input.ref);
+        const provided = ["value", "values", "checked"].filter((key) => input[key] !== void 0);
+        if (provided.length !== 1) throw new Error("form_input requires exactly one of value, values, or checked.");
+        const locator = this.refLocator(page, ref);
+        let info;
+        try {
+          info = await locator.evaluate(
+            (el) => {
+              const tag = el.tagName.toLowerCase();
+              if (el instanceof HTMLSelectElement) {
+                return {
+                  kind: el.multiple ? "multiselect" : "select",
+                  tag,
+                  options: [...el.options].map((option) => ({ value: option.value, label: option.label.trim() }))
+                };
+              }
+              if (el instanceof HTMLInputElement && (el.type === "checkbox" || el.type === "radio")) return { kind: "checkable", tag };
+              if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) return { kind: "text", tag };
+              if (el instanceof HTMLElement && el.isContentEditable) return { kind: "contenteditable", tag };
+              return { kind: "other", tag };
+            },
+            void 0,
+            { timeout: REF_ACTION_TIMEOUT_MS }
+          );
+        } catch (error) {
+          throw refActionError(error, ref, "form_input");
+        }
+        const expectField = (field) => {
+          if (provided[0] !== field) throw new Error(`Ref ${ref} is a ${info.kind} (${info.tag}); pass ${field}, not ${provided[0]}.`);
+        };
+        const matchOption = (entry) => {
+          const match = info.options.find((option) => option.value === entry) ?? info.options.find((option) => option.label === entry.trim());
+          if (!match) {
+            const available = info.options.slice(0, 20).map((option) => `${JSON.stringify(option.label)} (value ${JSON.stringify(option.value)})`).join(", ");
+            throw new Error(`No option matches ${JSON.stringify(entry)}. Available: ${available}`);
+          }
+          return match.value;
+        };
+        try {
+          switch (info.kind) {
+            case "select": {
+              expectField("value");
+              await locator.selectOption(matchOption(input.value), { timeout: REF_ACTION_TIMEOUT_MS });
+              break;
+            }
+            case "multiselect": {
+              expectField("values");
+              const entries = input.values;
+              if (!Array.isArray(entries) || entries.some((entry) => typeof entry !== "string")) {
+                throw new Error("form_input values must be an array of strings.");
+              }
+              await locator.selectOption(entries.map((entry) => matchOption(entry)), { timeout: REF_ACTION_TIMEOUT_MS });
+              break;
+            }
+            case "checkable":
+              expectField("checked");
+              await locator.setChecked(input.checked === true, { timeout: REF_ACTION_TIMEOUT_MS });
+              break;
+            case "text":
+            case "contenteditable":
+              expectField("value");
+              await locator.fill(input.value, { timeout: REF_ACTION_TIMEOUT_MS });
+              break;
+            default:
+              throw new Error(
+                `Ref ${ref} (${info.tag}) does not accept direct value input. Click it with browser_mouse_click and type with browser_keyboard_type mode "keys".`
+              );
+          }
+        } catch (error) {
+          if (error instanceof Error && !/Timeout|exceeded|waiting for/i.test(error.message)) throw error;
+          throw refActionError(error, ref, "form_input");
+        }
+        return { tabId, ref, kind: info.kind };
+      }
       case "keyboard": {
         const keys = Array.isArray(input.keys) ? input.keys : [];
         const modifiers = Array.isArray(input.modifiers) ? input.modifiers.filter((value) => typeof value === "string") : [];
@@ -269,47 +529,72 @@ class BrowserRuntime {
         }
         return { tabId, keys, modifiers };
       }
-      case "keyboard_type":
+      case "keyboard_type": {
         if (typeof input.text !== "string") throw new Error("keyboard_type requires text.");
+        if (input.mode === "keys") {
+          if (input.text.length > MAX_TYPED_KEYS_CHARS) {
+            throw new Error(
+              `keyboard_type mode "keys" is limited to ${MAX_TYPED_KEYS_CHARS} characters per call; split the text or use the default insert mode.`
+            );
+          }
+          const delay = clampInt(input.delayMs, 0, 100, 20);
+          if (input.text.length * delay > MAX_TYPED_KEYS_TOTAL_MS) {
+            throw new Error(`text.length x delayMs must stay at or below ${MAX_TYPED_KEYS_TOTAL_MS}ms; lower delayMs or split the text.`);
+          }
+          await page.keyboard.type(input.text, { delay });
+          return { tabId, characters: input.text.length, mode: "keys" };
+        }
         await page.keyboard.insertText(input.text);
         return { tabId, characters: input.text.length };
+      }
       case "run_js": {
         if (typeof input.code !== "string") throw new Error("run_js requires code.");
-        const evaluated = await page.evaluate(
-          async ({ code, maxBytes, maxCount }) => {
-            const artifacts = [];
-            let totalBytes = 0;
-            const r5dCreateArtifact = (filename, content) => {
-              if (typeof filename !== "string" || !filename.trim()) {
-                throw new TypeError("r5dCreateArtifact(filename, content): filename must be a non-empty string.");
-              }
-              if (typeof content !== "string") {
-                throw new TypeError(
-                  `r5dCreateArtifact(${JSON.stringify(filename)}, content): content must be a string. Use JSON.stringify(value) for objects.`
-                );
-              }
-              if (artifacts.length >= maxCount) {
-                throw new RangeError(`r5dCreateArtifact: at most ${maxCount} artifacts can be created in one browser_run_js call.`);
-              }
-              totalBytes += new TextEncoder().encode(content).length;
-              if (totalBytes > maxBytes) {
-                throw new RangeError(`r5dCreateArtifact: artifact content exceeded the ${maxBytes} byte budget for one browser_run_js call.`);
-              }
-              artifacts.push({ filename, content });
-              return filename;
-            };
-            const invoke = new Function("r5dCreateArtifact", `"use strict"; return (async () => {
+        let evaluated;
+        try {
+          evaluated = await page.evaluate(
+            async ({ code, maxBytes, maxCount }) => {
+              const artifacts = [];
+              let totalBytes = 0;
+              const r5dCreateArtifact = (filename, content) => {
+                if (typeof filename !== "string" || !filename.trim()) {
+                  throw new TypeError("r5dCreateArtifact(filename, content): filename must be a non-empty string.");
+                }
+                if (typeof content !== "string") {
+                  throw new TypeError(
+                    `r5dCreateArtifact(${JSON.stringify(filename)}, content): content must be a string. Use JSON.stringify(value) for objects.`
+                  );
+                }
+                if (artifacts.length >= maxCount) {
+                  throw new RangeError(`r5dCreateArtifact: at most ${maxCount} artifacts can be created in one browser_run_js call.`);
+                }
+                totalBytes += new TextEncoder().encode(content).length;
+                if (totalBytes > maxBytes) {
+                  throw new RangeError(`r5dCreateArtifact: artifact content exceeded the ${maxBytes} byte budget for one browser_run_js call.`);
+                }
+                artifacts.push({ filename, content });
+                return filename;
+              };
+              const invoke = new Function("r5dCreateArtifact", `"use strict"; return (async () => {
 ${code}
 })()`);
-            try {
-              return { result: await invoke(r5dCreateArtifact), artifacts };
-            } catch (error) {
-              if (artifacts.length === 0) throw error;
-              return { artifacts, failure: error instanceof Error ? error.message : String(error) };
-            }
-          },
-          { code: input.code, maxBytes: MAX_RUN_JS_ARTIFACT_BYTES, maxCount: MAX_RUN_JS_ARTIFACT_COUNT }
-        );
+              try {
+                return { result: await invoke(r5dCreateArtifact), artifacts };
+              } catch (error) {
+                if (artifacts.length === 0) throw error;
+                return { artifacts, failure: error instanceof Error ? error.message : String(error) };
+              }
+            },
+            { code: input.code, maxBytes: MAX_RUN_JS_ARTIFACT_BYTES, maxCount: MAX_RUN_JS_ARTIFACT_COUNT }
+          );
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          if (isDestroyedContext(message)) {
+            throw new Error(
+              "The page navigated or reloaded while the script was running, destroying its JavaScript context. The script was NOT retried because it may have side effects \u2014 if part of it ran (clicks, submits), verify the page state with browser_read_page before repeating anything. Wait out navigations with browser_wait_for, or restructure the script so navigation-triggering actions come last."
+            );
+          }
+          throw new Error(stripCallLog(message));
+        }
         const serialized = JSON.stringify(evaluated.result);
         if (serialized && Buffer.byteLength(serialized) > MAX_RUN_JS_RESULT_BYTES) {
           throw new Error("Browser JavaScript result exceeded the 1 MiB limit.");

package/dist/cjs/main.cjs

@@ -150,7 +150,16 @@ async function connectLoop(params) {
             version: findVersion(),
             chromiumVersion: params.chromiumVersion,
             profilePath: params.profilePath,
-            capabilities: { portForwarding: true, runJsArtifacts: true }
+            capabilities: {
+              portForwarding: true,
+              runJsArtifacts: true,
+              readPage: true,
+              refActions: true,
+              waitFor: true,
+              formInput: true,
+              typedKeys: true,
+              screenshotFullPage: true
+            }
           },
           tabs: await params.runtime.listTabs(),
           portForwards: params.portForwards.list()

package/dist/cjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-browser",
-  "version": "0.0.70",
+  "version": "0.0.72",
   "type": "commonjs"
 }

package/dist/mjs/browser-runtime.mjs

@@ -4,6 +4,38 @@ const MAX_DOWNLOAD_CHUNK_BYTES = 1024 * 1024;
 const MAX_RUN_JS_RESULT_BYTES = 1024 * 1024;
 const MAX_RUN_JS_ARTIFACT_BYTES = 4 * 1024 * 1024;
 const MAX_RUN_JS_ARTIFACT_COUNT = 32;
+const REF_ACTION_TIMEOUT_MS = 5e3;
+const NAVIGATION_TIMEOUT_MS = 2e4;
+const READ_PAGE_TIMEOUT_MS = 1e4;
+const MAX_FULL_PAGE_HEIGHT = 4e3;
+const MAX_FULL_PAGE_WIDTH = 4096;
+const MAX_TYPED_KEYS_CHARS = 500;
+const MAX_TYPED_KEYS_TOTAL_MS = 2e4;
+function clampInt(value, min, max, fallback) {
+  const num = value === void 0 ? fallback : Number(value);
+  if (!Number.isFinite(num)) return fallback;
+  return Math.min(max, Math.max(min, Math.round(num)));
+}
+function requireRef(value) {
+  if (typeof value !== "string" || !/^(f\d+)?e\d+$/.test(value)) {
+    throw new Error('ref must look like "e12" or "f1e3", exactly as printed by browser_read_page ([ref=e12]).');
+  }
+  return value;
+}
+function stripCallLog(message) {
+  const withoutCallLog = message.split(/\n\s*Call log:/)[0] ?? message;
+  return withoutCallLog.split("\n").filter((line) => !/^\s+at\s/.test(line)).join("\n").trim();
+}
+function isDestroyedContext(message) {
+  return /Execution context was destroyed|because of a navigation|Cannot find context with specified id|Target closed/i.test(message);
+}
+function refActionError(error, ref, verb) {
+  const message = stripCallLog(error instanceof Error ? error.message : String(error));
+  return new Error(
+    `${message}
+Ref ${ref} comes from the most recent browser_read_page snapshot of this tab. If the page navigated or re-rendered since, take a new snapshot and retry. If another element covers the target, dismiss the overlay before the ${verb}.`
+  );
+}
 class BrowserRuntime {
   constructor(context, downloadsPath) {
     this.context = context;
@@ -138,6 +170,18 @@ class BrowserRuntime {
     if (!page || page.isClosed()) throw new Error(`Browser tab ${tabId} is no longer open.`);
     return page;
   }
+  refLocator(page, ref) {
+    return page.locator(`aria-ref=${ref}`);
+  }
+  async gotoSettled(page, url) {
+    try {
+      await page.goto(url, { waitUntil: "domcontentloaded", timeout: NAVIGATION_TIMEOUT_MS });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      if (!/ERR_ABORTED|interrupted by another navigation/i.test(message)) throw new Error(stripCallLog(message));
+      await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => void 0);
+    }
+  }
   async tabInfo(page) {
     const tabId = this.track(page);
     let windowId = "unknown";
@@ -183,7 +227,7 @@ class BrowserRuntime {
         page2 = await this.context.newPage();
       }
       const url = typeof input.url === "string" ? input.url : "about:blank";
-      if (url !== "about:blank") await page2.goto(url, { waitUntil: "domcontentloaded" });
+      if (url !== "about:blank") await this.gotoSettled(page2, url);
       return { tab: await this.tabInfo(page2) };
     }
     const page = this.page(input.tabId);
@@ -193,10 +237,57 @@ class BrowserRuntime {
         await page.close();
         return { closed: true, tabId };
       case "navigate":
-        await page.goto(String(input.url), { waitUntil: "domcontentloaded" });
+        await this.gotoSettled(page, String(input.url));
         return { tab: await this.tabInfo(page) };
+      case "read_page": {
+        const maxChars = clampInt(input.maxChars, 1e3, 2e5, 5e4);
+        const depth = input.depth === void 0 ? void 0 : clampInt(input.depth, 1, 50, 50);
+        const options = { mode: "ai", boxes: true, timeout: READ_PAGE_TIMEOUT_MS, ...depth === void 0 ? {} : { depth } };
+        let snapshot;
+        try {
+          snapshot = await page.ariaSnapshot(options);
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          if (isDestroyedContext(message)) {
+            await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => void 0);
+            snapshot = await page.ariaSnapshot(options);
+          } else if (/Timeout/i.test(message)) {
+            throw new Error(
+              `read_page timed out after ${READ_PAGE_TIMEOUT_MS}ms building the outline. The page is very large or busy; retry with a depth limit (e.g. depth: 10).`
+            );
+          } else {
+            throw new Error(stripCallLog(message));
+          }
+        }
+        let text = snapshot;
+        let truncated = false;
+        if (text.length > maxChars) {
+          const cut = text.lastIndexOf("\n", maxChars);
+          text = text.slice(0, cut > 0 ? cut : maxChars);
+          truncated = true;
+        }
+        return {
+          tabId,
+          url: page.url(),
+          title: await page.title().catch(() => ""),
+          text,
+          truncated,
+          ...truncated ? { note: `Truncated at ${maxChars} of ${snapshot.length} chars. Raise maxChars, pass depth for an overview, or scroll and re-read.` } : {}
+        };
+      }
       case "screenshot": {
-        const bytes = await page.screenshot({ type: "png" });
+        if (input.fullPage === true) {
+          const metrics = await page.evaluate(() => ({
+            docWidth: Math.max(document.documentElement.scrollWidth, window.innerWidth),
+            docHeight: Math.max(document.documentElement.scrollHeight, window.innerHeight)
+          }));
+          const width = Math.min(metrics.docWidth, MAX_FULL_PAGE_WIDTH);
+          const height = Math.min(metrics.docHeight, MAX_FULL_PAGE_HEIGHT);
+          const truncated = metrics.docHeight > MAX_FULL_PAGE_HEIGHT || metrics.docWidth > MAX_FULL_PAGE_WIDTH;
+          const bytes2 = truncated ? await page.screenshot({ type: "png", scale: "css", timeout: 15e3, clip: { x: 0, y: 0, width, height } }) : await page.screenshot({ type: "png", scale: "css", timeout: 15e3, fullPage: true });
+          return { base64: bytes2.toString("base64"), width, height, fullPage: true, pageHeight: metrics.docHeight, truncated };
+        }
+        const bytes = await page.screenshot({ type: "png", scale: "css" });
         const viewport = page.viewportSize();
         const cursor = this.cursor.get(tabId);
         return {
@@ -216,12 +307,181 @@ class BrowserRuntime {
         return { tabId, x, y };
       }
       case "mouse_click": {
-        const cursor = this.cursor.get(tabId);
-        if (!cursor) throw new Error("Move the mouse in this tab before clicking.");
         const button = input.button === "middle" || input.button === "right" ? input.button : "left";
-        await page.mouse.click(cursor.x, cursor.y, { button, clickCount: input.clickCount === 2 ? 2 : 1 });
+        const clickCount = input.clickCount === 2 ? 2 : 1;
+        if (input.ref !== void 0) {
+          if (input.x !== void 0 || input.y !== void 0) throw new Error("mouse_click accepts ref or x/y, not both.");
+          const ref = requireRef(input.ref);
+          const locator = this.refLocator(page, ref);
+          try {
+            await locator.click({ button, clickCount, timeout: REF_ACTION_TIMEOUT_MS });
+          } catch (error) {
+            throw refActionError(error, ref, "click");
+          }
+          const box = await locator.boundingBox().catch(() => null);
+          if (box) this.cursor.set(tabId, { x: box.x + box.width / 2, y: box.y + box.height / 2 });
+          return {
+            tabId,
+            ref,
+            button,
+            clickCount,
+            ...box ? { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) } : {}
+          };
+        }
+        if (input.x !== void 0 || input.y !== void 0) {
+          const x = Number(input.x);
+          const y = Number(input.y);
+          if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error("mouse_click by coordinates requires numeric x and y.");
+          await page.mouse.move(x, y);
+          this.cursor.set(tabId, { x, y });
+          await page.mouse.click(x, y, { button, clickCount });
+          return { tabId, x, y, button, clickCount };
+        }
+        const cursor = this.cursor.get(tabId);
+        if (!cursor) throw new Error("Pass ref or x/y, or move the mouse in this tab before clicking.");
+        await page.mouse.click(cursor.x, cursor.y, { button, clickCount });
         return { tabId, ...cursor, button };
       }
+      case "scroll": {
+        if (input.ref !== void 0) {
+          if (input.direction !== void 0) throw new Error("scroll accepts ref or direction, not both.");
+          const ref = requireRef(input.ref);
+          const locator = this.refLocator(page, ref);
+          try {
+            await locator.scrollIntoViewIfNeeded({ timeout: REF_ACTION_TIMEOUT_MS });
+          } catch (error) {
+            throw refActionError(error, ref, "scroll");
+          }
+          const box = await locator.boundingBox().catch(() => null);
+          if (box) this.cursor.set(tabId, { x: box.x + box.width / 2, y: box.y + box.height / 2 });
+          return { tabId, ref, ...box ? { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) } : {} };
+        }
+        const direction = input.direction;
+        if (direction !== "up" && direction !== "down" && direction !== "left" && direction !== "right") {
+          throw new Error('scroll requires ref, or direction "up", "down", "left", or "right".');
+        }
+        const amount = clampInt(input.amount, 1, 2e4, 600);
+        let pointer;
+        if (input.x !== void 0 || input.y !== void 0) {
+          const x = Number(input.x);
+          const y = Number(input.y);
+          if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error("scroll requires numeric x and y when provided.");
+          pointer = { x, y };
+        } else {
+          pointer = this.cursor.get(tabId) ?? await page.evaluate(() => ({ x: Math.round(window.innerWidth / 2), y: Math.round(window.innerHeight / 2) }));
+        }
+        await page.mouse.move(pointer.x, pointer.y);
+        this.cursor.set(tabId, pointer);
+        const deltaX = direction === "left" ? -amount : direction === "right" ? amount : 0;
+        const deltaY = direction === "up" ? -amount : direction === "down" ? amount : 0;
+        await page.mouse.wheel(deltaX, deltaY);
+        return { tabId, direction, amount, ...pointer };
+      }
+      case "wait_for": {
+        const timeout = clampInt(input.timeoutMs, 500, 25e3, 1e4);
+        const provided = ["selector", "text", "url", "loadState"].filter((key) => input[key] !== void 0);
+        const condition = provided[0];
+        if (provided.length !== 1 || condition === void 0) {
+          throw new Error("wait_for requires exactly one of selector, text, url, or loadState.");
+        }
+        const state = input.state === "hidden" ? "hidden" : "visible";
+        const started = Date.now();
+        try {
+          if (typeof input.selector === "string") {
+            await page.locator(input.selector).first().waitFor({ state, timeout });
+          } else if (typeof input.text === "string") {
+            await page.getByText(input.text).first().waitFor({ state, timeout });
+          } else if (typeof input.url === "string") {
+            await page.waitForURL(input.url, { timeout, waitUntil: "domcontentloaded" });
+          } else if (input.loadState === "load" || input.loadState === "domcontentloaded" || input.loadState === "networkidle") {
+            await page.waitForLoadState(input.loadState, { timeout });
+          } else {
+            throw new Error('wait_for loadState must be "load", "domcontentloaded", or "networkidle".');
+          }
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          if (!/Timeout|exceeded/i.test(message)) throw new Error(stripCallLog(message));
+          throw new Error(
+            `wait_for gave up after ${timeout}ms waiting for ${condition} ${JSON.stringify(input[condition])}${condition === "selector" || condition === "text" ? ` to become ${state}` : ""}. Current URL: ${page.url()}. Take a browser_read_page snapshot to see the actual page state.`
+          );
+        }
+        return { tabId, matched: true, elapsedMs: Date.now() - started, url: page.url(), title: await page.title().catch(() => "") };
+      }
+      case "form_input": {
+        const ref = requireRef(input.ref);
+        const provided = ["value", "values", "checked"].filter((key) => input[key] !== void 0);
+        if (provided.length !== 1) throw new Error("form_input requires exactly one of value, values, or checked.");
+        const locator = this.refLocator(page, ref);
+        let info;
+        try {
+          info = await locator.evaluate(
+            (el) => {
+              const tag = el.tagName.toLowerCase();
+              if (el instanceof HTMLSelectElement) {
+                return {
+                  kind: el.multiple ? "multiselect" : "select",
+                  tag,
+                  options: [...el.options].map((option) => ({ value: option.value, label: option.label.trim() }))
+                };
+              }
+              if (el instanceof HTMLInputElement && (el.type === "checkbox" || el.type === "radio")) return { kind: "checkable", tag };
+              if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) return { kind: "text", tag };
+              if (el instanceof HTMLElement && el.isContentEditable) return { kind: "contenteditable", tag };
+              return { kind: "other", tag };
+            },
+            void 0,
+            { timeout: REF_ACTION_TIMEOUT_MS }
+          );
+        } catch (error) {
+          throw refActionError(error, ref, "form_input");
+        }
+        const expectField = (field) => {
+          if (provided[0] !== field) throw new Error(`Ref ${ref} is a ${info.kind} (${info.tag}); pass ${field}, not ${provided[0]}.`);
+        };
+        const matchOption = (entry) => {
+          const match = info.options.find((option) => option.value === entry) ?? info.options.find((option) => option.label === entry.trim());
+          if (!match) {
+            const available = info.options.slice(0, 20).map((option) => `${JSON.stringify(option.label)} (value ${JSON.stringify(option.value)})`).join(", ");
+            throw new Error(`No option matches ${JSON.stringify(entry)}. Available: ${available}`);
+          }
+          return match.value;
+        };
+        try {
+          switch (info.kind) {
+            case "select": {
+              expectField("value");
+              await locator.selectOption(matchOption(input.value), { timeout: REF_ACTION_TIMEOUT_MS });
+              break;
+            }
+            case "multiselect": {
+              expectField("values");
+              const entries = input.values;
+              if (!Array.isArray(entries) || entries.some((entry) => typeof entry !== "string")) {
+                throw new Error("form_input values must be an array of strings.");
+              }
+              await locator.selectOption(entries.map((entry) => matchOption(entry)), { timeout: REF_ACTION_TIMEOUT_MS });
+              break;
+            }
+            case "checkable":
+              expectField("checked");
+              await locator.setChecked(input.checked === true, { timeout: REF_ACTION_TIMEOUT_MS });
+              break;
+            case "text":
+            case "contenteditable":
+              expectField("value");
+              await locator.fill(input.value, { timeout: REF_ACTION_TIMEOUT_MS });
+              break;
+            default:
+              throw new Error(
+                `Ref ${ref} (${info.tag}) does not accept direct value input. Click it with browser_mouse_click and type with browser_keyboard_type mode "keys".`
+              );
+          }
+        } catch (error) {
+          if (error instanceof Error && !/Timeout|exceeded|waiting for/i.test(error.message)) throw error;
+          throw refActionError(error, ref, "form_input");
+        }
+        return { tabId, ref, kind: info.kind };
+      }
       case "keyboard": {
         const keys = Array.isArray(input.keys) ? input.keys : [];
         const modifiers = Array.isArray(input.modifiers) ? input.modifiers.filter((value) => typeof value === "string") : [];
@@ -236,47 +496,72 @@ class BrowserRuntime {
         }
         return { tabId, keys, modifiers };
       }
-      case "keyboard_type":
+      case "keyboard_type": {
         if (typeof input.text !== "string") throw new Error("keyboard_type requires text.");
+        if (input.mode === "keys") {
+          if (input.text.length > MAX_TYPED_KEYS_CHARS) {
+            throw new Error(
+              `keyboard_type mode "keys" is limited to ${MAX_TYPED_KEYS_CHARS} characters per call; split the text or use the default insert mode.`
+            );
+          }
+          const delay = clampInt(input.delayMs, 0, 100, 20);
+          if (input.text.length * delay > MAX_TYPED_KEYS_TOTAL_MS) {
+            throw new Error(`text.length x delayMs must stay at or below ${MAX_TYPED_KEYS_TOTAL_MS}ms; lower delayMs or split the text.`);
+          }
+          await page.keyboard.type(input.text, { delay });
+          return { tabId, characters: input.text.length, mode: "keys" };
+        }
         await page.keyboard.insertText(input.text);
         return { tabId, characters: input.text.length };
+      }
       case "run_js": {
         if (typeof input.code !== "string") throw new Error("run_js requires code.");
-        const evaluated = await page.evaluate(
-          async ({ code, maxBytes, maxCount }) => {
-            const artifacts = [];
-            let totalBytes = 0;
-            const r5dCreateArtifact = (filename, content) => {
-              if (typeof filename !== "string" || !filename.trim()) {
-                throw new TypeError("r5dCreateArtifact(filename, content): filename must be a non-empty string.");
-              }
-              if (typeof content !== "string") {
-                throw new TypeError(
-                  `r5dCreateArtifact(${JSON.stringify(filename)}, content): content must be a string. Use JSON.stringify(value) for objects.`
-                );
-              }
-              if (artifacts.length >= maxCount) {
-                throw new RangeError(`r5dCreateArtifact: at most ${maxCount} artifacts can be created in one browser_run_js call.`);
-              }
-              totalBytes += new TextEncoder().encode(content).length;
-              if (totalBytes > maxBytes) {
-                throw new RangeError(`r5dCreateArtifact: artifact content exceeded the ${maxBytes} byte budget for one browser_run_js call.`);
-              }
-              artifacts.push({ filename, content });
-              return filename;
-            };
-            const invoke = new Function("r5dCreateArtifact", `"use strict"; return (async () => {
+        let evaluated;
+        try {
+          evaluated = await page.evaluate(
+            async ({ code, maxBytes, maxCount }) => {
+              const artifacts = [];
+              let totalBytes = 0;
+              const r5dCreateArtifact = (filename, content) => {
+                if (typeof filename !== "string" || !filename.trim()) {
+                  throw new TypeError("r5dCreateArtifact(filename, content): filename must be a non-empty string.");
+                }
+                if (typeof content !== "string") {
+                  throw new TypeError(
+                    `r5dCreateArtifact(${JSON.stringify(filename)}, content): content must be a string. Use JSON.stringify(value) for objects.`
+                  );
+                }
+                if (artifacts.length >= maxCount) {
+                  throw new RangeError(`r5dCreateArtifact: at most ${maxCount} artifacts can be created in one browser_run_js call.`);
+                }
+                totalBytes += new TextEncoder().encode(content).length;
+                if (totalBytes > maxBytes) {
+                  throw new RangeError(`r5dCreateArtifact: artifact content exceeded the ${maxBytes} byte budget for one browser_run_js call.`);
+                }
+                artifacts.push({ filename, content });
+                return filename;
+              };
+              const invoke = new Function("r5dCreateArtifact", `"use strict"; return (async () => {
 ${code}
 })()`);
-            try {
-              return { result: await invoke(r5dCreateArtifact), artifacts };
-            } catch (error) {
-              if (artifacts.length === 0) throw error;
-              return { artifacts, failure: error instanceof Error ? error.message : String(error) };
-            }
-          },
-          { code: input.code, maxBytes: MAX_RUN_JS_ARTIFACT_BYTES, maxCount: MAX_RUN_JS_ARTIFACT_COUNT }
-        );
+              try {
+                return { result: await invoke(r5dCreateArtifact), artifacts };
+              } catch (error) {
+                if (artifacts.length === 0) throw error;
+                return { artifacts, failure: error instanceof Error ? error.message : String(error) };
+              }
+            },
+            { code: input.code, maxBytes: MAX_RUN_JS_ARTIFACT_BYTES, maxCount: MAX_RUN_JS_ARTIFACT_COUNT }
+          );
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          if (isDestroyedContext(message)) {
+            throw new Error(
+              "The page navigated or reloaded while the script was running, destroying its JavaScript context. The script was NOT retried because it may have side effects \u2014 if part of it ran (clicks, submits), verify the page state with browser_read_page before repeating anything. Wait out navigations with browser_wait_for, or restructure the script so navigation-triggering actions come last."
+            );
+          }
+          throw new Error(stripCallLog(message));
+        }
         const serialized = JSON.stringify(evaluated.result);
         if (serialized && Buffer.byteLength(serialized) > MAX_RUN_JS_RESULT_BYTES) {
           throw new Error("Browser JavaScript result exceeded the 1 MiB limit.");

package/dist/mjs/main.mjs

@@ -127,7 +127,16 @@ async function connectLoop(params) {
             version: findVersion(),
             chromiumVersion: params.chromiumVersion,
             profilePath: params.profilePath,
-            capabilities: { portForwarding: true, runJsArtifacts: true }
+            capabilities: {
+              portForwarding: true,
+              runJsArtifacts: true,
+              readPage: true,
+              refActions: true,
+              waitFor: true,
+              formInput: true,
+              typedKeys: true,
+              screenshotFullPage: true
+            }
           },
           tabs: await params.runtime.listTabs(),
           portForwards: params.portForwards.list()

package/dist/mjs/package.json

@@ -1,5 +1,5 @@
 {
   "name": "@ricsam/r5d-browser",
-  "version": "0.0.70",
+  "version": "0.0.72",
   "type": "module"
 }

package/dist/types/browser-runtime.d.ts

@@ -1,5 +1,5 @@
 import type { BrowserContext } from "playwright-core";
-export type BrowserAction = "list_tabs" | "list_downloads" | "get_download" | "open_tab" | "close_tab" | "navigate" | "screenshot" | "move_mouse" | "mouse_click" | "keyboard" | "keyboard_type" | "run_js";
+export type BrowserAction = "list_tabs" | "list_downloads" | "get_download" | "open_tab" | "close_tab" | "navigate" | "read_page" | "screenshot" | "move_mouse" | "mouse_click" | "scroll" | "wait_for" | "form_input" | "keyboard" | "keyboard_type" | "run_js";
 export type TabInfo = {
     tabId: string;
     windowId: string;
@@ -29,6 +29,8 @@ export declare class BrowserRuntime {
     private listDownloads;
     private getDownloadChunk;
     private page;
+    private refLocator;
+    private gotoSettled;
     private tabInfo;
     listTabs(): Promise<TabInfo[]>;
     drainDownloads(): Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ricsam/r5d-browser",
-  "version": "0.0.70",
+  "version": "0.0.72",
   "type": "module",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/mjs/main.mjs",