@agentium/browser 2.0.7 → 2.1.0

package/dist/index.js

@@ -1,5 +1,5 @@
 // src/browser-agent.ts
-import { EventBus, Logger, MemoryManager } from "@agentium/core";
+import { EventBus, Logger, MemoryManager, RunContext } from "@agentium/core";
 import { z } from "zod";
 
 // src/stealth.ts
@@ -112,7 +112,11 @@ function buildStealthContextOpts(config, viewport) {
     locale: config.locale ?? "en-US",
     timezoneId: config.timezone ?? "America/New_York",
     colorScheme: "light",
-    deviceScaleFactor: 2,
+    // Default to 1 — matches the most common real-user setup and avoids
+    // visual zoom/stretch in headed mode on non-Retina displays (the host
+    // OS would downsample a 2× rendered surface). Users on Retina-only
+    // deployments can opt back into DPR=2 for sharper screenshots.
+    deviceScaleFactor: config.deviceScaleFactor ?? 1,
     hasTouch: false,
     javaScriptEnabled: true,
     ignoreHTTPSErrors: config.ignoreHTTPSErrors ?? false
@@ -146,6 +150,13 @@ var BrowserProvider = class {
   _viewport;
   _videoDir;
   _humanize;
+  /**
+   * Most recent DOM snapshot (one per `extractDOM` call). Indexed actions
+   * (`clickByIndex`, `inputByIndex`, …) resolve their `index` against this.
+   */
+  _lastDom = [];
+  /** True if we connected over CDP (don't tear down the browser on close). */
+  _attached = false;
   constructor() {
     this._viewport = { width: 1280, height: 720 };
   }
@@ -154,8 +165,6 @@ var BrowserProvider = class {
     const pw = await import("playwright");
     const chromium = pw.chromium;
     this._viewport = opts?.viewport ?? { width: 1280, height: 720 };
-    const stealthEnabled = !!opts?.stealth;
-    const stealthCfg = typeof opts?.stealth === "object" ? opts.stealth : {};
     if (opts?.humanize) {
       const h = typeof opts.humanize === "object" ? opts.humanize : {};
       this._humanize = {
@@ -165,6 +174,20 @@ var BrowserProvider = class {
         mouseMovement: h.mouseMovement ?? true
       };
     }
+    if (opts?.cdpUrl) {
+      this.browser = await chromium.connectOverCDP(opts.cdpUrl);
+      this._attached = true;
+      const contexts = this.browser.contexts();
+      this.context = contexts.length > 0 ? contexts[0] : await this.browser.newContext({ viewport: this._viewport });
+      const existingPages = this.context.pages();
+      this.page = existingPages.length > 0 ? existingPages[0] : await this.context.newPage();
+      this.tabCounter = 0;
+      this.activeTabId = "tab-0";
+      this.pages.set("tab-0", this.page);
+      return;
+    }
+    const stealthEnabled = !!opts?.stealth;
+    const stealthCfg = typeof opts?.stealth === "object" ? opts.stealth : {};
     const launchOpts = {
       headless: opts?.headless ?? true
     };
@@ -231,7 +254,11 @@ var BrowserProvider = class {
   get viewport() {
     return this._viewport;
   }
-  // ── Interaction (with optional humanize) ─────────────────────────────
+  /** Most recent DOM snapshot. Each entry has a stable `index`. */
+  get lastDom() {
+    return this._lastDom;
+  }
+  // ── Coordinate-based Interaction ─────────────────────────────────────
   async click(x, y) {
     this.ensurePage();
     const [cx, cy] = this.clampToViewport(x, y);
@@ -261,6 +288,20 @@ var BrowserProvider = class {
     this.ensurePage();
     await this.page.keyboard.press(key);
   }
+  /**
+   * Send arbitrary keyboard keys / shortcuts. Accepts a single
+   * Playwright key spec (`"Enter"`, `"Control+l"`, `"Shift+ArrowDown"`)
+   * or a space-separated sequence (`"Tab Tab Enter"`).
+   */
+  async sendKeys(keys) {
+    this.ensurePage();
+    const tokens = keys.trim().split(/\s+/).filter(Boolean);
+    for (const token of tokens) {
+      await this.page.keyboard.press(token);
+      await this.sleep(this._humanize ? this.randInt(40, 120) : 30);
+    }
+    await this.humanPause();
+  }
   async scroll(direction, amount) {
     this.ensurePage();
     const base = amount ?? 400;
@@ -278,13 +319,262 @@ var BrowserProvider = class {
     }
     await this.humanPause();
   }
-  // ── DOM Extraction ───────────────────────────────────────────────────
+  // ── Indexed Interaction (preferred path) ─────────────────────────────
+  /**
+   * Build a Playwright locator for a DOM-snapshot index. Each `extractDOM`
+   * call tags surviving elements with `data-bua-idx="<n>"`; we resolve by
+   * that attribute. Returns null if the index is unknown.
+   */
+  locatorForIndex(index) {
+    if (!this.page) return null;
+    if (!this._lastDom.find((e) => e.index === index)) return null;
+    return this.page.locator(`[data-bua-idx="${index}"]`).first();
+  }
+  /**
+   * Click an element by its DOM-snapshot index. The most reliable click
+   * path on dynamic pages — survives layout shifts and DPR oddities.
+   */
+  async clickByIndex(index, opts) {
+    this.ensurePage();
+    const loc = this.locatorForIndex(index);
+    if (!loc) return false;
+    try {
+      await loc.scrollIntoViewIfNeeded({ timeout: 1500 }).catch(() => {
+      });
+      await loc.click({ timeout: opts?.timeout ?? 5e3 });
+      await this.humanPause();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Focus an indexed input, optionally clear it, and type. Returns false
+   * if the index couldn't be resolved or the input couldn't be focused.
+   */
+  async inputByIndex(index, text, opts) {
+    this.ensurePage();
+    const loc = this.locatorForIndex(index);
+    if (!loc) return false;
+    try {
+      const timeout = opts?.timeout ?? 5e3;
+      await loc.scrollIntoViewIfNeeded({ timeout: 1500 }).catch(() => {
+      });
+      if (opts?.clear !== false) {
+        await loc.fill("", { timeout });
+      }
+      const delay = this._humanize ? this.randInt(this._humanize.typingDelay[0], this._humanize.typingDelay[1]) : 30;
+      await loc.click({ timeout });
+      await loc.pressSequentially(text, { delay });
+      if (opts?.submit) {
+        await this.page.keyboard.press("Enter");
+      }
+      await this.humanPause();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  async uploadFileByIndex(index, path) {
+    this.ensurePage();
+    const loc = this.locatorForIndex(index);
+    if (!loc) return false;
+    try {
+      await loc.setInputFiles(path, { timeout: 5e3 });
+      await this.humanPause();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  /** Scroll the indexed element into view (no click). */
+  async scrollIntoViewByIndex(index) {
+    this.ensurePage();
+    const loc = this.locatorForIndex(index);
+    if (!loc) return false;
+    try {
+      await loc.scrollIntoViewIfNeeded({ timeout: 3e3 });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  // ── Text-based Interaction ───────────────────────────────────────────
+  /**
+   * Deterministic, DOM-based click using Playwright's text locator.
+   *
+   * Returns `true` if a matching, visible, clickable element was found and
+   * clicked within `timeout` ms; `false` otherwise (so the caller can fall
+   * back to coordinate clicking). Substring-matches by default — e.g.
+   * `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
+   */
+  async clickByText(keyword, opts) {
+    this.ensurePage();
+    const timeout = opts?.timeout ?? 3e3;
+    try {
+      const locator = this.page.locator(`text=${keyword}`).first();
+      await locator.click({ timeout });
+      await this.humanPause();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Scroll the first occurrence of `text` into view. Returns false if no
+   * match was found within the timeout.
+   */
+  async findText(text, opts) {
+    this.ensurePage();
+    const timeout = opts?.timeout ?? 3e3;
+    try {
+      const locator = this.page.getByText(text).first();
+      await locator.scrollIntoViewIfNeeded({ timeout });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  // ── Dropdowns ────────────────────────────────────────────────────────
+  /**
+   * Read the options of a native `<select>` at the given DOM-snapshot
+   * index. Returns `[]` if the element is not a `<select>`.
+   */
+  async dropdownOptions(index) {
+    this.ensurePage();
+    const loc = this.locatorForIndex(index);
+    if (!loc) return [];
+    try {
+      return await loc.evaluate((el) => {
+        if (!el || el.tagName !== "SELECT") return [];
+        return Array.from(el.options).map((opt) => ({
+          value: opt.value,
+          label: (opt.label || opt.textContent || "").trim(),
+          selected: !!opt.selected
+        }));
+      });
+    } catch {
+      return [];
+    }
+  }
+  /**
+   * Select an option in a native `<select>` by its visible text or value.
+   * Returns false if the element isn't a `<select>` or no option matched.
+   */
+  async selectDropdown(index, text) {
+    this.ensurePage();
+    const loc = this.locatorForIndex(index);
+    if (!loc) return false;
+    try {
+      await loc.selectOption({ label: text }, { timeout: 3e3 });
+      await this.humanPause();
+      return true;
+    } catch {
+      try {
+        await loc.selectOption({ value: text }, { timeout: 3e3 });
+        await this.humanPause();
+        return true;
+      } catch {
+        return false;
+      }
+    }
+  }
+  // ── JS Evaluation ────────────────────────────────────────────────────
+  /**
+   * Run arbitrary JS in the page context. The caller is responsible for
+   * gating this behind a config flag — the BrowserAgent only routes the
+   * `evaluate` action here when `allowEvaluate: true`.
+   *
+   * The code is wrapped in `(async () => { ... })()` and the return value
+   * is coerced to a string for the model.
+   */
+  async evaluate(code) {
+    this.ensurePage();
+    try {
+      const result = await this.page.evaluate(
+        // eslint-disable-next-line @typescript-eslint/no-implied-eval
+        new Function("return (async () => { " + code + " })()")
+      );
+      if (result === void 0) return "undefined";
+      if (result === null) return "null";
+      if (typeof result === "string") return result;
+      try {
+        return JSON.stringify(result);
+      } catch {
+        return String(result);
+      }
+    } catch (e) {
+      throw new Error(`evaluate failed: ${e?.message ?? e}`);
+    }
+  }
+  // ── Page text (for `extract`) ────────────────────────────────────────
+  /**
+   * Returns a clean text representation of the visible page body, with
+   * optional link extraction. Used by the BrowserAgent's `extract` action
+   * — the text is passed to a (usually cheap) LLM with the user's query.
+   */
+  async pageText(opts) {
+    this.ensurePage();
+    const maxChars = opts?.maxChars ?? 2e4;
+    const extractLinks = !!opts?.extractLinks;
+    const raw = await this.page.evaluate((withLinks) => {
+      const doc = globalThis.document;
+      const SKIP = /* @__PURE__ */ new Set(["SCRIPT", "STYLE", "NOSCRIPT", "TEMPLATE", "SVG"]);
+      const win = globalThis.window;
+      const lines = [];
+      function visit(node) {
+        if (!node) return;
+        if (node.nodeType === 3) {
+          const t = (node.nodeValue || "").trim();
+          if (t) lines.push(t);
+          return;
+        }
+        if (node.nodeType !== 1) return;
+        const tag = node.tagName.toUpperCase();
+        if (SKIP.has(tag)) return;
+        const style = win.getComputedStyle(node);
+        if (style && (style.visibility === "hidden" || style.display === "none")) return;
+        if (tag === "A" && withLinks) {
+          const href = node.getAttribute("href") || "";
+          const txt = (node.innerText || node.textContent || "").trim();
+          if (txt && href) {
+            lines.push(`[${txt}](${href})`);
+            return;
+          }
+        }
+        for (const child of node.childNodes) visit(child);
+      }
+      visit(doc.body);
+      return lines.join("\n");
+    }, extractLinks);
+    const collapsed = raw.replace(/\n{3,}/g, "\n\n").trim();
+    return collapsed.length > maxChars ? `${collapsed.slice(0, maxChars)}
+\u2026[truncated]` : collapsed;
+  }
+  // ── DOM Extraction (with index tagging) ──────────────────────────────
+  /**
+   * Snapshot the interactive elements visible in the viewport, tag each
+   * with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
+   * return BOTH a human-readable string (for the model) and the
+   * structured list (for the runtime).
+   *
+   * Three properties matter for accuracy:
+   *  - Hit-tested: each listed coordinate / index actually reaches the
+   *    labeled element (overlays / occlusion skip the entry).
+   *  - Visibility-filtered: invisible, zero-size, `pointer-events: none`,
+   *    and near-zero-opacity elements are excluded.
+   *  - `cursor: pointer` fallback pass: a second scan catches custom React
+   *    widgets that have no semantic role/href/onclick but are clickable.
+   */
   async extractDOM(opts) {
     this.ensurePage();
     const max = opts?.maxElements ?? 120;
-    const elements = await this.page.evaluate((limit) => {
+    const raw = await this.page.evaluate((limit) => {
       const doc = globalThis.document;
       const win = globalThis.window;
+      for (const stale of doc.querySelectorAll("[data-bua-idx]")) {
+        stale.removeAttribute("data-bua-idx");
+      }
       const selectors = [
         "a[href]",
         "button",
@@ -304,7 +594,24 @@ var BrowserProvider = class {
         "[contenteditable='true']",
         "[tabindex]:not([tabindex='-1'])"
       ];
-      const all = Array.from(doc.querySelectorAll(selectors.join(",")));
+      const semantic = Array.from(doc.querySelectorAll(selectors.join(",")));
+      const POINTER_SCAN_CAP = 2e3;
+      const pointerCandidates = [];
+      const allEls = doc.querySelectorAll("*");
+      for (let i = 0; i < allEls.length && pointerCandidates.length < POINTER_SCAN_CAP; i++) {
+        const el = allEls[i];
+        if (!el) continue;
+        const tag = el.tagName.toLowerCase();
+        if (tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
+        const rect = el.getBoundingClientRect();
+        if (rect.width < 4 || rect.height < 4) continue;
+        if (rect.bottom < 0 || rect.top > win.innerHeight) continue;
+        if (rect.right < 0 || rect.left > win.innerWidth) continue;
+        const style = win.getComputedStyle(el);
+        if (style.cursor !== "pointer") continue;
+        pointerCandidates.push(el);
+      }
+      const all = semantic.concat(pointerCandidates);
       const vh = win.innerHeight;
       const vw = win.innerWidth;
       const entries = [];
@@ -326,9 +633,17 @@ var BrowserProvider = class {
           if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
             continue;
           }
+          if (hit && el.contains(hit)) seen.add(hit);
         } catch {
         }
         seen.add(el);
+        entries.push({ el, cx, cy });
+      }
+      entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
+      const out = [];
+      let idx = 1;
+      for (const { el, cx, cy } of entries) {
+        if (out.length >= limit) break;
         const tag = el.tagName.toLowerCase();
         const role = el.getAttribute("role") || tag;
         const type = el.getAttribute("type") || "";
@@ -343,16 +658,29 @@ var BrowserProvider = class {
         let label = ariaLabel || text || placeholder || title || value || name;
         if (!label && href) label = href.slice(0, 60);
         if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
-        entries.push({
+        el.setAttribute("data-bua-idx", String(idx));
+        out.push({
+          index: idx,
           cx,
           cy,
-          line: `[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`
+          role,
+          type: type || void 0,
+          label,
+          tag,
+          isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
+          isSelect: tag === "select",
+          isFile: tag === "input" && type === "file"
         });
+        idx++;
       }
-      entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
-      return entries.slice(0, limit).map((e) => e.line).join("\n");
+      return out;
     }, max);
-    return elements;
+    this._lastDom = raw;
+    const lines = raw.map((e) => {
+      const typeSuffix = e.type ? `(${e.type})` : "";
+      return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}: "${e.label}"`;
+    });
+    return { text: lines.join("\n"), elements: raw };
   }
   // ── Page Info ────────────────────────────────────────────────────────
   async getPageInfo() {
@@ -437,12 +765,18 @@ var BrowserProvider = class {
   // ── Cleanup ──────────────────────────────────────────────────────────
   async close() {
     try {
-      if (this.context) await this.context.close();
+      if (this.context && !this._attached) await this.context.close();
     } catch (err) {
       console.warn("[agentium/browser] Error closing context:", err instanceof Error ? err.message : err);
     }
     try {
-      if (this.browser) await this.browser.close();
+      if (this.browser && !this._attached) await this.browser.close();
+      else if (this.browser && this._attached) {
+        try {
+          await this.browser.close();
+        } catch {
+        }
+      }
     } catch (err) {
       console.warn("[agentium/browser] Error closing browser:", err instanceof Error ? err.message : err);
     }
@@ -450,6 +784,7 @@ var BrowserProvider = class {
     this.context = null;
     this.browser = null;
     this.pages.clear();
+    this._lastDom = [];
   }
   // ── Private: Humanize helpers ────────────────────────────────────────
   /** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
@@ -471,8 +806,7 @@ var BrowserProvider = class {
     return [cx, cy];
   }
   /**
-   * Simulate human mouse movement using Bézier-like interpolation.
-   * Moves from the current mouse position to the target in small steps.
+   * Simulate human mouse movement using smoothstep interpolation.
    */
   async humanMouseMove(targetX, targetY) {
     const steps = this.randInt(5, 12);
@@ -509,89 +843,144 @@ var BrowserProvider = class {
 };
 
 // src/prompts.ts
-function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
-  const lines = [
-    `You are a browser automation agent. You receive a screenshot of a web browser and decide what action to take next to complete the user's task.`,
+function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options) {
+  const maxActions = options?.maxActionsPerStep ?? 3;
+  const allowEvaluate = !!options?.allowEvaluate;
+  const tools = options?.tools ?? [];
+  const useVision = options?.useVision ?? "auto";
+  if (options?.overrideSystemMessage) {
+    const lines2 = [options.overrideSystemMessage];
+    appendCredentials(lines2, credentialKeys);
+    lines2.push("", "## Response Format", responseFormatLines(maxActions).join("\n"));
+    return lines2.join("\n");
+  }
+  const lines = [];
+  lines.push(
+    `You are a browser automation agent. Your job is to complete the user's task by interacting with a real web page.`,
+    ``,
+    `## What you receive each step`,
+    `- The current URL and page title`,
+    `- A numbered list of interactive elements visible in the viewport (the "DOM snapshot"). Each entry looks like \`[idx] [cx,cy] role(type): "label"\` where \`idx\` is a stable per-step index, \`cx,cy\` are the CSS-pixel center coordinates, and \`label\` is the visible text / aria-label / placeholder. **Use the index to act on elements** \u2014 it is the most reliable handle.`
+  );
+  if (useVision !== false) {
+    lines.push(
+      `- A PNG screenshot of the same viewport at exactly ${viewport.width}\xD7${viewport.height} pixels. Use it for visual context (layout, icons, charts) when the DOM snapshot is ambiguous.`
+    );
+  } else {
+    lines.push(
+      `- (Vision is disabled \u2014 no screenshot. Rely entirely on the DOM snapshot and page text.)`
+    );
+  }
+  lines.push(
+    `- A short list of your previous actions (so you don't repeat yourself).`,
     ``,
-    `## Viewport & Coordinate System`,
-    `- The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels.`,
-    `- The screenshot you are given has EXACTLY the same dimensions: ${viewport.width}\xD7${viewport.height} pixels (top-left = 0,0).`,
-    `- ALL coordinates you return MUST be within 0..${viewport.width - 1} for x and 0..${viewport.height - 1} for y.`,
-    `- Coordinates outside this range will be rejected \u2014 your click will go nowhere.`,
-    `- If a "Interactive elements on page" list is provided below, PREFER its coordinates over your own visual estimation. They are exact element centers in the same coordinate system.`,
+    `## Coordinate System`,
+    `The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels, top-left origin. Coordinates outside that range are rejected.`,
     ``,
     `## Available Actions`,
-    `Respond with a JSON object containing one of these actions:`,
+    `Respond with a JSON object \u2014 or, when several independent actions can be safely batched (e.g. filling multiple form fields), an array of up to ${maxActions} JSON objects. The runtime executes them in order and stops early if the page navigates.`,
     ``,
-    `### click`,
-    `Click at a specific coordinate. Use for buttons, links, inputs, checkboxes, etc.`,
-    `\`{ "action": "click", "x": <number>, "y": <number>, "description": "<what you are clicking>" }\``,
+    `### click \u2014 by index (preferred)`,
+    `\`{ "action": "click", "index": <n>, "description": "<what you are clicking, ideally with the visible text in quotes>" }\``,
+    `If you cannot find a matching index (e.g. an element is partly off-screen and not in the snapshot), you may fall back to coordinates: \`{ "action": "click", "x": <n>, "y": <n>, "description": "Click on 'Cheapest' tab" }\`. The runtime will additionally try a Playwright text locator using the quoted phrase.`,
     ``,
-    `### type`,
-    `Type text. If x/y are provided, click that position first (to focus the input), then type. If omitted, types into the currently focused element. To press Enter after typing (e.g., to submit a search), append "\\n" to the text.`,
-    `\`{ "action": "type", "text": "<text to type>", "x": <number|optional>, "y": <number|optional> }\``,
-    `Example: \`{ "action": "type", "text": "search query\\n", "x": 640, "y": 300 }\` \u2014 clicks the search box, types, and presses Enter.`,
+    `### type \u2014 by index (preferred)`,
+    `Type into the element at \`index\`. The field is cleared first by default. Set \`"submit": true\` to press Enter after typing (e.g. submit a search).`,
+    `\`{ "action": "type", "index": <n>, "text": "<text>", "clear": true, "submit": false }\``,
+    `Fallback: \`{ "action": "type", "text": "<text>", "x": <n>, "y": <n> }\` clicks the coordinates first, then types.`,
     ``,
     `### scroll`,
-    `Scroll the page up or down. Use when content is below or above the visible area.`,
-    `\`{ "action": "scroll", "direction": "up"|"down", "amount": <pixels, optional, default 400> }\``,
+    `\`{ "action": "scroll", "direction": "down"|"up", "amount": 400 }\` or \`{ "action": "scroll", "index": <n> }\` to scroll a specific element into view.`,
     ``,
-    `### navigate`,
-    `Navigate to a specific URL. Use when you know the exact URL to visit.`,
-    `\`{ "action": "navigate", "url": "<full URL>" }\``,
+    `### find_text`,
+    `Scroll the first occurrence of a phrase into the viewport. Use this instead of repeated \`scroll\` actions when you know what you're hunting for.`,
+    `\`{ "action": "find_text", "text": "Annual report 2024" }\``,
     ``,
-    `### back`,
-    `Go back to the previous page.`,
-    `\`{ "action": "back" }\``,
+    `### send_keys`,
+    `Send arbitrary keys / shortcuts. Single key, combo (with \`+\`), or space-separated sequence.`,
+    `\`{ "action": "send_keys", "keys": "Tab Tab Enter" }\` \xB7 \`{ "action": "send_keys", "keys": "Control+l" }\` \xB7 \`{ "action": "send_keys", "keys": "Escape" }\``,
     ``,
-    `### wait`,
-    `Wait for the page to load or for a timed event. Use sparingly.`,
-    `\`{ "action": "wait", "ms": <milliseconds> }\``,
+    `### dropdown_options / select_dropdown`,
+    `Inspect or set a native \`<select>\` by index. **Do not click into native selects** \u2014 the OS overlay is not part of the DOM.`,
+    `\`{ "action": "dropdown_options", "index": <n> }\` returns the option list in the next observation.`,
+    `\`{ "action": "select_dropdown", "index": <n>, "text": "United States" }\` picks the matching option.`,
     ``,
-    `### done`,
-    `The task is complete. Provide a summary of what was accomplished.`,
-    `\`{ "action": "done", "result": "<summary of what was accomplished>" }\``,
+    `### upload_file`,
+    `Set a file on an \`<input type="file">\` by index. \`path\` must be a path the runtime can read; the agent does NOT have a filesystem of its own.`,
+    `\`{ "action": "upload_file", "index": <n>, "path": "/abs/path/to/file.pdf" }\``,
     ``,
-    `### fail`,
-    `The task cannot be completed. Explain why.`,
-    `\`{ "action": "fail", "reason": "<why the task failed>" }\``,
+    `### navigate / back`,
+    `\`{ "action": "navigate", "url": "<full URL>" }\` \xB7 \`{ "action": "back" }\``,
     ``,
-    `## Rules`,
-    `1. ALWAYS look at the screenshot carefully before deciding your action.`,
-    `2. Provide coordinates that target the CENTER of the element you want to interact with.`,
-    `3. When an "Interactive elements on page" list is provided, identify the target element in that list by its visible label (button text, aria-label, placeholder, link href) and use ITS coordinates \u2014 they are the ground truth. Only fall back to visual estimation when no matching element is listed.`,
-    `4. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
-    `5. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
-    `6. If your previous click landed on the wrong element (e.g. page didn't change or a different element became focused), the target was probably misaligned \u2014 re-read the elements list and pick a different coordinate; do NOT repeat the same click.`,
-    `7. If a page is loading or blank, use "wait" with a short delay and try again.`,
-    `8. If you see a cookie banner, consent dialog, or popup, dismiss it FIRST before proceeding with the task \u2014 these often intercept clicks on elements behind them.`,
-    `9. NEVER hallucinate content. Only report what you can actually see on the screen.`,
-    `10. When the task is fully complete, use "done" immediately with a comprehensive result.`,
-    `11. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
+    `### wait / screenshot`,
+    `\`{ "action": "wait", "ms": <ms \u2264 10000> }\` \xB7 \`{ "action": "screenshot" }\` (request a fresh image on the next step \u2014 only useful when vision mode is "auto").`,
     ``,
-    `## Response Format`,
-    `Respond with ONLY a valid JSON object. No markdown, no explanation, just the JSON action.`
-  ];
-  if (credentialKeys && credentialKeys.length > 0) {
+    `### extract`,
+    `Extract structured information from the current page using a secondary LLM. Cheaper than reasoning over multiple screenshots yourself when you just need facts/text. The extracted result is returned in the next observation as \`Last extract result:\`.`,
+    `\`{ "action": "extract", "query": "List the top 5 result titles and their prices", "extractLinks": false }\``
+  );
+  if (allowEvaluate) {
     lines.push(
       ``,
-      `## Secure Credentials`,
-      `The following credential placeholders are available for use in "type" actions:`,
-      ...credentialKeys.map((k) => `- \`{{${k}}}\``),
-      ``,
-      `When you need to fill in a login form or any field requiring these credentials,`,
-      `use the EXACT placeholder (e.g. \`{{email}}\`) as the "text" value in a type action.`,
-      `The system will securely replace them with real values at execution time.`,
-      `NEVER guess, invent, or ask the user for the actual credential values.`,
-      `NEVER include real credential values in "done" or "fail" results.`
+      `### evaluate (JS escape hatch)`,
+      `Run arbitrary JavaScript in the page context. Use this ONLY when no other action fits \u2014 e.g. shadow DOM access, complex selectors, custom widget state. The return value is stringified and returned in the next observation.`,
+      `\`{ "action": "evaluate", "code": "return document.title" }\``
     );
   }
+  if (tools.length > 0) {
+    lines.push(``, `### tool \u2014 invoke a custom tool`);
+    lines.push(
+      `In addition to browser actions, you have access to the following custom tools. Invoke them with \`{ "action": "tool", "name": "<tool>", "args": { ... } }\`. The result is returned in the next observation.`
+    );
+    for (const t of tools) {
+      lines.push(`- **${t.name}** \u2014 ${t.description}`);
+    }
+  }
+  lines.push(
+    ``,
+    `### done / fail`,
+    `\`{ "action": "done", "result": "<comprehensive summary of what was accomplished and any data the user asked for>" }\` \u2014 use when the task is complete.`,
+    `\`{ "action": "fail", "reason": "<why>" }\` \u2014 use when the task cannot be completed even after several attempts.`,
+    ``,
+    `## Rules`,
+    `1. ALWAYS check the DOM snapshot first. If the target element is listed, use its \`index\`. Coordinates are a fallback.`,
+    `2. For text-bearing targets, ALSO include the visible label in quotes in \`description\` \u2014 the runtime will use it as a third-tier fallback if the locator fails.`,
+    `3. Dismiss cookie banners, consent dialogs, and modal popups FIRST \u2014 they intercept clicks on elements behind them.`,
+    `4. If a previous action clearly hit the wrong thing, do NOT repeat it. Pick a different element or a different action.`,
+    `5. Batch independent actions when safe (e.g. filling 3 form fields). Don't batch when an action navigates or substantially changes the DOM.`,
+    `6. NEVER hallucinate. Only report data you can actually see in the snapshot, screenshot, or an extract result.`,
+    `7. When the task is fully complete, return \`done\` IMMEDIATELY with a thorough result \u2014 don't add extra confirmation steps.`,
+    `8. If after several attempts you cannot make progress, return \`fail\` with a clear reason.`
+  );
+  appendCredentials(lines, credentialKeys);
   if (extraInstructions) {
     lines.push(``, `## Additional Instructions`, extraInstructions);
   }
+  lines.push(``, `## Response Format`, responseFormatLines(maxActions).join("\n"));
   return lines.join("\n");
 }
-function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot) {
+function appendCredentials(lines, credentialKeys) {
+  if (!credentialKeys || credentialKeys.length === 0) return;
+  lines.push(
+    ``,
+    `## Secure Credentials`,
+    `The following credential placeholders are available for use in "type" actions:`,
+    ...credentialKeys.map((k) => `- \`{{${k}}}\``),
+    ``,
+    `When you need to fill in a login form or any field requiring these credentials,`,
+    `use the EXACT placeholder (e.g. \`{{email}}\`) as the "text" value in a type action.`,
+    `The system will securely replace them with real values at execution time.`,
+    `NEVER guess, invent, or ask the user for the actual credential values.`,
+    `NEVER include real credential values in "done" or "fail" results.`
+  );
+}
+function responseFormatLines(maxActions) {
+  return [
+    `Respond with ONLY valid JSON. No markdown, no commentary.`,
+    `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
+  ];
+}
+function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract) {
   const lines = [];
   lines.push(`**Task:** ${task}`);
   lines.push(`**Current URL:** ${pageUrl}`);
@@ -599,9 +988,14 @@ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, do
   lines.push(`**Step:** ${stepIndex + 1}`);
   if (domSnapshot) {
     lines.push(``);
-    lines.push(`**Interactive elements on page (format: [centerX,centerY] role: "label"):**`);
+    lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
     lines.push(domSnapshot);
   }
+  if (lastExtract) {
+    lines.push(``);
+    lines.push(`**Last extract result:**`);
+    lines.push(lastExtract);
+  }
   if (actionHistory.length > 0) {
     lines.push(``);
     lines.push(`**Previous actions:**`);
@@ -610,16 +1004,19 @@ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, do
     }
   }
   lines.push(``);
-  lines.push(`Look at the screenshot and decide the next action to complete the task.`);
+  lines.push(`Decide the next action(s) to complete the task.`);
   return lines.join("\n");
 }
 function summarizeAction(action) {
   switch (action.action) {
     case "click":
-      return `Clicked at (${action.x}, ${action.y}): ${action.description}`;
+      if (typeof action.index === "number") return `Clicked [${action.index}]: ${action.description ?? ""}`.trim();
+      return `Clicked at (${action.x}, ${action.y}): ${action.description ?? ""}`.trim();
     case "type":
+      if (typeof action.index === "number") return `Typed into [${action.index}]: "${action.text}"`;
       return action.x != null ? `Clicked (${action.x}, ${action.y}) and typed "${action.text}"` : `Typed "${action.text}"`;
     case "scroll":
+      if (typeof action.index === "number") return `Scrolled [${action.index}] into view`;
       return `Scrolled ${action.direction}${action.amount ? ` ${action.amount}px` : ""}`;
     case "navigate":
       return `Navigated to ${action.url}`;
@@ -628,7 +1025,23 @@ function summarizeAction(action) {
     case "wait":
       return `Waited ${action.ms}ms`;
     case "screenshot":
-      return `Took an extra screenshot`;
+      return `Requested a fresh screenshot`;
+    case "send_keys":
+      return `Sent keys: ${action.keys}`;
+    case "find_text":
+      return `Scrolled to text: "${action.text}"`;
+    case "evaluate":
+      return `Evaluated JS`;
+    case "dropdown_options":
+      return `Read dropdown options at [${action.index}]`;
+    case "select_dropdown":
+      return `Selected "${action.text}" in dropdown [${action.index}]`;
+    case "upload_file":
+      return `Uploaded file to [${action.index}]: ${action.path}`;
+    case "extract":
+      return `Extracted: "${action.query}"`;
+    case "tool":
+      return `Called tool "${action.name}"`;
     case "done":
       return `Done: ${action.result}`;
     case "fail":
@@ -643,19 +1056,32 @@ var BrowserAgent = class {
   name;
   eventBus;
   model;
+  pageExtractionLLM;
   instructions;
+  extendSystemMessage;
+  overrideSystemMessage;
   maxSteps;
+  maxFailures;
+  maxActionsPerStep;
+  initialActions;
+  useVision;
+  directlyOpenUrl;
   headless;
   viewport;
   defaultStartUrl;
   waitAfterAction;
   maxRepeats;
   useDOM;
+  allowEvaluate;
+  allowedDomains;
+  prohibitedDomains;
   storageState;
+  cdpUrl;
   recordVideo;
   credentials;
   stealth;
   humanize;
+  tools;
   costTracker;
   memoryManager = null;
   logger;
@@ -666,19 +1092,32 @@ var BrowserAgent = class {
   constructor(config) {
     this.name = config.name;
     this.model = config.model;
+    this.pageExtractionLLM = config.pageExtractionLLM ?? null;
     this.instructions = config.instructions;
+    this.extendSystemMessage = config.extendSystemMessage;
+    this.overrideSystemMessage = config.overrideSystemMessage;
     this.maxSteps = config.maxSteps ?? 30;
+    this.maxFailures = config.maxFailures ?? 3;
+    this.maxActionsPerStep = Math.max(1, config.maxActionsPerStep ?? 3);
+    this.initialActions = config.initialActions ?? [];
+    this.useVision = config.useVision ?? "auto";
+    this.directlyOpenUrl = config.directlyOpenUrl ?? true;
     this.headless = config.headless ?? true;
     this.viewport = config.viewport ?? { width: 1280, height: 720 };
     this.defaultStartUrl = config.startUrl;
     this.waitAfterAction = config.waitAfterAction ?? 1500;
     this.maxRepeats = config.maxRepeats ?? 3;
     this.useDOM = config.useDOM ?? true;
+    this.allowEvaluate = config.allowEvaluate ?? false;
+    this.allowedDomains = config.allowedDomains;
+    this.prohibitedDomains = config.prohibitedDomains;
     this.storageState = config.storageState;
+    this.cdpUrl = config.cdpUrl;
     this.recordVideo = config.recordVideo;
     this.credentials = config.credentials;
     this.stealth = config.stealth;
     this.humanize = config.humanize;
+    this.tools = config.tools ?? [];
     this.costTracker = config.costTracker ?? null;
     this.eventBus = config.eventBus ?? new EventBus();
     this.logger = new Logger({
@@ -691,13 +1130,20 @@ var BrowserAgent = class {
   }
   async run(task, opts) {
     const startTime = Date.now();
-    const startUrl = opts?.startUrl ?? this.defaultStartUrl;
+    const maxSteps = opts?.maxSteps ?? this.maxSteps;
     const sessionId = opts?.sessionId ?? `browser_${Date.now()}`;
     const userId = opts?.userId;
     const browser = new BrowserProvider();
     const steps = [];
     const actionHistory = [];
-    let extraInstructions = this.instructions ?? "";
+    const extractedContent = [];
+    let lastExtractResult;
+    let failureCount = 0;
+    let lastActionKey = "";
+    let repeatCount = 0;
+    let lastActionWasScreenshot = false;
+    const baseExtra = [this.instructions, this.extendSystemMessage].filter(Boolean).join("\n\n");
+    let extraInstructions = baseExtra;
     if (this.memoryManager) {
       await this.memoryManager.ensureReady();
       const memoryContext = await this.memoryManager.buildContext(sessionId, userId, task, this.name);
@@ -708,14 +1154,21 @@ ${memoryContext}` : memoryContext;
       }
     }
     const credentialKeys = this.credentials?.keys();
-    const systemPrompt = buildSystemPrompt(this.viewport, extraInstructions || void 0, credentialKeys);
-    let lastActionKey = "";
-    let repeatCount = 0;
+    const systemPrompt = buildSystemPrompt(this.viewport, extraInstructions || void 0, credentialKeys, {
+      overrideSystemMessage: this.overrideSystemMessage,
+      maxActionsPerStep: this.maxActionsPerStep,
+      allowEvaluate: this.allowEvaluate,
+      tools: this.tools,
+      useVision: this.useVision,
+      useDOM: this.useDOM
+    });
     try {
       this.logger.info("Launching browser", {
         headless: this.headless,
         viewport: this.viewport,
         useDOM: this.useDOM,
+        useVision: this.useVision,
+        cdpUrl: this.cdpUrl ?? void 0,
         recordVideo: !!this.recordVideo,
         stealth: !!this.stealth,
         humanize: !!this.humanize
@@ -726,42 +1179,72 @@ ${memoryContext}` : memoryContext;
         storageState: this.storageState,
         recordVideo: this.recordVideo,
         stealth: this.stealth,
-        humanize: this.humanize
+        humanize: this.humanize,
+        cdpUrl: this.cdpUrl
       });
+      const startUrl = opts?.startUrl ?? this.defaultStartUrl ?? this.detectUrlInTask(task);
       if (startUrl) {
         this.logger.info("Navigating to start URL", { url: startUrl });
+        this.assertDomainAllowed(startUrl);
         await browser.navigate(startUrl);
       }
-      for (let step = 0; step < this.maxSteps; step++) {
-        const screenshot = await browser.screenshot();
+      for (const ia of this.initialActions) {
+        try {
+          await this.executeAction(browser, ia, actionHistory, extractedContent);
+        } catch (e) {
+          this.logger.warn("initialAction failed", { action: ia, error: e?.message });
+        }
+        await this.sleep(this.waitAfterAction);
+      }
+      for (let step = 0; step < maxSteps; step++) {
         const pageInfo = await browser.getPageInfo();
         let domSnapshot;
         if (this.useDOM) {
-          domSnapshot = await browser.extractDOM();
+          const dom = await browser.extractDOM();
+          domSnapshot = dom.text;
+        }
+        const wantVision = this.shouldCaptureVision(step, lastActionWasScreenshot);
+        const screenshot = wantVision ? await browser.screenshot() : Buffer.alloc(0);
+        if (wantVision) this.eventBus.emit("browser.screenshot", { data: screenshot });
+        const userText = buildUserMessage(
+          task,
+          pageInfo.url,
+          pageInfo.title,
+          step,
+          actionHistory,
+          domSnapshot,
+          lastExtractResult
+        );
+        const content = [];
+        if (wantVision) {
+          content.push({ type: "image", data: screenshot.toString("base64"), mimeType: "image/png" });
         }
-        this.eventBus.emit("browser.screenshot", { data: screenshot });
-        const userText = buildUserMessage(task, pageInfo.url, pageInfo.title, step, actionHistory, domSnapshot);
+        content.push({ type: "text", text: userText });
         const messages = [
           { role: "system", content: systemPrompt },
-          {
-            role: "user",
-            content: [
-              {
-                type: "image",
-                data: screenshot.toString("base64"),
-                mimeType: "image/png"
-              },
-              { type: "text", text: userText }
-            ]
-          }
+          { role: "user", content }
         ];
-        this.logger.debug("Sending screenshot to vision model", { step, url: pageInfo.url });
-        const response = await this.model.generate(messages, {
-          temperature: 0.1,
-          maxTokens: 1024,
-          apiKey: opts?.apiKey,
-          responseFormat: "json"
-        });
+        this.logger.debug("Calling model", { step, url: pageInfo.url, vision: wantVision });
+        let response;
+        try {
+          response = await this.model.generate(messages, {
+            temperature: 0.1,
+            maxTokens: 1024,
+            apiKey: opts?.apiKey,
+            responseFormat: "json"
+          });
+        } catch (e) {
+          failureCount++;
+          this.logger.warn("Model call failed", { error: e?.message, failureCount });
+          if (failureCount > this.maxFailures) {
+            return await this.finalize(browser, steps, startTime, opts, extractedContent, {
+              result: `Model call failed ${failureCount} times: ${e?.message}`,
+              success: false
+            });
+          }
+          actionHistory.push(`(model call failed: ${e?.message ?? "unknown"} \u2014 retrying)`);
+          continue;
+        }
         if (this.costTracker && response.usage) {
           this.costTracker.track({
             runId: sessionId,
@@ -773,78 +1256,119 @@ ${memoryContext}` : memoryContext;
           });
         }
         const raw = typeof response.message.content === "string" ? response.message.content : "";
-        let action;
+        let parsed;
         try {
-          action = JSON.parse(raw);
+          parsed = JSON.parse(raw);
         } catch {
-          this.logger.warn("Failed to parse model response as JSON, retrying", { raw });
+          failureCount++;
+          this.logger.warn("Failed to parse model response as JSON", { raw, failureCount });
+          if (failureCount > this.maxFailures) {
+            return await this.finalize(browser, steps, startTime, opts, extractedContent, {
+              result: `Model returned invalid JSON ${failureCount} times. Last response: ${raw.slice(0, 200)}`,
+              success: false
+            });
+          }
           actionHistory.push("(invalid JSON response \u2014 retrying)");
           continue;
         }
-        const step_ = {
-          index: step,
-          action,
-          screenshot,
-          pageUrl: pageInfo.url,
-          pageTitle: pageInfo.title,
-          timestamp: /* @__PURE__ */ new Date(),
-          dom: domSnapshot
-        };
-        steps.push(step_);
-        let summary = summarizeAction(action);
-        if (this.credentials) {
-          summary = this.credentials.mask(summary);
-        }
-        const actionKey = JSON.stringify(action);
-        if (actionKey === lastActionKey) {
-          repeatCount++;
-        } else {
-          lastActionKey = actionKey;
-          repeatCount = 1;
-        }
-        if (repeatCount > this.maxRepeats && action.action !== "done" && action.action !== "fail") {
-          this.logger.warn("Stuck in a loop \u2014 same action repeated", {
-            action: action.action,
-            repeats: repeatCount,
-            maxRepeats: this.maxRepeats
-          });
-          actionHistory.push(
-            `\u26A0 LOOP DETECTED: "${summary}" repeated ${repeatCount} times. The agent was stuck and auto-stopped. Try a different approach or a different startUrl.`
-          );
-          return await this.finalize(browser, steps, startTime, opts, {
-            result: `Stuck in a loop: "${summary}" was repeated ${repeatCount} times. The page may have a popup, consent banner, or unexpected state blocking progress.`,
-            success: false
+        const actions = (Array.isArray(parsed) ? parsed : [parsed]).slice(
+          0,
+          this.maxActionsPerStep
+        );
+        let didTerminate = null;
+        let didNavigate = false;
+        lastActionWasScreenshot = false;
+        for (let ai = 0; ai < actions.length; ai++) {
+          const action = actions[ai];
+          let summary = summarizeAction(action);
+          if (this.credentials) summary = this.credentials.mask(summary);
+          const actionKey = JSON.stringify(action);
+          if (actionKey === lastActionKey) {
+            repeatCount++;
+          } else {
+            lastActionKey = actionKey;
+            repeatCount = 1;
+          }
+          if (repeatCount > this.maxRepeats && action.action !== "done" && action.action !== "fail") {
+            this.logger.warn("Stuck in a loop \u2014 same action repeated", {
+              action: action.action,
+              repeats: repeatCount,
+              maxRepeats: this.maxRepeats
+            });
+            actionHistory.push(
+              `\u26A0 LOOP DETECTED: "${summary}" repeated ${repeatCount} times. Auto-stopping.`
+            );
+            didTerminate = {
+              result: `Stuck in a loop: "${summary}" was repeated ${repeatCount} times. The page may have a popup, consent banner, or unexpected state blocking progress.`,
+              success: false
+            };
+            break;
+          }
+          actionHistory.push(summary);
+          this.logger.info(`Step ${step + 1}.${ai + 1}: ${summary}`);
+          this.eventBus.emit("browser.action", { action });
+          if (action.action === "done") {
+            const result = this.credentials ? this.credentials.mask(action.result) : action.result;
+            didTerminate = { result, success: true };
+            break;
+          }
+          if (action.action === "fail") {
+            const result = this.credentials ? this.credentials.mask(action.reason) : action.reason;
+            didTerminate = { result, success: false };
+            break;
+          }
+          let stepOk = true;
+          let stepOutput;
+          try {
+            const exec = await this.executeAction(browser, action, actionHistory, extractedContent);
+            stepOutput = exec?.output;
+            if (exec?.didNavigate) didNavigate = true;
+            if (action.action === "extract" && exec?.output) {
+              lastExtractResult = exec.output;
+            }
+            if (action.action === "screenshot") {
+              lastActionWasScreenshot = true;
+            }
+          } catch (e) {
+            stepOk = false;
+            failureCount++;
+            this.logger.warn("Action failed", { action: action.action, error: e?.message, failureCount });
+            actionHistory.push(`(error: ${e?.message ?? "unknown"})`);
+            if (failureCount > this.maxFailures) {
+              didTerminate = {
+                result: `Action ${action.action} failed ${failureCount} times. Last error: ${e?.message}`,
+                success: false
+              };
+              break;
+            }
+          }
+          steps.push({
+            index: steps.length,
+            action,
+            screenshot,
+            pageUrl: pageInfo.url,
+            pageTitle: pageInfo.title,
+            timestamp: /* @__PURE__ */ new Date(),
+            dom: domSnapshot,
+            output: stepOutput,
+            ok: stepOk
           });
-        }
-        actionHistory.push(summary);
-        this.logger.info(`Step ${step + 1}: ${summary}`);
-        this.eventBus.emit("browser.action", { action });
-        this.eventBus.emit("browser.step", {
-          index: step,
-          action,
-          pageUrl: pageInfo.url,
-          screenshot
-        });
-        if (action.action === "done") {
-          const result = this.credentials ? this.credentials.mask(action.result) : action.result;
-          return await this.finalize(browser, steps, startTime, opts, {
-            result,
-            success: true
+          this.eventBus.emit("browser.step", {
+            index: steps.length - 1,
+            action,
+            pageUrl: pageInfo.url,
+            screenshot
           });
+          await this.sleep(this.waitAfterAction);
+          if (didNavigate) break;
         }
-        if (action.action === "fail") {
-          const result = this.credentials ? this.credentials.mask(action.reason) : action.reason;
-          return await this.finalize(browser, steps, startTime, opts, {
-            result,
-            success: false
-          });
+        if (didTerminate) {
+          return await this.finalize(browser, steps, startTime, opts, extractedContent, didTerminate);
         }
-        await this.executeAction(browser, action);
-        await this.sleep(this.waitAfterAction);
       }
-      this.logger.warn("Max steps reached without completing task", { maxSteps: this.maxSteps });
-      return await this.finalize(browser, steps, startTime, opts, {
-        result: `Task not completed within ${this.maxSteps} steps. Last actions: ${actionHistory.slice(-3).join("; ")}`,
+      this.logger.warn("Max steps reached without completing task", { maxSteps });
+      return await this.finalize(browser, steps, startTime, opts, extractedContent, {
+        result: `Task not completed within ${maxSteps} steps. Last actions: ${actionHistory.slice(-3).join("; ")}`,
         success: false
       });
     } catch (error) {
@@ -857,7 +1381,8 @@ ${memoryContext}` : memoryContext;
         steps,
         finalUrl: "",
         finalScreenshot: Buffer.alloc(0),
-        durationMs: Date.now() - startTime
+        durationMs: Date.now() - startTime,
+        extractedContent
       };
     }
   }
@@ -880,9 +1405,43 @@ ${memoryContext}` : memoryContext;
     };
   }
   // ── Private helpers ──────────────────────────────────────────────────
-  async finalize(browser, steps, startTime, opts, outcome) {
-    const finalScreenshot = await browser.screenshot();
-    const finalInfo = await browser.getPageInfo();
+  shouldCaptureVision(step, lastActionWasScreenshot) {
+    if (this.useVision === false) return false;
+    if (this.useVision === true) return true;
+    if (step === 0) return true;
+    if (!this.useDOM) return true;
+    return lastActionWasScreenshot;
+  }
+  detectUrlInTask(task) {
+    if (!this.directlyOpenUrl) return void 0;
+    const match = task.match(/https?:\/\/[^\s)<>"']+/);
+    return match ? match[0] : void 0;
+  }
+  assertDomainAllowed(url) {
+    if (!this.allowedDomains?.length && !this.prohibitedDomains?.length) return;
+    let host;
+    try {
+      host = new URL(url).hostname.toLowerCase();
+    } catch {
+      throw new Error(`Cannot parse URL for domain check: ${url}`);
+    }
+    if (this.allowedDomains?.length && !this.allowedDomains.some((p) => matchDomain(host, p))) {
+      throw new Error(`Navigation blocked: ${host} is not in allowedDomains`);
+    }
+    if (this.prohibitedDomains?.length && this.prohibitedDomains.some((p) => matchDomain(host, p))) {
+      throw new Error(`Navigation blocked: ${host} is in prohibitedDomains`);
+    }
+  }
+  async finalize(browser, steps, startTime, opts, extractedContent, outcome) {
+    let finalScreenshot;
+    let finalInfo;
+    try {
+      finalScreenshot = await browser.screenshot();
+      finalInfo = await browser.getPageInfo();
+    } catch {
+      finalScreenshot = Buffer.alloc(0);
+      finalInfo = { url: "", title: "" };
+    }
     if (opts?.saveStorageState) {
       try {
         await browser.saveStorageState(opts.saveStorageState);
@@ -903,7 +1462,8 @@ ${memoryContext}` : memoryContext;
       finalUrl: finalInfo.url,
       finalScreenshot,
       durationMs: Date.now() - startTime,
-      videoPath
+      videoPath,
+      extractedContent
     };
     if (this.memoryManager) {
       const sessionId = opts?.sessionId ?? `browser_${startTime}`;
@@ -927,49 +1487,194 @@ ${memoryContext}` : memoryContext;
     });
     return output;
   }
-  async executeAction(browser, action) {
-    try {
-      switch (action.action) {
-        case "click":
+  /**
+   * Execute a single action. Returns `{ output?, didNavigate? }` for the
+   * caller's state tracking. May throw — the loop handles failure-budget
+   * accounting in that case.
+   */
+  async executeAction(browser, action, _actionHistory, extractedContent) {
+    switch (action.action) {
+      case "click": {
+        if (typeof action.index === "number") {
+          const ok = await browser.clickByIndex(action.index);
+          if (ok) return {};
+        }
+        const keyword = this.extractClickKeyword(action.description);
+        if (keyword && await browser.clickByText(keyword)) {
+          this.logger.debug("Clicked by text", { keyword });
+          return {};
+        }
+        if (typeof action.x === "number" && typeof action.y === "number") {
           await browser.click(action.x, action.y);
-          break;
-        case "type": {
-          const resolvedText = this.credentials ? this.credentials.resolve(action.text) : action.text;
-          if (action.x != null && action.y != null) {
-            await browser.clickAndType(action.x, action.y, resolvedText);
-          } else {
-            await browser.type(resolvedText);
-          }
-          if (resolvedText.includes("\n")) {
-            await browser.pressKey("Enter");
-          }
-          break;
+          return {};
         }
-        case "scroll":
-          await browser.scroll(action.direction, action.amount);
-          break;
-        case "navigate":
-          await browser.navigate(action.url);
-          break;
-        case "back":
-          await browser.back();
-          break;
-        case "wait":
-          await this.sleep(Math.min(action.ms, 1e4));
-          break;
-        case "screenshot":
-          break;
-        default:
-          this.logger.warn("Unknown action", { action });
+        throw new Error(`click action has no resolvable target (index/text/coordinates)`);
       }
-    } catch (error) {
-      this.logger.warn("Action execution failed", { action: action.action, error: error.message });
+      case "type": {
+        const resolvedText = this.credentials ? this.credentials.resolve(action.text) : action.text;
+        const submit = action.submit ?? resolvedText.includes("\n");
+        const cleanText = submit ? resolvedText.replace(/\n+$/, "") : resolvedText;
+        if (typeof action.index === "number") {
+          const ok = await browser.inputByIndex(action.index, cleanText, {
+            clear: action.clear,
+            submit
+          });
+          if (ok) return {};
+        }
+        if (typeof action.x === "number" && typeof action.y === "number") {
+          await browser.clickAndType(action.x, action.y, cleanText);
+          if (submit) await browser.pressKey("Enter");
+          return {};
+        }
+        await browser.type(cleanText);
+        if (submit) await browser.pressKey("Enter");
+        return {};
+      }
+      case "scroll": {
+        if (typeof action.index === "number") {
+          await browser.scrollIntoViewByIndex(action.index);
+          return {};
+        }
+        await browser.scroll(action.direction, action.amount);
+        return {};
+      }
+      case "navigate": {
+        this.assertDomainAllowed(action.url);
+        await browser.navigate(action.url);
+        return { didNavigate: true };
+      }
+      case "back":
+        await browser.back();
+        return { didNavigate: true };
+      case "wait":
+        await this.sleep(Math.min(action.ms, 1e4));
+        return {};
+      case "screenshot":
+        return {};
+      case "send_keys":
+        await browser.sendKeys(action.keys);
+        return {};
+      case "find_text":
+        await browser.findText(action.text);
+        return {};
+      case "evaluate": {
+        if (!this.allowEvaluate) {
+          throw new Error("evaluate is disabled. Set allowEvaluate: true on BrowserAgent to enable.");
+        }
+        const out = await browser.evaluate(action.code);
+        return { output: `evaluate \u2192 ${out}` };
+      }
+      case "dropdown_options": {
+        const options = await browser.dropdownOptions(action.index);
+        const formatted = options.map((o, i) => `${i + 1}. "${o.label}" (value="${o.value}")${o.selected ? " [selected]" : ""}`).join("\n");
+        return { output: `Dropdown [${action.index}] options:
+${formatted || "(no options found)"}` };
+      }
+      case "select_dropdown": {
+        const ok = await browser.selectDropdown(action.index, action.text);
+        if (!ok) throw new Error(`Could not select "${action.text}" on dropdown [${action.index}]`);
+        return {};
+      }
+      case "upload_file": {
+        const ok = await browser.uploadFileByIndex(action.index, action.path);
+        if (!ok) throw new Error(`Could not upload "${action.path}" to [${action.index}]`);
+        return {};
+      }
+      case "extract": {
+        const pageText = await browser.pageText({ extractLinks: action.extractLinks });
+        const model = this.pageExtractionLLM ?? this.model;
+        const messages = [
+          {
+            role: "system",
+            content: "You extract information from web pages. Use ONLY the page content provided. If the requested information is not present, say so explicitly. Be concise and structured (lists/tables) when appropriate."
+          },
+          {
+            role: "user",
+            content: `Query: ${action.query}
+
+Page content:
+${pageText}`
+          }
+        ];
+        const response = await model.generate(messages, { temperature: 0, maxTokens: 2048 });
+        const out = typeof response.message.content === "string" ? response.message.content : "";
+        const masked = this.credentials ? this.credentials.mask(out) : out;
+        extractedContent.push(masked);
+        return { output: masked };
+      }
+      case "tool": {
+        const tool = this.tools.find((t) => t.name === action.name);
+        if (!tool) throw new Error(`Tool "${action.name}" is not registered on this BrowserAgent`);
+        const ctx = new RunContext({
+          sessionId: `browser_${this.name}_${Date.now()}`,
+          eventBus: this.eventBus
+        });
+        const result = await tool.execute(action.args ?? {}, ctx);
+        const out = typeof result === "string" ? result : JSON.stringify(result);
+        return { output: `${action.name} \u2192 ${out}` };
+      }
+      // `done` and `fail` are intercepted by the run loop before we get
+      // here. Listing them keeps the discriminated union exhaustive.
+      case "done":
+      case "fail":
+        return {};
     }
   }
   sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
+  /**
+   * Parse a quoted target keyword from a click action's `description`.
+   * Returns `undefined` for generic / ambiguous labels (login buttons,
+   * close, OK, etc.) where a substring text match could fire on the
+   * wrong element.
+   */
+  extractClickKeyword(description) {
+    if (!description) return void 0;
+    const match = description.match(/['"\u2018\u2019\u201C\u201D]([^'"\u2018\u2019\u201C\u201D]{1,80})['"\u2018\u2019\u201C\u201D]/);
+    if (!match) return void 0;
+    const keyword = match[1].trim();
+    if (!keyword || keyword.length < 2) return void 0;
+    const skip = /* @__PURE__ */ new Set([
+      "log in",
+      "login",
+      "sign in",
+      "sign up",
+      "submit",
+      "close",
+      "ok",
+      "okay",
+      "cancel",
+      "yes",
+      "no",
+      "x",
+      "continue",
+      "next",
+      "back",
+      "accept",
+      "dismiss",
+      "got it",
+      "agree",
+      "i agree",
+      "allow",
+      "deny"
+    ]);
+    if (skip.has(keyword.toLowerCase())) return void 0;
+    return keyword;
+  }
 };
+function matchDomain(host, pattern) {
+  const h = host.toLowerCase();
+  let p = pattern.toLowerCase().replace(/^https?\*?:\/\//, "").replace(/^\/+/, "");
+  if (p.includes("/")) p = p.split("/")[0];
+  if (p === "*") return true;
+  if (p === h) return true;
+  if (p.startsWith("*.")) {
+    const base = p.slice(2);
+    return h === base || h.endsWith(`.${base}`);
+  }
+  return false;
+}
 
 // src/credential-vault.ts
 var CredentialVault = class {