@agentium/browser 2.1.1 → 2.2.1

package/dist/index.js

@@ -571,132 +571,286 @@ var BrowserProvider = class {
   /**
    * Snapshot the interactive elements visible in the viewport, tag each
    * with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
-   * return BOTH a human-readable string (for the model) and the
-   * structured list (for the runtime).
+   * return:
+   *   - `text`: a human-readable string fed to the model
+   *   - `elements`: the structured list with stable indices
+   *   - `scroll`: spatial context (pages above/below, hidden interactive count)
    *
-   * Three properties matter for accuracy:
-   *  - Hit-tested: each listed coordinate / index actually reaches the
+   * Five properties matter for accuracy:
+   *  - **Hit-tested**: each listed coordinate / index actually reaches the
    *    labeled element (overlays / occlusion skip the entry).
-   *  - Visibility-filtered: invisible, zero-size, `pointer-events: none`,
-   *    and near-zero-opacity elements are excluded.
-   *  - `cursor: pointer` fallback pass: a second scan catches custom React
-   *    widgets that have no semantic role/href/onclick but are clickable.
+   *  - **Visibility filtered (with parent chain)**: an element is dropped
+   *    if itself OR any ancestor is `display:none`, `visibility:hidden`,
+   *    `pointer-events:none`, or near-zero opacity.
+   *  - **Shadow DOM piercing**: traverses open shadow roots so custom
+   *    elements / web components are visible to the agent.
+   *  - **Same-origin iframes**: walks into each accessible iframe and
+   *    includes its interactive elements (offset by the iframe's screen
+   *    position so the coordinates the model sees are still viewport-
+   *    relative).
+   *  - **`cursor: pointer` fallback pass**: catches custom React widgets
+   *    that have no semantic role/href/onclick but are clickable.
    */
   async extractDOM(opts) {
     this.ensurePage();
-    const max = opts?.maxElements ?? 120;
-    const raw = await this.page.evaluate((limit) => {
-      const doc = globalThis.document;
-      const win = globalThis.window;
-      for (const stale of doc.querySelectorAll("[data-bua-idx]")) {
-        stale.removeAttribute("data-bua-idx");
-      }
-      const selectors = [
-        "a[href]",
-        "button",
-        "input",
-        "textarea",
-        "select",
-        "[role='button']",
-        "[role='link']",
-        "[role='tab']",
-        "[role='menuitem']",
-        "[role='checkbox']",
-        "[role='radio']",
-        "[role='switch']",
-        "[role='combobox']",
-        "[role='option']",
-        "[onclick]",
-        "[contenteditable='true']",
-        "[tabindex]:not([tabindex='-1'])"
-      ];
-      const semantic = Array.from(doc.querySelectorAll(selectors.join(",")));
-      const POINTER_SCAN_CAP = 2e3;
-      const pointerCandidates = [];
-      const allEls = doc.querySelectorAll("*");
-      for (let i = 0; i < allEls.length && pointerCandidates.length < POINTER_SCAN_CAP; i++) {
-        const el = allEls[i];
-        if (!el) continue;
-        const tag = el.tagName.toLowerCase();
-        if (tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
-        const rect = el.getBoundingClientRect();
-        if (rect.width < 4 || rect.height < 4) continue;
-        if (rect.bottom < 0 || rect.top > win.innerHeight) continue;
-        if (rect.right < 0 || rect.left > win.innerWidth) continue;
-        const style = win.getComputedStyle(el);
-        if (style.cursor !== "pointer") continue;
-        pointerCandidates.push(el);
-      }
-      const all = semantic.concat(pointerCandidates);
-      const vh = win.innerHeight;
-      const vw = win.innerWidth;
-      const entries = [];
-      const seen = /* @__PURE__ */ new Set();
-      for (const el of all) {
-        if (seen.has(el)) continue;
-        const rect = el.getBoundingClientRect();
-        if (rect.width < 4 || rect.height < 4) continue;
-        if (rect.bottom < 0 || rect.top > vh) continue;
-        if (rect.right < 0 || rect.left > vw) continue;
-        const style = win.getComputedStyle(el);
-        if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
+    const max = opts?.maxElements ?? 200;
+    const mainResult = await this.page.evaluate((limit) => {
+      return globalThis.__buaExtract(limit, 0, 0, "main");
+    }, max).catch(async () => {
+      await this.installExtractorScript();
+      return await this.page.evaluate((limit) => {
+        return globalThis.__buaExtract(limit, 0, 0, "main");
+      }, max);
+    });
+    let collected = mainResult.elements;
+    const scroll = mainResult.scroll;
+    try {
+      const frames = this.page.frames();
+      for (const frame of frames) {
+        if (frame === this.page.mainFrame()) continue;
+        if (collected.length >= max) break;
+        let bbox = null;
+        try {
+          const owner = await frame.frameElement();
+          if (owner) {
+            const rect = await owner.boundingBox();
+            if (rect) bbox = { x: rect.x, y: rect.y };
+          }
+        } catch {
           continue;
         }
-        const cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
-        const cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
+        if (!bbox) continue;
+        let iframeRes = null;
         try {
-          const hit = doc.elementFromPoint(cx, cy);
-          if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
-            continue;
-          }
-          if (hit && el.contains(hit)) seen.add(hit);
+          iframeRes = await frame.evaluate(
+            (args) => {
+              return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
+            },
+            { limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
+          );
         } catch {
+          try {
+            await frame.evaluate(this.extractorScriptSource());
+            iframeRes = await frame.evaluate(
+              (args) => {
+                return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
+              },
+              { limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
+            );
+          } catch {
+          }
+        }
+        if (iframeRes?.elements?.length) {
+          const offset = collected.length;
+          for (let i = 0; i < iframeRes.elements.length; i++) {
+            const e = iframeRes.elements[i];
+            e.index = offset + i + 1;
+            collected.push(e);
+            if (collected.length >= max) break;
+          }
         }
-        seen.add(el);
-        entries.push({ el, cx, cy });
-      }
-      entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
-      const out = [];
-      let idx = 1;
-      for (const { el, cx, cy } of entries) {
-        if (out.length >= limit) break;
-        const tag = el.tagName.toLowerCase();
-        const role = el.getAttribute("role") || tag;
-        const type = el.getAttribute("type") || "";
-        const innerText = el.innerText || el.textContent || "";
-        const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
-        const placeholder = el.getAttribute("placeholder") || "";
-        const ariaLabel = el.getAttribute("aria-label") || "";
-        const title = el.getAttribute("title") || "";
-        const name = el.getAttribute("name") || "";
-        const href = el.getAttribute("href") || "";
-        const value = el.value || "";
-        let label = ariaLabel || text || placeholder || title || value || name;
-        if (!label && href) label = href.slice(0, 60);
-        if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
-        el.setAttribute("data-bua-idx", String(idx));
-        out.push({
-          index: idx,
-          cx,
-          cy,
-          role,
-          type: type || void 0,
-          label,
-          tag,
-          isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
-          isSelect: tag === "select",
-          isFile: tag === "input" && type === "file"
-        });
-        idx++;
       }
-      return out;
-    }, max);
-    this._lastDom = raw;
-    const lines = raw.map((e) => {
+    } catch {
+    }
+    this._lastDom = collected;
+    const lines = collected.map((e) => {
       const typeSuffix = e.type ? `(${e.type})` : "";
-      return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}: "${e.label}"`;
+      const frameSuffix = e.frame && e.frame !== "main" ? ` [frame]` : "";
+      return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}${frameSuffix}: "${e.label}"`;
     });
-    return { text: lines.join("\n"), elements: raw };
+    return { text: lines.join("\n"), elements: collected, scroll };
+  }
+  /**
+   * Install the `__buaExtract` global on the main page. Idempotent —
+   * subsequent calls are no-ops.
+   */
+  async installExtractorScript() {
+    if (!this.page) return;
+    await this.page.evaluate(this.extractorScriptSource());
+  }
+  /**
+   * The extractor source. Lives in its own method so we can also inject
+   * it into iframes that haven't yet had it loaded.
+   *
+   * This function intentionally runs entirely in the page context. It:
+   *   - traverses the regular DOM + open shadow roots (deep)
+   *   - applies a parent-chain visibility filter
+   *   - applies a `cursor:pointer` second pass for custom widgets
+   *   - hit-tests each candidate at its center to avoid overlay collisions
+   *   - returns scroll context (pages above/below, hidden counts)
+   *   - tags survivors with `data-bua-idx` for indexed actions
+   */
+  extractorScriptSource() {
+    return (
+      /* js */
+      `
+      (function () {
+        if (typeof window.__buaExtract === "function") return;
+
+        function isAncestorVisible(el) {
+          let cur = el;
+          while (cur && cur !== document.body && cur !== document.documentElement) {
+            const s = window.getComputedStyle(cur);
+            if (!s) return false;
+            if (s.display === "none" || s.visibility === "hidden" || s.visibility === "collapse") return false;
+            if (parseFloat(s.opacity || "1") < 0.05) return false;
+            cur = cur.parentElement || (cur.getRootNode && cur.getRootNode().host) || null;
+          }
+          return true;
+        }
+
+        function collectSemantic(root, out) {
+          if (!root || !root.querySelectorAll) return;
+          var sel = [
+            "a[href]", "button", "input", "textarea", "select",
+            "[role='button']", "[role='link']", "[role='tab']",
+            "[role='menuitem']", "[role='checkbox']", "[role='radio']",
+            "[role='switch']", "[role='combobox']", "[role='option']",
+            "[onclick]", "[contenteditable='true']", "[tabindex]:not([tabindex='-1'])"
+          ].join(",");
+          var found = root.querySelectorAll(sel);
+          for (var i = 0; i < found.length; i++) out.push(found[i]);
+          // shadow DOM piercing
+          var all = root.querySelectorAll("*");
+          for (var j = 0; j < all.length; j++) {
+            var n = all[j];
+            if (n.shadowRoot) collectSemantic(n.shadowRoot, out);
+          }
+        }
+
+        function collectPointer(root, out, cap) {
+          if (!root || !root.querySelectorAll) return;
+          var all = root.querySelectorAll("*");
+          var vh = window.innerHeight, vw = window.innerWidth;
+          for (var i = 0; i < all.length && out.length < cap; i++) {
+            var el = all[i];
+            var tag = (el.tagName || "").toLowerCase();
+            if (!tag || tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
+            var rect = el.getBoundingClientRect();
+            if (rect.width < 4 || rect.height < 4) continue;
+            if (rect.bottom < 0 || rect.top > vh) continue;
+            if (rect.right < 0 || rect.left > vw) continue;
+            var s = window.getComputedStyle(el);
+            if (s.cursor !== "pointer") continue;
+            out.push(el);
+            if (el.shadowRoot) collectPointer(el.shadowRoot, out, cap);
+          }
+        }
+
+        function countHiddenInteractive(root, vh, vw) {
+          if (!root || !root.querySelectorAll) return 0;
+          var sel = ["a[href]","button","input","textarea","select","[role='button']","[role='link']","[role='tab']","[role='menuitem']"].join(",");
+          var n = 0;
+          var found = root.querySelectorAll(sel);
+          for (var i = 0; i < found.length; i++) {
+            var r = found[i].getBoundingClientRect();
+            if (r.bottom < 0 || r.top > vh || r.right < 0 || r.left > vw) n++;
+          }
+          return n;
+        }
+
+        window.__buaExtract = function (limit, offsetX, offsetY, frameName) {
+          // Clear stale markers each call.
+          var stale = document.querySelectorAll("[data-bua-idx]");
+          for (var s = 0; s < stale.length; s++) stale[s].removeAttribute("data-bua-idx");
+
+          var vh = window.innerHeight, vw = window.innerWidth;
+          var semantic = []; collectSemantic(document, semantic);
+          var pointer = []; collectPointer(document, pointer, 2000);
+          var all = semantic.concat(pointer);
+
+          var entries = [];
+          var seen = new Set();
+
+          for (var i = 0; i < all.length; i++) {
+            var el = all[i];
+            if (seen.has(el)) continue;
+            var rect = el.getBoundingClientRect();
+            if (rect.width < 4 || rect.height < 4) continue;
+            if (rect.bottom < 0 || rect.top > vh) continue;
+            if (rect.right < 0 || rect.left > vw) continue;
+
+            var style = window.getComputedStyle(el);
+            if (style.visibility === "hidden" || style.display === "none" ||
+                style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) continue;
+            if (!isAncestorVisible(el)) continue;
+
+            var cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
+            var cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
+
+            try {
+              var hit = document.elementFromPoint(cx, cy);
+              if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) continue;
+              if (hit && el.contains(hit)) seen.add(hit);
+            } catch (_) {}
+            seen.add(el);
+            entries.push({ el: el, cx: cx, cy: cy });
+          }
+
+          entries.sort(function (a, b) { return a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy; });
+
+          var out = [];
+          var idx = 1;
+          for (var k = 0; k < entries.length && out.length < limit; k++) {
+            var ent = entries[k];
+            var el = ent.el;
+            var tag = (el.tagName || "").toLowerCase();
+            var role = el.getAttribute("role") || tag;
+            var type = el.getAttribute("type") || "";
+            var innerText = (el.innerText || el.textContent || "");
+            var text = innerText.trim().replace(/\\s+/g, " ").slice(0, 80);
+            var placeholder = el.getAttribute("placeholder") || "";
+            var ariaLabel = el.getAttribute("aria-label") || "";
+            var title = el.getAttribute("title") || "";
+            var name = el.getAttribute("name") || "";
+            var href = el.getAttribute("href") || "";
+            var value = el.value || "";
+            var label = ariaLabel || text || placeholder || title || value || name;
+            if (!label && href) label = href.slice(0, 60);
+            if (!label) label = "(" + tag + (type ? (" type=" + type) : "") + ")";
+
+            el.setAttribute("data-bua-idx", String(idx));
+            out.push({
+              index: idx,
+              cx: ent.cx + offsetX,
+              cy: ent.cy + offsetY,
+              role: role,
+              type: type || undefined,
+              label: label,
+              tag: tag,
+              isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
+              isSelect: tag === "select",
+              isFile: tag === "input" && type === "file",
+              frame: frameName,
+            });
+            idx++;
+          }
+
+          // Scroll context (only meaningful for the main frame; iframes pass through).
+          var doc = document.scrollingElement || document.documentElement || document.body;
+          var pagesAbove = 0, pagesBelow = 0;
+          if (doc && vh > 0) {
+            pagesAbove = Math.max(0, Math.round(doc.scrollTop / vh));
+            var below = Math.max(0, doc.scrollHeight - doc.scrollTop - vh);
+            pagesBelow = Math.round(below / vh);
+          }
+
+          // Hidden interactive count over the WHOLE document tree.
+          var hidden = countHiddenInteractive(document, vh, vw);
+
+          return {
+            elements: out,
+            scroll: {
+              pagesAbove: pagesAbove,
+              pagesBelow: pagesBelow,
+              totalInteractive: out.length + hidden,
+              hiddenInteractive: hidden,
+            },
+          };
+        };
+      })();
+    `
+    );
   }
   // ── Page Info ────────────────────────────────────────────────────────
   async getPageInfo() {
@@ -858,16 +1012,169 @@ var BrowserProvider = class {
   }
 };
 
+// src/loop-detector.ts
+var LoopDetector = class {
+  actionWindow = [];
+  pageHistory = [];
+  actionWindowSize;
+  pageHistorySize;
+  /** Thresholds: repeat counts at which each severity fires. */
+  actionThresholds;
+  /** Thresholds: consecutive unchanged-page counts at which each fires. */
+  pageThresholds;
+  constructor(opts) {
+    this.actionWindowSize = opts?.actionWindowSize ?? 20;
+    this.pageHistorySize = opts?.pageHistorySize ?? 10;
+    this.actionThresholds = {
+      warn: opts?.actionThresholds?.warn ?? 5,
+      escalate: opts?.actionThresholds?.escalate ?? 8,
+      abort: opts?.actionThresholds?.abort ?? 12
+    };
+    this.pageThresholds = {
+      warn: opts?.pageThresholds?.warn ?? 5,
+      escalate: opts?.pageThresholds?.escalate ?? 8,
+      abort: opts?.pageThresholds?.abort ?? 12
+    };
+  }
+  /**
+   * Record an action and return advice for the runtime. `severity` of
+   * `abort` means the loop should terminate now.
+   */
+  recordAction(action) {
+    const key = normalizeAction(action);
+    this.actionWindow.push(key);
+    if (this.actionWindow.length > this.actionWindowSize) this.actionWindow.shift();
+    let count = 0;
+    for (const k of this.actionWindow) if (k === key) count++;
+    if (count >= this.actionThresholds.abort) {
+      return {
+        severity: "abort",
+        message: `Auto-stopped: the action "${key}" has been repeated ${count} times in the last ${this.actionWindow.length} steps with no apparent progress.`
+      };
+    }
+    if (count >= this.actionThresholds.escalate) {
+      return {
+        severity: "escalate",
+        message: `You have repeated essentially the same action ${count} times. The current approach is NOT working \u2014 pick a different strategy: scroll, navigate elsewhere, dismiss a popup, or use "fail" if the task truly cannot be done.`
+      };
+    }
+    if (count >= this.actionThresholds.warn) {
+      return {
+        severity: "warn",
+        message: `You've repeated this action ${count} times \u2014 if the page hasn't changed, try a different approach.`
+      };
+    }
+    return { severity: "none" };
+  }
+  /**
+   * Record a page fingerprint and return advice if the page has been
+   * stagnant. Counts consecutive identical fingerprints at the END of
+   * the history (so a single navigation resets the counter).
+   */
+  recordPage(fp) {
+    this.pageHistory.push(fp);
+    if (this.pageHistory.length > this.pageHistorySize) this.pageHistory.shift();
+    let stagnantCount = 1;
+    for (let i = this.pageHistory.length - 2; i >= 0; i--) {
+      const prev = this.pageHistory[i];
+      if (prev.url === fp.url && prev.interactiveCount === fp.interactiveCount && prev.textHash === fp.textHash) {
+        stagnantCount++;
+      } else {
+        break;
+      }
+    }
+    if (stagnantCount >= this.pageThresholds.abort) {
+      return {
+        severity: "abort",
+        message: `Auto-stopped: the page hasn't changed in ${stagnantCount} consecutive steps. Likely a blocking overlay or dead control.`
+      };
+    }
+    if (stagnantCount >= this.pageThresholds.escalate) {
+      return {
+        severity: "escalate",
+        message: `The page has not changed in ${stagnantCount} steps. Something is blocking progress \u2014 try dismissing popups, navigating elsewhere, or scrolling.`
+      };
+    }
+    if (stagnantCount >= this.pageThresholds.warn) {
+      return {
+        severity: "warn",
+        message: `Page hasn't changed in ${stagnantCount} steps \u2014 verify your action is taking effect.`
+      };
+    }
+    return { severity: "none" };
+  }
+  /** Combine two advices, returning the more severe one. */
+  static combine(a, b) {
+    const rank = { none: 0, warn: 1, escalate: 2, abort: 3 };
+    return rank[a.severity] >= rank[b.severity] ? a : b;
+  }
+};
+function normalizeAction(action) {
+  switch (action.action) {
+    case "click":
+      if (typeof action.index === "number") return `click:${action.index}`;
+      if (typeof action.x === "number" && typeof action.y === "number") {
+        return `click:xy:${Math.round(action.x / 16)},${Math.round(action.y / 16)}`;
+      }
+      return `click:?`;
+    case "type":
+      if (typeof action.index === "number") return `type:${action.index}:${truncate(action.text, 40)}`;
+      return `type:?:${truncate(action.text, 40)}`;
+    case "scroll":
+      return `scroll:${action.direction}:${action.amount ?? "default"}${action.index != null ? `:${action.index}` : ""}`;
+    case "navigate":
+      return `navigate:${action.url}`;
+    case "back":
+      return "back";
+    case "wait":
+      return `wait:${Math.round(action.ms / 1e3)}`;
+    case "screenshot":
+      return "screenshot";
+    case "send_keys":
+      return `send_keys:${action.keys}`;
+    case "find_text":
+      return `find_text:${truncate(action.text, 40)}`;
+    case "evaluate":
+      return `evaluate:${truncate(action.code, 80)}`;
+    case "dropdown_options":
+      return `dropdown_options:${action.index}`;
+    case "select_dropdown":
+      return `select_dropdown:${action.index}:${truncate(action.text, 40)}`;
+    case "upload_file":
+      return `upload_file:${action.index}:${action.path}`;
+    case "extract":
+      return `extract:${truncate(action.query, 80)}`;
+    case "tool":
+      return `tool:${action.name}`;
+    case "done":
+    case "fail":
+      return action.action;
+  }
+}
+function fnvHash(s) {
+  let h = 2166136261;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = h * 16777619 >>> 0;
+  }
+  return h;
+}
+function truncate(s, n) {
+  if (!s) return "";
+  return s.length <= n ? s : `${s.slice(0, n)}\u2026`;
+}
+
 // src/prompts.ts
 function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options) {
   const maxActions = options?.maxActionsPerStep ?? 3;
   const allowEvaluate = !!options?.allowEvaluate;
   const tools = options?.tools ?? [];
   const useVision = options?.useVision ?? "auto";
+  const useThinking = options?.useThinking ?? true;
   if (options?.overrideSystemMessage) {
     const lines2 = [options.overrideSystemMessage];
     appendCredentials(lines2, credentialKeys);
-    lines2.push("", "## Response Format", responseFormatLines(maxActions).join("\n"));
+    lines2.push("", "## Response Format", responseFormatLines(maxActions, useThinking).join("\n"));
     return lines2.join("\n");
   }
   const lines = [];
@@ -972,7 +1279,7 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options)
   if (extraInstructions) {
     lines.push(``, `## Additional Instructions`, extraInstructions);
   }
-  lines.push(``, `## Response Format`, responseFormatLines(maxActions).join("\n"));
+  lines.push(``, `## Response Format`, responseFormatLines(maxActions, useThinking).join("\n"));
   return lines.join("\n");
 }
 function appendCredentials(lines, credentialKeys) {
@@ -990,22 +1297,61 @@ function appendCredentials(lines, credentialKeys) {
     `NEVER include real credential values in "done" or "fail" results.`
   );
 }
-function responseFormatLines(maxActions) {
+function responseFormatLines(maxActions, useThinking) {
+  if (!useThinking) {
+    return [
+      `Respond with ONLY valid JSON. No markdown, no commentary.`,
+      `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
+    ];
+  }
   return [
-    `Respond with ONLY valid JSON. No markdown, no commentary.`,
-    `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
+    `Respond with ONLY a single valid JSON object. No markdown, no commentary.`,
+    ``,
+    `Use this exact shape:`,
+    "```json",
+    `{`,
+    `  "thinking": "Short chain-of-thought: what do I see, what's my plan?",`,
+    `  "evaluation_previous_goal": "Did the previous action succeed? success/partial/failure + 1 line",`,
+    `  "memory": "Compact bullet list of facts to remember across steps (URLs, found data, \u2026)",`,
+    `  "next_goal": "What I want to accomplish in THIS step",`,
+    `  "action": <single action object OR array of up to ${maxActions} action objects>`,
+    `}`,
+    "```",
+    ``,
+    `On the very first step, set "evaluation_previous_goal" to "n/a \u2014 first step".`,
+    `Keep each text field to one short paragraph or a few lines. Be concrete.`,
+    `Only the "action" field is executed; the others help you self-correct over multiple steps.`
   ];
 }
-function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract) {
+function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget) {
   const lines = [];
   lines.push(`**Task:** ${task}`);
   lines.push(`**Current URL:** ${pageUrl}`);
   if (pageTitle) lines.push(`**Page Title:** ${pageTitle}`);
-  lines.push(`**Step:** ${stepIndex + 1}`);
-  if (domSnapshot) {
-    lines.push(``);
-    lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
-    lines.push(domSnapshot);
+  if (stepBudget) {
+    lines.push(`**Step:** ${stepIndex + 1} of ${stepBudget.max} (${stepBudget.max - stepIndex - 1} remaining)`);
+  } else {
+    lines.push(`**Step:** ${stepIndex + 1}`);
+  }
+  if (scroll) {
+    const parts = [];
+    parts.push(`${scroll.totalInteractive} interactive elements (${scroll.hiddenInteractive} hidden)`);
+    if (scroll.pagesAbove > 0) parts.push(`${scroll.pagesAbove} page${scroll.pagesAbove === 1 ? "" : "s"} above`);
+    if (scroll.pagesBelow > 0) parts.push(`${scroll.pagesBelow} page${scroll.pagesBelow === 1 ? "" : "s"} below`);
+    if (scroll.pagesAbove === 0 && scroll.pagesBelow === 0) parts.push("fits in viewport");
+    lines.push(`**Page stats:** ${parts.join(" \xB7 ")}`);
+  }
+  if (domSnapshot !== void 0) {
+    if (domSnapshot.trim().length === 0) {
+      lines.push(``);
+      lines.push(
+        `**\u26A0 Empty page / no interactive elements detected.** The page may be blank, blocked by a captcha/anti-bot wall, mid-load, or rendered with shadow DOM in an unsupported way. Consider: \`wait\` + retry, \`navigate\` to a different URL, or \`fail\` if the site is blocking access.`
+      );
+    } else {
+      lines.push(``);
+      lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
+      lines.push(domSnapshot);
+    }
   }
   if (lastExtract) {
     lines.push(``);
@@ -1019,6 +1365,10 @@ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, do
       lines.push(`- ${entry}`);
     }
   }
+  if (nudge) {
+    lines.push(``);
+    lines.push(`**\u26A0 Runtime hint:** ${nudge}`);
+  }
   lines.push(``);
   lines.push(`Decide the next action(s) to complete the task.`);
   return lines.join("\n");
@@ -1073,6 +1423,9 @@ var BrowserAgent = class {
   eventBus;
   model;
   pageExtractionLLM;
+  fallbackModel;
+  useThinking;
+  historyWindow;
   instructions;
   extendSystemMessage;
   overrideSystemMessage;
@@ -1109,6 +1462,9 @@ var BrowserAgent = class {
     this.name = config.name;
     this.model = config.model;
     this.pageExtractionLLM = config.pageExtractionLLM ?? null;
+    this.fallbackModel = config.fallbackModel ?? null;
+    this.useThinking = config.useThinking ?? true;
+    this.historyWindow = Math.max(0, config.historyWindow ?? 6);
     this.instructions = config.instructions;
     this.extendSystemMessage = config.extendSystemMessage;
     this.overrideSystemMessage = config.overrideSystemMessage;
@@ -1154,10 +1510,10 @@ var BrowserAgent = class {
     const actionHistory = [];
     const extractedContent = [];
     let lastExtractResult;
-    let failureCount = 0;
-    let lastActionKey = "";
-    let repeatCount = 0;
+    let consecutiveFailures = 0;
     let lastActionWasScreenshot = false;
+    const loop = new LoopDetector();
+    const historyTurns = [];
     const baseExtra = [this.instructions, this.extendSystemMessage].filter(Boolean).join("\n\n");
     let extraInstructions = baseExtra;
     if (this.memoryManager) {
@@ -1176,7 +1532,8 @@ ${memoryContext}` : memoryContext;
       allowEvaluate: this.allowEvaluate,
       tools: this.tools,
       useVision: this.useVision,
-      useDOM: this.useDOM
+      useDOM: this.useDOM,
+      useThinking: this.useThinking
     });
     try {
       this.logger.info("Launching browser", {
@@ -1184,6 +1541,7 @@ ${memoryContext}` : memoryContext;
         viewport: this.viewport,
         useDOM: this.useDOM,
         useVision: this.useVision,
+        useThinking: this.useThinking,
         cdpUrl: this.cdpUrl ?? void 0,
         recordVideo: !!this.recordVideo,
         stealth: !!this.stealth,
@@ -1203,6 +1561,7 @@ ${memoryContext}` : memoryContext;
         this.logger.info("Navigating to start URL", { url: startUrl });
         this.assertDomainAllowed(startUrl);
         await browser.navigate(startUrl);
+        await this.navigationHealthCheck(browser, startUrl);
       }
       for (const ia of this.initialActions) {
         try {
@@ -1215,13 +1574,35 @@ ${memoryContext}` : memoryContext;
       for (let step = 0; step < maxSteps; step++) {
         const pageInfo = await browser.getPageInfo();
         let domSnapshot;
+        let scrollCtx;
         if (this.useDOM) {
           const dom = await browser.extractDOM();
           domSnapshot = dom.text;
+          scrollCtx = dom.scroll;
+        }
+        const pageAdvice = loop.recordPage({
+          url: pageInfo.url,
+          interactiveCount: scrollCtx?.totalInteractive ?? 0,
+          textHash: fnvHash(domSnapshot ?? "")
+        });
+        if (pageAdvice.severity === "abort") {
+          return await this.finalize(browser, steps, startTime, opts, extractedContent, {
+            result: pageAdvice.message ?? "Auto-stopped: page is stagnant.",
+            success: false
+          });
         }
         const wantVision = this.shouldCaptureVision(step, lastActionWasScreenshot);
         const screenshot = wantVision ? await browser.screenshot() : Buffer.alloc(0);
         if (wantVision) this.eventBus.emit("browser.screenshot", { data: screenshot });
+        const isLastStep = step === maxSteps - 1;
+        const nudgeParts = [];
+        if (pageAdvice.severity !== "none" && pageAdvice.message) nudgeParts.push(pageAdvice.message);
+        if (isLastStep) {
+          nudgeParts.push(
+            "This is your FINAL step. Return a `done` action right now summarizing whatever you have, even if partial."
+          );
+        }
+        const nudge = nudgeParts.length > 0 ? nudgeParts.join(" ") : void 0;
         const userText = buildUserMessage(
           task,
           pageInfo.url,
@@ -1229,65 +1610,49 @@ ${memoryContext}` : memoryContext;
           step,
           actionHistory,
           domSnapshot,
-          lastExtractResult
+          lastExtractResult,
+          scrollCtx,
+          nudge,
+          { current: step, max: maxSteps }
         );
-        const content = [];
-        if (wantVision) {
-          content.push({ type: "image", data: screenshot.toString("base64"), mimeType: "image/png" });
-        }
-        content.push({ type: "text", text: userText });
-        const messages = [
-          { role: "system", content: systemPrompt },
-          { role: "user", content }
-        ];
+        const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
         this.logger.debug("Calling model", { step, url: pageInfo.url, vision: wantVision });
-        let response;
-        try {
-          response = await this.model.generate(messages, {
-            temperature: 0.1,
-            maxTokens: 1024,
-            apiKey: opts?.apiKey,
-            responseFormat: "json"
-          });
-        } catch (e) {
-          failureCount++;
-          this.logger.warn("Model call failed", { error: e?.message, failureCount });
-          if (failureCount > this.maxFailures) {
-            return await this.finalize(browser, steps, startTime, opts, extractedContent, {
-              result: `Model call failed ${failureCount} times: ${e?.message}`,
-              success: false
-            });
+        const { response, modelUsed } = await this.callModelWithFallback(messages, opts?.apiKey);
+        if (!response) {
+          consecutiveFailures++;
+          actionHistory.push(`(model call failed \u2014 retrying, ${consecutiveFailures}/${this.maxFailures})`);
+          if (consecutiveFailures > this.maxFailures) {
+            return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
           }
-          actionHistory.push(`(model call failed: ${e?.message ?? "unknown"} \u2014 retrying)`);
           continue;
         }
         if (this.costTracker && response.usage) {
           this.costTracker.track({
             runId: sessionId,
             agentName: this.name,
-            modelId: this.model.modelId,
+            modelId: modelUsed.modelId,
             usage: response.usage,
             sessionId,
             userId
           });
         }
         const raw = typeof response.message.content === "string" ? response.message.content : "";
-        let parsed;
-        try {
-          parsed = JSON.parse(raw);
-        } catch {
-          failureCount++;
-          this.logger.warn("Failed to parse model response as JSON", { raw, failureCount });
-          if (failureCount > this.maxFailures) {
-            return await this.finalize(browser, steps, startTime, opts, extractedContent, {
-              result: `Model returned invalid JSON ${failureCount} times. Last response: ${raw.slice(0, 200)}`,
-              success: false
-            });
+        const envelope = this.parseEnvelope(raw);
+        if (!envelope) {
+          consecutiveFailures++;
+          this.logger.warn("Failed to parse model response", { raw: raw.slice(0, 200), consecutiveFailures });
+          if (consecutiveFailures > this.maxFailures) {
+            return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "parse");
           }
           actionHistory.push("(invalid JSON response \u2014 retrying)");
           continue;
         }
-        const actions = (Array.isArray(parsed) ? parsed : [parsed]).slice(
+        consecutiveFailures = 0;
+        historyTurns.push({ userText, hasScreenshot: wantVision, envelope });
+        if (this.historyWindow > 0 && historyTurns.length > this.historyWindow) {
+          historyTurns.splice(0, historyTurns.length - this.historyWindow);
+        }
+        const actions = (Array.isArray(envelope.action) ? envelope.action : [envelope.action]).slice(
           0,
           this.maxActionsPerStep
         );
@@ -1298,24 +1663,12 @@ ${memoryContext}` : memoryContext;
           const action = actions[ai];
           let summary = summarizeAction(action);
           if (this.credentials) summary = this.credentials.mask(summary);
-          const actionKey = JSON.stringify(action);
-          if (actionKey === lastActionKey) {
-            repeatCount++;
-          } else {
-            lastActionKey = actionKey;
-            repeatCount = 1;
-          }
-          if (repeatCount > this.maxRepeats && action.action !== "done" && action.action !== "fail") {
-            this.logger.warn("Stuck in a loop \u2014 same action repeated", {
-              action: action.action,
-              repeats: repeatCount,
-              maxRepeats: this.maxRepeats
-            });
-            actionHistory.push(
-              `\u26A0 LOOP DETECTED: "${summary}" repeated ${repeatCount} times. Auto-stopping.`
-            );
+          const advice = loop.recordAction(action);
+          if (advice.severity === "abort" && action.action !== "done" && action.action !== "fail") {
+            this.logger.warn("Loop detector aborting run", { advice });
+            actionHistory.push(`\u26A0 ${advice.message ?? "loop detected \u2014 auto-stopping"}`);
             didTerminate = {
-              result: `Stuck in a loop: "${summary}" was repeated ${repeatCount} times. The page may have a popup, consent banner, or unexpected state blocking progress.`,
+              result: advice.message ?? "Stuck in a loop \u2014 auto-stopped.",
               success: false
             };
             break;
@@ -1338,21 +1691,20 @@ ${memoryContext}` : memoryContext;
           try {
             const exec = await this.executeAction(browser, action, actionHistory, extractedContent);
             stepOutput = exec?.output;
-            if (exec?.didNavigate) didNavigate = true;
-            if (action.action === "extract" && exec?.output) {
-              lastExtractResult = exec.output;
-            }
-            if (action.action === "screenshot") {
-              lastActionWasScreenshot = true;
+            if (exec?.didNavigate) {
+              didNavigate = true;
+              await this.navigationHealthCheck(browser, pageInfo.url);
             }
+            if (action.action === "extract" && exec?.output) lastExtractResult = exec.output;
+            if (action.action === "screenshot") lastActionWasScreenshot = true;
           } catch (e) {
             stepOk = false;
-            failureCount++;
-            this.logger.warn("Action failed", { action: action.action, error: e?.message, failureCount });
+            consecutiveFailures++;
+            this.logger.warn("Action failed", { action: action.action, error: e?.message, consecutiveFailures });
             actionHistory.push(`(error: ${e?.message ?? "unknown"})`);
-            if (failureCount > this.maxFailures) {
+            if (consecutiveFailures > this.maxFailures) {
               didTerminate = {
-                result: `Action ${action.action} failed ${failureCount} times. Last error: ${e?.message}`,
+                result: `Action ${action.action} failed ${consecutiveFailures} times. Last error: ${e?.message}`,
                 success: false
               };
               break;
@@ -1367,7 +1719,11 @@ ${memoryContext}` : memoryContext;
             timestamp: /* @__PURE__ */ new Date(),
             dom: domSnapshot,
             output: stepOutput,
-            ok: stepOk
+            ok: stepOk,
+            thinking: envelope.thinking,
+            evaluationPreviousGoal: envelope.evaluationPreviousGoal,
+            memory: envelope.memory,
+            nextGoal: envelope.nextGoal
           });
           this.eventBus.emit("browser.step", {
             index: steps.length - 1,
@@ -1382,11 +1738,8 @@ ${memoryContext}` : memoryContext;
           return await this.finalize(browser, steps, startTime, opts, extractedContent, didTerminate);
         }
       }
-      this.logger.warn("Max steps reached without completing task", { maxSteps });
-      return await this.finalize(browser, steps, startTime, opts, extractedContent, {
-        result: `Task not completed within ${maxSteps} steps. Last actions: ${actionHistory.slice(-3).join("; ")}`,
-        success: false
-      });
+      this.logger.warn("Max steps reached without explicit done", { maxSteps });
+      return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "max_steps");
     } catch (error) {
       this.logger.error("Browser agent error", { error: error.message });
       this.eventBus.emit("browser.error", { error });
@@ -1448,6 +1801,163 @@ ${memoryContext}` : memoryContext;
       throw new Error(`Navigation blocked: ${host} is in prohibitedDomains`);
     }
   }
+  /**
+   * Build the message array sent to the model: system prompt + a compact
+   * summary of older turns (if any) + the most recent `historyWindow`
+   * turns verbatim + the current step's user message.
+   *
+   * Inspired by browser-use's history compaction. Keeps tokens bounded
+   * while giving the model meaningful context about what it already
+   * tried.
+   */
+  buildMessages(systemPrompt, historyTurns, currentUserText, currentScreenshot) {
+    const msgs = [{ role: "system", content: systemPrompt }];
+    if (this.historyWindow > 0 && historyTurns.length > 0) {
+      for (const turn of historyTurns) {
+        const env = turn.envelope;
+        const replay = [];
+        if (env.thinking) replay.push(`thinking: ${env.thinking}`);
+        if (env.evaluationPreviousGoal) replay.push(`evaluation: ${env.evaluationPreviousGoal}`);
+        if (env.memory) replay.push(`memory: ${env.memory}`);
+        if (env.nextGoal) replay.push(`next_goal: ${env.nextGoal}`);
+        replay.push(`action: ${JSON.stringify(env.action)}`);
+        msgs.push({ role: "assistant", content: replay.join("\n") });
+      }
+    }
+    const content = [];
+    if (currentScreenshot && currentScreenshot.length > 0) {
+      content.push({
+        type: "image",
+        data: currentScreenshot.toString("base64"),
+        mimeType: "image/png"
+      });
+    }
+    content.push({ type: "text", text: currentUserText });
+    msgs.push({ role: "user", content });
+    return msgs;
+  }
+  /**
+   * Call the primary model, retrying once with `fallbackModel` on
+   * transient errors (5xx, 429, network). Returns the response or
+   * `null` if both models failed.
+   */
+  async callModelWithFallback(messages, apiKey) {
+    const reqOpts = { temperature: 0.1, maxTokens: 1024, apiKey, responseFormat: "json" };
+    try {
+      const r = await this.model.generate(messages, reqOpts);
+      return { response: r, modelUsed: this.model };
+    } catch (e) {
+      if (this.fallbackModel && this.isTransientError(e)) {
+        this.logger.warn("Primary model failed; trying fallbackModel", {
+          error: e?.message,
+          primary: this.model.modelId,
+          fallback: this.fallbackModel.modelId
+        });
+        try {
+          const r = await this.fallbackModel.generate(messages, reqOpts);
+          return { response: r, modelUsed: this.fallbackModel };
+        } catch (e2) {
+          this.logger.warn("Fallback model also failed", { error: e2?.message });
+          return { response: null, modelUsed: this.model };
+        }
+      }
+      this.logger.warn("Model call failed (no fallback configured)", { error: e?.message });
+      return { response: null, modelUsed: this.model };
+    }
+  }
+  isTransientError(e) {
+    const msg = String(e?.message ?? e).toLowerCase();
+    if (msg.includes("rate limit") || msg.includes("429")) return true;
+    if (msg.includes("timeout") || msg.includes("etimedout")) return true;
+    if (msg.includes("econnreset") || msg.includes("network")) return true;
+    if (/\b5\d\d\b/.test(msg)) return true;
+    if (msg.includes("401") || msg.includes("402") || msg.includes("auth")) return true;
+    return false;
+  }
+  /**
+   * Parse the model's raw response into an `AgentOutput`. Tolerant to
+   * three shapes:
+   *  - Full envelope: { thinking, evaluation_previous_goal, action, ... }
+   *  - Raw action object (legacy / `useThinking: false`)
+   *  - Raw action array
+   *
+   * Also strips ```json fences the model occasionally adds.
+   */
+  parseEnvelope(raw) {
+    if (!raw) return null;
+    let s = raw.trim();
+    const fence = s.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
+    if (fence) s = fence[1].trim();
+    let json;
+    try {
+      json = JSON.parse(s);
+    } catch {
+      const m = s.match(/[\[{][\s\S]*[\]}]/);
+      if (!m) return null;
+      try {
+        json = JSON.parse(m[0]);
+      } catch {
+        return null;
+      }
+    }
+    if (Array.isArray(json)) return { action: json };
+    if (json && typeof json === "object" && typeof json.action === "string") {
+      return { action: json };
+    }
+    if (json && typeof json === "object" && "action" in json) {
+      const out = { action: json.action };
+      out.thinking = json.thinking ?? json.reasoning ?? void 0;
+      out.evaluationPreviousGoal = json.evaluationPreviousGoal ?? json.evaluation_previous_goal ?? void 0;
+      out.memory = json.memory ?? void 0;
+      out.nextGoal = json.nextGoal ?? json.next_goal ?? void 0;
+      return out;
+    }
+    return null;
+  }
+  /**
+   * Quick post-navigation health check. If the page came back blank
+   * (no body text and no interactive elements), reload once and wait
+   * for stable. This catches the "FreightOS half-loaded font test" class
+   * of failure before the LLM ever sees it.
+   */
+  async navigationHealthCheck(browser, url) {
+    try {
+      const ok = await browser.pageText({ maxChars: 200 }).catch(() => "");
+      if (ok && ok.trim().length > 5) return;
+      this.logger.warn("Navigation produced an empty page; reloading once", { url });
+      await this.sleep(800);
+      try {
+        await browser.navigate(url);
+      } catch {
+      }
+    } catch {
+    }
+  }
+  /**
+   * Force-finalize the run with whatever partial data the agent has.
+   * Called when:
+   *   - `maxSteps` is exhausted without an explicit `done`,
+   *   - `maxFailures` is exceeded,
+   *   - the model can't produce parseable JSON enough times to make
+   *     forward progress.
+   * The result is composed from `extractedContent` + the last few
+   * action summaries so the caller gets something useful instead of
+   * just a one-line error.
+   */
+  async forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, reason) {
+    const reasonLine = reason === "model" ? "Model call failed too many consecutive times." : reason === "parse" ? "Model produced invalid JSON too many times." : "Max step budget exhausted before an explicit `done`.";
+    const tail = actionHistory.slice(-5).join(" \u2192 ");
+    const extracts = extractedContent.length > 0 ? `
+
+Extracts so far:
+${extractedContent.join("\n---\n")}` : "";
+    const result = `${reasonLine}
+Last actions: ${tail || "(none)"}${extracts}`;
+    return await this.finalize(browser, steps, startTime, opts, extractedContent, {
+      result,
+      success: false
+    });
+  }
   async finalize(browser, steps, startTime, opts, extractedContent, outcome) {
     let finalScreenshot;
     let finalInfo;