@agentium/browser 2.1.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -118,29 +118,39 @@ function getStealthScript() {
118
118
  };
119
119
  }
120
120
 
121
- // \u2500\u2500 Prevent iframe detection of automation \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
122
- Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', {
123
- get: function() {
124
- return window;
125
- },
126
- });
127
-
128
- // \u2500\u2500 Remove "cdc_" Playwright/ChromeDriver markers from DOM \u2500\u2500\u2500\u2500\u2500\u2500\u2500
129
- const observer = new MutationObserver((mutations) => {
130
- for (const mutation of mutations) {
131
- for (const node of mutation.addedNodes) {
132
- if (node.nodeType === 1) {
133
- const el = node;
134
- for (const attr of [...el.attributes]) {
135
- if (attr.name.startsWith('cdc_') || attr.name.startsWith('__playwright')) {
136
- el.removeAttribute(attr.name);
121
+ // \u2500\u2500 Remove "cdc_" Playwright/ChromeDriver markers from new elements \u2500
122
+ // Only watches childList additions, NOT every attribute mutation \u2014
123
+ // attributes:true subtree:true is O(every DOM write) on heavy SPAs
124
+ // like FreightOS and noticeably slows the page.
125
+ try {
126
+ const observer = new MutationObserver((mutations) => {
127
+ for (const mutation of mutations) {
128
+ for (const node of mutation.addedNodes) {
129
+ if (node && node.nodeType === 1) {
130
+ const el = node;
131
+ if (!el.attributes) continue;
132
+ for (const attr of [...el.attributes]) {
133
+ if (attr.name.startsWith('cdc_') || attr.name.startsWith('__playwright')) {
134
+ el.removeAttribute(attr.name);
135
+ }
137
136
  }
138
137
  }
139
138
  }
140
139
  }
140
+ });
141
+ if (document.documentElement) {
142
+ observer.observe(document.documentElement, { childList: true, subtree: true });
141
143
  }
142
- });
143
- observer.observe(document.documentElement, { attributes: true, childList: true, subtree: true });
144
+ } catch (_e) { /* never break the page over a stealth patch */ }
145
+
146
+ // NOTE on iframe.contentWindow:
147
+ // We deliberately do NOT override HTMLIFrameElement.prototype.contentWindow.
148
+ // An earlier version returned the parent window, which breaks legitimate
149
+ // page JS on any site that uses iframes (checkout widgets, embedded
150
+ // videos, A/B test scripts, analytics). The "anti-detection" value was
151
+ // minimal \u2014 modern detectors don't rely on that property anyway \u2014 and
152
+ // the breakage caused pages to fail to boot, leaving font-test residue
153
+ // on screen.
144
154
  `;
145
155
  }
146
156
  function buildStealthContextOpts(config, viewport) {
@@ -226,14 +236,20 @@ var BrowserProvider = class {
226
236
  }
227
237
  const stealthEnabled = !!opts?.stealth;
228
238
  const stealthCfg = typeof opts?.stealth === "object" ? opts.stealth : {};
229
- const launchOpts = {
230
- headless: opts?.headless ?? true
231
- };
239
+ const headless = opts?.headless ?? true;
240
+ const launchOpts = { headless };
241
+ const args = [];
232
242
  if (stealthEnabled) {
233
- const { args, proxy } = buildStealthLaunchArgs(stealthCfg);
234
- launchOpts.args = args;
235
- if (proxy) launchOpts.proxy = proxy;
243
+ const stealth = buildStealthLaunchArgs(stealthCfg);
244
+ args.push(...stealth.args);
245
+ if (stealth.proxy) launchOpts.proxy = stealth.proxy;
246
+ }
247
+ if (!headless) {
248
+ const CHROME_VERTICAL_CHROME_PX = 140;
249
+ args.push(`--window-size=${this._viewport.width},${this._viewport.height + CHROME_VERTICAL_CHROME_PX}`);
250
+ args.push("--window-position=0,0");
236
251
  }
252
+ if (args.length) launchOpts.args = args;
237
253
  this.browser = await chromium.launch(launchOpts);
238
254
  let contextOpts;
239
255
  if (stealthEnabled) {
@@ -593,132 +609,286 @@ var BrowserProvider = class {
593
609
  /**
594
610
  * Snapshot the interactive elements visible in the viewport, tag each
595
611
  * with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
596
- * return BOTH a human-readable string (for the model) and the
597
- * structured list (for the runtime).
612
+ * return:
613
+ * - `text`: a human-readable string fed to the model
614
+ * - `elements`: the structured list with stable indices
615
+ * - `scroll`: spatial context (pages above/below, hidden interactive count)
598
616
  *
599
- * Three properties matter for accuracy:
600
- * - Hit-tested: each listed coordinate / index actually reaches the
617
+ * Five properties matter for accuracy:
618
+ * - **Hit-tested**: each listed coordinate / index actually reaches the
601
619
  * labeled element (overlays / occlusion skip the entry).
602
- * - Visibility-filtered: invisible, zero-size, `pointer-events: none`,
603
- * and near-zero-opacity elements are excluded.
604
- * - `cursor: pointer` fallback pass: a second scan catches custom React
605
- * widgets that have no semantic role/href/onclick but are clickable.
620
+ * - **Visibility filtered (with parent chain)**: an element is dropped
621
+ * if itself OR any ancestor is `display:none`, `visibility:hidden`,
622
+ * `pointer-events:none`, or near-zero opacity.
623
+ * - **Shadow DOM piercing**: traverses open shadow roots so custom
624
+ * elements / web components are visible to the agent.
625
+ * - **Same-origin iframes**: walks into each accessible iframe and
626
+ * includes its interactive elements (offset by the iframe's screen
627
+ * position so the coordinates the model sees are still viewport-
628
+ * relative).
629
+ * - **`cursor: pointer` fallback pass**: catches custom React widgets
630
+ * that have no semantic role/href/onclick but are clickable.
606
631
  */
607
632
  async extractDOM(opts) {
608
633
  this.ensurePage();
609
- const max = opts?.maxElements ?? 120;
610
- const raw = await this.page.evaluate((limit) => {
611
- const doc = globalThis.document;
612
- const win = globalThis.window;
613
- for (const stale of doc.querySelectorAll("[data-bua-idx]")) {
614
- stale.removeAttribute("data-bua-idx");
615
- }
616
- const selectors = [
617
- "a[href]",
618
- "button",
619
- "input",
620
- "textarea",
621
- "select",
622
- "[role='button']",
623
- "[role='link']",
624
- "[role='tab']",
625
- "[role='menuitem']",
626
- "[role='checkbox']",
627
- "[role='radio']",
628
- "[role='switch']",
629
- "[role='combobox']",
630
- "[role='option']",
631
- "[onclick]",
632
- "[contenteditable='true']",
633
- "[tabindex]:not([tabindex='-1'])"
634
- ];
635
- const semantic = Array.from(doc.querySelectorAll(selectors.join(",")));
636
- const POINTER_SCAN_CAP = 2e3;
637
- const pointerCandidates = [];
638
- const allEls = doc.querySelectorAll("*");
639
- for (let i = 0; i < allEls.length && pointerCandidates.length < POINTER_SCAN_CAP; i++) {
640
- const el = allEls[i];
641
- if (!el) continue;
642
- const tag = el.tagName.toLowerCase();
643
- if (tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
644
- const rect = el.getBoundingClientRect();
645
- if (rect.width < 4 || rect.height < 4) continue;
646
- if (rect.bottom < 0 || rect.top > win.innerHeight) continue;
647
- if (rect.right < 0 || rect.left > win.innerWidth) continue;
648
- const style = win.getComputedStyle(el);
649
- if (style.cursor !== "pointer") continue;
650
- pointerCandidates.push(el);
651
- }
652
- const all = semantic.concat(pointerCandidates);
653
- const vh = win.innerHeight;
654
- const vw = win.innerWidth;
655
- const entries = [];
656
- const seen = /* @__PURE__ */ new Set();
657
- for (const el of all) {
658
- if (seen.has(el)) continue;
659
- const rect = el.getBoundingClientRect();
660
- if (rect.width < 4 || rect.height < 4) continue;
661
- if (rect.bottom < 0 || rect.top > vh) continue;
662
- if (rect.right < 0 || rect.left > vw) continue;
663
- const style = win.getComputedStyle(el);
664
- if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
634
+ const max = opts?.maxElements ?? 200;
635
+ const mainResult = await this.page.evaluate((limit) => {
636
+ return globalThis.__buaExtract(limit, 0, 0, "main");
637
+ }, max).catch(async () => {
638
+ await this.installExtractorScript();
639
+ return await this.page.evaluate((limit) => {
640
+ return globalThis.__buaExtract(limit, 0, 0, "main");
641
+ }, max);
642
+ });
643
+ let collected = mainResult.elements;
644
+ const scroll = mainResult.scroll;
645
+ try {
646
+ const frames = this.page.frames();
647
+ for (const frame of frames) {
648
+ if (frame === this.page.mainFrame()) continue;
649
+ if (collected.length >= max) break;
650
+ let bbox = null;
651
+ try {
652
+ const owner = await frame.frameElement();
653
+ if (owner) {
654
+ const rect = await owner.boundingBox();
655
+ if (rect) bbox = { x: rect.x, y: rect.y };
656
+ }
657
+ } catch {
665
658
  continue;
666
659
  }
667
- const cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
668
- const cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
660
+ if (!bbox) continue;
661
+ let iframeRes = null;
669
662
  try {
670
- const hit = doc.elementFromPoint(cx, cy);
671
- if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
672
- continue;
673
- }
674
- if (hit && el.contains(hit)) seen.add(hit);
663
+ iframeRes = await frame.evaluate(
664
+ (args) => {
665
+ return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
666
+ },
667
+ { limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
668
+ );
675
669
  } catch {
670
+ try {
671
+ await frame.evaluate(this.extractorScriptSource());
672
+ iframeRes = await frame.evaluate(
673
+ (args) => {
674
+ return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
675
+ },
676
+ { limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
677
+ );
678
+ } catch {
679
+ }
680
+ }
681
+ if (iframeRes?.elements?.length) {
682
+ const offset = collected.length;
683
+ for (let i = 0; i < iframeRes.elements.length; i++) {
684
+ const e = iframeRes.elements[i];
685
+ e.index = offset + i + 1;
686
+ collected.push(e);
687
+ if (collected.length >= max) break;
688
+ }
676
689
  }
677
- seen.add(el);
678
- entries.push({ el, cx, cy });
679
- }
680
- entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
681
- const out = [];
682
- let idx = 1;
683
- for (const { el, cx, cy } of entries) {
684
- if (out.length >= limit) break;
685
- const tag = el.tagName.toLowerCase();
686
- const role = el.getAttribute("role") || tag;
687
- const type = el.getAttribute("type") || "";
688
- const innerText = el.innerText || el.textContent || "";
689
- const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
690
- const placeholder = el.getAttribute("placeholder") || "";
691
- const ariaLabel = el.getAttribute("aria-label") || "";
692
- const title = el.getAttribute("title") || "";
693
- const name = el.getAttribute("name") || "";
694
- const href = el.getAttribute("href") || "";
695
- const value = el.value || "";
696
- let label = ariaLabel || text || placeholder || title || value || name;
697
- if (!label && href) label = href.slice(0, 60);
698
- if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
699
- el.setAttribute("data-bua-idx", String(idx));
700
- out.push({
701
- index: idx,
702
- cx,
703
- cy,
704
- role,
705
- type: type || void 0,
706
- label,
707
- tag,
708
- isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
709
- isSelect: tag === "select",
710
- isFile: tag === "input" && type === "file"
711
- });
712
- idx++;
713
690
  }
714
- return out;
715
- }, max);
716
- this._lastDom = raw;
717
- const lines = raw.map((e) => {
691
+ } catch {
692
+ }
693
+ this._lastDom = collected;
694
+ const lines = collected.map((e) => {
718
695
  const typeSuffix = e.type ? `(${e.type})` : "";
719
- return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}: "${e.label}"`;
696
+ const frameSuffix = e.frame && e.frame !== "main" ? ` [frame]` : "";
697
+ return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}${frameSuffix}: "${e.label}"`;
720
698
  });
721
- return { text: lines.join("\n"), elements: raw };
699
+ return { text: lines.join("\n"), elements: collected, scroll };
700
+ }
701
+ /**
702
+ * Install the `__buaExtract` global on the main page. Idempotent —
703
+ * subsequent calls are no-ops.
704
+ */
705
+ async installExtractorScript() {
706
+ if (!this.page) return;
707
+ await this.page.evaluate(this.extractorScriptSource());
708
+ }
709
+ /**
710
+ * The extractor source. Lives in its own method so we can also inject
711
+ * it into iframes that haven't yet had it loaded.
712
+ *
713
+ * This function intentionally runs entirely in the page context. It:
714
+ * - traverses the regular DOM + open shadow roots (deep)
715
+ * - applies a parent-chain visibility filter
716
+ * - applies a `cursor:pointer` second pass for custom widgets
717
+ * - hit-tests each candidate at its center to avoid overlay collisions
718
+ * - returns scroll context (pages above/below, hidden counts)
719
+ * - tags survivors with `data-bua-idx` for indexed actions
720
+ */
721
+ extractorScriptSource() {
722
+ return (
723
+ /* js */
724
+ `
725
+ (function () {
726
+ if (typeof window.__buaExtract === "function") return;
727
+
728
+ function isAncestorVisible(el) {
729
+ let cur = el;
730
+ while (cur && cur !== document.body && cur !== document.documentElement) {
731
+ const s = window.getComputedStyle(cur);
732
+ if (!s) return false;
733
+ if (s.display === "none" || s.visibility === "hidden" || s.visibility === "collapse") return false;
734
+ if (parseFloat(s.opacity || "1") < 0.05) return false;
735
+ cur = cur.parentElement || (cur.getRootNode && cur.getRootNode().host) || null;
736
+ }
737
+ return true;
738
+ }
739
+
740
+ function collectSemantic(root, out) {
741
+ if (!root || !root.querySelectorAll) return;
742
+ var sel = [
743
+ "a[href]", "button", "input", "textarea", "select",
744
+ "[role='button']", "[role='link']", "[role='tab']",
745
+ "[role='menuitem']", "[role='checkbox']", "[role='radio']",
746
+ "[role='switch']", "[role='combobox']", "[role='option']",
747
+ "[onclick]", "[contenteditable='true']", "[tabindex]:not([tabindex='-1'])"
748
+ ].join(",");
749
+ var found = root.querySelectorAll(sel);
750
+ for (var i = 0; i < found.length; i++) out.push(found[i]);
751
+ // shadow DOM piercing
752
+ var all = root.querySelectorAll("*");
753
+ for (var j = 0; j < all.length; j++) {
754
+ var n = all[j];
755
+ if (n.shadowRoot) collectSemantic(n.shadowRoot, out);
756
+ }
757
+ }
758
+
759
+ function collectPointer(root, out, cap) {
760
+ if (!root || !root.querySelectorAll) return;
761
+ var all = root.querySelectorAll("*");
762
+ var vh = window.innerHeight, vw = window.innerWidth;
763
+ for (var i = 0; i < all.length && out.length < cap; i++) {
764
+ var el = all[i];
765
+ var tag = (el.tagName || "").toLowerCase();
766
+ if (!tag || tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
767
+ var rect = el.getBoundingClientRect();
768
+ if (rect.width < 4 || rect.height < 4) continue;
769
+ if (rect.bottom < 0 || rect.top > vh) continue;
770
+ if (rect.right < 0 || rect.left > vw) continue;
771
+ var s = window.getComputedStyle(el);
772
+ if (s.cursor !== "pointer") continue;
773
+ out.push(el);
774
+ if (el.shadowRoot) collectPointer(el.shadowRoot, out, cap);
775
+ }
776
+ }
777
+
778
+ function countHiddenInteractive(root, vh, vw) {
779
+ if (!root || !root.querySelectorAll) return 0;
780
+ var sel = ["a[href]","button","input","textarea","select","[role='button']","[role='link']","[role='tab']","[role='menuitem']"].join(",");
781
+ var n = 0;
782
+ var found = root.querySelectorAll(sel);
783
+ for (var i = 0; i < found.length; i++) {
784
+ var r = found[i].getBoundingClientRect();
785
+ if (r.bottom < 0 || r.top > vh || r.right < 0 || r.left > vw) n++;
786
+ }
787
+ return n;
788
+ }
789
+
790
+ window.__buaExtract = function (limit, offsetX, offsetY, frameName) {
791
+ // Clear stale markers each call.
792
+ var stale = document.querySelectorAll("[data-bua-idx]");
793
+ for (var s = 0; s < stale.length; s++) stale[s].removeAttribute("data-bua-idx");
794
+
795
+ var vh = window.innerHeight, vw = window.innerWidth;
796
+ var semantic = []; collectSemantic(document, semantic);
797
+ var pointer = []; collectPointer(document, pointer, 2000);
798
+ var all = semantic.concat(pointer);
799
+
800
+ var entries = [];
801
+ var seen = new Set();
802
+
803
+ for (var i = 0; i < all.length; i++) {
804
+ var el = all[i];
805
+ if (seen.has(el)) continue;
806
+ var rect = el.getBoundingClientRect();
807
+ if (rect.width < 4 || rect.height < 4) continue;
808
+ if (rect.bottom < 0 || rect.top > vh) continue;
809
+ if (rect.right < 0 || rect.left > vw) continue;
810
+
811
+ var style = window.getComputedStyle(el);
812
+ if (style.visibility === "hidden" || style.display === "none" ||
813
+ style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) continue;
814
+ if (!isAncestorVisible(el)) continue;
815
+
816
+ var cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
817
+ var cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
818
+
819
+ try {
820
+ var hit = document.elementFromPoint(cx, cy);
821
+ if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) continue;
822
+ if (hit && el.contains(hit)) seen.add(hit);
823
+ } catch (_) {}
824
+ seen.add(el);
825
+ entries.push({ el: el, cx: cx, cy: cy });
826
+ }
827
+
828
+ entries.sort(function (a, b) { return a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy; });
829
+
830
+ var out = [];
831
+ var idx = 1;
832
+ for (var k = 0; k < entries.length && out.length < limit; k++) {
833
+ var ent = entries[k];
834
+ var el = ent.el;
835
+ var tag = (el.tagName || "").toLowerCase();
836
+ var role = el.getAttribute("role") || tag;
837
+ var type = el.getAttribute("type") || "";
838
+ var innerText = (el.innerText || el.textContent || "");
839
+ var text = innerText.trim().replace(/\\s+/g, " ").slice(0, 80);
840
+ var placeholder = el.getAttribute("placeholder") || "";
841
+ var ariaLabel = el.getAttribute("aria-label") || "";
842
+ var title = el.getAttribute("title") || "";
843
+ var name = el.getAttribute("name") || "";
844
+ var href = el.getAttribute("href") || "";
845
+ var value = el.value || "";
846
+ var label = ariaLabel || text || placeholder || title || value || name;
847
+ if (!label && href) label = href.slice(0, 60);
848
+ if (!label) label = "(" + tag + (type ? (" type=" + type) : "") + ")";
849
+
850
+ el.setAttribute("data-bua-idx", String(idx));
851
+ out.push({
852
+ index: idx,
853
+ cx: ent.cx + offsetX,
854
+ cy: ent.cy + offsetY,
855
+ role: role,
856
+ type: type || undefined,
857
+ label: label,
858
+ tag: tag,
859
+ isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
860
+ isSelect: tag === "select",
861
+ isFile: tag === "input" && type === "file",
862
+ frame: frameName,
863
+ });
864
+ idx++;
865
+ }
866
+
867
+ // Scroll context (only meaningful for the main frame; iframes pass through).
868
+ var doc = document.scrollingElement || document.documentElement || document.body;
869
+ var pagesAbove = 0, pagesBelow = 0;
870
+ if (doc && vh > 0) {
871
+ pagesAbove = Math.max(0, Math.round(doc.scrollTop / vh));
872
+ var below = Math.max(0, doc.scrollHeight - doc.scrollTop - vh);
873
+ pagesBelow = Math.round(below / vh);
874
+ }
875
+
876
+ // Hidden interactive count over the WHOLE document tree.
877
+ var hidden = countHiddenInteractive(document, vh, vw);
878
+
879
+ return {
880
+ elements: out,
881
+ scroll: {
882
+ pagesAbove: pagesAbove,
883
+ pagesBelow: pagesBelow,
884
+ totalInteractive: out.length + hidden,
885
+ hiddenInteractive: hidden,
886
+ },
887
+ };
888
+ };
889
+ })();
890
+ `
891
+ );
722
892
  }
723
893
  // ── Page Info ────────────────────────────────────────────────────────
724
894
  async getPageInfo() {
@@ -880,16 +1050,169 @@ var BrowserProvider = class {
880
1050
  }
881
1051
  };
882
1052
 
1053
+ // src/loop-detector.ts
1054
+ var LoopDetector = class {
1055
+ actionWindow = [];
1056
+ pageHistory = [];
1057
+ actionWindowSize;
1058
+ pageHistorySize;
1059
+ /** Thresholds: repeat counts at which each severity fires. */
1060
+ actionThresholds;
1061
+ /** Thresholds: consecutive unchanged-page counts at which each fires. */
1062
+ pageThresholds;
1063
+ constructor(opts) {
1064
+ this.actionWindowSize = opts?.actionWindowSize ?? 20;
1065
+ this.pageHistorySize = opts?.pageHistorySize ?? 10;
1066
+ this.actionThresholds = {
1067
+ warn: opts?.actionThresholds?.warn ?? 5,
1068
+ escalate: opts?.actionThresholds?.escalate ?? 8,
1069
+ abort: opts?.actionThresholds?.abort ?? 12
1070
+ };
1071
+ this.pageThresholds = {
1072
+ warn: opts?.pageThresholds?.warn ?? 5,
1073
+ escalate: opts?.pageThresholds?.escalate ?? 8,
1074
+ abort: opts?.pageThresholds?.abort ?? 12
1075
+ };
1076
+ }
1077
+ /**
1078
+ * Record an action and return advice for the runtime. `severity` of
1079
+ * `abort` means the loop should terminate now.
1080
+ */
1081
+ recordAction(action) {
1082
+ const key = normalizeAction(action);
1083
+ this.actionWindow.push(key);
1084
+ if (this.actionWindow.length > this.actionWindowSize) this.actionWindow.shift();
1085
+ let count = 0;
1086
+ for (const k of this.actionWindow) if (k === key) count++;
1087
+ if (count >= this.actionThresholds.abort) {
1088
+ return {
1089
+ severity: "abort",
1090
+ message: `Auto-stopped: the action "${key}" has been repeated ${count} times in the last ${this.actionWindow.length} steps with no apparent progress.`
1091
+ };
1092
+ }
1093
+ if (count >= this.actionThresholds.escalate) {
1094
+ return {
1095
+ severity: "escalate",
1096
+ message: `You have repeated essentially the same action ${count} times. The current approach is NOT working \u2014 pick a different strategy: scroll, navigate elsewhere, dismiss a popup, or use "fail" if the task truly cannot be done.`
1097
+ };
1098
+ }
1099
+ if (count >= this.actionThresholds.warn) {
1100
+ return {
1101
+ severity: "warn",
1102
+ message: `You've repeated this action ${count} times \u2014 if the page hasn't changed, try a different approach.`
1103
+ };
1104
+ }
1105
+ return { severity: "none" };
1106
+ }
1107
+ /**
1108
+ * Record a page fingerprint and return advice if the page has been
1109
+ * stagnant. Counts consecutive identical fingerprints at the END of
1110
+ * the history (so a single navigation resets the counter).
1111
+ */
1112
+ recordPage(fp) {
1113
+ this.pageHistory.push(fp);
1114
+ if (this.pageHistory.length > this.pageHistorySize) this.pageHistory.shift();
1115
+ let stagnantCount = 1;
1116
+ for (let i = this.pageHistory.length - 2; i >= 0; i--) {
1117
+ const prev = this.pageHistory[i];
1118
+ if (prev.url === fp.url && prev.interactiveCount === fp.interactiveCount && prev.textHash === fp.textHash) {
1119
+ stagnantCount++;
1120
+ } else {
1121
+ break;
1122
+ }
1123
+ }
1124
+ if (stagnantCount >= this.pageThresholds.abort) {
1125
+ return {
1126
+ severity: "abort",
1127
+ message: `Auto-stopped: the page hasn't changed in ${stagnantCount} consecutive steps. Likely a blocking overlay or dead control.`
1128
+ };
1129
+ }
1130
+ if (stagnantCount >= this.pageThresholds.escalate) {
1131
+ return {
1132
+ severity: "escalate",
1133
+ message: `The page has not changed in ${stagnantCount} steps. Something is blocking progress \u2014 try dismissing popups, navigating elsewhere, or scrolling.`
1134
+ };
1135
+ }
1136
+ if (stagnantCount >= this.pageThresholds.warn) {
1137
+ return {
1138
+ severity: "warn",
1139
+ message: `Page hasn't changed in ${stagnantCount} steps \u2014 verify your action is taking effect.`
1140
+ };
1141
+ }
1142
+ return { severity: "none" };
1143
+ }
1144
+ /** Combine two advices, returning the more severe one. */
1145
+ static combine(a, b) {
1146
+ const rank = { none: 0, warn: 1, escalate: 2, abort: 3 };
1147
+ return rank[a.severity] >= rank[b.severity] ? a : b;
1148
+ }
1149
+ };
1150
+ function normalizeAction(action) {
1151
+ switch (action.action) {
1152
+ case "click":
1153
+ if (typeof action.index === "number") return `click:${action.index}`;
1154
+ if (typeof action.x === "number" && typeof action.y === "number") {
1155
+ return `click:xy:${Math.round(action.x / 16)},${Math.round(action.y / 16)}`;
1156
+ }
1157
+ return `click:?`;
1158
+ case "type":
1159
+ if (typeof action.index === "number") return `type:${action.index}:${truncate(action.text, 40)}`;
1160
+ return `type:?:${truncate(action.text, 40)}`;
1161
+ case "scroll":
1162
+ return `scroll:${action.direction}:${action.amount ?? "default"}${action.index != null ? `:${action.index}` : ""}`;
1163
+ case "navigate":
1164
+ return `navigate:${action.url}`;
1165
+ case "back":
1166
+ return "back";
1167
+ case "wait":
1168
+ return `wait:${Math.round(action.ms / 1e3)}`;
1169
+ case "screenshot":
1170
+ return "screenshot";
1171
+ case "send_keys":
1172
+ return `send_keys:${action.keys}`;
1173
+ case "find_text":
1174
+ return `find_text:${truncate(action.text, 40)}`;
1175
+ case "evaluate":
1176
+ return `evaluate:${truncate(action.code, 80)}`;
1177
+ case "dropdown_options":
1178
+ return `dropdown_options:${action.index}`;
1179
+ case "select_dropdown":
1180
+ return `select_dropdown:${action.index}:${truncate(action.text, 40)}`;
1181
+ case "upload_file":
1182
+ return `upload_file:${action.index}:${action.path}`;
1183
+ case "extract":
1184
+ return `extract:${truncate(action.query, 80)}`;
1185
+ case "tool":
1186
+ return `tool:${action.name}`;
1187
+ case "done":
1188
+ case "fail":
1189
+ return action.action;
1190
+ }
1191
+ }
1192
+ function fnvHash(s) {
1193
+ let h = 2166136261;
1194
+ for (let i = 0; i < s.length; i++) {
1195
+ h ^= s.charCodeAt(i);
1196
+ h = h * 16777619 >>> 0;
1197
+ }
1198
+ return h;
1199
+ }
1200
+ function truncate(s, n) {
1201
+ if (!s) return "";
1202
+ return s.length <= n ? s : `${s.slice(0, n)}\u2026`;
1203
+ }
1204
+
883
1205
  // src/prompts.ts
884
1206
  function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options) {
885
1207
  const maxActions = options?.maxActionsPerStep ?? 3;
886
1208
  const allowEvaluate = !!options?.allowEvaluate;
887
1209
  const tools = options?.tools ?? [];
888
1210
  const useVision = options?.useVision ?? "auto";
1211
+ const useThinking = options?.useThinking ?? true;
889
1212
  if (options?.overrideSystemMessage) {
890
1213
  const lines2 = [options.overrideSystemMessage];
891
1214
  appendCredentials(lines2, credentialKeys);
892
- lines2.push("", "## Response Format", responseFormatLines(maxActions).join("\n"));
1215
+ lines2.push("", "## Response Format", responseFormatLines(maxActions, useThinking).join("\n"));
893
1216
  return lines2.join("\n");
894
1217
  }
895
1218
  const lines = [];
@@ -994,7 +1317,7 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options)
994
1317
  if (extraInstructions) {
995
1318
  lines.push(``, `## Additional Instructions`, extraInstructions);
996
1319
  }
997
- lines.push(``, `## Response Format`, responseFormatLines(maxActions).join("\n"));
1320
+ lines.push(``, `## Response Format`, responseFormatLines(maxActions, useThinking).join("\n"));
998
1321
  return lines.join("\n");
999
1322
  }
1000
1323
  function appendCredentials(lines, credentialKeys) {
@@ -1012,22 +1335,61 @@ function appendCredentials(lines, credentialKeys) {
1012
1335
  `NEVER include real credential values in "done" or "fail" results.`
1013
1336
  );
1014
1337
  }
1015
- function responseFormatLines(maxActions) {
1338
+ function responseFormatLines(maxActions, useThinking) {
1339
+ if (!useThinking) {
1340
+ return [
1341
+ `Respond with ONLY valid JSON. No markdown, no commentary.`,
1342
+ `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
1343
+ ];
1344
+ }
1016
1345
  return [
1017
- `Respond with ONLY valid JSON. No markdown, no commentary.`,
1018
- `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
1346
+ `Respond with ONLY a single valid JSON object. No markdown, no commentary.`,
1347
+ ``,
1348
+ `Use this exact shape:`,
1349
+ "```json",
1350
+ `{`,
1351
+ ` "thinking": "Short chain-of-thought: what do I see, what's my plan?",`,
1352
+ ` "evaluation_previous_goal": "Did the previous action succeed? success/partial/failure + 1 line",`,
1353
+ ` "memory": "Compact bullet list of facts to remember across steps (URLs, found data, \u2026)",`,
1354
+ ` "next_goal": "What I want to accomplish in THIS step",`,
1355
+ ` "action": <single action object OR array of up to ${maxActions} action objects>`,
1356
+ `}`,
1357
+ "```",
1358
+ ``,
1359
+ `On the very first step, set "evaluation_previous_goal" to "n/a \u2014 first step".`,
1360
+ `Keep each text field to one short paragraph or a few lines. Be concrete.`,
1361
+ `Only the "action" field is executed; the others help you self-correct over multiple steps.`
1019
1362
  ];
1020
1363
  }
1021
- function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract) {
1364
+ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget) {
1022
1365
  const lines = [];
1023
1366
  lines.push(`**Task:** ${task}`);
1024
1367
  lines.push(`**Current URL:** ${pageUrl}`);
1025
1368
  if (pageTitle) lines.push(`**Page Title:** ${pageTitle}`);
1026
- lines.push(`**Step:** ${stepIndex + 1}`);
1027
- if (domSnapshot) {
1028
- lines.push(``);
1029
- lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
1030
- lines.push(domSnapshot);
1369
+ if (stepBudget) {
1370
+ lines.push(`**Step:** ${stepIndex + 1} of ${stepBudget.max} (${stepBudget.max - stepIndex - 1} remaining)`);
1371
+ } else {
1372
+ lines.push(`**Step:** ${stepIndex + 1}`);
1373
+ }
1374
+ if (scroll) {
1375
+ const parts = [];
1376
+ parts.push(`${scroll.totalInteractive} interactive elements (${scroll.hiddenInteractive} hidden)`);
1377
+ if (scroll.pagesAbove > 0) parts.push(`${scroll.pagesAbove} page${scroll.pagesAbove === 1 ? "" : "s"} above`);
1378
+ if (scroll.pagesBelow > 0) parts.push(`${scroll.pagesBelow} page${scroll.pagesBelow === 1 ? "" : "s"} below`);
1379
+ if (scroll.pagesAbove === 0 && scroll.pagesBelow === 0) parts.push("fits in viewport");
1380
+ lines.push(`**Page stats:** ${parts.join(" \xB7 ")}`);
1381
+ }
1382
+ if (domSnapshot !== void 0) {
1383
+ if (domSnapshot.trim().length === 0) {
1384
+ lines.push(``);
1385
+ lines.push(
1386
+ `**\u26A0 Empty page / no interactive elements detected.** The page may be blank, blocked by a captcha/anti-bot wall, mid-load, or rendered with shadow DOM in an unsupported way. Consider: \`wait\` + retry, \`navigate\` to a different URL, or \`fail\` if the site is blocking access.`
1387
+ );
1388
+ } else {
1389
+ lines.push(``);
1390
+ lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
1391
+ lines.push(domSnapshot);
1392
+ }
1031
1393
  }
1032
1394
  if (lastExtract) {
1033
1395
  lines.push(``);
@@ -1041,6 +1403,10 @@ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, do
1041
1403
  lines.push(`- ${entry}`);
1042
1404
  }
1043
1405
  }
1406
+ if (nudge) {
1407
+ lines.push(``);
1408
+ lines.push(`**\u26A0 Runtime hint:** ${nudge}`);
1409
+ }
1044
1410
  lines.push(``);
1045
1411
  lines.push(`Decide the next action(s) to complete the task.`);
1046
1412
  return lines.join("\n");
@@ -1095,6 +1461,9 @@ var BrowserAgent = class {
1095
1461
  eventBus;
1096
1462
  model;
1097
1463
  pageExtractionLLM;
1464
+ fallbackModel;
1465
+ useThinking;
1466
+ historyWindow;
1098
1467
  instructions;
1099
1468
  extendSystemMessage;
1100
1469
  overrideSystemMessage;
@@ -1131,6 +1500,9 @@ var BrowserAgent = class {
1131
1500
  this.name = config.name;
1132
1501
  this.model = config.model;
1133
1502
  this.pageExtractionLLM = config.pageExtractionLLM ?? null;
1503
+ this.fallbackModel = config.fallbackModel ?? null;
1504
+ this.useThinking = config.useThinking ?? true;
1505
+ this.historyWindow = Math.max(0, config.historyWindow ?? 6);
1134
1506
  this.instructions = config.instructions;
1135
1507
  this.extendSystemMessage = config.extendSystemMessage;
1136
1508
  this.overrideSystemMessage = config.overrideSystemMessage;
@@ -1176,10 +1548,10 @@ var BrowserAgent = class {
1176
1548
  const actionHistory = [];
1177
1549
  const extractedContent = [];
1178
1550
  let lastExtractResult;
1179
- let failureCount = 0;
1180
- let lastActionKey = "";
1181
- let repeatCount = 0;
1551
+ let consecutiveFailures = 0;
1182
1552
  let lastActionWasScreenshot = false;
1553
+ const loop = new LoopDetector();
1554
+ const historyTurns = [];
1183
1555
  const baseExtra = [this.instructions, this.extendSystemMessage].filter(Boolean).join("\n\n");
1184
1556
  let extraInstructions = baseExtra;
1185
1557
  if (this.memoryManager) {
@@ -1198,7 +1570,8 @@ ${memoryContext}` : memoryContext;
1198
1570
  allowEvaluate: this.allowEvaluate,
1199
1571
  tools: this.tools,
1200
1572
  useVision: this.useVision,
1201
- useDOM: this.useDOM
1573
+ useDOM: this.useDOM,
1574
+ useThinking: this.useThinking
1202
1575
  });
1203
1576
  try {
1204
1577
  this.logger.info("Launching browser", {
@@ -1206,6 +1579,7 @@ ${memoryContext}` : memoryContext;
1206
1579
  viewport: this.viewport,
1207
1580
  useDOM: this.useDOM,
1208
1581
  useVision: this.useVision,
1582
+ useThinking: this.useThinking,
1209
1583
  cdpUrl: this.cdpUrl ?? void 0,
1210
1584
  recordVideo: !!this.recordVideo,
1211
1585
  stealth: !!this.stealth,
@@ -1225,6 +1599,7 @@ ${memoryContext}` : memoryContext;
1225
1599
  this.logger.info("Navigating to start URL", { url: startUrl });
1226
1600
  this.assertDomainAllowed(startUrl);
1227
1601
  await browser.navigate(startUrl);
1602
+ await this.navigationHealthCheck(browser, startUrl);
1228
1603
  }
1229
1604
  for (const ia of this.initialActions) {
1230
1605
  try {
@@ -1237,13 +1612,35 @@ ${memoryContext}` : memoryContext;
1237
1612
  for (let step = 0; step < maxSteps; step++) {
1238
1613
  const pageInfo = await browser.getPageInfo();
1239
1614
  let domSnapshot;
1615
+ let scrollCtx;
1240
1616
  if (this.useDOM) {
1241
1617
  const dom = await browser.extractDOM();
1242
1618
  domSnapshot = dom.text;
1619
+ scrollCtx = dom.scroll;
1620
+ }
1621
+ const pageAdvice = loop.recordPage({
1622
+ url: pageInfo.url,
1623
+ interactiveCount: scrollCtx?.totalInteractive ?? 0,
1624
+ textHash: fnvHash(domSnapshot ?? "")
1625
+ });
1626
+ if (pageAdvice.severity === "abort") {
1627
+ return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1628
+ result: pageAdvice.message ?? "Auto-stopped: page is stagnant.",
1629
+ success: false
1630
+ });
1243
1631
  }
1244
1632
  const wantVision = this.shouldCaptureVision(step, lastActionWasScreenshot);
1245
1633
  const screenshot = wantVision ? await browser.screenshot() : Buffer.alloc(0);
1246
1634
  if (wantVision) this.eventBus.emit("browser.screenshot", { data: screenshot });
1635
+ const isLastStep = step === maxSteps - 1;
1636
+ const nudgeParts = [];
1637
+ if (pageAdvice.severity !== "none" && pageAdvice.message) nudgeParts.push(pageAdvice.message);
1638
+ if (isLastStep) {
1639
+ nudgeParts.push(
1640
+ "This is your FINAL step. Return a `done` action right now summarizing whatever you have, even if partial."
1641
+ );
1642
+ }
1643
+ const nudge = nudgeParts.length > 0 ? nudgeParts.join(" ") : void 0;
1247
1644
  const userText = buildUserMessage(
1248
1645
  task,
1249
1646
  pageInfo.url,
@@ -1251,65 +1648,49 @@ ${memoryContext}` : memoryContext;
1251
1648
  step,
1252
1649
  actionHistory,
1253
1650
  domSnapshot,
1254
- lastExtractResult
1651
+ lastExtractResult,
1652
+ scrollCtx,
1653
+ nudge,
1654
+ { current: step, max: maxSteps }
1255
1655
  );
1256
- const content = [];
1257
- if (wantVision) {
1258
- content.push({ type: "image", data: screenshot.toString("base64"), mimeType: "image/png" });
1259
- }
1260
- content.push({ type: "text", text: userText });
1261
- const messages = [
1262
- { role: "system", content: systemPrompt },
1263
- { role: "user", content }
1264
- ];
1656
+ const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
1265
1657
  this.logger.debug("Calling model", { step, url: pageInfo.url, vision: wantVision });
1266
- let response;
1267
- try {
1268
- response = await this.model.generate(messages, {
1269
- temperature: 0.1,
1270
- maxTokens: 1024,
1271
- apiKey: opts?.apiKey,
1272
- responseFormat: "json"
1273
- });
1274
- } catch (e) {
1275
- failureCount++;
1276
- this.logger.warn("Model call failed", { error: e?.message, failureCount });
1277
- if (failureCount > this.maxFailures) {
1278
- return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1279
- result: `Model call failed ${failureCount} times: ${e?.message}`,
1280
- success: false
1281
- });
1658
+ const { response, modelUsed } = await this.callModelWithFallback(messages, opts?.apiKey);
1659
+ if (!response) {
1660
+ consecutiveFailures++;
1661
+ actionHistory.push(`(model call failed \u2014 retrying, ${consecutiveFailures}/${this.maxFailures})`);
1662
+ if (consecutiveFailures > this.maxFailures) {
1663
+ return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
1282
1664
  }
1283
- actionHistory.push(`(model call failed: ${e?.message ?? "unknown"} \u2014 retrying)`);
1284
1665
  continue;
1285
1666
  }
1286
1667
  if (this.costTracker && response.usage) {
1287
1668
  this.costTracker.track({
1288
1669
  runId: sessionId,
1289
1670
  agentName: this.name,
1290
- modelId: this.model.modelId,
1671
+ modelId: modelUsed.modelId,
1291
1672
  usage: response.usage,
1292
1673
  sessionId,
1293
1674
  userId
1294
1675
  });
1295
1676
  }
1296
1677
  const raw = typeof response.message.content === "string" ? response.message.content : "";
1297
- let parsed;
1298
- try {
1299
- parsed = JSON.parse(raw);
1300
- } catch {
1301
- failureCount++;
1302
- this.logger.warn("Failed to parse model response as JSON", { raw, failureCount });
1303
- if (failureCount > this.maxFailures) {
1304
- return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1305
- result: `Model returned invalid JSON ${failureCount} times. Last response: ${raw.slice(0, 200)}`,
1306
- success: false
1307
- });
1678
+ const envelope = this.parseEnvelope(raw);
1679
+ if (!envelope) {
1680
+ consecutiveFailures++;
1681
+ this.logger.warn("Failed to parse model response", { raw: raw.slice(0, 200), consecutiveFailures });
1682
+ if (consecutiveFailures > this.maxFailures) {
1683
+ return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "parse");
1308
1684
  }
1309
1685
  actionHistory.push("(invalid JSON response \u2014 retrying)");
1310
1686
  continue;
1311
1687
  }
1312
- const actions = (Array.isArray(parsed) ? parsed : [parsed]).slice(
1688
+ consecutiveFailures = 0;
1689
+ historyTurns.push({ userText, hasScreenshot: wantVision, envelope });
1690
+ if (this.historyWindow > 0 && historyTurns.length > this.historyWindow) {
1691
+ historyTurns.splice(0, historyTurns.length - this.historyWindow);
1692
+ }
1693
+ const actions = (Array.isArray(envelope.action) ? envelope.action : [envelope.action]).slice(
1313
1694
  0,
1314
1695
  this.maxActionsPerStep
1315
1696
  );
@@ -1320,24 +1701,12 @@ ${memoryContext}` : memoryContext;
1320
1701
  const action = actions[ai];
1321
1702
  let summary = summarizeAction(action);
1322
1703
  if (this.credentials) summary = this.credentials.mask(summary);
1323
- const actionKey = JSON.stringify(action);
1324
- if (actionKey === lastActionKey) {
1325
- repeatCount++;
1326
- } else {
1327
- lastActionKey = actionKey;
1328
- repeatCount = 1;
1329
- }
1330
- if (repeatCount > this.maxRepeats && action.action !== "done" && action.action !== "fail") {
1331
- this.logger.warn("Stuck in a loop \u2014 same action repeated", {
1332
- action: action.action,
1333
- repeats: repeatCount,
1334
- maxRepeats: this.maxRepeats
1335
- });
1336
- actionHistory.push(
1337
- `\u26A0 LOOP DETECTED: "${summary}" repeated ${repeatCount} times. Auto-stopping.`
1338
- );
1704
+ const advice = loop.recordAction(action);
1705
+ if (advice.severity === "abort" && action.action !== "done" && action.action !== "fail") {
1706
+ this.logger.warn("Loop detector aborting run", { advice });
1707
+ actionHistory.push(`\u26A0 ${advice.message ?? "loop detected \u2014 auto-stopping"}`);
1339
1708
  didTerminate = {
1340
- result: `Stuck in a loop: "${summary}" was repeated ${repeatCount} times. The page may have a popup, consent banner, or unexpected state blocking progress.`,
1709
+ result: advice.message ?? "Stuck in a loop \u2014 auto-stopped.",
1341
1710
  success: false
1342
1711
  };
1343
1712
  break;
@@ -1360,21 +1729,20 @@ ${memoryContext}` : memoryContext;
1360
1729
  try {
1361
1730
  const exec = await this.executeAction(browser, action, actionHistory, extractedContent);
1362
1731
  stepOutput = exec?.output;
1363
- if (exec?.didNavigate) didNavigate = true;
1364
- if (action.action === "extract" && exec?.output) {
1365
- lastExtractResult = exec.output;
1366
- }
1367
- if (action.action === "screenshot") {
1368
- lastActionWasScreenshot = true;
1732
+ if (exec?.didNavigate) {
1733
+ didNavigate = true;
1734
+ await this.navigationHealthCheck(browser, pageInfo.url);
1369
1735
  }
1736
+ if (action.action === "extract" && exec?.output) lastExtractResult = exec.output;
1737
+ if (action.action === "screenshot") lastActionWasScreenshot = true;
1370
1738
  } catch (e) {
1371
1739
  stepOk = false;
1372
- failureCount++;
1373
- this.logger.warn("Action failed", { action: action.action, error: e?.message, failureCount });
1740
+ consecutiveFailures++;
1741
+ this.logger.warn("Action failed", { action: action.action, error: e?.message, consecutiveFailures });
1374
1742
  actionHistory.push(`(error: ${e?.message ?? "unknown"})`);
1375
- if (failureCount > this.maxFailures) {
1743
+ if (consecutiveFailures > this.maxFailures) {
1376
1744
  didTerminate = {
1377
- result: `Action ${action.action} failed ${failureCount} times. Last error: ${e?.message}`,
1745
+ result: `Action ${action.action} failed ${consecutiveFailures} times. Last error: ${e?.message}`,
1378
1746
  success: false
1379
1747
  };
1380
1748
  break;
@@ -1389,7 +1757,11 @@ ${memoryContext}` : memoryContext;
1389
1757
  timestamp: /* @__PURE__ */ new Date(),
1390
1758
  dom: domSnapshot,
1391
1759
  output: stepOutput,
1392
- ok: stepOk
1760
+ ok: stepOk,
1761
+ thinking: envelope.thinking,
1762
+ evaluationPreviousGoal: envelope.evaluationPreviousGoal,
1763
+ memory: envelope.memory,
1764
+ nextGoal: envelope.nextGoal
1393
1765
  });
1394
1766
  this.eventBus.emit("browser.step", {
1395
1767
  index: steps.length - 1,
@@ -1404,11 +1776,8 @@ ${memoryContext}` : memoryContext;
1404
1776
  return await this.finalize(browser, steps, startTime, opts, extractedContent, didTerminate);
1405
1777
  }
1406
1778
  }
1407
- this.logger.warn("Max steps reached without completing task", { maxSteps });
1408
- return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1409
- result: `Task not completed within ${maxSteps} steps. Last actions: ${actionHistory.slice(-3).join("; ")}`,
1410
- success: false
1411
- });
1779
+ this.logger.warn("Max steps reached without explicit done", { maxSteps });
1780
+ return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "max_steps");
1412
1781
  } catch (error) {
1413
1782
  this.logger.error("Browser agent error", { error: error.message });
1414
1783
  this.eventBus.emit("browser.error", { error });
@@ -1470,6 +1839,163 @@ ${memoryContext}` : memoryContext;
1470
1839
  throw new Error(`Navigation blocked: ${host} is in prohibitedDomains`);
1471
1840
  }
1472
1841
  }
1842
+ /**
1843
+ * Build the message array sent to the model: system prompt + a compact
1844
+ * summary of older turns (if any) + the most recent `historyWindow`
1845
+ * turns verbatim + the current step's user message.
1846
+ *
1847
+ * Inspired by browser-use's history compaction. Keeps tokens bounded
1848
+ * while giving the model meaningful context about what it already
1849
+ * tried.
1850
+ */
1851
+ buildMessages(systemPrompt, historyTurns, currentUserText, currentScreenshot) {
1852
+ const msgs = [{ role: "system", content: systemPrompt }];
1853
+ if (this.historyWindow > 0 && historyTurns.length > 0) {
1854
+ for (const turn of historyTurns) {
1855
+ const env = turn.envelope;
1856
+ const replay = [];
1857
+ if (env.thinking) replay.push(`thinking: ${env.thinking}`);
1858
+ if (env.evaluationPreviousGoal) replay.push(`evaluation: ${env.evaluationPreviousGoal}`);
1859
+ if (env.memory) replay.push(`memory: ${env.memory}`);
1860
+ if (env.nextGoal) replay.push(`next_goal: ${env.nextGoal}`);
1861
+ replay.push(`action: ${JSON.stringify(env.action)}`);
1862
+ msgs.push({ role: "assistant", content: replay.join("\n") });
1863
+ }
1864
+ }
1865
+ const content = [];
1866
+ if (currentScreenshot && currentScreenshot.length > 0) {
1867
+ content.push({
1868
+ type: "image",
1869
+ data: currentScreenshot.toString("base64"),
1870
+ mimeType: "image/png"
1871
+ });
1872
+ }
1873
+ content.push({ type: "text", text: currentUserText });
1874
+ msgs.push({ role: "user", content });
1875
+ return msgs;
1876
+ }
1877
+ /**
1878
+ * Call the primary model, retrying once with `fallbackModel` on
1879
+ * transient errors (5xx, 429, network). Returns the response or
1880
+ * `null` if both models failed.
1881
+ */
1882
+ async callModelWithFallback(messages, apiKey) {
1883
+ const reqOpts = { temperature: 0.1, maxTokens: 1024, apiKey, responseFormat: "json" };
1884
+ try {
1885
+ const r = await this.model.generate(messages, reqOpts);
1886
+ return { response: r, modelUsed: this.model };
1887
+ } catch (e) {
1888
+ if (this.fallbackModel && this.isTransientError(e)) {
1889
+ this.logger.warn("Primary model failed; trying fallbackModel", {
1890
+ error: e?.message,
1891
+ primary: this.model.modelId,
1892
+ fallback: this.fallbackModel.modelId
1893
+ });
1894
+ try {
1895
+ const r = await this.fallbackModel.generate(messages, reqOpts);
1896
+ return { response: r, modelUsed: this.fallbackModel };
1897
+ } catch (e2) {
1898
+ this.logger.warn("Fallback model also failed", { error: e2?.message });
1899
+ return { response: null, modelUsed: this.model };
1900
+ }
1901
+ }
1902
+ this.logger.warn("Model call failed (no fallback configured)", { error: e?.message });
1903
+ return { response: null, modelUsed: this.model };
1904
+ }
1905
+ }
1906
+ isTransientError(e) {
1907
+ const msg = String(e?.message ?? e).toLowerCase();
1908
+ if (msg.includes("rate limit") || msg.includes("429")) return true;
1909
+ if (msg.includes("timeout") || msg.includes("etimedout")) return true;
1910
+ if (msg.includes("econnreset") || msg.includes("network")) return true;
1911
+ if (/\b5\d\d\b/.test(msg)) return true;
1912
+ if (msg.includes("401") || msg.includes("402") || msg.includes("auth")) return true;
1913
+ return false;
1914
+ }
1915
+ /**
1916
+ * Parse the model's raw response into an `AgentOutput`. Tolerant to
1917
+ * three shapes:
1918
+ * - Full envelope: { thinking, evaluation_previous_goal, action, ... }
1919
+ * - Raw action object (legacy / `useThinking: false`)
1920
+ * - Raw action array
1921
+ *
1922
+ * Also strips ```json fences the model occasionally adds.
1923
+ */
1924
+ parseEnvelope(raw) {
1925
+ if (!raw) return null;
1926
+ let s = raw.trim();
1927
+ const fence = s.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
1928
+ if (fence) s = fence[1].trim();
1929
+ let json;
1930
+ try {
1931
+ json = JSON.parse(s);
1932
+ } catch {
1933
+ const m = s.match(/[\[{][\s\S]*[\]}]/);
1934
+ if (!m) return null;
1935
+ try {
1936
+ json = JSON.parse(m[0]);
1937
+ } catch {
1938
+ return null;
1939
+ }
1940
+ }
1941
+ if (Array.isArray(json)) return { action: json };
1942
+ if (json && typeof json === "object" && typeof json.action === "string") {
1943
+ return { action: json };
1944
+ }
1945
+ if (json && typeof json === "object" && "action" in json) {
1946
+ const out = { action: json.action };
1947
+ out.thinking = json.thinking ?? json.reasoning ?? void 0;
1948
+ out.evaluationPreviousGoal = json.evaluationPreviousGoal ?? json.evaluation_previous_goal ?? void 0;
1949
+ out.memory = json.memory ?? void 0;
1950
+ out.nextGoal = json.nextGoal ?? json.next_goal ?? void 0;
1951
+ return out;
1952
+ }
1953
+ return null;
1954
+ }
1955
+ /**
1956
+ * Quick post-navigation health check. If the page came back blank
1957
+ * (no body text and no interactive elements), reload once and wait
1958
+ * for stable. This catches the "FreightOS half-loaded font test" class
1959
+ * of failure before the LLM ever sees it.
1960
+ */
1961
+ async navigationHealthCheck(browser, url) {
1962
+ try {
1963
+ const ok = await browser.pageText({ maxChars: 200 }).catch(() => "");
1964
+ if (ok && ok.trim().length > 5) return;
1965
+ this.logger.warn("Navigation produced an empty page; reloading once", { url });
1966
+ await this.sleep(800);
1967
+ try {
1968
+ await browser.navigate(url);
1969
+ } catch {
1970
+ }
1971
+ } catch {
1972
+ }
1973
+ }
1974
+ /**
1975
+ * Force-finalize the run with whatever partial data the agent has.
1976
+ * Called when:
1977
+ * - `maxSteps` is exhausted without an explicit `done`,
1978
+ * - `maxFailures` is exceeded,
1979
+ * - the model can't produce parseable JSON enough times to make
1980
+ * forward progress.
1981
+ * The result is composed from `extractedContent` + the last few
1982
+ * action summaries so the caller gets something useful instead of
1983
+ * just a one-line error.
1984
+ */
1985
+ async forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, reason) {
1986
+ const reasonLine = reason === "model" ? "Model call failed too many consecutive times." : reason === "parse" ? "Model produced invalid JSON too many times." : "Max step budget exhausted before an explicit `done`.";
1987
+ const tail = actionHistory.slice(-5).join(" \u2192 ");
1988
+ const extracts = extractedContent.length > 0 ? `
1989
+
1990
+ Extracts so far:
1991
+ ${extractedContent.join("\n---\n")}` : "";
1992
+ const result = `${reasonLine}
1993
+ Last actions: ${tail || "(none)"}${extracts}`;
1994
+ return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1995
+ result,
1996
+ success: false
1997
+ });
1998
+ }
1473
1999
  async finalize(browser, steps, startTime, opts, extractedContent, outcome) {
1474
2000
  let finalScreenshot;
1475
2001
  let finalInfo;