@agentium/browser 2.1.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -80,29 +80,39 @@ function getStealthScript() {
80
80
  };
81
81
  }
82
82
 
83
- // \u2500\u2500 Prevent iframe detection of automation \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
84
- Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', {
85
- get: function() {
86
- return window;
87
- },
88
- });
89
-
90
- // \u2500\u2500 Remove "cdc_" Playwright/ChromeDriver markers from DOM \u2500\u2500\u2500\u2500\u2500\u2500\u2500
91
- const observer = new MutationObserver((mutations) => {
92
- for (const mutation of mutations) {
93
- for (const node of mutation.addedNodes) {
94
- if (node.nodeType === 1) {
95
- const el = node;
96
- for (const attr of [...el.attributes]) {
97
- if (attr.name.startsWith('cdc_') || attr.name.startsWith('__playwright')) {
98
- el.removeAttribute(attr.name);
83
+ // \u2500\u2500 Remove "cdc_" Playwright/ChromeDriver markers from new elements \u2500
84
+ // Only watches childList additions, NOT every attribute mutation \u2014
85
+ // attributes:true subtree:true is O(every DOM write) on heavy SPAs
86
+ // like FreightOS and noticeably slows the page.
87
+ try {
88
+ const observer = new MutationObserver((mutations) => {
89
+ for (const mutation of mutations) {
90
+ for (const node of mutation.addedNodes) {
91
+ if (node && node.nodeType === 1) {
92
+ const el = node;
93
+ if (!el.attributes) continue;
94
+ for (const attr of [...el.attributes]) {
95
+ if (attr.name.startsWith('cdc_') || attr.name.startsWith('__playwright')) {
96
+ el.removeAttribute(attr.name);
97
+ }
99
98
  }
100
99
  }
101
100
  }
102
101
  }
102
+ });
103
+ if (document.documentElement) {
104
+ observer.observe(document.documentElement, { childList: true, subtree: true });
103
105
  }
104
- });
105
- observer.observe(document.documentElement, { attributes: true, childList: true, subtree: true });
106
+ } catch (_e) { /* never break the page over a stealth patch */ }
107
+
108
+ // NOTE on iframe.contentWindow:
109
+ // We deliberately do NOT override HTMLIFrameElement.prototype.contentWindow.
110
+ // An earlier version returned the parent window, which breaks legitimate
111
+ // page JS on any site that uses iframes (checkout widgets, embedded
112
+ // videos, A/B test scripts, analytics). The "anti-detection" value was
113
+ // minimal \u2014 modern detectors don't rely on that property anyway \u2014 and
114
+ // the breakage caused pages to fail to boot, leaving font-test residue
115
+ // on screen.
106
116
  `;
107
117
  }
108
118
  function buildStealthContextOpts(config, viewport) {
@@ -188,14 +198,20 @@ var BrowserProvider = class {
188
198
  }
189
199
  const stealthEnabled = !!opts?.stealth;
190
200
  const stealthCfg = typeof opts?.stealth === "object" ? opts.stealth : {};
191
- const launchOpts = {
192
- headless: opts?.headless ?? true
193
- };
201
+ const headless = opts?.headless ?? true;
202
+ const launchOpts = { headless };
203
+ const args = [];
194
204
  if (stealthEnabled) {
195
- const { args, proxy } = buildStealthLaunchArgs(stealthCfg);
196
- launchOpts.args = args;
197
- if (proxy) launchOpts.proxy = proxy;
205
+ const stealth = buildStealthLaunchArgs(stealthCfg);
206
+ args.push(...stealth.args);
207
+ if (stealth.proxy) launchOpts.proxy = stealth.proxy;
208
+ }
209
+ if (!headless) {
210
+ const CHROME_VERTICAL_CHROME_PX = 140;
211
+ args.push(`--window-size=${this._viewport.width},${this._viewport.height + CHROME_VERTICAL_CHROME_PX}`);
212
+ args.push("--window-position=0,0");
198
213
  }
214
+ if (args.length) launchOpts.args = args;
199
215
  this.browser = await chromium.launch(launchOpts);
200
216
  let contextOpts;
201
217
  if (stealthEnabled) {
@@ -555,132 +571,286 @@ var BrowserProvider = class {
555
571
  /**
556
572
  * Snapshot the interactive elements visible in the viewport, tag each
557
573
  * with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
558
- * return BOTH a human-readable string (for the model) and the
559
- * structured list (for the runtime).
574
+ * return:
575
+ * - `text`: a human-readable string fed to the model
576
+ * - `elements`: the structured list with stable indices
577
+ * - `scroll`: spatial context (pages above/below, hidden interactive count)
560
578
  *
561
- * Three properties matter for accuracy:
562
- * - Hit-tested: each listed coordinate / index actually reaches the
579
+ * Five properties matter for accuracy:
580
+ * - **Hit-tested**: each listed coordinate / index actually reaches the
563
581
  * labeled element (overlays / occlusion skip the entry).
564
- * - Visibility-filtered: invisible, zero-size, `pointer-events: none`,
565
- * and near-zero-opacity elements are excluded.
566
- * - `cursor: pointer` fallback pass: a second scan catches custom React
567
- * widgets that have no semantic role/href/onclick but are clickable.
582
+ * - **Visibility filtered (with parent chain)**: an element is dropped
583
+ * if itself OR any ancestor is `display:none`, `visibility:hidden`,
584
+ * `pointer-events:none`, or near-zero opacity.
585
+ * - **Shadow DOM piercing**: traverses open shadow roots so custom
586
+ * elements / web components are visible to the agent.
587
+ * - **Same-origin iframes**: walks into each accessible iframe and
588
+ * includes its interactive elements (offset by the iframe's screen
589
+ * position so the coordinates the model sees are still viewport-
590
+ * relative).
591
+ * - **`cursor: pointer` fallback pass**: catches custom React widgets
592
+ * that have no semantic role/href/onclick but are clickable.
568
593
  */
569
594
  async extractDOM(opts) {
570
595
  this.ensurePage();
571
- const max = opts?.maxElements ?? 120;
572
- const raw = await this.page.evaluate((limit) => {
573
- const doc = globalThis.document;
574
- const win = globalThis.window;
575
- for (const stale of doc.querySelectorAll("[data-bua-idx]")) {
576
- stale.removeAttribute("data-bua-idx");
577
- }
578
- const selectors = [
579
- "a[href]",
580
- "button",
581
- "input",
582
- "textarea",
583
- "select",
584
- "[role='button']",
585
- "[role='link']",
586
- "[role='tab']",
587
- "[role='menuitem']",
588
- "[role='checkbox']",
589
- "[role='radio']",
590
- "[role='switch']",
591
- "[role='combobox']",
592
- "[role='option']",
593
- "[onclick]",
594
- "[contenteditable='true']",
595
- "[tabindex]:not([tabindex='-1'])"
596
- ];
597
- const semantic = Array.from(doc.querySelectorAll(selectors.join(",")));
598
- const POINTER_SCAN_CAP = 2e3;
599
- const pointerCandidates = [];
600
- const allEls = doc.querySelectorAll("*");
601
- for (let i = 0; i < allEls.length && pointerCandidates.length < POINTER_SCAN_CAP; i++) {
602
- const el = allEls[i];
603
- if (!el) continue;
604
- const tag = el.tagName.toLowerCase();
605
- if (tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
606
- const rect = el.getBoundingClientRect();
607
- if (rect.width < 4 || rect.height < 4) continue;
608
- if (rect.bottom < 0 || rect.top > win.innerHeight) continue;
609
- if (rect.right < 0 || rect.left > win.innerWidth) continue;
610
- const style = win.getComputedStyle(el);
611
- if (style.cursor !== "pointer") continue;
612
- pointerCandidates.push(el);
613
- }
614
- const all = semantic.concat(pointerCandidates);
615
- const vh = win.innerHeight;
616
- const vw = win.innerWidth;
617
- const entries = [];
618
- const seen = /* @__PURE__ */ new Set();
619
- for (const el of all) {
620
- if (seen.has(el)) continue;
621
- const rect = el.getBoundingClientRect();
622
- if (rect.width < 4 || rect.height < 4) continue;
623
- if (rect.bottom < 0 || rect.top > vh) continue;
624
- if (rect.right < 0 || rect.left > vw) continue;
625
- const style = win.getComputedStyle(el);
626
- if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
596
+ const max = opts?.maxElements ?? 200;
597
+ const mainResult = await this.page.evaluate((limit) => {
598
+ return globalThis.__buaExtract(limit, 0, 0, "main");
599
+ }, max).catch(async () => {
600
+ await this.installExtractorScript();
601
+ return await this.page.evaluate((limit) => {
602
+ return globalThis.__buaExtract(limit, 0, 0, "main");
603
+ }, max);
604
+ });
605
+ let collected = mainResult.elements;
606
+ const scroll = mainResult.scroll;
607
+ try {
608
+ const frames = this.page.frames();
609
+ for (const frame of frames) {
610
+ if (frame === this.page.mainFrame()) continue;
611
+ if (collected.length >= max) break;
612
+ let bbox = null;
613
+ try {
614
+ const owner = await frame.frameElement();
615
+ if (owner) {
616
+ const rect = await owner.boundingBox();
617
+ if (rect) bbox = { x: rect.x, y: rect.y };
618
+ }
619
+ } catch {
627
620
  continue;
628
621
  }
629
- const cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
630
- const cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
622
+ if (!bbox) continue;
623
+ let iframeRes = null;
631
624
  try {
632
- const hit = doc.elementFromPoint(cx, cy);
633
- if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
634
- continue;
635
- }
636
- if (hit && el.contains(hit)) seen.add(hit);
625
+ iframeRes = await frame.evaluate(
626
+ (args) => {
627
+ return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
628
+ },
629
+ { limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
630
+ );
637
631
  } catch {
632
+ try {
633
+ await frame.evaluate(this.extractorScriptSource());
634
+ iframeRes = await frame.evaluate(
635
+ (args) => {
636
+ return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
637
+ },
638
+ { limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
639
+ );
640
+ } catch {
641
+ }
642
+ }
643
+ if (iframeRes?.elements?.length) {
644
+ const offset = collected.length;
645
+ for (let i = 0; i < iframeRes.elements.length; i++) {
646
+ const e = iframeRes.elements[i];
647
+ e.index = offset + i + 1;
648
+ collected.push(e);
649
+ if (collected.length >= max) break;
650
+ }
638
651
  }
639
- seen.add(el);
640
- entries.push({ el, cx, cy });
641
- }
642
- entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
643
- const out = [];
644
- let idx = 1;
645
- for (const { el, cx, cy } of entries) {
646
- if (out.length >= limit) break;
647
- const tag = el.tagName.toLowerCase();
648
- const role = el.getAttribute("role") || tag;
649
- const type = el.getAttribute("type") || "";
650
- const innerText = el.innerText || el.textContent || "";
651
- const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
652
- const placeholder = el.getAttribute("placeholder") || "";
653
- const ariaLabel = el.getAttribute("aria-label") || "";
654
- const title = el.getAttribute("title") || "";
655
- const name = el.getAttribute("name") || "";
656
- const href = el.getAttribute("href") || "";
657
- const value = el.value || "";
658
- let label = ariaLabel || text || placeholder || title || value || name;
659
- if (!label && href) label = href.slice(0, 60);
660
- if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
661
- el.setAttribute("data-bua-idx", String(idx));
662
- out.push({
663
- index: idx,
664
- cx,
665
- cy,
666
- role,
667
- type: type || void 0,
668
- label,
669
- tag,
670
- isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
671
- isSelect: tag === "select",
672
- isFile: tag === "input" && type === "file"
673
- });
674
- idx++;
675
652
  }
676
- return out;
677
- }, max);
678
- this._lastDom = raw;
679
- const lines = raw.map((e) => {
653
+ } catch {
654
+ }
655
+ this._lastDom = collected;
656
+ const lines = collected.map((e) => {
680
657
  const typeSuffix = e.type ? `(${e.type})` : "";
681
- return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}: "${e.label}"`;
658
+ const frameSuffix = e.frame && e.frame !== "main" ? ` [frame]` : "";
659
+ return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}${frameSuffix}: "${e.label}"`;
682
660
  });
683
- return { text: lines.join("\n"), elements: raw };
661
+ return { text: lines.join("\n"), elements: collected, scroll };
662
+ }
663
+ /**
664
+ * Install the `__buaExtract` global on the main page. Idempotent —
665
+ * subsequent calls are no-ops.
666
+ */
667
+ async installExtractorScript() {
668
+ if (!this.page) return;
669
+ await this.page.evaluate(this.extractorScriptSource());
670
+ }
671
+ /**
672
+ * The extractor source. Lives in its own method so we can also inject
673
+ * it into iframes that haven't yet had it loaded.
674
+ *
675
+ * This function intentionally runs entirely in the page context. It:
676
+ * - traverses the regular DOM + open shadow roots (deep)
677
+ * - applies a parent-chain visibility filter
678
+ * - applies a `cursor:pointer` second pass for custom widgets
679
+ * - hit-tests each candidate at its center to avoid overlay collisions
680
+ * - returns scroll context (pages above/below, hidden counts)
681
+ * - tags survivors with `data-bua-idx` for indexed actions
682
+ */
683
+ extractorScriptSource() {
684
+ return (
685
+ /* js */
686
+ `
687
+ (function () {
688
+ if (typeof window.__buaExtract === "function") return;
689
+
690
+ function isAncestorVisible(el) {
691
+ let cur = el;
692
+ while (cur && cur !== document.body && cur !== document.documentElement) {
693
+ const s = window.getComputedStyle(cur);
694
+ if (!s) return false;
695
+ if (s.display === "none" || s.visibility === "hidden" || s.visibility === "collapse") return false;
696
+ if (parseFloat(s.opacity || "1") < 0.05) return false;
697
+ cur = cur.parentElement || (cur.getRootNode && cur.getRootNode().host) || null;
698
+ }
699
+ return true;
700
+ }
701
+
702
+ function collectSemantic(root, out) {
703
+ if (!root || !root.querySelectorAll) return;
704
+ var sel = [
705
+ "a[href]", "button", "input", "textarea", "select",
706
+ "[role='button']", "[role='link']", "[role='tab']",
707
+ "[role='menuitem']", "[role='checkbox']", "[role='radio']",
708
+ "[role='switch']", "[role='combobox']", "[role='option']",
709
+ "[onclick]", "[contenteditable='true']", "[tabindex]:not([tabindex='-1'])"
710
+ ].join(",");
711
+ var found = root.querySelectorAll(sel);
712
+ for (var i = 0; i < found.length; i++) out.push(found[i]);
713
+ // shadow DOM piercing
714
+ var all = root.querySelectorAll("*");
715
+ for (var j = 0; j < all.length; j++) {
716
+ var n = all[j];
717
+ if (n.shadowRoot) collectSemantic(n.shadowRoot, out);
718
+ }
719
+ }
720
+
721
+ function collectPointer(root, out, cap) {
722
+ if (!root || !root.querySelectorAll) return;
723
+ var all = root.querySelectorAll("*");
724
+ var vh = window.innerHeight, vw = window.innerWidth;
725
+ for (var i = 0; i < all.length && out.length < cap; i++) {
726
+ var el = all[i];
727
+ var tag = (el.tagName || "").toLowerCase();
728
+ if (!tag || tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
729
+ var rect = el.getBoundingClientRect();
730
+ if (rect.width < 4 || rect.height < 4) continue;
731
+ if (rect.bottom < 0 || rect.top > vh) continue;
732
+ if (rect.right < 0 || rect.left > vw) continue;
733
+ var s = window.getComputedStyle(el);
734
+ if (s.cursor !== "pointer") continue;
735
+ out.push(el);
736
+ if (el.shadowRoot) collectPointer(el.shadowRoot, out, cap);
737
+ }
738
+ }
739
+
740
+ function countHiddenInteractive(root, vh, vw) {
741
+ if (!root || !root.querySelectorAll) return 0;
742
+ var sel = ["a[href]","button","input","textarea","select","[role='button']","[role='link']","[role='tab']","[role='menuitem']"].join(",");
743
+ var n = 0;
744
+ var found = root.querySelectorAll(sel);
745
+ for (var i = 0; i < found.length; i++) {
746
+ var r = found[i].getBoundingClientRect();
747
+ if (r.bottom < 0 || r.top > vh || r.right < 0 || r.left > vw) n++;
748
+ }
749
+ return n;
750
+ }
751
+
752
+ window.__buaExtract = function (limit, offsetX, offsetY, frameName) {
753
+ // Clear stale markers each call.
754
+ var stale = document.querySelectorAll("[data-bua-idx]");
755
+ for (var s = 0; s < stale.length; s++) stale[s].removeAttribute("data-bua-idx");
756
+
757
+ var vh = window.innerHeight, vw = window.innerWidth;
758
+ var semantic = []; collectSemantic(document, semantic);
759
+ var pointer = []; collectPointer(document, pointer, 2000);
760
+ var all = semantic.concat(pointer);
761
+
762
+ var entries = [];
763
+ var seen = new Set();
764
+
765
+ for (var i = 0; i < all.length; i++) {
766
+ var el = all[i];
767
+ if (seen.has(el)) continue;
768
+ var rect = el.getBoundingClientRect();
769
+ if (rect.width < 4 || rect.height < 4) continue;
770
+ if (rect.bottom < 0 || rect.top > vh) continue;
771
+ if (rect.right < 0 || rect.left > vw) continue;
772
+
773
+ var style = window.getComputedStyle(el);
774
+ if (style.visibility === "hidden" || style.display === "none" ||
775
+ style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) continue;
776
+ if (!isAncestorVisible(el)) continue;
777
+
778
+ var cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
779
+ var cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
780
+
781
+ try {
782
+ var hit = document.elementFromPoint(cx, cy);
783
+ if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) continue;
784
+ if (hit && el.contains(hit)) seen.add(hit);
785
+ } catch (_) {}
786
+ seen.add(el);
787
+ entries.push({ el: el, cx: cx, cy: cy });
788
+ }
789
+
790
+ entries.sort(function (a, b) { return a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy; });
791
+
792
+ var out = [];
793
+ var idx = 1;
794
+ for (var k = 0; k < entries.length && out.length < limit; k++) {
795
+ var ent = entries[k];
796
+ var el = ent.el;
797
+ var tag = (el.tagName || "").toLowerCase();
798
+ var role = el.getAttribute("role") || tag;
799
+ var type = el.getAttribute("type") || "";
800
+ var innerText = (el.innerText || el.textContent || "");
801
+ var text = innerText.trim().replace(/\\s+/g, " ").slice(0, 80);
802
+ var placeholder = el.getAttribute("placeholder") || "";
803
+ var ariaLabel = el.getAttribute("aria-label") || "";
804
+ var title = el.getAttribute("title") || "";
805
+ var name = el.getAttribute("name") || "";
806
+ var href = el.getAttribute("href") || "";
807
+ var value = el.value || "";
808
+ var label = ariaLabel || text || placeholder || title || value || name;
809
+ if (!label && href) label = href.slice(0, 60);
810
+ if (!label) label = "(" + tag + (type ? (" type=" + type) : "") + ")";
811
+
812
+ el.setAttribute("data-bua-idx", String(idx));
813
+ out.push({
814
+ index: idx,
815
+ cx: ent.cx + offsetX,
816
+ cy: ent.cy + offsetY,
817
+ role: role,
818
+ type: type || undefined,
819
+ label: label,
820
+ tag: tag,
821
+ isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
822
+ isSelect: tag === "select",
823
+ isFile: tag === "input" && type === "file",
824
+ frame: frameName,
825
+ });
826
+ idx++;
827
+ }
828
+
829
+ // Scroll context (only meaningful for the main frame; iframes pass through).
830
+ var doc = document.scrollingElement || document.documentElement || document.body;
831
+ var pagesAbove = 0, pagesBelow = 0;
832
+ if (doc && vh > 0) {
833
+ pagesAbove = Math.max(0, Math.round(doc.scrollTop / vh));
834
+ var below = Math.max(0, doc.scrollHeight - doc.scrollTop - vh);
835
+ pagesBelow = Math.round(below / vh);
836
+ }
837
+
838
+ // Hidden interactive count over the WHOLE document tree.
839
+ var hidden = countHiddenInteractive(document, vh, vw);
840
+
841
+ return {
842
+ elements: out,
843
+ scroll: {
844
+ pagesAbove: pagesAbove,
845
+ pagesBelow: pagesBelow,
846
+ totalInteractive: out.length + hidden,
847
+ hiddenInteractive: hidden,
848
+ },
849
+ };
850
+ };
851
+ })();
852
+ `
853
+ );
684
854
  }
685
855
  // ── Page Info ────────────────────────────────────────────────────────
686
856
  async getPageInfo() {
@@ -842,16 +1012,169 @@ var BrowserProvider = class {
842
1012
  }
843
1013
  };
844
1014
 
1015
+ // src/loop-detector.ts
1016
+ var LoopDetector = class {
1017
+ actionWindow = [];
1018
+ pageHistory = [];
1019
+ actionWindowSize;
1020
+ pageHistorySize;
1021
+ /** Thresholds: repeat counts at which each severity fires. */
1022
+ actionThresholds;
1023
+ /** Thresholds: consecutive unchanged-page counts at which each fires. */
1024
+ pageThresholds;
1025
+ constructor(opts) {
1026
+ this.actionWindowSize = opts?.actionWindowSize ?? 20;
1027
+ this.pageHistorySize = opts?.pageHistorySize ?? 10;
1028
+ this.actionThresholds = {
1029
+ warn: opts?.actionThresholds?.warn ?? 5,
1030
+ escalate: opts?.actionThresholds?.escalate ?? 8,
1031
+ abort: opts?.actionThresholds?.abort ?? 12
1032
+ };
1033
+ this.pageThresholds = {
1034
+ warn: opts?.pageThresholds?.warn ?? 5,
1035
+ escalate: opts?.pageThresholds?.escalate ?? 8,
1036
+ abort: opts?.pageThresholds?.abort ?? 12
1037
+ };
1038
+ }
1039
+ /**
1040
+ * Record an action and return advice for the runtime. `severity` of
1041
+ * `abort` means the loop should terminate now.
1042
+ */
1043
+ recordAction(action) {
1044
+ const key = normalizeAction(action);
1045
+ this.actionWindow.push(key);
1046
+ if (this.actionWindow.length > this.actionWindowSize) this.actionWindow.shift();
1047
+ let count = 0;
1048
+ for (const k of this.actionWindow) if (k === key) count++;
1049
+ if (count >= this.actionThresholds.abort) {
1050
+ return {
1051
+ severity: "abort",
1052
+ message: `Auto-stopped: the action "${key}" has been repeated ${count} times in the last ${this.actionWindow.length} steps with no apparent progress.`
1053
+ };
1054
+ }
1055
+ if (count >= this.actionThresholds.escalate) {
1056
+ return {
1057
+ severity: "escalate",
1058
+ message: `You have repeated essentially the same action ${count} times. The current approach is NOT working \u2014 pick a different strategy: scroll, navigate elsewhere, dismiss a popup, or use "fail" if the task truly cannot be done.`
1059
+ };
1060
+ }
1061
+ if (count >= this.actionThresholds.warn) {
1062
+ return {
1063
+ severity: "warn",
1064
+ message: `You've repeated this action ${count} times \u2014 if the page hasn't changed, try a different approach.`
1065
+ };
1066
+ }
1067
+ return { severity: "none" };
1068
+ }
1069
+ /**
1070
+ * Record a page fingerprint and return advice if the page has been
1071
+ * stagnant. Counts consecutive identical fingerprints at the END of
1072
+ * the history (so a single navigation resets the counter).
1073
+ */
1074
+ recordPage(fp) {
1075
+ this.pageHistory.push(fp);
1076
+ if (this.pageHistory.length > this.pageHistorySize) this.pageHistory.shift();
1077
+ let stagnantCount = 1;
1078
+ for (let i = this.pageHistory.length - 2; i >= 0; i--) {
1079
+ const prev = this.pageHistory[i];
1080
+ if (prev.url === fp.url && prev.interactiveCount === fp.interactiveCount && prev.textHash === fp.textHash) {
1081
+ stagnantCount++;
1082
+ } else {
1083
+ break;
1084
+ }
1085
+ }
1086
+ if (stagnantCount >= this.pageThresholds.abort) {
1087
+ return {
1088
+ severity: "abort",
1089
+ message: `Auto-stopped: the page hasn't changed in ${stagnantCount} consecutive steps. Likely a blocking overlay or dead control.`
1090
+ };
1091
+ }
1092
+ if (stagnantCount >= this.pageThresholds.escalate) {
1093
+ return {
1094
+ severity: "escalate",
1095
+ message: `The page has not changed in ${stagnantCount} steps. Something is blocking progress \u2014 try dismissing popups, navigating elsewhere, or scrolling.`
1096
+ };
1097
+ }
1098
+ if (stagnantCount >= this.pageThresholds.warn) {
1099
+ return {
1100
+ severity: "warn",
1101
+ message: `Page hasn't changed in ${stagnantCount} steps \u2014 verify your action is taking effect.`
1102
+ };
1103
+ }
1104
+ return { severity: "none" };
1105
+ }
1106
+ /** Combine two advices, returning the more severe one. */
1107
+ static combine(a, b) {
1108
+ const rank = { none: 0, warn: 1, escalate: 2, abort: 3 };
1109
+ return rank[a.severity] >= rank[b.severity] ? a : b;
1110
+ }
1111
+ };
1112
+ function normalizeAction(action) {
1113
+ switch (action.action) {
1114
+ case "click":
1115
+ if (typeof action.index === "number") return `click:${action.index}`;
1116
+ if (typeof action.x === "number" && typeof action.y === "number") {
1117
+ return `click:xy:${Math.round(action.x / 16)},${Math.round(action.y / 16)}`;
1118
+ }
1119
+ return `click:?`;
1120
+ case "type":
1121
+ if (typeof action.index === "number") return `type:${action.index}:${truncate(action.text, 40)}`;
1122
+ return `type:?:${truncate(action.text, 40)}`;
1123
+ case "scroll":
1124
+ return `scroll:${action.direction}:${action.amount ?? "default"}${action.index != null ? `:${action.index}` : ""}`;
1125
+ case "navigate":
1126
+ return `navigate:${action.url}`;
1127
+ case "back":
1128
+ return "back";
1129
+ case "wait":
1130
+ return `wait:${Math.round(action.ms / 1e3)}`;
1131
+ case "screenshot":
1132
+ return "screenshot";
1133
+ case "send_keys":
1134
+ return `send_keys:${action.keys}`;
1135
+ case "find_text":
1136
+ return `find_text:${truncate(action.text, 40)}`;
1137
+ case "evaluate":
1138
+ return `evaluate:${truncate(action.code, 80)}`;
1139
+ case "dropdown_options":
1140
+ return `dropdown_options:${action.index}`;
1141
+ case "select_dropdown":
1142
+ return `select_dropdown:${action.index}:${truncate(action.text, 40)}`;
1143
+ case "upload_file":
1144
+ return `upload_file:${action.index}:${action.path}`;
1145
+ case "extract":
1146
+ return `extract:${truncate(action.query, 80)}`;
1147
+ case "tool":
1148
+ return `tool:${action.name}`;
1149
+ case "done":
1150
+ case "fail":
1151
+ return action.action;
1152
+ }
1153
+ }
1154
+ function fnvHash(s) {
1155
+ let h = 2166136261;
1156
+ for (let i = 0; i < s.length; i++) {
1157
+ h ^= s.charCodeAt(i);
1158
+ h = h * 16777619 >>> 0;
1159
+ }
1160
+ return h;
1161
+ }
1162
+ function truncate(s, n) {
1163
+ if (!s) return "";
1164
+ return s.length <= n ? s : `${s.slice(0, n)}\u2026`;
1165
+ }
1166
+
845
1167
  // src/prompts.ts
846
1168
  function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options) {
847
1169
  const maxActions = options?.maxActionsPerStep ?? 3;
848
1170
  const allowEvaluate = !!options?.allowEvaluate;
849
1171
  const tools = options?.tools ?? [];
850
1172
  const useVision = options?.useVision ?? "auto";
1173
+ const useThinking = options?.useThinking ?? true;
851
1174
  if (options?.overrideSystemMessage) {
852
1175
  const lines2 = [options.overrideSystemMessage];
853
1176
  appendCredentials(lines2, credentialKeys);
854
- lines2.push("", "## Response Format", responseFormatLines(maxActions).join("\n"));
1177
+ lines2.push("", "## Response Format", responseFormatLines(maxActions, useThinking).join("\n"));
855
1178
  return lines2.join("\n");
856
1179
  }
857
1180
  const lines = [];
@@ -956,7 +1279,7 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options)
956
1279
  if (extraInstructions) {
957
1280
  lines.push(``, `## Additional Instructions`, extraInstructions);
958
1281
  }
959
- lines.push(``, `## Response Format`, responseFormatLines(maxActions).join("\n"));
1282
+ lines.push(``, `## Response Format`, responseFormatLines(maxActions, useThinking).join("\n"));
960
1283
  return lines.join("\n");
961
1284
  }
962
1285
  function appendCredentials(lines, credentialKeys) {
@@ -974,22 +1297,61 @@ function appendCredentials(lines, credentialKeys) {
974
1297
  `NEVER include real credential values in "done" or "fail" results.`
975
1298
  );
976
1299
  }
977
- function responseFormatLines(maxActions) {
1300
+ function responseFormatLines(maxActions, useThinking) {
1301
+ if (!useThinking) {
1302
+ return [
1303
+ `Respond with ONLY valid JSON. No markdown, no commentary.`,
1304
+ `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
1305
+ ];
1306
+ }
978
1307
  return [
979
- `Respond with ONLY valid JSON. No markdown, no commentary.`,
980
- `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
1308
+ `Respond with ONLY a single valid JSON object. No markdown, no commentary.`,
1309
+ ``,
1310
+ `Use this exact shape:`,
1311
+ "```json",
1312
+ `{`,
1313
+ ` "thinking": "Short chain-of-thought: what do I see, what's my plan?",`,
1314
+ ` "evaluation_previous_goal": "Did the previous action succeed? success/partial/failure + 1 line",`,
1315
+ ` "memory": "Compact bullet list of facts to remember across steps (URLs, found data, \u2026)",`,
1316
+ ` "next_goal": "What I want to accomplish in THIS step",`,
1317
+ ` "action": <single action object OR array of up to ${maxActions} action objects>`,
1318
+ `}`,
1319
+ "```",
1320
+ ``,
1321
+ `On the very first step, set "evaluation_previous_goal" to "n/a \u2014 first step".`,
1322
+ `Keep each text field to one short paragraph or a few lines. Be concrete.`,
1323
+ `Only the "action" field is executed; the others help you self-correct over multiple steps.`
981
1324
  ];
982
1325
  }
983
- function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract) {
1326
+ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget) {
984
1327
  const lines = [];
985
1328
  lines.push(`**Task:** ${task}`);
986
1329
  lines.push(`**Current URL:** ${pageUrl}`);
987
1330
  if (pageTitle) lines.push(`**Page Title:** ${pageTitle}`);
988
- lines.push(`**Step:** ${stepIndex + 1}`);
989
- if (domSnapshot) {
990
- lines.push(``);
991
- lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
992
- lines.push(domSnapshot);
1331
+ if (stepBudget) {
1332
+ lines.push(`**Step:** ${stepIndex + 1} of ${stepBudget.max} (${stepBudget.max - stepIndex - 1} remaining)`);
1333
+ } else {
1334
+ lines.push(`**Step:** ${stepIndex + 1}`);
1335
+ }
1336
+ if (scroll) {
1337
+ const parts = [];
1338
+ parts.push(`${scroll.totalInteractive} interactive elements (${scroll.hiddenInteractive} hidden)`);
1339
+ if (scroll.pagesAbove > 0) parts.push(`${scroll.pagesAbove} page${scroll.pagesAbove === 1 ? "" : "s"} above`);
1340
+ if (scroll.pagesBelow > 0) parts.push(`${scroll.pagesBelow} page${scroll.pagesBelow === 1 ? "" : "s"} below`);
1341
+ if (scroll.pagesAbove === 0 && scroll.pagesBelow === 0) parts.push("fits in viewport");
1342
+ lines.push(`**Page stats:** ${parts.join(" \xB7 ")}`);
1343
+ }
1344
+ if (domSnapshot !== void 0) {
1345
+ if (domSnapshot.trim().length === 0) {
1346
+ lines.push(``);
1347
+ lines.push(
1348
+ `**\u26A0 Empty page / no interactive elements detected.** The page may be blank, blocked by a captcha/anti-bot wall, mid-load, or rendered with shadow DOM in an unsupported way. Consider: \`wait\` + retry, \`navigate\` to a different URL, or \`fail\` if the site is blocking access.`
1349
+ );
1350
+ } else {
1351
+ lines.push(``);
1352
+ lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
1353
+ lines.push(domSnapshot);
1354
+ }
993
1355
  }
994
1356
  if (lastExtract) {
995
1357
  lines.push(``);
@@ -1003,6 +1365,10 @@ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, do
1003
1365
  lines.push(`- ${entry}`);
1004
1366
  }
1005
1367
  }
1368
+ if (nudge) {
1369
+ lines.push(``);
1370
+ lines.push(`**\u26A0 Runtime hint:** ${nudge}`);
1371
+ }
1006
1372
  lines.push(``);
1007
1373
  lines.push(`Decide the next action(s) to complete the task.`);
1008
1374
  return lines.join("\n");
@@ -1057,6 +1423,9 @@ var BrowserAgent = class {
1057
1423
  eventBus;
1058
1424
  model;
1059
1425
  pageExtractionLLM;
1426
+ fallbackModel;
1427
+ useThinking;
1428
+ historyWindow;
1060
1429
  instructions;
1061
1430
  extendSystemMessage;
1062
1431
  overrideSystemMessage;
@@ -1093,6 +1462,9 @@ var BrowserAgent = class {
1093
1462
  this.name = config.name;
1094
1463
  this.model = config.model;
1095
1464
  this.pageExtractionLLM = config.pageExtractionLLM ?? null;
1465
+ this.fallbackModel = config.fallbackModel ?? null;
1466
+ this.useThinking = config.useThinking ?? true;
1467
+ this.historyWindow = Math.max(0, config.historyWindow ?? 6);
1096
1468
  this.instructions = config.instructions;
1097
1469
  this.extendSystemMessage = config.extendSystemMessage;
1098
1470
  this.overrideSystemMessage = config.overrideSystemMessage;
@@ -1138,10 +1510,10 @@ var BrowserAgent = class {
1138
1510
  const actionHistory = [];
1139
1511
  const extractedContent = [];
1140
1512
  let lastExtractResult;
1141
- let failureCount = 0;
1142
- let lastActionKey = "";
1143
- let repeatCount = 0;
1513
+ let consecutiveFailures = 0;
1144
1514
  let lastActionWasScreenshot = false;
1515
+ const loop = new LoopDetector();
1516
+ const historyTurns = [];
1145
1517
  const baseExtra = [this.instructions, this.extendSystemMessage].filter(Boolean).join("\n\n");
1146
1518
  let extraInstructions = baseExtra;
1147
1519
  if (this.memoryManager) {
@@ -1160,7 +1532,8 @@ ${memoryContext}` : memoryContext;
1160
1532
  allowEvaluate: this.allowEvaluate,
1161
1533
  tools: this.tools,
1162
1534
  useVision: this.useVision,
1163
- useDOM: this.useDOM
1535
+ useDOM: this.useDOM,
1536
+ useThinking: this.useThinking
1164
1537
  });
1165
1538
  try {
1166
1539
  this.logger.info("Launching browser", {
@@ -1168,6 +1541,7 @@ ${memoryContext}` : memoryContext;
1168
1541
  viewport: this.viewport,
1169
1542
  useDOM: this.useDOM,
1170
1543
  useVision: this.useVision,
1544
+ useThinking: this.useThinking,
1171
1545
  cdpUrl: this.cdpUrl ?? void 0,
1172
1546
  recordVideo: !!this.recordVideo,
1173
1547
  stealth: !!this.stealth,
@@ -1187,6 +1561,7 @@ ${memoryContext}` : memoryContext;
1187
1561
  this.logger.info("Navigating to start URL", { url: startUrl });
1188
1562
  this.assertDomainAllowed(startUrl);
1189
1563
  await browser.navigate(startUrl);
1564
+ await this.navigationHealthCheck(browser, startUrl);
1190
1565
  }
1191
1566
  for (const ia of this.initialActions) {
1192
1567
  try {
@@ -1199,13 +1574,35 @@ ${memoryContext}` : memoryContext;
1199
1574
  for (let step = 0; step < maxSteps; step++) {
1200
1575
  const pageInfo = await browser.getPageInfo();
1201
1576
  let domSnapshot;
1577
+ let scrollCtx;
1202
1578
  if (this.useDOM) {
1203
1579
  const dom = await browser.extractDOM();
1204
1580
  domSnapshot = dom.text;
1581
+ scrollCtx = dom.scroll;
1582
+ }
1583
+ const pageAdvice = loop.recordPage({
1584
+ url: pageInfo.url,
1585
+ interactiveCount: scrollCtx?.totalInteractive ?? 0,
1586
+ textHash: fnvHash(domSnapshot ?? "")
1587
+ });
1588
+ if (pageAdvice.severity === "abort") {
1589
+ return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1590
+ result: pageAdvice.message ?? "Auto-stopped: page is stagnant.",
1591
+ success: false
1592
+ });
1205
1593
  }
1206
1594
  const wantVision = this.shouldCaptureVision(step, lastActionWasScreenshot);
1207
1595
  const screenshot = wantVision ? await browser.screenshot() : Buffer.alloc(0);
1208
1596
  if (wantVision) this.eventBus.emit("browser.screenshot", { data: screenshot });
1597
+ const isLastStep = step === maxSteps - 1;
1598
+ const nudgeParts = [];
1599
+ if (pageAdvice.severity !== "none" && pageAdvice.message) nudgeParts.push(pageAdvice.message);
1600
+ if (isLastStep) {
1601
+ nudgeParts.push(
1602
+ "This is your FINAL step. Return a `done` action right now summarizing whatever you have, even if partial."
1603
+ );
1604
+ }
1605
+ const nudge = nudgeParts.length > 0 ? nudgeParts.join(" ") : void 0;
1209
1606
  const userText = buildUserMessage(
1210
1607
  task,
1211
1608
  pageInfo.url,
@@ -1213,65 +1610,49 @@ ${memoryContext}` : memoryContext;
1213
1610
  step,
1214
1611
  actionHistory,
1215
1612
  domSnapshot,
1216
- lastExtractResult
1613
+ lastExtractResult,
1614
+ scrollCtx,
1615
+ nudge,
1616
+ { current: step, max: maxSteps }
1217
1617
  );
1218
- const content = [];
1219
- if (wantVision) {
1220
- content.push({ type: "image", data: screenshot.toString("base64"), mimeType: "image/png" });
1221
- }
1222
- content.push({ type: "text", text: userText });
1223
- const messages = [
1224
- { role: "system", content: systemPrompt },
1225
- { role: "user", content }
1226
- ];
1618
+ const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
1227
1619
  this.logger.debug("Calling model", { step, url: pageInfo.url, vision: wantVision });
1228
- let response;
1229
- try {
1230
- response = await this.model.generate(messages, {
1231
- temperature: 0.1,
1232
- maxTokens: 1024,
1233
- apiKey: opts?.apiKey,
1234
- responseFormat: "json"
1235
- });
1236
- } catch (e) {
1237
- failureCount++;
1238
- this.logger.warn("Model call failed", { error: e?.message, failureCount });
1239
- if (failureCount > this.maxFailures) {
1240
- return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1241
- result: `Model call failed ${failureCount} times: ${e?.message}`,
1242
- success: false
1243
- });
1620
+ const { response, modelUsed } = await this.callModelWithFallback(messages, opts?.apiKey);
1621
+ if (!response) {
1622
+ consecutiveFailures++;
1623
+ actionHistory.push(`(model call failed \u2014 retrying, ${consecutiveFailures}/${this.maxFailures})`);
1624
+ if (consecutiveFailures > this.maxFailures) {
1625
+ return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
1244
1626
  }
1245
- actionHistory.push(`(model call failed: ${e?.message ?? "unknown"} \u2014 retrying)`);
1246
1627
  continue;
1247
1628
  }
1248
1629
  if (this.costTracker && response.usage) {
1249
1630
  this.costTracker.track({
1250
1631
  runId: sessionId,
1251
1632
  agentName: this.name,
1252
- modelId: this.model.modelId,
1633
+ modelId: modelUsed.modelId,
1253
1634
  usage: response.usage,
1254
1635
  sessionId,
1255
1636
  userId
1256
1637
  });
1257
1638
  }
1258
1639
  const raw = typeof response.message.content === "string" ? response.message.content : "";
1259
- let parsed;
1260
- try {
1261
- parsed = JSON.parse(raw);
1262
- } catch {
1263
- failureCount++;
1264
- this.logger.warn("Failed to parse model response as JSON", { raw, failureCount });
1265
- if (failureCount > this.maxFailures) {
1266
- return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1267
- result: `Model returned invalid JSON ${failureCount} times. Last response: ${raw.slice(0, 200)}`,
1268
- success: false
1269
- });
1640
+ const envelope = this.parseEnvelope(raw);
1641
+ if (!envelope) {
1642
+ consecutiveFailures++;
1643
+ this.logger.warn("Failed to parse model response", { raw: raw.slice(0, 200), consecutiveFailures });
1644
+ if (consecutiveFailures > this.maxFailures) {
1645
+ return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "parse");
1270
1646
  }
1271
1647
  actionHistory.push("(invalid JSON response \u2014 retrying)");
1272
1648
  continue;
1273
1649
  }
1274
- const actions = (Array.isArray(parsed) ? parsed : [parsed]).slice(
1650
+ consecutiveFailures = 0;
1651
+ historyTurns.push({ userText, hasScreenshot: wantVision, envelope });
1652
+ if (this.historyWindow > 0 && historyTurns.length > this.historyWindow) {
1653
+ historyTurns.splice(0, historyTurns.length - this.historyWindow);
1654
+ }
1655
+ const actions = (Array.isArray(envelope.action) ? envelope.action : [envelope.action]).slice(
1275
1656
  0,
1276
1657
  this.maxActionsPerStep
1277
1658
  );
@@ -1282,24 +1663,12 @@ ${memoryContext}` : memoryContext;
1282
1663
  const action = actions[ai];
1283
1664
  let summary = summarizeAction(action);
1284
1665
  if (this.credentials) summary = this.credentials.mask(summary);
1285
- const actionKey = JSON.stringify(action);
1286
- if (actionKey === lastActionKey) {
1287
- repeatCount++;
1288
- } else {
1289
- lastActionKey = actionKey;
1290
- repeatCount = 1;
1291
- }
1292
- if (repeatCount > this.maxRepeats && action.action !== "done" && action.action !== "fail") {
1293
- this.logger.warn("Stuck in a loop \u2014 same action repeated", {
1294
- action: action.action,
1295
- repeats: repeatCount,
1296
- maxRepeats: this.maxRepeats
1297
- });
1298
- actionHistory.push(
1299
- `\u26A0 LOOP DETECTED: "${summary}" repeated ${repeatCount} times. Auto-stopping.`
1300
- );
1666
+ const advice = loop.recordAction(action);
1667
+ if (advice.severity === "abort" && action.action !== "done" && action.action !== "fail") {
1668
+ this.logger.warn("Loop detector aborting run", { advice });
1669
+ actionHistory.push(`\u26A0 ${advice.message ?? "loop detected \u2014 auto-stopping"}`);
1301
1670
  didTerminate = {
1302
- result: `Stuck in a loop: "${summary}" was repeated ${repeatCount} times. The page may have a popup, consent banner, or unexpected state blocking progress.`,
1671
+ result: advice.message ?? "Stuck in a loop \u2014 auto-stopped.",
1303
1672
  success: false
1304
1673
  };
1305
1674
  break;
@@ -1322,21 +1691,20 @@ ${memoryContext}` : memoryContext;
1322
1691
  try {
1323
1692
  const exec = await this.executeAction(browser, action, actionHistory, extractedContent);
1324
1693
  stepOutput = exec?.output;
1325
- if (exec?.didNavigate) didNavigate = true;
1326
- if (action.action === "extract" && exec?.output) {
1327
- lastExtractResult = exec.output;
1328
- }
1329
- if (action.action === "screenshot") {
1330
- lastActionWasScreenshot = true;
1694
+ if (exec?.didNavigate) {
1695
+ didNavigate = true;
1696
+ await this.navigationHealthCheck(browser, pageInfo.url);
1331
1697
  }
1698
+ if (action.action === "extract" && exec?.output) lastExtractResult = exec.output;
1699
+ if (action.action === "screenshot") lastActionWasScreenshot = true;
1332
1700
  } catch (e) {
1333
1701
  stepOk = false;
1334
- failureCount++;
1335
- this.logger.warn("Action failed", { action: action.action, error: e?.message, failureCount });
1702
+ consecutiveFailures++;
1703
+ this.logger.warn("Action failed", { action: action.action, error: e?.message, consecutiveFailures });
1336
1704
  actionHistory.push(`(error: ${e?.message ?? "unknown"})`);
1337
- if (failureCount > this.maxFailures) {
1705
+ if (consecutiveFailures > this.maxFailures) {
1338
1706
  didTerminate = {
1339
- result: `Action ${action.action} failed ${failureCount} times. Last error: ${e?.message}`,
1707
+ result: `Action ${action.action} failed ${consecutiveFailures} times. Last error: ${e?.message}`,
1340
1708
  success: false
1341
1709
  };
1342
1710
  break;
@@ -1351,7 +1719,11 @@ ${memoryContext}` : memoryContext;
1351
1719
  timestamp: /* @__PURE__ */ new Date(),
1352
1720
  dom: domSnapshot,
1353
1721
  output: stepOutput,
1354
- ok: stepOk
1722
+ ok: stepOk,
1723
+ thinking: envelope.thinking,
1724
+ evaluationPreviousGoal: envelope.evaluationPreviousGoal,
1725
+ memory: envelope.memory,
1726
+ nextGoal: envelope.nextGoal
1355
1727
  });
1356
1728
  this.eventBus.emit("browser.step", {
1357
1729
  index: steps.length - 1,
@@ -1366,11 +1738,8 @@ ${memoryContext}` : memoryContext;
1366
1738
  return await this.finalize(browser, steps, startTime, opts, extractedContent, didTerminate);
1367
1739
  }
1368
1740
  }
1369
- this.logger.warn("Max steps reached without completing task", { maxSteps });
1370
- return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1371
- result: `Task not completed within ${maxSteps} steps. Last actions: ${actionHistory.slice(-3).join("; ")}`,
1372
- success: false
1373
- });
1741
+ this.logger.warn("Max steps reached without explicit done", { maxSteps });
1742
+ return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "max_steps");
1374
1743
  } catch (error) {
1375
1744
  this.logger.error("Browser agent error", { error: error.message });
1376
1745
  this.eventBus.emit("browser.error", { error });
@@ -1432,6 +1801,163 @@ ${memoryContext}` : memoryContext;
1432
1801
  throw new Error(`Navigation blocked: ${host} is in prohibitedDomains`);
1433
1802
  }
1434
1803
  }
1804
+ /**
1805
+ * Build the message array sent to the model: system prompt + a compact
1806
+ * summary of older turns (if any) + the most recent `historyWindow`
1807
+ * turns verbatim + the current step's user message.
1808
+ *
1809
+ * Inspired by browser-use's history compaction. Keeps tokens bounded
1810
+ * while giving the model meaningful context about what it already
1811
+ * tried.
1812
+ */
1813
+ buildMessages(systemPrompt, historyTurns, currentUserText, currentScreenshot) {
1814
+ const msgs = [{ role: "system", content: systemPrompt }];
1815
+ if (this.historyWindow > 0 && historyTurns.length > 0) {
1816
+ for (const turn of historyTurns) {
1817
+ const env = turn.envelope;
1818
+ const replay = [];
1819
+ if (env.thinking) replay.push(`thinking: ${env.thinking}`);
1820
+ if (env.evaluationPreviousGoal) replay.push(`evaluation: ${env.evaluationPreviousGoal}`);
1821
+ if (env.memory) replay.push(`memory: ${env.memory}`);
1822
+ if (env.nextGoal) replay.push(`next_goal: ${env.nextGoal}`);
1823
+ replay.push(`action: ${JSON.stringify(env.action)}`);
1824
+ msgs.push({ role: "assistant", content: replay.join("\n") });
1825
+ }
1826
+ }
1827
+ const content = [];
1828
+ if (currentScreenshot && currentScreenshot.length > 0) {
1829
+ content.push({
1830
+ type: "image",
1831
+ data: currentScreenshot.toString("base64"),
1832
+ mimeType: "image/png"
1833
+ });
1834
+ }
1835
+ content.push({ type: "text", text: currentUserText });
1836
+ msgs.push({ role: "user", content });
1837
+ return msgs;
1838
+ }
1839
+ /**
1840
+ * Call the primary model, retrying once with `fallbackModel` on
1841
+ * transient errors (5xx, 429, network). Returns the response or
1842
+ * `null` if both models failed.
1843
+ */
1844
+ async callModelWithFallback(messages, apiKey) {
1845
+ const reqOpts = { temperature: 0.1, maxTokens: 1024, apiKey, responseFormat: "json" };
1846
+ try {
1847
+ const r = await this.model.generate(messages, reqOpts);
1848
+ return { response: r, modelUsed: this.model };
1849
+ } catch (e) {
1850
+ if (this.fallbackModel && this.isTransientError(e)) {
1851
+ this.logger.warn("Primary model failed; trying fallbackModel", {
1852
+ error: e?.message,
1853
+ primary: this.model.modelId,
1854
+ fallback: this.fallbackModel.modelId
1855
+ });
1856
+ try {
1857
+ const r = await this.fallbackModel.generate(messages, reqOpts);
1858
+ return { response: r, modelUsed: this.fallbackModel };
1859
+ } catch (e2) {
1860
+ this.logger.warn("Fallback model also failed", { error: e2?.message });
1861
+ return { response: null, modelUsed: this.model };
1862
+ }
1863
+ }
1864
+ this.logger.warn("Model call failed (no fallback configured)", { error: e?.message });
1865
+ return { response: null, modelUsed: this.model };
1866
+ }
1867
+ }
1868
+ isTransientError(e) {
1869
+ const msg = String(e?.message ?? e).toLowerCase();
1870
+ if (msg.includes("rate limit") || msg.includes("429")) return true;
1871
+ if (msg.includes("timeout") || msg.includes("etimedout")) return true;
1872
+ if (msg.includes("econnreset") || msg.includes("network")) return true;
1873
+ if (/\b5\d\d\b/.test(msg)) return true;
1874
+ if (msg.includes("401") || msg.includes("402") || msg.includes("auth")) return true;
1875
+ return false;
1876
+ }
1877
+ /**
1878
+ * Parse the model's raw response into an `AgentOutput`. Tolerant to
1879
+ * three shapes:
1880
+ * - Full envelope: { thinking, evaluation_previous_goal, action, ... }
1881
+ * - Raw action object (legacy / `useThinking: false`)
1882
+ * - Raw action array
1883
+ *
1884
+ * Also strips ```json fences the model occasionally adds.
1885
+ */
1886
+ parseEnvelope(raw) {
1887
+ if (!raw) return null;
1888
+ let s = raw.trim();
1889
+ const fence = s.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
1890
+ if (fence) s = fence[1].trim();
1891
+ let json;
1892
+ try {
1893
+ json = JSON.parse(s);
1894
+ } catch {
1895
+ const m = s.match(/[\[{][\s\S]*[\]}]/);
1896
+ if (!m) return null;
1897
+ try {
1898
+ json = JSON.parse(m[0]);
1899
+ } catch {
1900
+ return null;
1901
+ }
1902
+ }
1903
+ if (Array.isArray(json)) return { action: json };
1904
+ if (json && typeof json === "object" && typeof json.action === "string") {
1905
+ return { action: json };
1906
+ }
1907
+ if (json && typeof json === "object" && "action" in json) {
1908
+ const out = { action: json.action };
1909
+ out.thinking = json.thinking ?? json.reasoning ?? void 0;
1910
+ out.evaluationPreviousGoal = json.evaluationPreviousGoal ?? json.evaluation_previous_goal ?? void 0;
1911
+ out.memory = json.memory ?? void 0;
1912
+ out.nextGoal = json.nextGoal ?? json.next_goal ?? void 0;
1913
+ return out;
1914
+ }
1915
+ return null;
1916
+ }
1917
+ /**
1918
+ * Quick post-navigation health check. If the page came back blank
1919
+ * (no body text and no interactive elements), reload once and wait
1920
+ * for stable. This catches the "FreightOS half-loaded font test" class
1921
+ * of failure before the LLM ever sees it.
1922
+ */
1923
+ async navigationHealthCheck(browser, url) {
1924
+ try {
1925
+ const ok = await browser.pageText({ maxChars: 200 }).catch(() => "");
1926
+ if (ok && ok.trim().length > 5) return;
1927
+ this.logger.warn("Navigation produced an empty page; reloading once", { url });
1928
+ await this.sleep(800);
1929
+ try {
1930
+ await browser.navigate(url);
1931
+ } catch {
1932
+ }
1933
+ } catch {
1934
+ }
1935
+ }
1936
+ /**
1937
+ * Force-finalize the run with whatever partial data the agent has.
1938
+ * Called when:
1939
+ * - `maxSteps` is exhausted without an explicit `done`,
1940
+ * - `maxFailures` is exceeded,
1941
+ * - the model can't produce parseable JSON enough times to make
1942
+ * forward progress.
1943
+ * The result is composed from `extractedContent` + the last few
1944
+ * action summaries so the caller gets something useful instead of
1945
+ * just a one-line error.
1946
+ */
1947
+ async forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, reason) {
1948
+ const reasonLine = reason === "model" ? "Model call failed too many consecutive times." : reason === "parse" ? "Model produced invalid JSON too many times." : "Max step budget exhausted before an explicit `done`.";
1949
+ const tail = actionHistory.slice(-5).join(" \u2192 ");
1950
+ const extracts = extractedContent.length > 0 ? `
1951
+
1952
+ Extracts so far:
1953
+ ${extractedContent.join("\n---\n")}` : "";
1954
+ const result = `${reasonLine}
1955
+ Last actions: ${tail || "(none)"}${extracts}`;
1956
+ return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1957
+ result,
1958
+ success: false
1959
+ });
1960
+ }
1435
1961
  async finalize(browser, steps, startTime, opts, extractedContent, outcome) {
1436
1962
  let finalScreenshot;
1437
1963
  let finalInfo;