@agentium/browser 2.1.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +751 -225
- package/dist/index.d.cts +153 -13
- package/dist/index.d.ts +153 -13
- package/dist/index.js +751 -225
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -118,29 +118,39 @@ function getStealthScript() {
|
|
|
118
118
|
};
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
-
// \u2500\u2500
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
el.removeAttribute(attr.name);
|
|
121
|
+
// \u2500\u2500 Remove "cdc_" Playwright/ChromeDriver markers from new elements \u2500
|
|
122
|
+
// Only watches childList additions, NOT every attribute mutation \u2014
|
|
123
|
+
// attributes:true subtree:true is O(every DOM write) on heavy SPAs
|
|
124
|
+
// like FreightOS and noticeably slows the page.
|
|
125
|
+
try {
|
|
126
|
+
const observer = new MutationObserver((mutations) => {
|
|
127
|
+
for (const mutation of mutations) {
|
|
128
|
+
for (const node of mutation.addedNodes) {
|
|
129
|
+
if (node && node.nodeType === 1) {
|
|
130
|
+
const el = node;
|
|
131
|
+
if (!el.attributes) continue;
|
|
132
|
+
for (const attr of [...el.attributes]) {
|
|
133
|
+
if (attr.name.startsWith('cdc_') || attr.name.startsWith('__playwright')) {
|
|
134
|
+
el.removeAttribute(attr.name);
|
|
135
|
+
}
|
|
137
136
|
}
|
|
138
137
|
}
|
|
139
138
|
}
|
|
140
139
|
}
|
|
140
|
+
});
|
|
141
|
+
if (document.documentElement) {
|
|
142
|
+
observer.observe(document.documentElement, { childList: true, subtree: true });
|
|
141
143
|
}
|
|
142
|
-
})
|
|
143
|
-
|
|
144
|
+
} catch (_e) { /* never break the page over a stealth patch */ }
|
|
145
|
+
|
|
146
|
+
// NOTE on iframe.contentWindow:
|
|
147
|
+
// We deliberately do NOT override HTMLIFrameElement.prototype.contentWindow.
|
|
148
|
+
// An earlier version returned the parent window, which breaks legitimate
|
|
149
|
+
// page JS on any site that uses iframes (checkout widgets, embedded
|
|
150
|
+
// videos, A/B test scripts, analytics). The "anti-detection" value was
|
|
151
|
+
// minimal \u2014 modern detectors don't rely on that property anyway \u2014 and
|
|
152
|
+
// the breakage caused pages to fail to boot, leaving font-test residue
|
|
153
|
+
// on screen.
|
|
144
154
|
`;
|
|
145
155
|
}
|
|
146
156
|
function buildStealthContextOpts(config, viewport) {
|
|
@@ -226,14 +236,20 @@ var BrowserProvider = class {
|
|
|
226
236
|
}
|
|
227
237
|
const stealthEnabled = !!opts?.stealth;
|
|
228
238
|
const stealthCfg = typeof opts?.stealth === "object" ? opts.stealth : {};
|
|
229
|
-
const
|
|
230
|
-
|
|
231
|
-
|
|
239
|
+
const headless = opts?.headless ?? true;
|
|
240
|
+
const launchOpts = { headless };
|
|
241
|
+
const args = [];
|
|
232
242
|
if (stealthEnabled) {
|
|
233
|
-
const
|
|
234
|
-
|
|
235
|
-
if (proxy) launchOpts.proxy = proxy;
|
|
243
|
+
const stealth = buildStealthLaunchArgs(stealthCfg);
|
|
244
|
+
args.push(...stealth.args);
|
|
245
|
+
if (stealth.proxy) launchOpts.proxy = stealth.proxy;
|
|
246
|
+
}
|
|
247
|
+
if (!headless) {
|
|
248
|
+
const CHROME_VERTICAL_CHROME_PX = 140;
|
|
249
|
+
args.push(`--window-size=${this._viewport.width},${this._viewport.height + CHROME_VERTICAL_CHROME_PX}`);
|
|
250
|
+
args.push("--window-position=0,0");
|
|
236
251
|
}
|
|
252
|
+
if (args.length) launchOpts.args = args;
|
|
237
253
|
this.browser = await chromium.launch(launchOpts);
|
|
238
254
|
let contextOpts;
|
|
239
255
|
if (stealthEnabled) {
|
|
@@ -593,132 +609,286 @@ var BrowserProvider = class {
|
|
|
593
609
|
/**
|
|
594
610
|
* Snapshot the interactive elements visible in the viewport, tag each
|
|
595
611
|
* with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
|
|
596
|
-
* return
|
|
597
|
-
*
|
|
612
|
+
* return:
|
|
613
|
+
* - `text`: a human-readable string fed to the model
|
|
614
|
+
* - `elements`: the structured list with stable indices
|
|
615
|
+
* - `scroll`: spatial context (pages above/below, hidden interactive count)
|
|
598
616
|
*
|
|
599
|
-
*
|
|
600
|
-
* - Hit-tested
|
|
617
|
+
* Five properties matter for accuracy:
|
|
618
|
+
* - **Hit-tested**: each listed coordinate / index actually reaches the
|
|
601
619
|
* labeled element (overlays / occlusion skip the entry).
|
|
602
|
-
* - Visibility
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
620
|
+
* - **Visibility filtered (with parent chain)**: an element is dropped
|
|
621
|
+
* if itself OR any ancestor is `display:none`, `visibility:hidden`,
|
|
622
|
+
* `pointer-events:none`, or near-zero opacity.
|
|
623
|
+
* - **Shadow DOM piercing**: traverses open shadow roots so custom
|
|
624
|
+
* elements / web components are visible to the agent.
|
|
625
|
+
* - **Same-origin iframes**: walks into each accessible iframe and
|
|
626
|
+
* includes its interactive elements (offset by the iframe's screen
|
|
627
|
+
* position so the coordinates the model sees are still viewport-
|
|
628
|
+
* relative).
|
|
629
|
+
* - **`cursor: pointer` fallback pass**: catches custom React widgets
|
|
630
|
+
* that have no semantic role/href/onclick but are clickable.
|
|
606
631
|
*/
|
|
607
632
|
async extractDOM(opts) {
|
|
608
633
|
this.ensurePage();
|
|
609
|
-
const max = opts?.maxElements ??
|
|
610
|
-
const
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
"[tabindex]:not([tabindex='-1'])"
|
|
634
|
-
];
|
|
635
|
-
const semantic = Array.from(doc.querySelectorAll(selectors.join(",")));
|
|
636
|
-
const POINTER_SCAN_CAP = 2e3;
|
|
637
|
-
const pointerCandidates = [];
|
|
638
|
-
const allEls = doc.querySelectorAll("*");
|
|
639
|
-
for (let i = 0; i < allEls.length && pointerCandidates.length < POINTER_SCAN_CAP; i++) {
|
|
640
|
-
const el = allEls[i];
|
|
641
|
-
if (!el) continue;
|
|
642
|
-
const tag = el.tagName.toLowerCase();
|
|
643
|
-
if (tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
|
|
644
|
-
const rect = el.getBoundingClientRect();
|
|
645
|
-
if (rect.width < 4 || rect.height < 4) continue;
|
|
646
|
-
if (rect.bottom < 0 || rect.top > win.innerHeight) continue;
|
|
647
|
-
if (rect.right < 0 || rect.left > win.innerWidth) continue;
|
|
648
|
-
const style = win.getComputedStyle(el);
|
|
649
|
-
if (style.cursor !== "pointer") continue;
|
|
650
|
-
pointerCandidates.push(el);
|
|
651
|
-
}
|
|
652
|
-
const all = semantic.concat(pointerCandidates);
|
|
653
|
-
const vh = win.innerHeight;
|
|
654
|
-
const vw = win.innerWidth;
|
|
655
|
-
const entries = [];
|
|
656
|
-
const seen = /* @__PURE__ */ new Set();
|
|
657
|
-
for (const el of all) {
|
|
658
|
-
if (seen.has(el)) continue;
|
|
659
|
-
const rect = el.getBoundingClientRect();
|
|
660
|
-
if (rect.width < 4 || rect.height < 4) continue;
|
|
661
|
-
if (rect.bottom < 0 || rect.top > vh) continue;
|
|
662
|
-
if (rect.right < 0 || rect.left > vw) continue;
|
|
663
|
-
const style = win.getComputedStyle(el);
|
|
664
|
-
if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
|
|
634
|
+
const max = opts?.maxElements ?? 200;
|
|
635
|
+
const mainResult = await this.page.evaluate((limit) => {
|
|
636
|
+
return globalThis.__buaExtract(limit, 0, 0, "main");
|
|
637
|
+
}, max).catch(async () => {
|
|
638
|
+
await this.installExtractorScript();
|
|
639
|
+
return await this.page.evaluate((limit) => {
|
|
640
|
+
return globalThis.__buaExtract(limit, 0, 0, "main");
|
|
641
|
+
}, max);
|
|
642
|
+
});
|
|
643
|
+
let collected = mainResult.elements;
|
|
644
|
+
const scroll = mainResult.scroll;
|
|
645
|
+
try {
|
|
646
|
+
const frames = this.page.frames();
|
|
647
|
+
for (const frame of frames) {
|
|
648
|
+
if (frame === this.page.mainFrame()) continue;
|
|
649
|
+
if (collected.length >= max) break;
|
|
650
|
+
let bbox = null;
|
|
651
|
+
try {
|
|
652
|
+
const owner = await frame.frameElement();
|
|
653
|
+
if (owner) {
|
|
654
|
+
const rect = await owner.boundingBox();
|
|
655
|
+
if (rect) bbox = { x: rect.x, y: rect.y };
|
|
656
|
+
}
|
|
657
|
+
} catch {
|
|
665
658
|
continue;
|
|
666
659
|
}
|
|
667
|
-
|
|
668
|
-
|
|
660
|
+
if (!bbox) continue;
|
|
661
|
+
let iframeRes = null;
|
|
669
662
|
try {
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
663
|
+
iframeRes = await frame.evaluate(
|
|
664
|
+
(args) => {
|
|
665
|
+
return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
|
|
666
|
+
},
|
|
667
|
+
{ limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
|
|
668
|
+
);
|
|
675
669
|
} catch {
|
|
670
|
+
try {
|
|
671
|
+
await frame.evaluate(this.extractorScriptSource());
|
|
672
|
+
iframeRes = await frame.evaluate(
|
|
673
|
+
(args) => {
|
|
674
|
+
return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
|
|
675
|
+
},
|
|
676
|
+
{ limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
|
|
677
|
+
);
|
|
678
|
+
} catch {
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
if (iframeRes?.elements?.length) {
|
|
682
|
+
const offset = collected.length;
|
|
683
|
+
for (let i = 0; i < iframeRes.elements.length; i++) {
|
|
684
|
+
const e = iframeRes.elements[i];
|
|
685
|
+
e.index = offset + i + 1;
|
|
686
|
+
collected.push(e);
|
|
687
|
+
if (collected.length >= max) break;
|
|
688
|
+
}
|
|
676
689
|
}
|
|
677
|
-
seen.add(el);
|
|
678
|
-
entries.push({ el, cx, cy });
|
|
679
|
-
}
|
|
680
|
-
entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
|
|
681
|
-
const out = [];
|
|
682
|
-
let idx = 1;
|
|
683
|
-
for (const { el, cx, cy } of entries) {
|
|
684
|
-
if (out.length >= limit) break;
|
|
685
|
-
const tag = el.tagName.toLowerCase();
|
|
686
|
-
const role = el.getAttribute("role") || tag;
|
|
687
|
-
const type = el.getAttribute("type") || "";
|
|
688
|
-
const innerText = el.innerText || el.textContent || "";
|
|
689
|
-
const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
|
|
690
|
-
const placeholder = el.getAttribute("placeholder") || "";
|
|
691
|
-
const ariaLabel = el.getAttribute("aria-label") || "";
|
|
692
|
-
const title = el.getAttribute("title") || "";
|
|
693
|
-
const name = el.getAttribute("name") || "";
|
|
694
|
-
const href = el.getAttribute("href") || "";
|
|
695
|
-
const value = el.value || "";
|
|
696
|
-
let label = ariaLabel || text || placeholder || title || value || name;
|
|
697
|
-
if (!label && href) label = href.slice(0, 60);
|
|
698
|
-
if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
|
|
699
|
-
el.setAttribute("data-bua-idx", String(idx));
|
|
700
|
-
out.push({
|
|
701
|
-
index: idx,
|
|
702
|
-
cx,
|
|
703
|
-
cy,
|
|
704
|
-
role,
|
|
705
|
-
type: type || void 0,
|
|
706
|
-
label,
|
|
707
|
-
tag,
|
|
708
|
-
isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
|
|
709
|
-
isSelect: tag === "select",
|
|
710
|
-
isFile: tag === "input" && type === "file"
|
|
711
|
-
});
|
|
712
|
-
idx++;
|
|
713
690
|
}
|
|
714
|
-
|
|
715
|
-
}
|
|
716
|
-
this._lastDom =
|
|
717
|
-
const lines =
|
|
691
|
+
} catch {
|
|
692
|
+
}
|
|
693
|
+
this._lastDom = collected;
|
|
694
|
+
const lines = collected.map((e) => {
|
|
718
695
|
const typeSuffix = e.type ? `(${e.type})` : "";
|
|
719
|
-
|
|
696
|
+
const frameSuffix = e.frame && e.frame !== "main" ? ` [frame]` : "";
|
|
697
|
+
return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}${frameSuffix}: "${e.label}"`;
|
|
720
698
|
});
|
|
721
|
-
return { text: lines.join("\n"), elements:
|
|
699
|
+
return { text: lines.join("\n"), elements: collected, scroll };
|
|
700
|
+
}
|
|
701
|
+
/**
|
|
702
|
+
* Install the `__buaExtract` global on the main page. Idempotent —
|
|
703
|
+
* subsequent calls are no-ops.
|
|
704
|
+
*/
|
|
705
|
+
async installExtractorScript() {
|
|
706
|
+
if (!this.page) return;
|
|
707
|
+
await this.page.evaluate(this.extractorScriptSource());
|
|
708
|
+
}
|
|
709
|
+
/**
|
|
710
|
+
* The extractor source. Lives in its own method so we can also inject
|
|
711
|
+
* it into iframes that haven't yet had it loaded.
|
|
712
|
+
*
|
|
713
|
+
* This function intentionally runs entirely in the page context. It:
|
|
714
|
+
* - traverses the regular DOM + open shadow roots (deep)
|
|
715
|
+
* - applies a parent-chain visibility filter
|
|
716
|
+
* - applies a `cursor:pointer` second pass for custom widgets
|
|
717
|
+
* - hit-tests each candidate at its center to avoid overlay collisions
|
|
718
|
+
* - returns scroll context (pages above/below, hidden counts)
|
|
719
|
+
* - tags survivors with `data-bua-idx` for indexed actions
|
|
720
|
+
*/
|
|
721
|
+
extractorScriptSource() {
|
|
722
|
+
return (
|
|
723
|
+
/* js */
|
|
724
|
+
`
|
|
725
|
+
(function () {
|
|
726
|
+
if (typeof window.__buaExtract === "function") return;
|
|
727
|
+
|
|
728
|
+
function isAncestorVisible(el) {
|
|
729
|
+
let cur = el;
|
|
730
|
+
while (cur && cur !== document.body && cur !== document.documentElement) {
|
|
731
|
+
const s = window.getComputedStyle(cur);
|
|
732
|
+
if (!s) return false;
|
|
733
|
+
if (s.display === "none" || s.visibility === "hidden" || s.visibility === "collapse") return false;
|
|
734
|
+
if (parseFloat(s.opacity || "1") < 0.05) return false;
|
|
735
|
+
cur = cur.parentElement || (cur.getRootNode && cur.getRootNode().host) || null;
|
|
736
|
+
}
|
|
737
|
+
return true;
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
function collectSemantic(root, out) {
|
|
741
|
+
if (!root || !root.querySelectorAll) return;
|
|
742
|
+
var sel = [
|
|
743
|
+
"a[href]", "button", "input", "textarea", "select",
|
|
744
|
+
"[role='button']", "[role='link']", "[role='tab']",
|
|
745
|
+
"[role='menuitem']", "[role='checkbox']", "[role='radio']",
|
|
746
|
+
"[role='switch']", "[role='combobox']", "[role='option']",
|
|
747
|
+
"[onclick]", "[contenteditable='true']", "[tabindex]:not([tabindex='-1'])"
|
|
748
|
+
].join(",");
|
|
749
|
+
var found = root.querySelectorAll(sel);
|
|
750
|
+
for (var i = 0; i < found.length; i++) out.push(found[i]);
|
|
751
|
+
// shadow DOM piercing
|
|
752
|
+
var all = root.querySelectorAll("*");
|
|
753
|
+
for (var j = 0; j < all.length; j++) {
|
|
754
|
+
var n = all[j];
|
|
755
|
+
if (n.shadowRoot) collectSemantic(n.shadowRoot, out);
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
function collectPointer(root, out, cap) {
|
|
760
|
+
if (!root || !root.querySelectorAll) return;
|
|
761
|
+
var all = root.querySelectorAll("*");
|
|
762
|
+
var vh = window.innerHeight, vw = window.innerWidth;
|
|
763
|
+
for (var i = 0; i < all.length && out.length < cap; i++) {
|
|
764
|
+
var el = all[i];
|
|
765
|
+
var tag = (el.tagName || "").toLowerCase();
|
|
766
|
+
if (!tag || tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
|
|
767
|
+
var rect = el.getBoundingClientRect();
|
|
768
|
+
if (rect.width < 4 || rect.height < 4) continue;
|
|
769
|
+
if (rect.bottom < 0 || rect.top > vh) continue;
|
|
770
|
+
if (rect.right < 0 || rect.left > vw) continue;
|
|
771
|
+
var s = window.getComputedStyle(el);
|
|
772
|
+
if (s.cursor !== "pointer") continue;
|
|
773
|
+
out.push(el);
|
|
774
|
+
if (el.shadowRoot) collectPointer(el.shadowRoot, out, cap);
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
function countHiddenInteractive(root, vh, vw) {
|
|
779
|
+
if (!root || !root.querySelectorAll) return 0;
|
|
780
|
+
var sel = ["a[href]","button","input","textarea","select","[role='button']","[role='link']","[role='tab']","[role='menuitem']"].join(",");
|
|
781
|
+
var n = 0;
|
|
782
|
+
var found = root.querySelectorAll(sel);
|
|
783
|
+
for (var i = 0; i < found.length; i++) {
|
|
784
|
+
var r = found[i].getBoundingClientRect();
|
|
785
|
+
if (r.bottom < 0 || r.top > vh || r.right < 0 || r.left > vw) n++;
|
|
786
|
+
}
|
|
787
|
+
return n;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
window.__buaExtract = function (limit, offsetX, offsetY, frameName) {
|
|
791
|
+
// Clear stale markers each call.
|
|
792
|
+
var stale = document.querySelectorAll("[data-bua-idx]");
|
|
793
|
+
for (var s = 0; s < stale.length; s++) stale[s].removeAttribute("data-bua-idx");
|
|
794
|
+
|
|
795
|
+
var vh = window.innerHeight, vw = window.innerWidth;
|
|
796
|
+
var semantic = []; collectSemantic(document, semantic);
|
|
797
|
+
var pointer = []; collectPointer(document, pointer, 2000);
|
|
798
|
+
var all = semantic.concat(pointer);
|
|
799
|
+
|
|
800
|
+
var entries = [];
|
|
801
|
+
var seen = new Set();
|
|
802
|
+
|
|
803
|
+
for (var i = 0; i < all.length; i++) {
|
|
804
|
+
var el = all[i];
|
|
805
|
+
if (seen.has(el)) continue;
|
|
806
|
+
var rect = el.getBoundingClientRect();
|
|
807
|
+
if (rect.width < 4 || rect.height < 4) continue;
|
|
808
|
+
if (rect.bottom < 0 || rect.top > vh) continue;
|
|
809
|
+
if (rect.right < 0 || rect.left > vw) continue;
|
|
810
|
+
|
|
811
|
+
var style = window.getComputedStyle(el);
|
|
812
|
+
if (style.visibility === "hidden" || style.display === "none" ||
|
|
813
|
+
style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) continue;
|
|
814
|
+
if (!isAncestorVisible(el)) continue;
|
|
815
|
+
|
|
816
|
+
var cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
|
|
817
|
+
var cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
|
|
818
|
+
|
|
819
|
+
try {
|
|
820
|
+
var hit = document.elementFromPoint(cx, cy);
|
|
821
|
+
if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) continue;
|
|
822
|
+
if (hit && el.contains(hit)) seen.add(hit);
|
|
823
|
+
} catch (_) {}
|
|
824
|
+
seen.add(el);
|
|
825
|
+
entries.push({ el: el, cx: cx, cy: cy });
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
entries.sort(function (a, b) { return a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy; });
|
|
829
|
+
|
|
830
|
+
var out = [];
|
|
831
|
+
var idx = 1;
|
|
832
|
+
for (var k = 0; k < entries.length && out.length < limit; k++) {
|
|
833
|
+
var ent = entries[k];
|
|
834
|
+
var el = ent.el;
|
|
835
|
+
var tag = (el.tagName || "").toLowerCase();
|
|
836
|
+
var role = el.getAttribute("role") || tag;
|
|
837
|
+
var type = el.getAttribute("type") || "";
|
|
838
|
+
var innerText = (el.innerText || el.textContent || "");
|
|
839
|
+
var text = innerText.trim().replace(/\\s+/g, " ").slice(0, 80);
|
|
840
|
+
var placeholder = el.getAttribute("placeholder") || "";
|
|
841
|
+
var ariaLabel = el.getAttribute("aria-label") || "";
|
|
842
|
+
var title = el.getAttribute("title") || "";
|
|
843
|
+
var name = el.getAttribute("name") || "";
|
|
844
|
+
var href = el.getAttribute("href") || "";
|
|
845
|
+
var value = el.value || "";
|
|
846
|
+
var label = ariaLabel || text || placeholder || title || value || name;
|
|
847
|
+
if (!label && href) label = href.slice(0, 60);
|
|
848
|
+
if (!label) label = "(" + tag + (type ? (" type=" + type) : "") + ")";
|
|
849
|
+
|
|
850
|
+
el.setAttribute("data-bua-idx", String(idx));
|
|
851
|
+
out.push({
|
|
852
|
+
index: idx,
|
|
853
|
+
cx: ent.cx + offsetX,
|
|
854
|
+
cy: ent.cy + offsetY,
|
|
855
|
+
role: role,
|
|
856
|
+
type: type || undefined,
|
|
857
|
+
label: label,
|
|
858
|
+
tag: tag,
|
|
859
|
+
isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
|
|
860
|
+
isSelect: tag === "select",
|
|
861
|
+
isFile: tag === "input" && type === "file",
|
|
862
|
+
frame: frameName,
|
|
863
|
+
});
|
|
864
|
+
idx++;
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
// Scroll context (only meaningful for the main frame; iframes pass through).
|
|
868
|
+
var doc = document.scrollingElement || document.documentElement || document.body;
|
|
869
|
+
var pagesAbove = 0, pagesBelow = 0;
|
|
870
|
+
if (doc && vh > 0) {
|
|
871
|
+
pagesAbove = Math.max(0, Math.round(doc.scrollTop / vh));
|
|
872
|
+
var below = Math.max(0, doc.scrollHeight - doc.scrollTop - vh);
|
|
873
|
+
pagesBelow = Math.round(below / vh);
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
// Hidden interactive count over the WHOLE document tree.
|
|
877
|
+
var hidden = countHiddenInteractive(document, vh, vw);
|
|
878
|
+
|
|
879
|
+
return {
|
|
880
|
+
elements: out,
|
|
881
|
+
scroll: {
|
|
882
|
+
pagesAbove: pagesAbove,
|
|
883
|
+
pagesBelow: pagesBelow,
|
|
884
|
+
totalInteractive: out.length + hidden,
|
|
885
|
+
hiddenInteractive: hidden,
|
|
886
|
+
},
|
|
887
|
+
};
|
|
888
|
+
};
|
|
889
|
+
})();
|
|
890
|
+
`
|
|
891
|
+
);
|
|
722
892
|
}
|
|
723
893
|
// ── Page Info ────────────────────────────────────────────────────────
|
|
724
894
|
async getPageInfo() {
|
|
@@ -880,16 +1050,169 @@ var BrowserProvider = class {
|
|
|
880
1050
|
}
|
|
881
1051
|
};
|
|
882
1052
|
|
|
1053
|
+
// src/loop-detector.ts
|
|
1054
|
+
var LoopDetector = class {
|
|
1055
|
+
actionWindow = [];
|
|
1056
|
+
pageHistory = [];
|
|
1057
|
+
actionWindowSize;
|
|
1058
|
+
pageHistorySize;
|
|
1059
|
+
/** Thresholds: repeat counts at which each severity fires. */
|
|
1060
|
+
actionThresholds;
|
|
1061
|
+
/** Thresholds: consecutive unchanged-page counts at which each fires. */
|
|
1062
|
+
pageThresholds;
|
|
1063
|
+
constructor(opts) {
|
|
1064
|
+
this.actionWindowSize = opts?.actionWindowSize ?? 20;
|
|
1065
|
+
this.pageHistorySize = opts?.pageHistorySize ?? 10;
|
|
1066
|
+
this.actionThresholds = {
|
|
1067
|
+
warn: opts?.actionThresholds?.warn ?? 5,
|
|
1068
|
+
escalate: opts?.actionThresholds?.escalate ?? 8,
|
|
1069
|
+
abort: opts?.actionThresholds?.abort ?? 12
|
|
1070
|
+
};
|
|
1071
|
+
this.pageThresholds = {
|
|
1072
|
+
warn: opts?.pageThresholds?.warn ?? 5,
|
|
1073
|
+
escalate: opts?.pageThresholds?.escalate ?? 8,
|
|
1074
|
+
abort: opts?.pageThresholds?.abort ?? 12
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
/**
|
|
1078
|
+
* Record an action and return advice for the runtime. `severity` of
|
|
1079
|
+
* `abort` means the loop should terminate now.
|
|
1080
|
+
*/
|
|
1081
|
+
recordAction(action) {
|
|
1082
|
+
const key = normalizeAction(action);
|
|
1083
|
+
this.actionWindow.push(key);
|
|
1084
|
+
if (this.actionWindow.length > this.actionWindowSize) this.actionWindow.shift();
|
|
1085
|
+
let count = 0;
|
|
1086
|
+
for (const k of this.actionWindow) if (k === key) count++;
|
|
1087
|
+
if (count >= this.actionThresholds.abort) {
|
|
1088
|
+
return {
|
|
1089
|
+
severity: "abort",
|
|
1090
|
+
message: `Auto-stopped: the action "${key}" has been repeated ${count} times in the last ${this.actionWindow.length} steps with no apparent progress.`
|
|
1091
|
+
};
|
|
1092
|
+
}
|
|
1093
|
+
if (count >= this.actionThresholds.escalate) {
|
|
1094
|
+
return {
|
|
1095
|
+
severity: "escalate",
|
|
1096
|
+
message: `You have repeated essentially the same action ${count} times. The current approach is NOT working \u2014 pick a different strategy: scroll, navigate elsewhere, dismiss a popup, or use "fail" if the task truly cannot be done.`
|
|
1097
|
+
};
|
|
1098
|
+
}
|
|
1099
|
+
if (count >= this.actionThresholds.warn) {
|
|
1100
|
+
return {
|
|
1101
|
+
severity: "warn",
|
|
1102
|
+
message: `You've repeated this action ${count} times \u2014 if the page hasn't changed, try a different approach.`
|
|
1103
|
+
};
|
|
1104
|
+
}
|
|
1105
|
+
return { severity: "none" };
|
|
1106
|
+
}
|
|
1107
|
+
/**
|
|
1108
|
+
* Record a page fingerprint and return advice if the page has been
|
|
1109
|
+
* stagnant. Counts consecutive identical fingerprints at the END of
|
|
1110
|
+
* the history (so a single navigation resets the counter).
|
|
1111
|
+
*/
|
|
1112
|
+
recordPage(fp) {
|
|
1113
|
+
this.pageHistory.push(fp);
|
|
1114
|
+
if (this.pageHistory.length > this.pageHistorySize) this.pageHistory.shift();
|
|
1115
|
+
let stagnantCount = 1;
|
|
1116
|
+
for (let i = this.pageHistory.length - 2; i >= 0; i--) {
|
|
1117
|
+
const prev = this.pageHistory[i];
|
|
1118
|
+
if (prev.url === fp.url && prev.interactiveCount === fp.interactiveCount && prev.textHash === fp.textHash) {
|
|
1119
|
+
stagnantCount++;
|
|
1120
|
+
} else {
|
|
1121
|
+
break;
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
if (stagnantCount >= this.pageThresholds.abort) {
|
|
1125
|
+
return {
|
|
1126
|
+
severity: "abort",
|
|
1127
|
+
message: `Auto-stopped: the page hasn't changed in ${stagnantCount} consecutive steps. Likely a blocking overlay or dead control.`
|
|
1128
|
+
};
|
|
1129
|
+
}
|
|
1130
|
+
if (stagnantCount >= this.pageThresholds.escalate) {
|
|
1131
|
+
return {
|
|
1132
|
+
severity: "escalate",
|
|
1133
|
+
message: `The page has not changed in ${stagnantCount} steps. Something is blocking progress \u2014 try dismissing popups, navigating elsewhere, or scrolling.`
|
|
1134
|
+
};
|
|
1135
|
+
}
|
|
1136
|
+
if (stagnantCount >= this.pageThresholds.warn) {
|
|
1137
|
+
return {
|
|
1138
|
+
severity: "warn",
|
|
1139
|
+
message: `Page hasn't changed in ${stagnantCount} steps \u2014 verify your action is taking effect.`
|
|
1140
|
+
};
|
|
1141
|
+
}
|
|
1142
|
+
return { severity: "none" };
|
|
1143
|
+
}
|
|
1144
|
+
/** Combine two advices, returning the more severe one. */
|
|
1145
|
+
static combine(a, b) {
|
|
1146
|
+
const rank = { none: 0, warn: 1, escalate: 2, abort: 3 };
|
|
1147
|
+
return rank[a.severity] >= rank[b.severity] ? a : b;
|
|
1148
|
+
}
|
|
1149
|
+
};
|
|
1150
|
+
function normalizeAction(action) {
|
|
1151
|
+
switch (action.action) {
|
|
1152
|
+
case "click":
|
|
1153
|
+
if (typeof action.index === "number") return `click:${action.index}`;
|
|
1154
|
+
if (typeof action.x === "number" && typeof action.y === "number") {
|
|
1155
|
+
return `click:xy:${Math.round(action.x / 16)},${Math.round(action.y / 16)}`;
|
|
1156
|
+
}
|
|
1157
|
+
return `click:?`;
|
|
1158
|
+
case "type":
|
|
1159
|
+
if (typeof action.index === "number") return `type:${action.index}:${truncate(action.text, 40)}`;
|
|
1160
|
+
return `type:?:${truncate(action.text, 40)}`;
|
|
1161
|
+
case "scroll":
|
|
1162
|
+
return `scroll:${action.direction}:${action.amount ?? "default"}${action.index != null ? `:${action.index}` : ""}`;
|
|
1163
|
+
case "navigate":
|
|
1164
|
+
return `navigate:${action.url}`;
|
|
1165
|
+
case "back":
|
|
1166
|
+
return "back";
|
|
1167
|
+
case "wait":
|
|
1168
|
+
return `wait:${Math.round(action.ms / 1e3)}`;
|
|
1169
|
+
case "screenshot":
|
|
1170
|
+
return "screenshot";
|
|
1171
|
+
case "send_keys":
|
|
1172
|
+
return `send_keys:${action.keys}`;
|
|
1173
|
+
case "find_text":
|
|
1174
|
+
return `find_text:${truncate(action.text, 40)}`;
|
|
1175
|
+
case "evaluate":
|
|
1176
|
+
return `evaluate:${truncate(action.code, 80)}`;
|
|
1177
|
+
case "dropdown_options":
|
|
1178
|
+
return `dropdown_options:${action.index}`;
|
|
1179
|
+
case "select_dropdown":
|
|
1180
|
+
return `select_dropdown:${action.index}:${truncate(action.text, 40)}`;
|
|
1181
|
+
case "upload_file":
|
|
1182
|
+
return `upload_file:${action.index}:${action.path}`;
|
|
1183
|
+
case "extract":
|
|
1184
|
+
return `extract:${truncate(action.query, 80)}`;
|
|
1185
|
+
case "tool":
|
|
1186
|
+
return `tool:${action.name}`;
|
|
1187
|
+
case "done":
|
|
1188
|
+
case "fail":
|
|
1189
|
+
return action.action;
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
function fnvHash(s) {
|
|
1193
|
+
let h = 2166136261;
|
|
1194
|
+
for (let i = 0; i < s.length; i++) {
|
|
1195
|
+
h ^= s.charCodeAt(i);
|
|
1196
|
+
h = h * 16777619 >>> 0;
|
|
1197
|
+
}
|
|
1198
|
+
return h;
|
|
1199
|
+
}
|
|
1200
|
+
function truncate(s, n) {
|
|
1201
|
+
if (!s) return "";
|
|
1202
|
+
return s.length <= n ? s : `${s.slice(0, n)}\u2026`;
|
|
1203
|
+
}
|
|
1204
|
+
|
|
883
1205
|
// src/prompts.ts
|
|
884
1206
|
function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options) {
|
|
885
1207
|
const maxActions = options?.maxActionsPerStep ?? 3;
|
|
886
1208
|
const allowEvaluate = !!options?.allowEvaluate;
|
|
887
1209
|
const tools = options?.tools ?? [];
|
|
888
1210
|
const useVision = options?.useVision ?? "auto";
|
|
1211
|
+
const useThinking = options?.useThinking ?? true;
|
|
889
1212
|
if (options?.overrideSystemMessage) {
|
|
890
1213
|
const lines2 = [options.overrideSystemMessage];
|
|
891
1214
|
appendCredentials(lines2, credentialKeys);
|
|
892
|
-
lines2.push("", "## Response Format", responseFormatLines(maxActions).join("\n"));
|
|
1215
|
+
lines2.push("", "## Response Format", responseFormatLines(maxActions, useThinking).join("\n"));
|
|
893
1216
|
return lines2.join("\n");
|
|
894
1217
|
}
|
|
895
1218
|
const lines = [];
|
|
@@ -994,7 +1317,7 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options)
|
|
|
994
1317
|
if (extraInstructions) {
|
|
995
1318
|
lines.push(``, `## Additional Instructions`, extraInstructions);
|
|
996
1319
|
}
|
|
997
|
-
lines.push(``, `## Response Format`, responseFormatLines(maxActions).join("\n"));
|
|
1320
|
+
lines.push(``, `## Response Format`, responseFormatLines(maxActions, useThinking).join("\n"));
|
|
998
1321
|
return lines.join("\n");
|
|
999
1322
|
}
|
|
1000
1323
|
function appendCredentials(lines, credentialKeys) {
|
|
@@ -1012,22 +1335,61 @@ function appendCredentials(lines, credentialKeys) {
|
|
|
1012
1335
|
`NEVER include real credential values in "done" or "fail" results.`
|
|
1013
1336
|
);
|
|
1014
1337
|
}
|
|
1015
|
-
function responseFormatLines(maxActions) {
|
|
1338
|
+
function responseFormatLines(maxActions, useThinking) {
|
|
1339
|
+
if (!useThinking) {
|
|
1340
|
+
return [
|
|
1341
|
+
`Respond with ONLY valid JSON. No markdown, no commentary.`,
|
|
1342
|
+
`Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
|
|
1343
|
+
];
|
|
1344
|
+
}
|
|
1016
1345
|
return [
|
|
1017
|
-
`Respond with ONLY valid JSON. No markdown, no commentary.`,
|
|
1018
|
-
|
|
1346
|
+
`Respond with ONLY a single valid JSON object. No markdown, no commentary.`,
|
|
1347
|
+
``,
|
|
1348
|
+
`Use this exact shape:`,
|
|
1349
|
+
"```json",
|
|
1350
|
+
`{`,
|
|
1351
|
+
` "thinking": "Short chain-of-thought: what do I see, what's my plan?",`,
|
|
1352
|
+
` "evaluation_previous_goal": "Did the previous action succeed? success/partial/failure + 1 line",`,
|
|
1353
|
+
` "memory": "Compact bullet list of facts to remember across steps (URLs, found data, \u2026)",`,
|
|
1354
|
+
` "next_goal": "What I want to accomplish in THIS step",`,
|
|
1355
|
+
` "action": <single action object OR array of up to ${maxActions} action objects>`,
|
|
1356
|
+
`}`,
|
|
1357
|
+
"```",
|
|
1358
|
+
``,
|
|
1359
|
+
`On the very first step, set "evaluation_previous_goal" to "n/a \u2014 first step".`,
|
|
1360
|
+
`Keep each text field to one short paragraph or a few lines. Be concrete.`,
|
|
1361
|
+
`Only the "action" field is executed; the others help you self-correct over multiple steps.`
|
|
1019
1362
|
];
|
|
1020
1363
|
}
|
|
1021
|
-
function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract) {
|
|
1364
|
+
function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget) {
|
|
1022
1365
|
const lines = [];
|
|
1023
1366
|
lines.push(`**Task:** ${task}`);
|
|
1024
1367
|
lines.push(`**Current URL:** ${pageUrl}`);
|
|
1025
1368
|
if (pageTitle) lines.push(`**Page Title:** ${pageTitle}`);
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
lines.push(`**
|
|
1030
|
-
|
|
1369
|
+
if (stepBudget) {
|
|
1370
|
+
lines.push(`**Step:** ${stepIndex + 1} of ${stepBudget.max} (${stepBudget.max - stepIndex - 1} remaining)`);
|
|
1371
|
+
} else {
|
|
1372
|
+
lines.push(`**Step:** ${stepIndex + 1}`);
|
|
1373
|
+
}
|
|
1374
|
+
if (scroll) {
|
|
1375
|
+
const parts = [];
|
|
1376
|
+
parts.push(`${scroll.totalInteractive} interactive elements (${scroll.hiddenInteractive} hidden)`);
|
|
1377
|
+
if (scroll.pagesAbove > 0) parts.push(`${scroll.pagesAbove} page${scroll.pagesAbove === 1 ? "" : "s"} above`);
|
|
1378
|
+
if (scroll.pagesBelow > 0) parts.push(`${scroll.pagesBelow} page${scroll.pagesBelow === 1 ? "" : "s"} below`);
|
|
1379
|
+
if (scroll.pagesAbove === 0 && scroll.pagesBelow === 0) parts.push("fits in viewport");
|
|
1380
|
+
lines.push(`**Page stats:** ${parts.join(" \xB7 ")}`);
|
|
1381
|
+
}
|
|
1382
|
+
if (domSnapshot !== void 0) {
|
|
1383
|
+
if (domSnapshot.trim().length === 0) {
|
|
1384
|
+
lines.push(``);
|
|
1385
|
+
lines.push(
|
|
1386
|
+
`**\u26A0 Empty page / no interactive elements detected.** The page may be blank, blocked by a captcha/anti-bot wall, mid-load, or rendered with shadow DOM in an unsupported way. Consider: \`wait\` + retry, \`navigate\` to a different URL, or \`fail\` if the site is blocking access.`
|
|
1387
|
+
);
|
|
1388
|
+
} else {
|
|
1389
|
+
lines.push(``);
|
|
1390
|
+
lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
|
|
1391
|
+
lines.push(domSnapshot);
|
|
1392
|
+
}
|
|
1031
1393
|
}
|
|
1032
1394
|
if (lastExtract) {
|
|
1033
1395
|
lines.push(``);
|
|
@@ -1041,6 +1403,10 @@ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, do
|
|
|
1041
1403
|
lines.push(`- ${entry}`);
|
|
1042
1404
|
}
|
|
1043
1405
|
}
|
|
1406
|
+
if (nudge) {
|
|
1407
|
+
lines.push(``);
|
|
1408
|
+
lines.push(`**\u26A0 Runtime hint:** ${nudge}`);
|
|
1409
|
+
}
|
|
1044
1410
|
lines.push(``);
|
|
1045
1411
|
lines.push(`Decide the next action(s) to complete the task.`);
|
|
1046
1412
|
return lines.join("\n");
|
|
@@ -1095,6 +1461,9 @@ var BrowserAgent = class {
|
|
|
1095
1461
|
eventBus;
|
|
1096
1462
|
model;
|
|
1097
1463
|
pageExtractionLLM;
|
|
1464
|
+
fallbackModel;
|
|
1465
|
+
useThinking;
|
|
1466
|
+
historyWindow;
|
|
1098
1467
|
instructions;
|
|
1099
1468
|
extendSystemMessage;
|
|
1100
1469
|
overrideSystemMessage;
|
|
@@ -1131,6 +1500,9 @@ var BrowserAgent = class {
|
|
|
1131
1500
|
this.name = config.name;
|
|
1132
1501
|
this.model = config.model;
|
|
1133
1502
|
this.pageExtractionLLM = config.pageExtractionLLM ?? null;
|
|
1503
|
+
this.fallbackModel = config.fallbackModel ?? null;
|
|
1504
|
+
this.useThinking = config.useThinking ?? true;
|
|
1505
|
+
this.historyWindow = Math.max(0, config.historyWindow ?? 6);
|
|
1134
1506
|
this.instructions = config.instructions;
|
|
1135
1507
|
this.extendSystemMessage = config.extendSystemMessage;
|
|
1136
1508
|
this.overrideSystemMessage = config.overrideSystemMessage;
|
|
@@ -1176,10 +1548,10 @@ var BrowserAgent = class {
|
|
|
1176
1548
|
const actionHistory = [];
|
|
1177
1549
|
const extractedContent = [];
|
|
1178
1550
|
let lastExtractResult;
|
|
1179
|
-
let
|
|
1180
|
-
let lastActionKey = "";
|
|
1181
|
-
let repeatCount = 0;
|
|
1551
|
+
let consecutiveFailures = 0;
|
|
1182
1552
|
let lastActionWasScreenshot = false;
|
|
1553
|
+
const loop = new LoopDetector();
|
|
1554
|
+
const historyTurns = [];
|
|
1183
1555
|
const baseExtra = [this.instructions, this.extendSystemMessage].filter(Boolean).join("\n\n");
|
|
1184
1556
|
let extraInstructions = baseExtra;
|
|
1185
1557
|
if (this.memoryManager) {
|
|
@@ -1198,7 +1570,8 @@ ${memoryContext}` : memoryContext;
|
|
|
1198
1570
|
allowEvaluate: this.allowEvaluate,
|
|
1199
1571
|
tools: this.tools,
|
|
1200
1572
|
useVision: this.useVision,
|
|
1201
|
-
useDOM: this.useDOM
|
|
1573
|
+
useDOM: this.useDOM,
|
|
1574
|
+
useThinking: this.useThinking
|
|
1202
1575
|
});
|
|
1203
1576
|
try {
|
|
1204
1577
|
this.logger.info("Launching browser", {
|
|
@@ -1206,6 +1579,7 @@ ${memoryContext}` : memoryContext;
|
|
|
1206
1579
|
viewport: this.viewport,
|
|
1207
1580
|
useDOM: this.useDOM,
|
|
1208
1581
|
useVision: this.useVision,
|
|
1582
|
+
useThinking: this.useThinking,
|
|
1209
1583
|
cdpUrl: this.cdpUrl ?? void 0,
|
|
1210
1584
|
recordVideo: !!this.recordVideo,
|
|
1211
1585
|
stealth: !!this.stealth,
|
|
@@ -1225,6 +1599,7 @@ ${memoryContext}` : memoryContext;
|
|
|
1225
1599
|
this.logger.info("Navigating to start URL", { url: startUrl });
|
|
1226
1600
|
this.assertDomainAllowed(startUrl);
|
|
1227
1601
|
await browser.navigate(startUrl);
|
|
1602
|
+
await this.navigationHealthCheck(browser, startUrl);
|
|
1228
1603
|
}
|
|
1229
1604
|
for (const ia of this.initialActions) {
|
|
1230
1605
|
try {
|
|
@@ -1237,13 +1612,35 @@ ${memoryContext}` : memoryContext;
|
|
|
1237
1612
|
for (let step = 0; step < maxSteps; step++) {
|
|
1238
1613
|
const pageInfo = await browser.getPageInfo();
|
|
1239
1614
|
let domSnapshot;
|
|
1615
|
+
let scrollCtx;
|
|
1240
1616
|
if (this.useDOM) {
|
|
1241
1617
|
const dom = await browser.extractDOM();
|
|
1242
1618
|
domSnapshot = dom.text;
|
|
1619
|
+
scrollCtx = dom.scroll;
|
|
1620
|
+
}
|
|
1621
|
+
const pageAdvice = loop.recordPage({
|
|
1622
|
+
url: pageInfo.url,
|
|
1623
|
+
interactiveCount: scrollCtx?.totalInteractive ?? 0,
|
|
1624
|
+
textHash: fnvHash(domSnapshot ?? "")
|
|
1625
|
+
});
|
|
1626
|
+
if (pageAdvice.severity === "abort") {
|
|
1627
|
+
return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
1628
|
+
result: pageAdvice.message ?? "Auto-stopped: page is stagnant.",
|
|
1629
|
+
success: false
|
|
1630
|
+
});
|
|
1243
1631
|
}
|
|
1244
1632
|
const wantVision = this.shouldCaptureVision(step, lastActionWasScreenshot);
|
|
1245
1633
|
const screenshot = wantVision ? await browser.screenshot() : Buffer.alloc(0);
|
|
1246
1634
|
if (wantVision) this.eventBus.emit("browser.screenshot", { data: screenshot });
|
|
1635
|
+
const isLastStep = step === maxSteps - 1;
|
|
1636
|
+
const nudgeParts = [];
|
|
1637
|
+
if (pageAdvice.severity !== "none" && pageAdvice.message) nudgeParts.push(pageAdvice.message);
|
|
1638
|
+
if (isLastStep) {
|
|
1639
|
+
nudgeParts.push(
|
|
1640
|
+
"This is your FINAL step. Return a `done` action right now summarizing whatever you have, even if partial."
|
|
1641
|
+
);
|
|
1642
|
+
}
|
|
1643
|
+
const nudge = nudgeParts.length > 0 ? nudgeParts.join(" ") : void 0;
|
|
1247
1644
|
const userText = buildUserMessage(
|
|
1248
1645
|
task,
|
|
1249
1646
|
pageInfo.url,
|
|
@@ -1251,65 +1648,49 @@ ${memoryContext}` : memoryContext;
|
|
|
1251
1648
|
step,
|
|
1252
1649
|
actionHistory,
|
|
1253
1650
|
domSnapshot,
|
|
1254
|
-
lastExtractResult
|
|
1651
|
+
lastExtractResult,
|
|
1652
|
+
scrollCtx,
|
|
1653
|
+
nudge,
|
|
1654
|
+
{ current: step, max: maxSteps }
|
|
1255
1655
|
);
|
|
1256
|
-
const
|
|
1257
|
-
if (wantVision) {
|
|
1258
|
-
content.push({ type: "image", data: screenshot.toString("base64"), mimeType: "image/png" });
|
|
1259
|
-
}
|
|
1260
|
-
content.push({ type: "text", text: userText });
|
|
1261
|
-
const messages = [
|
|
1262
|
-
{ role: "system", content: systemPrompt },
|
|
1263
|
-
{ role: "user", content }
|
|
1264
|
-
];
|
|
1656
|
+
const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
|
|
1265
1657
|
this.logger.debug("Calling model", { step, url: pageInfo.url, vision: wantVision });
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
responseFormat: "json"
|
|
1273
|
-
});
|
|
1274
|
-
} catch (e) {
|
|
1275
|
-
failureCount++;
|
|
1276
|
-
this.logger.warn("Model call failed", { error: e?.message, failureCount });
|
|
1277
|
-
if (failureCount > this.maxFailures) {
|
|
1278
|
-
return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
1279
|
-
result: `Model call failed ${failureCount} times: ${e?.message}`,
|
|
1280
|
-
success: false
|
|
1281
|
-
});
|
|
1658
|
+
const { response, modelUsed } = await this.callModelWithFallback(messages, opts?.apiKey);
|
|
1659
|
+
if (!response) {
|
|
1660
|
+
consecutiveFailures++;
|
|
1661
|
+
actionHistory.push(`(model call failed \u2014 retrying, ${consecutiveFailures}/${this.maxFailures})`);
|
|
1662
|
+
if (consecutiveFailures > this.maxFailures) {
|
|
1663
|
+
return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
|
|
1282
1664
|
}
|
|
1283
|
-
actionHistory.push(`(model call failed: ${e?.message ?? "unknown"} \u2014 retrying)`);
|
|
1284
1665
|
continue;
|
|
1285
1666
|
}
|
|
1286
1667
|
if (this.costTracker && response.usage) {
|
|
1287
1668
|
this.costTracker.track({
|
|
1288
1669
|
runId: sessionId,
|
|
1289
1670
|
agentName: this.name,
|
|
1290
|
-
modelId:
|
|
1671
|
+
modelId: modelUsed.modelId,
|
|
1291
1672
|
usage: response.usage,
|
|
1292
1673
|
sessionId,
|
|
1293
1674
|
userId
|
|
1294
1675
|
});
|
|
1295
1676
|
}
|
|
1296
1677
|
const raw = typeof response.message.content === "string" ? response.message.content : "";
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
if (failureCount > this.maxFailures) {
|
|
1304
|
-
return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
1305
|
-
result: `Model returned invalid JSON ${failureCount} times. Last response: ${raw.slice(0, 200)}`,
|
|
1306
|
-
success: false
|
|
1307
|
-
});
|
|
1678
|
+
const envelope = this.parseEnvelope(raw);
|
|
1679
|
+
if (!envelope) {
|
|
1680
|
+
consecutiveFailures++;
|
|
1681
|
+
this.logger.warn("Failed to parse model response", { raw: raw.slice(0, 200), consecutiveFailures });
|
|
1682
|
+
if (consecutiveFailures > this.maxFailures) {
|
|
1683
|
+
return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "parse");
|
|
1308
1684
|
}
|
|
1309
1685
|
actionHistory.push("(invalid JSON response \u2014 retrying)");
|
|
1310
1686
|
continue;
|
|
1311
1687
|
}
|
|
1312
|
-
|
|
1688
|
+
consecutiveFailures = 0;
|
|
1689
|
+
historyTurns.push({ userText, hasScreenshot: wantVision, envelope });
|
|
1690
|
+
if (this.historyWindow > 0 && historyTurns.length > this.historyWindow) {
|
|
1691
|
+
historyTurns.splice(0, historyTurns.length - this.historyWindow);
|
|
1692
|
+
}
|
|
1693
|
+
const actions = (Array.isArray(envelope.action) ? envelope.action : [envelope.action]).slice(
|
|
1313
1694
|
0,
|
|
1314
1695
|
this.maxActionsPerStep
|
|
1315
1696
|
);
|
|
@@ -1320,24 +1701,12 @@ ${memoryContext}` : memoryContext;
|
|
|
1320
1701
|
const action = actions[ai];
|
|
1321
1702
|
let summary = summarizeAction(action);
|
|
1322
1703
|
if (this.credentials) summary = this.credentials.mask(summary);
|
|
1323
|
-
const
|
|
1324
|
-
if (
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
lastActionKey = actionKey;
|
|
1328
|
-
repeatCount = 1;
|
|
1329
|
-
}
|
|
1330
|
-
if (repeatCount > this.maxRepeats && action.action !== "done" && action.action !== "fail") {
|
|
1331
|
-
this.logger.warn("Stuck in a loop \u2014 same action repeated", {
|
|
1332
|
-
action: action.action,
|
|
1333
|
-
repeats: repeatCount,
|
|
1334
|
-
maxRepeats: this.maxRepeats
|
|
1335
|
-
});
|
|
1336
|
-
actionHistory.push(
|
|
1337
|
-
`\u26A0 LOOP DETECTED: "${summary}" repeated ${repeatCount} times. Auto-stopping.`
|
|
1338
|
-
);
|
|
1704
|
+
const advice = loop.recordAction(action);
|
|
1705
|
+
if (advice.severity === "abort" && action.action !== "done" && action.action !== "fail") {
|
|
1706
|
+
this.logger.warn("Loop detector aborting run", { advice });
|
|
1707
|
+
actionHistory.push(`\u26A0 ${advice.message ?? "loop detected \u2014 auto-stopping"}`);
|
|
1339
1708
|
didTerminate = {
|
|
1340
|
-
result:
|
|
1709
|
+
result: advice.message ?? "Stuck in a loop \u2014 auto-stopped.",
|
|
1341
1710
|
success: false
|
|
1342
1711
|
};
|
|
1343
1712
|
break;
|
|
@@ -1360,21 +1729,20 @@ ${memoryContext}` : memoryContext;
|
|
|
1360
1729
|
try {
|
|
1361
1730
|
const exec = await this.executeAction(browser, action, actionHistory, extractedContent);
|
|
1362
1731
|
stepOutput = exec?.output;
|
|
1363
|
-
if (exec?.didNavigate)
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
}
|
|
1367
|
-
if (action.action === "screenshot") {
|
|
1368
|
-
lastActionWasScreenshot = true;
|
|
1732
|
+
if (exec?.didNavigate) {
|
|
1733
|
+
didNavigate = true;
|
|
1734
|
+
await this.navigationHealthCheck(browser, pageInfo.url);
|
|
1369
1735
|
}
|
|
1736
|
+
if (action.action === "extract" && exec?.output) lastExtractResult = exec.output;
|
|
1737
|
+
if (action.action === "screenshot") lastActionWasScreenshot = true;
|
|
1370
1738
|
} catch (e) {
|
|
1371
1739
|
stepOk = false;
|
|
1372
|
-
|
|
1373
|
-
this.logger.warn("Action failed", { action: action.action, error: e?.message,
|
|
1740
|
+
consecutiveFailures++;
|
|
1741
|
+
this.logger.warn("Action failed", { action: action.action, error: e?.message, consecutiveFailures });
|
|
1374
1742
|
actionHistory.push(`(error: ${e?.message ?? "unknown"})`);
|
|
1375
|
-
if (
|
|
1743
|
+
if (consecutiveFailures > this.maxFailures) {
|
|
1376
1744
|
didTerminate = {
|
|
1377
|
-
result: `Action ${action.action} failed ${
|
|
1745
|
+
result: `Action ${action.action} failed ${consecutiveFailures} times. Last error: ${e?.message}`,
|
|
1378
1746
|
success: false
|
|
1379
1747
|
};
|
|
1380
1748
|
break;
|
|
@@ -1389,7 +1757,11 @@ ${memoryContext}` : memoryContext;
|
|
|
1389
1757
|
timestamp: /* @__PURE__ */ new Date(),
|
|
1390
1758
|
dom: domSnapshot,
|
|
1391
1759
|
output: stepOutput,
|
|
1392
|
-
ok: stepOk
|
|
1760
|
+
ok: stepOk,
|
|
1761
|
+
thinking: envelope.thinking,
|
|
1762
|
+
evaluationPreviousGoal: envelope.evaluationPreviousGoal,
|
|
1763
|
+
memory: envelope.memory,
|
|
1764
|
+
nextGoal: envelope.nextGoal
|
|
1393
1765
|
});
|
|
1394
1766
|
this.eventBus.emit("browser.step", {
|
|
1395
1767
|
index: steps.length - 1,
|
|
@@ -1404,11 +1776,8 @@ ${memoryContext}` : memoryContext;
|
|
|
1404
1776
|
return await this.finalize(browser, steps, startTime, opts, extractedContent, didTerminate);
|
|
1405
1777
|
}
|
|
1406
1778
|
}
|
|
1407
|
-
this.logger.warn("Max steps reached without
|
|
1408
|
-
return await this.
|
|
1409
|
-
result: `Task not completed within ${maxSteps} steps. Last actions: ${actionHistory.slice(-3).join("; ")}`,
|
|
1410
|
-
success: false
|
|
1411
|
-
});
|
|
1779
|
+
this.logger.warn("Max steps reached without explicit done", { maxSteps });
|
|
1780
|
+
return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "max_steps");
|
|
1412
1781
|
} catch (error) {
|
|
1413
1782
|
this.logger.error("Browser agent error", { error: error.message });
|
|
1414
1783
|
this.eventBus.emit("browser.error", { error });
|
|
@@ -1470,6 +1839,163 @@ ${memoryContext}` : memoryContext;
|
|
|
1470
1839
|
throw new Error(`Navigation blocked: ${host} is in prohibitedDomains`);
|
|
1471
1840
|
}
|
|
1472
1841
|
}
|
|
1842
|
+
/**
|
|
1843
|
+
* Build the message array sent to the model: system prompt + a compact
|
|
1844
|
+
* summary of older turns (if any) + the most recent `historyWindow`
|
|
1845
|
+
* turns verbatim + the current step's user message.
|
|
1846
|
+
*
|
|
1847
|
+
* Inspired by browser-use's history compaction. Keeps tokens bounded
|
|
1848
|
+
* while giving the model meaningful context about what it already
|
|
1849
|
+
* tried.
|
|
1850
|
+
*/
|
|
1851
|
+
buildMessages(systemPrompt, historyTurns, currentUserText, currentScreenshot) {
|
|
1852
|
+
const msgs = [{ role: "system", content: systemPrompt }];
|
|
1853
|
+
if (this.historyWindow > 0 && historyTurns.length > 0) {
|
|
1854
|
+
for (const turn of historyTurns) {
|
|
1855
|
+
const env = turn.envelope;
|
|
1856
|
+
const replay = [];
|
|
1857
|
+
if (env.thinking) replay.push(`thinking: ${env.thinking}`);
|
|
1858
|
+
if (env.evaluationPreviousGoal) replay.push(`evaluation: ${env.evaluationPreviousGoal}`);
|
|
1859
|
+
if (env.memory) replay.push(`memory: ${env.memory}`);
|
|
1860
|
+
if (env.nextGoal) replay.push(`next_goal: ${env.nextGoal}`);
|
|
1861
|
+
replay.push(`action: ${JSON.stringify(env.action)}`);
|
|
1862
|
+
msgs.push({ role: "assistant", content: replay.join("\n") });
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
const content = [];
|
|
1866
|
+
if (currentScreenshot && currentScreenshot.length > 0) {
|
|
1867
|
+
content.push({
|
|
1868
|
+
type: "image",
|
|
1869
|
+
data: currentScreenshot.toString("base64"),
|
|
1870
|
+
mimeType: "image/png"
|
|
1871
|
+
});
|
|
1872
|
+
}
|
|
1873
|
+
content.push({ type: "text", text: currentUserText });
|
|
1874
|
+
msgs.push({ role: "user", content });
|
|
1875
|
+
return msgs;
|
|
1876
|
+
}
|
|
1877
|
+
/**
|
|
1878
|
+
* Call the primary model, retrying once with `fallbackModel` on
|
|
1879
|
+
* transient errors (5xx, 429, network). Returns the response or
|
|
1880
|
+
* `null` if both models failed.
|
|
1881
|
+
*/
|
|
1882
|
+
async callModelWithFallback(messages, apiKey) {
|
|
1883
|
+
const reqOpts = { temperature: 0.1, maxTokens: 1024, apiKey, responseFormat: "json" };
|
|
1884
|
+
try {
|
|
1885
|
+
const r = await this.model.generate(messages, reqOpts);
|
|
1886
|
+
return { response: r, modelUsed: this.model };
|
|
1887
|
+
} catch (e) {
|
|
1888
|
+
if (this.fallbackModel && this.isTransientError(e)) {
|
|
1889
|
+
this.logger.warn("Primary model failed; trying fallbackModel", {
|
|
1890
|
+
error: e?.message,
|
|
1891
|
+
primary: this.model.modelId,
|
|
1892
|
+
fallback: this.fallbackModel.modelId
|
|
1893
|
+
});
|
|
1894
|
+
try {
|
|
1895
|
+
const r = await this.fallbackModel.generate(messages, reqOpts);
|
|
1896
|
+
return { response: r, modelUsed: this.fallbackModel };
|
|
1897
|
+
} catch (e2) {
|
|
1898
|
+
this.logger.warn("Fallback model also failed", { error: e2?.message });
|
|
1899
|
+
return { response: null, modelUsed: this.model };
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
this.logger.warn("Model call failed (no fallback configured)", { error: e?.message });
|
|
1903
|
+
return { response: null, modelUsed: this.model };
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1906
|
+
isTransientError(e) {
|
|
1907
|
+
const msg = String(e?.message ?? e).toLowerCase();
|
|
1908
|
+
if (msg.includes("rate limit") || msg.includes("429")) return true;
|
|
1909
|
+
if (msg.includes("timeout") || msg.includes("etimedout")) return true;
|
|
1910
|
+
if (msg.includes("econnreset") || msg.includes("network")) return true;
|
|
1911
|
+
if (/\b5\d\d\b/.test(msg)) return true;
|
|
1912
|
+
if (msg.includes("401") || msg.includes("402") || msg.includes("auth")) return true;
|
|
1913
|
+
return false;
|
|
1914
|
+
}
|
|
1915
|
+
/**
|
|
1916
|
+
* Parse the model's raw response into an `AgentOutput`. Tolerant to
|
|
1917
|
+
* three shapes:
|
|
1918
|
+
* - Full envelope: { thinking, evaluation_previous_goal, action, ... }
|
|
1919
|
+
* - Raw action object (legacy / `useThinking: false`)
|
|
1920
|
+
* - Raw action array
|
|
1921
|
+
*
|
|
1922
|
+
* Also strips ```json fences the model occasionally adds.
|
|
1923
|
+
*/
|
|
1924
|
+
parseEnvelope(raw) {
|
|
1925
|
+
if (!raw) return null;
|
|
1926
|
+
let s = raw.trim();
|
|
1927
|
+
const fence = s.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
|
|
1928
|
+
if (fence) s = fence[1].trim();
|
|
1929
|
+
let json;
|
|
1930
|
+
try {
|
|
1931
|
+
json = JSON.parse(s);
|
|
1932
|
+
} catch {
|
|
1933
|
+
const m = s.match(/[\[{][\s\S]*[\]}]/);
|
|
1934
|
+
if (!m) return null;
|
|
1935
|
+
try {
|
|
1936
|
+
json = JSON.parse(m[0]);
|
|
1937
|
+
} catch {
|
|
1938
|
+
return null;
|
|
1939
|
+
}
|
|
1940
|
+
}
|
|
1941
|
+
if (Array.isArray(json)) return { action: json };
|
|
1942
|
+
if (json && typeof json === "object" && typeof json.action === "string") {
|
|
1943
|
+
return { action: json };
|
|
1944
|
+
}
|
|
1945
|
+
if (json && typeof json === "object" && "action" in json) {
|
|
1946
|
+
const out = { action: json.action };
|
|
1947
|
+
out.thinking = json.thinking ?? json.reasoning ?? void 0;
|
|
1948
|
+
out.evaluationPreviousGoal = json.evaluationPreviousGoal ?? json.evaluation_previous_goal ?? void 0;
|
|
1949
|
+
out.memory = json.memory ?? void 0;
|
|
1950
|
+
out.nextGoal = json.nextGoal ?? json.next_goal ?? void 0;
|
|
1951
|
+
return out;
|
|
1952
|
+
}
|
|
1953
|
+
return null;
|
|
1954
|
+
}
|
|
1955
|
+
/**
|
|
1956
|
+
* Quick post-navigation health check. If the page came back blank
|
|
1957
|
+
* (no body text and no interactive elements), reload once and wait
|
|
1958
|
+
* for stable. This catches the "FreightOS half-loaded font test" class
|
|
1959
|
+
* of failure before the LLM ever sees it.
|
|
1960
|
+
*/
|
|
1961
|
+
async navigationHealthCheck(browser, url) {
|
|
1962
|
+
try {
|
|
1963
|
+
const ok = await browser.pageText({ maxChars: 200 }).catch(() => "");
|
|
1964
|
+
if (ok && ok.trim().length > 5) return;
|
|
1965
|
+
this.logger.warn("Navigation produced an empty page; reloading once", { url });
|
|
1966
|
+
await this.sleep(800);
|
|
1967
|
+
try {
|
|
1968
|
+
await browser.navigate(url);
|
|
1969
|
+
} catch {
|
|
1970
|
+
}
|
|
1971
|
+
} catch {
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
/**
|
|
1975
|
+
* Force-finalize the run with whatever partial data the agent has.
|
|
1976
|
+
* Called when:
|
|
1977
|
+
* - `maxSteps` is exhausted without an explicit `done`,
|
|
1978
|
+
* - `maxFailures` is exceeded,
|
|
1979
|
+
* - the model can't produce parseable JSON enough times to make
|
|
1980
|
+
* forward progress.
|
|
1981
|
+
* The result is composed from `extractedContent` + the last few
|
|
1982
|
+
* action summaries so the caller gets something useful instead of
|
|
1983
|
+
* just a one-line error.
|
|
1984
|
+
*/
|
|
1985
|
+
async forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, reason) {
|
|
1986
|
+
const reasonLine = reason === "model" ? "Model call failed too many consecutive times." : reason === "parse" ? "Model produced invalid JSON too many times." : "Max step budget exhausted before an explicit `done`.";
|
|
1987
|
+
const tail = actionHistory.slice(-5).join(" \u2192 ");
|
|
1988
|
+
const extracts = extractedContent.length > 0 ? `
|
|
1989
|
+
|
|
1990
|
+
Extracts so far:
|
|
1991
|
+
${extractedContent.join("\n---\n")}` : "";
|
|
1992
|
+
const result = `${reasonLine}
|
|
1993
|
+
Last actions: ${tail || "(none)"}${extracts}`;
|
|
1994
|
+
return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
1995
|
+
result,
|
|
1996
|
+
success: false
|
|
1997
|
+
});
|
|
1998
|
+
}
|
|
1473
1999
|
async finalize(browser, steps, startTime, opts, extractedContent, outcome) {
|
|
1474
2000
|
let finalScreenshot;
|
|
1475
2001
|
let finalInfo;
|