@agentium/browser 2.1.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +751 -225
- package/dist/index.d.cts +153 -13
- package/dist/index.d.ts +153 -13
- package/dist/index.js +751 -225
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -80,29 +80,39 @@ function getStealthScript() {
|
|
|
80
80
|
};
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
// \u2500\u2500
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
el.removeAttribute(attr.name);
|
|
83
|
+
// \u2500\u2500 Remove "cdc_" Playwright/ChromeDriver markers from new elements \u2500
|
|
84
|
+
// Only watches childList additions, NOT every attribute mutation \u2014
|
|
85
|
+
// attributes:true subtree:true is O(every DOM write) on heavy SPAs
|
|
86
|
+
// like FreightOS and noticeably slows the page.
|
|
87
|
+
try {
|
|
88
|
+
const observer = new MutationObserver((mutations) => {
|
|
89
|
+
for (const mutation of mutations) {
|
|
90
|
+
for (const node of mutation.addedNodes) {
|
|
91
|
+
if (node && node.nodeType === 1) {
|
|
92
|
+
const el = node;
|
|
93
|
+
if (!el.attributes) continue;
|
|
94
|
+
for (const attr of [...el.attributes]) {
|
|
95
|
+
if (attr.name.startsWith('cdc_') || attr.name.startsWith('__playwright')) {
|
|
96
|
+
el.removeAttribute(attr.name);
|
|
97
|
+
}
|
|
99
98
|
}
|
|
100
99
|
}
|
|
101
100
|
}
|
|
102
101
|
}
|
|
102
|
+
});
|
|
103
|
+
if (document.documentElement) {
|
|
104
|
+
observer.observe(document.documentElement, { childList: true, subtree: true });
|
|
103
105
|
}
|
|
104
|
-
})
|
|
105
|
-
|
|
106
|
+
} catch (_e) { /* never break the page over a stealth patch */ }
|
|
107
|
+
|
|
108
|
+
// NOTE on iframe.contentWindow:
|
|
109
|
+
// We deliberately do NOT override HTMLIFrameElement.prototype.contentWindow.
|
|
110
|
+
// An earlier version returned the parent window, which breaks legitimate
|
|
111
|
+
// page JS on any site that uses iframes (checkout widgets, embedded
|
|
112
|
+
// videos, A/B test scripts, analytics). The "anti-detection" value was
|
|
113
|
+
// minimal \u2014 modern detectors don't rely on that property anyway \u2014 and
|
|
114
|
+
// the breakage caused pages to fail to boot, leaving font-test residue
|
|
115
|
+
// on screen.
|
|
106
116
|
`;
|
|
107
117
|
}
|
|
108
118
|
function buildStealthContextOpts(config, viewport) {
|
|
@@ -188,14 +198,20 @@ var BrowserProvider = class {
|
|
|
188
198
|
}
|
|
189
199
|
const stealthEnabled = !!opts?.stealth;
|
|
190
200
|
const stealthCfg = typeof opts?.stealth === "object" ? opts.stealth : {};
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
|
|
201
|
+
const headless = opts?.headless ?? true;
|
|
202
|
+
const launchOpts = { headless };
|
|
203
|
+
const args = [];
|
|
194
204
|
if (stealthEnabled) {
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
if (proxy) launchOpts.proxy = proxy;
|
|
205
|
+
const stealth = buildStealthLaunchArgs(stealthCfg);
|
|
206
|
+
args.push(...stealth.args);
|
|
207
|
+
if (stealth.proxy) launchOpts.proxy = stealth.proxy;
|
|
208
|
+
}
|
|
209
|
+
if (!headless) {
|
|
210
|
+
const CHROME_VERTICAL_CHROME_PX = 140;
|
|
211
|
+
args.push(`--window-size=${this._viewport.width},${this._viewport.height + CHROME_VERTICAL_CHROME_PX}`);
|
|
212
|
+
args.push("--window-position=0,0");
|
|
198
213
|
}
|
|
214
|
+
if (args.length) launchOpts.args = args;
|
|
199
215
|
this.browser = await chromium.launch(launchOpts);
|
|
200
216
|
let contextOpts;
|
|
201
217
|
if (stealthEnabled) {
|
|
@@ -555,132 +571,286 @@ var BrowserProvider = class {
|
|
|
555
571
|
/**
|
|
556
572
|
* Snapshot the interactive elements visible in the viewport, tag each
|
|
557
573
|
* with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
|
|
558
|
-
* return
|
|
559
|
-
*
|
|
574
|
+
* return:
|
|
575
|
+
* - `text`: a human-readable string fed to the model
|
|
576
|
+
* - `elements`: the structured list with stable indices
|
|
577
|
+
* - `scroll`: spatial context (pages above/below, hidden interactive count)
|
|
560
578
|
*
|
|
561
|
-
*
|
|
562
|
-
* - Hit-tested
|
|
579
|
+
* Five properties matter for accuracy:
|
|
580
|
+
* - **Hit-tested**: each listed coordinate / index actually reaches the
|
|
563
581
|
* labeled element (overlays / occlusion skip the entry).
|
|
564
|
-
* - Visibility
|
|
565
|
-
*
|
|
566
|
-
*
|
|
567
|
-
*
|
|
582
|
+
* - **Visibility filtered (with parent chain)**: an element is dropped
|
|
583
|
+
* if itself OR any ancestor is `display:none`, `visibility:hidden`,
|
|
584
|
+
* `pointer-events:none`, or near-zero opacity.
|
|
585
|
+
* - **Shadow DOM piercing**: traverses open shadow roots so custom
|
|
586
|
+
* elements / web components are visible to the agent.
|
|
587
|
+
* - **Same-origin iframes**: walks into each accessible iframe and
|
|
588
|
+
* includes its interactive elements (offset by the iframe's screen
|
|
589
|
+
* position so the coordinates the model sees are still viewport-
|
|
590
|
+
* relative).
|
|
591
|
+
* - **`cursor: pointer` fallback pass**: catches custom React widgets
|
|
592
|
+
* that have no semantic role/href/onclick but are clickable.
|
|
568
593
|
*/
|
|
569
594
|
async extractDOM(opts) {
|
|
570
595
|
this.ensurePage();
|
|
571
|
-
const max = opts?.maxElements ??
|
|
572
|
-
const
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
"[tabindex]:not([tabindex='-1'])"
|
|
596
|
-
];
|
|
597
|
-
const semantic = Array.from(doc.querySelectorAll(selectors.join(",")));
|
|
598
|
-
const POINTER_SCAN_CAP = 2e3;
|
|
599
|
-
const pointerCandidates = [];
|
|
600
|
-
const allEls = doc.querySelectorAll("*");
|
|
601
|
-
for (let i = 0; i < allEls.length && pointerCandidates.length < POINTER_SCAN_CAP; i++) {
|
|
602
|
-
const el = allEls[i];
|
|
603
|
-
if (!el) continue;
|
|
604
|
-
const tag = el.tagName.toLowerCase();
|
|
605
|
-
if (tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
|
|
606
|
-
const rect = el.getBoundingClientRect();
|
|
607
|
-
if (rect.width < 4 || rect.height < 4) continue;
|
|
608
|
-
if (rect.bottom < 0 || rect.top > win.innerHeight) continue;
|
|
609
|
-
if (rect.right < 0 || rect.left > win.innerWidth) continue;
|
|
610
|
-
const style = win.getComputedStyle(el);
|
|
611
|
-
if (style.cursor !== "pointer") continue;
|
|
612
|
-
pointerCandidates.push(el);
|
|
613
|
-
}
|
|
614
|
-
const all = semantic.concat(pointerCandidates);
|
|
615
|
-
const vh = win.innerHeight;
|
|
616
|
-
const vw = win.innerWidth;
|
|
617
|
-
const entries = [];
|
|
618
|
-
const seen = /* @__PURE__ */ new Set();
|
|
619
|
-
for (const el of all) {
|
|
620
|
-
if (seen.has(el)) continue;
|
|
621
|
-
const rect = el.getBoundingClientRect();
|
|
622
|
-
if (rect.width < 4 || rect.height < 4) continue;
|
|
623
|
-
if (rect.bottom < 0 || rect.top > vh) continue;
|
|
624
|
-
if (rect.right < 0 || rect.left > vw) continue;
|
|
625
|
-
const style = win.getComputedStyle(el);
|
|
626
|
-
if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
|
|
596
|
+
const max = opts?.maxElements ?? 200;
|
|
597
|
+
const mainResult = await this.page.evaluate((limit) => {
|
|
598
|
+
return globalThis.__buaExtract(limit, 0, 0, "main");
|
|
599
|
+
}, max).catch(async () => {
|
|
600
|
+
await this.installExtractorScript();
|
|
601
|
+
return await this.page.evaluate((limit) => {
|
|
602
|
+
return globalThis.__buaExtract(limit, 0, 0, "main");
|
|
603
|
+
}, max);
|
|
604
|
+
});
|
|
605
|
+
let collected = mainResult.elements;
|
|
606
|
+
const scroll = mainResult.scroll;
|
|
607
|
+
try {
|
|
608
|
+
const frames = this.page.frames();
|
|
609
|
+
for (const frame of frames) {
|
|
610
|
+
if (frame === this.page.mainFrame()) continue;
|
|
611
|
+
if (collected.length >= max) break;
|
|
612
|
+
let bbox = null;
|
|
613
|
+
try {
|
|
614
|
+
const owner = await frame.frameElement();
|
|
615
|
+
if (owner) {
|
|
616
|
+
const rect = await owner.boundingBox();
|
|
617
|
+
if (rect) bbox = { x: rect.x, y: rect.y };
|
|
618
|
+
}
|
|
619
|
+
} catch {
|
|
627
620
|
continue;
|
|
628
621
|
}
|
|
629
|
-
|
|
630
|
-
|
|
622
|
+
if (!bbox) continue;
|
|
623
|
+
let iframeRes = null;
|
|
631
624
|
try {
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
625
|
+
iframeRes = await frame.evaluate(
|
|
626
|
+
(args) => {
|
|
627
|
+
return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
|
|
628
|
+
},
|
|
629
|
+
{ limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
|
|
630
|
+
);
|
|
637
631
|
} catch {
|
|
632
|
+
try {
|
|
633
|
+
await frame.evaluate(this.extractorScriptSource());
|
|
634
|
+
iframeRes = await frame.evaluate(
|
|
635
|
+
(args) => {
|
|
636
|
+
return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
|
|
637
|
+
},
|
|
638
|
+
{ limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
|
|
639
|
+
);
|
|
640
|
+
} catch {
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
if (iframeRes?.elements?.length) {
|
|
644
|
+
const offset = collected.length;
|
|
645
|
+
for (let i = 0; i < iframeRes.elements.length; i++) {
|
|
646
|
+
const e = iframeRes.elements[i];
|
|
647
|
+
e.index = offset + i + 1;
|
|
648
|
+
collected.push(e);
|
|
649
|
+
if (collected.length >= max) break;
|
|
650
|
+
}
|
|
638
651
|
}
|
|
639
|
-
seen.add(el);
|
|
640
|
-
entries.push({ el, cx, cy });
|
|
641
|
-
}
|
|
642
|
-
entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
|
|
643
|
-
const out = [];
|
|
644
|
-
let idx = 1;
|
|
645
|
-
for (const { el, cx, cy } of entries) {
|
|
646
|
-
if (out.length >= limit) break;
|
|
647
|
-
const tag = el.tagName.toLowerCase();
|
|
648
|
-
const role = el.getAttribute("role") || tag;
|
|
649
|
-
const type = el.getAttribute("type") || "";
|
|
650
|
-
const innerText = el.innerText || el.textContent || "";
|
|
651
|
-
const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
|
|
652
|
-
const placeholder = el.getAttribute("placeholder") || "";
|
|
653
|
-
const ariaLabel = el.getAttribute("aria-label") || "";
|
|
654
|
-
const title = el.getAttribute("title") || "";
|
|
655
|
-
const name = el.getAttribute("name") || "";
|
|
656
|
-
const href = el.getAttribute("href") || "";
|
|
657
|
-
const value = el.value || "";
|
|
658
|
-
let label = ariaLabel || text || placeholder || title || value || name;
|
|
659
|
-
if (!label && href) label = href.slice(0, 60);
|
|
660
|
-
if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
|
|
661
|
-
el.setAttribute("data-bua-idx", String(idx));
|
|
662
|
-
out.push({
|
|
663
|
-
index: idx,
|
|
664
|
-
cx,
|
|
665
|
-
cy,
|
|
666
|
-
role,
|
|
667
|
-
type: type || void 0,
|
|
668
|
-
label,
|
|
669
|
-
tag,
|
|
670
|
-
isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
|
|
671
|
-
isSelect: tag === "select",
|
|
672
|
-
isFile: tag === "input" && type === "file"
|
|
673
|
-
});
|
|
674
|
-
idx++;
|
|
675
652
|
}
|
|
676
|
-
|
|
677
|
-
}
|
|
678
|
-
this._lastDom =
|
|
679
|
-
const lines =
|
|
653
|
+
} catch {
|
|
654
|
+
}
|
|
655
|
+
this._lastDom = collected;
|
|
656
|
+
const lines = collected.map((e) => {
|
|
680
657
|
const typeSuffix = e.type ? `(${e.type})` : "";
|
|
681
|
-
|
|
658
|
+
const frameSuffix = e.frame && e.frame !== "main" ? ` [frame]` : "";
|
|
659
|
+
return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}${frameSuffix}: "${e.label}"`;
|
|
682
660
|
});
|
|
683
|
-
return { text: lines.join("\n"), elements:
|
|
661
|
+
return { text: lines.join("\n"), elements: collected, scroll };
|
|
662
|
+
}
|
|
663
|
+
/**
|
|
664
|
+
* Install the `__buaExtract` global on the main page. Idempotent —
|
|
665
|
+
* subsequent calls are no-ops.
|
|
666
|
+
*/
|
|
667
|
+
async installExtractorScript() {
|
|
668
|
+
if (!this.page) return;
|
|
669
|
+
await this.page.evaluate(this.extractorScriptSource());
|
|
670
|
+
}
|
|
671
|
+
/**
|
|
672
|
+
* The extractor source. Lives in its own method so we can also inject
|
|
673
|
+
* it into iframes that haven't yet had it loaded.
|
|
674
|
+
*
|
|
675
|
+
* This function intentionally runs entirely in the page context. It:
|
|
676
|
+
* - traverses the regular DOM + open shadow roots (deep)
|
|
677
|
+
* - applies a parent-chain visibility filter
|
|
678
|
+
* - applies a `cursor:pointer` second pass for custom widgets
|
|
679
|
+
* - hit-tests each candidate at its center to avoid overlay collisions
|
|
680
|
+
* - returns scroll context (pages above/below, hidden counts)
|
|
681
|
+
* - tags survivors with `data-bua-idx` for indexed actions
|
|
682
|
+
*/
|
|
683
|
+
extractorScriptSource() {
|
|
684
|
+
return (
|
|
685
|
+
/* js */
|
|
686
|
+
`
|
|
687
|
+
(function () {
|
|
688
|
+
if (typeof window.__buaExtract === "function") return;
|
|
689
|
+
|
|
690
|
+
function isAncestorVisible(el) {
|
|
691
|
+
let cur = el;
|
|
692
|
+
while (cur && cur !== document.body && cur !== document.documentElement) {
|
|
693
|
+
const s = window.getComputedStyle(cur);
|
|
694
|
+
if (!s) return false;
|
|
695
|
+
if (s.display === "none" || s.visibility === "hidden" || s.visibility === "collapse") return false;
|
|
696
|
+
if (parseFloat(s.opacity || "1") < 0.05) return false;
|
|
697
|
+
cur = cur.parentElement || (cur.getRootNode && cur.getRootNode().host) || null;
|
|
698
|
+
}
|
|
699
|
+
return true;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
function collectSemantic(root, out) {
|
|
703
|
+
if (!root || !root.querySelectorAll) return;
|
|
704
|
+
var sel = [
|
|
705
|
+
"a[href]", "button", "input", "textarea", "select",
|
|
706
|
+
"[role='button']", "[role='link']", "[role='tab']",
|
|
707
|
+
"[role='menuitem']", "[role='checkbox']", "[role='radio']",
|
|
708
|
+
"[role='switch']", "[role='combobox']", "[role='option']",
|
|
709
|
+
"[onclick]", "[contenteditable='true']", "[tabindex]:not([tabindex='-1'])"
|
|
710
|
+
].join(",");
|
|
711
|
+
var found = root.querySelectorAll(sel);
|
|
712
|
+
for (var i = 0; i < found.length; i++) out.push(found[i]);
|
|
713
|
+
// shadow DOM piercing
|
|
714
|
+
var all = root.querySelectorAll("*");
|
|
715
|
+
for (var j = 0; j < all.length; j++) {
|
|
716
|
+
var n = all[j];
|
|
717
|
+
if (n.shadowRoot) collectSemantic(n.shadowRoot, out);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
function collectPointer(root, out, cap) {
|
|
722
|
+
if (!root || !root.querySelectorAll) return;
|
|
723
|
+
var all = root.querySelectorAll("*");
|
|
724
|
+
var vh = window.innerHeight, vw = window.innerWidth;
|
|
725
|
+
for (var i = 0; i < all.length && out.length < cap; i++) {
|
|
726
|
+
var el = all[i];
|
|
727
|
+
var tag = (el.tagName || "").toLowerCase();
|
|
728
|
+
if (!tag || tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
|
|
729
|
+
var rect = el.getBoundingClientRect();
|
|
730
|
+
if (rect.width < 4 || rect.height < 4) continue;
|
|
731
|
+
if (rect.bottom < 0 || rect.top > vh) continue;
|
|
732
|
+
if (rect.right < 0 || rect.left > vw) continue;
|
|
733
|
+
var s = window.getComputedStyle(el);
|
|
734
|
+
if (s.cursor !== "pointer") continue;
|
|
735
|
+
out.push(el);
|
|
736
|
+
if (el.shadowRoot) collectPointer(el.shadowRoot, out, cap);
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
function countHiddenInteractive(root, vh, vw) {
|
|
741
|
+
if (!root || !root.querySelectorAll) return 0;
|
|
742
|
+
var sel = ["a[href]","button","input","textarea","select","[role='button']","[role='link']","[role='tab']","[role='menuitem']"].join(",");
|
|
743
|
+
var n = 0;
|
|
744
|
+
var found = root.querySelectorAll(sel);
|
|
745
|
+
for (var i = 0; i < found.length; i++) {
|
|
746
|
+
var r = found[i].getBoundingClientRect();
|
|
747
|
+
if (r.bottom < 0 || r.top > vh || r.right < 0 || r.left > vw) n++;
|
|
748
|
+
}
|
|
749
|
+
return n;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
window.__buaExtract = function (limit, offsetX, offsetY, frameName) {
|
|
753
|
+
// Clear stale markers each call.
|
|
754
|
+
var stale = document.querySelectorAll("[data-bua-idx]");
|
|
755
|
+
for (var s = 0; s < stale.length; s++) stale[s].removeAttribute("data-bua-idx");
|
|
756
|
+
|
|
757
|
+
var vh = window.innerHeight, vw = window.innerWidth;
|
|
758
|
+
var semantic = []; collectSemantic(document, semantic);
|
|
759
|
+
var pointer = []; collectPointer(document, pointer, 2000);
|
|
760
|
+
var all = semantic.concat(pointer);
|
|
761
|
+
|
|
762
|
+
var entries = [];
|
|
763
|
+
var seen = new Set();
|
|
764
|
+
|
|
765
|
+
for (var i = 0; i < all.length; i++) {
|
|
766
|
+
var el = all[i];
|
|
767
|
+
if (seen.has(el)) continue;
|
|
768
|
+
var rect = el.getBoundingClientRect();
|
|
769
|
+
if (rect.width < 4 || rect.height < 4) continue;
|
|
770
|
+
if (rect.bottom < 0 || rect.top > vh) continue;
|
|
771
|
+
if (rect.right < 0 || rect.left > vw) continue;
|
|
772
|
+
|
|
773
|
+
var style = window.getComputedStyle(el);
|
|
774
|
+
if (style.visibility === "hidden" || style.display === "none" ||
|
|
775
|
+
style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) continue;
|
|
776
|
+
if (!isAncestorVisible(el)) continue;
|
|
777
|
+
|
|
778
|
+
var cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
|
|
779
|
+
var cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
|
|
780
|
+
|
|
781
|
+
try {
|
|
782
|
+
var hit = document.elementFromPoint(cx, cy);
|
|
783
|
+
if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) continue;
|
|
784
|
+
if (hit && el.contains(hit)) seen.add(hit);
|
|
785
|
+
} catch (_) {}
|
|
786
|
+
seen.add(el);
|
|
787
|
+
entries.push({ el: el, cx: cx, cy: cy });
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
entries.sort(function (a, b) { return a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy; });
|
|
791
|
+
|
|
792
|
+
var out = [];
|
|
793
|
+
var idx = 1;
|
|
794
|
+
for (var k = 0; k < entries.length && out.length < limit; k++) {
|
|
795
|
+
var ent = entries[k];
|
|
796
|
+
var el = ent.el;
|
|
797
|
+
var tag = (el.tagName || "").toLowerCase();
|
|
798
|
+
var role = el.getAttribute("role") || tag;
|
|
799
|
+
var type = el.getAttribute("type") || "";
|
|
800
|
+
var innerText = (el.innerText || el.textContent || "");
|
|
801
|
+
var text = innerText.trim().replace(/\\s+/g, " ").slice(0, 80);
|
|
802
|
+
var placeholder = el.getAttribute("placeholder") || "";
|
|
803
|
+
var ariaLabel = el.getAttribute("aria-label") || "";
|
|
804
|
+
var title = el.getAttribute("title") || "";
|
|
805
|
+
var name = el.getAttribute("name") || "";
|
|
806
|
+
var href = el.getAttribute("href") || "";
|
|
807
|
+
var value = el.value || "";
|
|
808
|
+
var label = ariaLabel || text || placeholder || title || value || name;
|
|
809
|
+
if (!label && href) label = href.slice(0, 60);
|
|
810
|
+
if (!label) label = "(" + tag + (type ? (" type=" + type) : "") + ")";
|
|
811
|
+
|
|
812
|
+
el.setAttribute("data-bua-idx", String(idx));
|
|
813
|
+
out.push({
|
|
814
|
+
index: idx,
|
|
815
|
+
cx: ent.cx + offsetX,
|
|
816
|
+
cy: ent.cy + offsetY,
|
|
817
|
+
role: role,
|
|
818
|
+
type: type || undefined,
|
|
819
|
+
label: label,
|
|
820
|
+
tag: tag,
|
|
821
|
+
isInput: tag === "input" || tag === "textarea" || el.isContentEditable === true,
|
|
822
|
+
isSelect: tag === "select",
|
|
823
|
+
isFile: tag === "input" && type === "file",
|
|
824
|
+
frame: frameName,
|
|
825
|
+
});
|
|
826
|
+
idx++;
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
// Scroll context (only meaningful for the main frame; iframes pass through).
|
|
830
|
+
var doc = document.scrollingElement || document.documentElement || document.body;
|
|
831
|
+
var pagesAbove = 0, pagesBelow = 0;
|
|
832
|
+
if (doc && vh > 0) {
|
|
833
|
+
pagesAbove = Math.max(0, Math.round(doc.scrollTop / vh));
|
|
834
|
+
var below = Math.max(0, doc.scrollHeight - doc.scrollTop - vh);
|
|
835
|
+
pagesBelow = Math.round(below / vh);
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
// Hidden interactive count over the WHOLE document tree.
|
|
839
|
+
var hidden = countHiddenInteractive(document, vh, vw);
|
|
840
|
+
|
|
841
|
+
return {
|
|
842
|
+
elements: out,
|
|
843
|
+
scroll: {
|
|
844
|
+
pagesAbove: pagesAbove,
|
|
845
|
+
pagesBelow: pagesBelow,
|
|
846
|
+
totalInteractive: out.length + hidden,
|
|
847
|
+
hiddenInteractive: hidden,
|
|
848
|
+
},
|
|
849
|
+
};
|
|
850
|
+
};
|
|
851
|
+
})();
|
|
852
|
+
`
|
|
853
|
+
);
|
|
684
854
|
}
|
|
685
855
|
// ── Page Info ────────────────────────────────────────────────────────
|
|
686
856
|
async getPageInfo() {
|
|
@@ -842,16 +1012,169 @@ var BrowserProvider = class {
|
|
|
842
1012
|
}
|
|
843
1013
|
};
|
|
844
1014
|
|
|
1015
|
+
// src/loop-detector.ts
|
|
1016
|
+
var LoopDetector = class {
|
|
1017
|
+
actionWindow = [];
|
|
1018
|
+
pageHistory = [];
|
|
1019
|
+
actionWindowSize;
|
|
1020
|
+
pageHistorySize;
|
|
1021
|
+
/** Thresholds: repeat counts at which each severity fires. */
|
|
1022
|
+
actionThresholds;
|
|
1023
|
+
/** Thresholds: consecutive unchanged-page counts at which each fires. */
|
|
1024
|
+
pageThresholds;
|
|
1025
|
+
constructor(opts) {
|
|
1026
|
+
this.actionWindowSize = opts?.actionWindowSize ?? 20;
|
|
1027
|
+
this.pageHistorySize = opts?.pageHistorySize ?? 10;
|
|
1028
|
+
this.actionThresholds = {
|
|
1029
|
+
warn: opts?.actionThresholds?.warn ?? 5,
|
|
1030
|
+
escalate: opts?.actionThresholds?.escalate ?? 8,
|
|
1031
|
+
abort: opts?.actionThresholds?.abort ?? 12
|
|
1032
|
+
};
|
|
1033
|
+
this.pageThresholds = {
|
|
1034
|
+
warn: opts?.pageThresholds?.warn ?? 5,
|
|
1035
|
+
escalate: opts?.pageThresholds?.escalate ?? 8,
|
|
1036
|
+
abort: opts?.pageThresholds?.abort ?? 12
|
|
1037
|
+
};
|
|
1038
|
+
}
|
|
1039
|
+
/**
|
|
1040
|
+
* Record an action and return advice for the runtime. `severity` of
|
|
1041
|
+
* `abort` means the loop should terminate now.
|
|
1042
|
+
*/
|
|
1043
|
+
recordAction(action) {
|
|
1044
|
+
const key = normalizeAction(action);
|
|
1045
|
+
this.actionWindow.push(key);
|
|
1046
|
+
if (this.actionWindow.length > this.actionWindowSize) this.actionWindow.shift();
|
|
1047
|
+
let count = 0;
|
|
1048
|
+
for (const k of this.actionWindow) if (k === key) count++;
|
|
1049
|
+
if (count >= this.actionThresholds.abort) {
|
|
1050
|
+
return {
|
|
1051
|
+
severity: "abort",
|
|
1052
|
+
message: `Auto-stopped: the action "${key}" has been repeated ${count} times in the last ${this.actionWindow.length} steps with no apparent progress.`
|
|
1053
|
+
};
|
|
1054
|
+
}
|
|
1055
|
+
if (count >= this.actionThresholds.escalate) {
|
|
1056
|
+
return {
|
|
1057
|
+
severity: "escalate",
|
|
1058
|
+
message: `You have repeated essentially the same action ${count} times. The current approach is NOT working \u2014 pick a different strategy: scroll, navigate elsewhere, dismiss a popup, or use "fail" if the task truly cannot be done.`
|
|
1059
|
+
};
|
|
1060
|
+
}
|
|
1061
|
+
if (count >= this.actionThresholds.warn) {
|
|
1062
|
+
return {
|
|
1063
|
+
severity: "warn",
|
|
1064
|
+
message: `You've repeated this action ${count} times \u2014 if the page hasn't changed, try a different approach.`
|
|
1065
|
+
};
|
|
1066
|
+
}
|
|
1067
|
+
return { severity: "none" };
|
|
1068
|
+
}
|
|
1069
|
+
/**
|
|
1070
|
+
* Record a page fingerprint and return advice if the page has been
|
|
1071
|
+
* stagnant. Counts consecutive identical fingerprints at the END of
|
|
1072
|
+
* the history (so a single navigation resets the counter).
|
|
1073
|
+
*/
|
|
1074
|
+
recordPage(fp) {
|
|
1075
|
+
this.pageHistory.push(fp);
|
|
1076
|
+
if (this.pageHistory.length > this.pageHistorySize) this.pageHistory.shift();
|
|
1077
|
+
let stagnantCount = 1;
|
|
1078
|
+
for (let i = this.pageHistory.length - 2; i >= 0; i--) {
|
|
1079
|
+
const prev = this.pageHistory[i];
|
|
1080
|
+
if (prev.url === fp.url && prev.interactiveCount === fp.interactiveCount && prev.textHash === fp.textHash) {
|
|
1081
|
+
stagnantCount++;
|
|
1082
|
+
} else {
|
|
1083
|
+
break;
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
if (stagnantCount >= this.pageThresholds.abort) {
|
|
1087
|
+
return {
|
|
1088
|
+
severity: "abort",
|
|
1089
|
+
message: `Auto-stopped: the page hasn't changed in ${stagnantCount} consecutive steps. Likely a blocking overlay or dead control.`
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
if (stagnantCount >= this.pageThresholds.escalate) {
|
|
1093
|
+
return {
|
|
1094
|
+
severity: "escalate",
|
|
1095
|
+
message: `The page has not changed in ${stagnantCount} steps. Something is blocking progress \u2014 try dismissing popups, navigating elsewhere, or scrolling.`
|
|
1096
|
+
};
|
|
1097
|
+
}
|
|
1098
|
+
if (stagnantCount >= this.pageThresholds.warn) {
|
|
1099
|
+
return {
|
|
1100
|
+
severity: "warn",
|
|
1101
|
+
message: `Page hasn't changed in ${stagnantCount} steps \u2014 verify your action is taking effect.`
|
|
1102
|
+
};
|
|
1103
|
+
}
|
|
1104
|
+
return { severity: "none" };
|
|
1105
|
+
}
|
|
1106
|
+
/** Combine two advices, returning the more severe one. */
|
|
1107
|
+
static combine(a, b) {
|
|
1108
|
+
const rank = { none: 0, warn: 1, escalate: 2, abort: 3 };
|
|
1109
|
+
return rank[a.severity] >= rank[b.severity] ? a : b;
|
|
1110
|
+
}
|
|
1111
|
+
};
|
|
1112
|
+
function normalizeAction(action) {
|
|
1113
|
+
switch (action.action) {
|
|
1114
|
+
case "click":
|
|
1115
|
+
if (typeof action.index === "number") return `click:${action.index}`;
|
|
1116
|
+
if (typeof action.x === "number" && typeof action.y === "number") {
|
|
1117
|
+
return `click:xy:${Math.round(action.x / 16)},${Math.round(action.y / 16)}`;
|
|
1118
|
+
}
|
|
1119
|
+
return `click:?`;
|
|
1120
|
+
case "type":
|
|
1121
|
+
if (typeof action.index === "number") return `type:${action.index}:${truncate(action.text, 40)}`;
|
|
1122
|
+
return `type:?:${truncate(action.text, 40)}`;
|
|
1123
|
+
case "scroll":
|
|
1124
|
+
return `scroll:${action.direction}:${action.amount ?? "default"}${action.index != null ? `:${action.index}` : ""}`;
|
|
1125
|
+
case "navigate":
|
|
1126
|
+
return `navigate:${action.url}`;
|
|
1127
|
+
case "back":
|
|
1128
|
+
return "back";
|
|
1129
|
+
case "wait":
|
|
1130
|
+
return `wait:${Math.round(action.ms / 1e3)}`;
|
|
1131
|
+
case "screenshot":
|
|
1132
|
+
return "screenshot";
|
|
1133
|
+
case "send_keys":
|
|
1134
|
+
return `send_keys:${action.keys}`;
|
|
1135
|
+
case "find_text":
|
|
1136
|
+
return `find_text:${truncate(action.text, 40)}`;
|
|
1137
|
+
case "evaluate":
|
|
1138
|
+
return `evaluate:${truncate(action.code, 80)}`;
|
|
1139
|
+
case "dropdown_options":
|
|
1140
|
+
return `dropdown_options:${action.index}`;
|
|
1141
|
+
case "select_dropdown":
|
|
1142
|
+
return `select_dropdown:${action.index}:${truncate(action.text, 40)}`;
|
|
1143
|
+
case "upload_file":
|
|
1144
|
+
return `upload_file:${action.index}:${action.path}`;
|
|
1145
|
+
case "extract":
|
|
1146
|
+
return `extract:${truncate(action.query, 80)}`;
|
|
1147
|
+
case "tool":
|
|
1148
|
+
return `tool:${action.name}`;
|
|
1149
|
+
case "done":
|
|
1150
|
+
case "fail":
|
|
1151
|
+
return action.action;
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
function fnvHash(s) {
|
|
1155
|
+
let h = 2166136261;
|
|
1156
|
+
for (let i = 0; i < s.length; i++) {
|
|
1157
|
+
h ^= s.charCodeAt(i);
|
|
1158
|
+
h = h * 16777619 >>> 0;
|
|
1159
|
+
}
|
|
1160
|
+
return h;
|
|
1161
|
+
}
|
|
1162
|
+
function truncate(s, n) {
|
|
1163
|
+
if (!s) return "";
|
|
1164
|
+
return s.length <= n ? s : `${s.slice(0, n)}\u2026`;
|
|
1165
|
+
}
|
|
1166
|
+
|
|
845
1167
|
// src/prompts.ts
|
|
846
1168
|
function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options) {
|
|
847
1169
|
const maxActions = options?.maxActionsPerStep ?? 3;
|
|
848
1170
|
const allowEvaluate = !!options?.allowEvaluate;
|
|
849
1171
|
const tools = options?.tools ?? [];
|
|
850
1172
|
const useVision = options?.useVision ?? "auto";
|
|
1173
|
+
const useThinking = options?.useThinking ?? true;
|
|
851
1174
|
if (options?.overrideSystemMessage) {
|
|
852
1175
|
const lines2 = [options.overrideSystemMessage];
|
|
853
1176
|
appendCredentials(lines2, credentialKeys);
|
|
854
|
-
lines2.push("", "## Response Format", responseFormatLines(maxActions).join("\n"));
|
|
1177
|
+
lines2.push("", "## Response Format", responseFormatLines(maxActions, useThinking).join("\n"));
|
|
855
1178
|
return lines2.join("\n");
|
|
856
1179
|
}
|
|
857
1180
|
const lines = [];
|
|
@@ -956,7 +1279,7 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options)
|
|
|
956
1279
|
if (extraInstructions) {
|
|
957
1280
|
lines.push(``, `## Additional Instructions`, extraInstructions);
|
|
958
1281
|
}
|
|
959
|
-
lines.push(``, `## Response Format`, responseFormatLines(maxActions).join("\n"));
|
|
1282
|
+
lines.push(``, `## Response Format`, responseFormatLines(maxActions, useThinking).join("\n"));
|
|
960
1283
|
return lines.join("\n");
|
|
961
1284
|
}
|
|
962
1285
|
function appendCredentials(lines, credentialKeys) {
|
|
@@ -974,22 +1297,61 @@ function appendCredentials(lines, credentialKeys) {
|
|
|
974
1297
|
`NEVER include real credential values in "done" or "fail" results.`
|
|
975
1298
|
);
|
|
976
1299
|
}
|
|
977
|
-
function responseFormatLines(maxActions) {
|
|
1300
|
+
function responseFormatLines(maxActions, useThinking) {
|
|
1301
|
+
if (!useThinking) {
|
|
1302
|
+
return [
|
|
1303
|
+
`Respond with ONLY valid JSON. No markdown, no commentary.`,
|
|
1304
|
+
`Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
|
|
1305
|
+
];
|
|
1306
|
+
}
|
|
978
1307
|
return [
|
|
979
|
-
`Respond with ONLY valid JSON. No markdown, no commentary.`,
|
|
980
|
-
|
|
1308
|
+
`Respond with ONLY a single valid JSON object. No markdown, no commentary.`,
|
|
1309
|
+
``,
|
|
1310
|
+
`Use this exact shape:`,
|
|
1311
|
+
"```json",
|
|
1312
|
+
`{`,
|
|
1313
|
+
` "thinking": "Short chain-of-thought: what do I see, what's my plan?",`,
|
|
1314
|
+
` "evaluation_previous_goal": "Did the previous action succeed? success/partial/failure + 1 line",`,
|
|
1315
|
+
` "memory": "Compact bullet list of facts to remember across steps (URLs, found data, \u2026)",`,
|
|
1316
|
+
` "next_goal": "What I want to accomplish in THIS step",`,
|
|
1317
|
+
` "action": <single action object OR array of up to ${maxActions} action objects>`,
|
|
1318
|
+
`}`,
|
|
1319
|
+
"```",
|
|
1320
|
+
``,
|
|
1321
|
+
`On the very first step, set "evaluation_previous_goal" to "n/a \u2014 first step".`,
|
|
1322
|
+
`Keep each text field to one short paragraph or a few lines. Be concrete.`,
|
|
1323
|
+
`Only the "action" field is executed; the others help you self-correct over multiple steps.`
|
|
981
1324
|
];
|
|
982
1325
|
}
|
|
983
|
-
function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract) {
|
|
1326
|
+
function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget) {
|
|
984
1327
|
const lines = [];
|
|
985
1328
|
lines.push(`**Task:** ${task}`);
|
|
986
1329
|
lines.push(`**Current URL:** ${pageUrl}`);
|
|
987
1330
|
if (pageTitle) lines.push(`**Page Title:** ${pageTitle}`);
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
lines.push(`**
|
|
992
|
-
|
|
1331
|
+
if (stepBudget) {
|
|
1332
|
+
lines.push(`**Step:** ${stepIndex + 1} of ${stepBudget.max} (${stepBudget.max - stepIndex - 1} remaining)`);
|
|
1333
|
+
} else {
|
|
1334
|
+
lines.push(`**Step:** ${stepIndex + 1}`);
|
|
1335
|
+
}
|
|
1336
|
+
if (scroll) {
|
|
1337
|
+
const parts = [];
|
|
1338
|
+
parts.push(`${scroll.totalInteractive} interactive elements (${scroll.hiddenInteractive} hidden)`);
|
|
1339
|
+
if (scroll.pagesAbove > 0) parts.push(`${scroll.pagesAbove} page${scroll.pagesAbove === 1 ? "" : "s"} above`);
|
|
1340
|
+
if (scroll.pagesBelow > 0) parts.push(`${scroll.pagesBelow} page${scroll.pagesBelow === 1 ? "" : "s"} below`);
|
|
1341
|
+
if (scroll.pagesAbove === 0 && scroll.pagesBelow === 0) parts.push("fits in viewport");
|
|
1342
|
+
lines.push(`**Page stats:** ${parts.join(" \xB7 ")}`);
|
|
1343
|
+
}
|
|
1344
|
+
if (domSnapshot !== void 0) {
|
|
1345
|
+
if (domSnapshot.trim().length === 0) {
|
|
1346
|
+
lines.push(``);
|
|
1347
|
+
lines.push(
|
|
1348
|
+
`**\u26A0 Empty page / no interactive elements detected.** The page may be blank, blocked by a captcha/anti-bot wall, mid-load, or rendered with shadow DOM in an unsupported way. Consider: \`wait\` + retry, \`navigate\` to a different URL, or \`fail\` if the site is blocking access.`
|
|
1349
|
+
);
|
|
1350
|
+
} else {
|
|
1351
|
+
lines.push(``);
|
|
1352
|
+
lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
|
|
1353
|
+
lines.push(domSnapshot);
|
|
1354
|
+
}
|
|
993
1355
|
}
|
|
994
1356
|
if (lastExtract) {
|
|
995
1357
|
lines.push(``);
|
|
@@ -1003,6 +1365,10 @@ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, do
|
|
|
1003
1365
|
lines.push(`- ${entry}`);
|
|
1004
1366
|
}
|
|
1005
1367
|
}
|
|
1368
|
+
if (nudge) {
|
|
1369
|
+
lines.push(``);
|
|
1370
|
+
lines.push(`**\u26A0 Runtime hint:** ${nudge}`);
|
|
1371
|
+
}
|
|
1006
1372
|
lines.push(``);
|
|
1007
1373
|
lines.push(`Decide the next action(s) to complete the task.`);
|
|
1008
1374
|
return lines.join("\n");
|
|
@@ -1057,6 +1423,9 @@ var BrowserAgent = class {
|
|
|
1057
1423
|
eventBus;
|
|
1058
1424
|
model;
|
|
1059
1425
|
pageExtractionLLM;
|
|
1426
|
+
fallbackModel;
|
|
1427
|
+
useThinking;
|
|
1428
|
+
historyWindow;
|
|
1060
1429
|
instructions;
|
|
1061
1430
|
extendSystemMessage;
|
|
1062
1431
|
overrideSystemMessage;
|
|
@@ -1093,6 +1462,9 @@ var BrowserAgent = class {
|
|
|
1093
1462
|
this.name = config.name;
|
|
1094
1463
|
this.model = config.model;
|
|
1095
1464
|
this.pageExtractionLLM = config.pageExtractionLLM ?? null;
|
|
1465
|
+
this.fallbackModel = config.fallbackModel ?? null;
|
|
1466
|
+
this.useThinking = config.useThinking ?? true;
|
|
1467
|
+
this.historyWindow = Math.max(0, config.historyWindow ?? 6);
|
|
1096
1468
|
this.instructions = config.instructions;
|
|
1097
1469
|
this.extendSystemMessage = config.extendSystemMessage;
|
|
1098
1470
|
this.overrideSystemMessage = config.overrideSystemMessage;
|
|
@@ -1138,10 +1510,10 @@ var BrowserAgent = class {
|
|
|
1138
1510
|
const actionHistory = [];
|
|
1139
1511
|
const extractedContent = [];
|
|
1140
1512
|
let lastExtractResult;
|
|
1141
|
-
let
|
|
1142
|
-
let lastActionKey = "";
|
|
1143
|
-
let repeatCount = 0;
|
|
1513
|
+
let consecutiveFailures = 0;
|
|
1144
1514
|
let lastActionWasScreenshot = false;
|
|
1515
|
+
const loop = new LoopDetector();
|
|
1516
|
+
const historyTurns = [];
|
|
1145
1517
|
const baseExtra = [this.instructions, this.extendSystemMessage].filter(Boolean).join("\n\n");
|
|
1146
1518
|
let extraInstructions = baseExtra;
|
|
1147
1519
|
if (this.memoryManager) {
|
|
@@ -1160,7 +1532,8 @@ ${memoryContext}` : memoryContext;
|
|
|
1160
1532
|
allowEvaluate: this.allowEvaluate,
|
|
1161
1533
|
tools: this.tools,
|
|
1162
1534
|
useVision: this.useVision,
|
|
1163
|
-
useDOM: this.useDOM
|
|
1535
|
+
useDOM: this.useDOM,
|
|
1536
|
+
useThinking: this.useThinking
|
|
1164
1537
|
});
|
|
1165
1538
|
try {
|
|
1166
1539
|
this.logger.info("Launching browser", {
|
|
@@ -1168,6 +1541,7 @@ ${memoryContext}` : memoryContext;
|
|
|
1168
1541
|
viewport: this.viewport,
|
|
1169
1542
|
useDOM: this.useDOM,
|
|
1170
1543
|
useVision: this.useVision,
|
|
1544
|
+
useThinking: this.useThinking,
|
|
1171
1545
|
cdpUrl: this.cdpUrl ?? void 0,
|
|
1172
1546
|
recordVideo: !!this.recordVideo,
|
|
1173
1547
|
stealth: !!this.stealth,
|
|
@@ -1187,6 +1561,7 @@ ${memoryContext}` : memoryContext;
|
|
|
1187
1561
|
this.logger.info("Navigating to start URL", { url: startUrl });
|
|
1188
1562
|
this.assertDomainAllowed(startUrl);
|
|
1189
1563
|
await browser.navigate(startUrl);
|
|
1564
|
+
await this.navigationHealthCheck(browser, startUrl);
|
|
1190
1565
|
}
|
|
1191
1566
|
for (const ia of this.initialActions) {
|
|
1192
1567
|
try {
|
|
@@ -1199,13 +1574,35 @@ ${memoryContext}` : memoryContext;
|
|
|
1199
1574
|
for (let step = 0; step < maxSteps; step++) {
|
|
1200
1575
|
const pageInfo = await browser.getPageInfo();
|
|
1201
1576
|
let domSnapshot;
|
|
1577
|
+
let scrollCtx;
|
|
1202
1578
|
if (this.useDOM) {
|
|
1203
1579
|
const dom = await browser.extractDOM();
|
|
1204
1580
|
domSnapshot = dom.text;
|
|
1581
|
+
scrollCtx = dom.scroll;
|
|
1582
|
+
}
|
|
1583
|
+
const pageAdvice = loop.recordPage({
|
|
1584
|
+
url: pageInfo.url,
|
|
1585
|
+
interactiveCount: scrollCtx?.totalInteractive ?? 0,
|
|
1586
|
+
textHash: fnvHash(domSnapshot ?? "")
|
|
1587
|
+
});
|
|
1588
|
+
if (pageAdvice.severity === "abort") {
|
|
1589
|
+
return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
1590
|
+
result: pageAdvice.message ?? "Auto-stopped: page is stagnant.",
|
|
1591
|
+
success: false
|
|
1592
|
+
});
|
|
1205
1593
|
}
|
|
1206
1594
|
const wantVision = this.shouldCaptureVision(step, lastActionWasScreenshot);
|
|
1207
1595
|
const screenshot = wantVision ? await browser.screenshot() : Buffer.alloc(0);
|
|
1208
1596
|
if (wantVision) this.eventBus.emit("browser.screenshot", { data: screenshot });
|
|
1597
|
+
const isLastStep = step === maxSteps - 1;
|
|
1598
|
+
const nudgeParts = [];
|
|
1599
|
+
if (pageAdvice.severity !== "none" && pageAdvice.message) nudgeParts.push(pageAdvice.message);
|
|
1600
|
+
if (isLastStep) {
|
|
1601
|
+
nudgeParts.push(
|
|
1602
|
+
"This is your FINAL step. Return a `done` action right now summarizing whatever you have, even if partial."
|
|
1603
|
+
);
|
|
1604
|
+
}
|
|
1605
|
+
const nudge = nudgeParts.length > 0 ? nudgeParts.join(" ") : void 0;
|
|
1209
1606
|
const userText = buildUserMessage(
|
|
1210
1607
|
task,
|
|
1211
1608
|
pageInfo.url,
|
|
@@ -1213,65 +1610,49 @@ ${memoryContext}` : memoryContext;
|
|
|
1213
1610
|
step,
|
|
1214
1611
|
actionHistory,
|
|
1215
1612
|
domSnapshot,
|
|
1216
|
-
lastExtractResult
|
|
1613
|
+
lastExtractResult,
|
|
1614
|
+
scrollCtx,
|
|
1615
|
+
nudge,
|
|
1616
|
+
{ current: step, max: maxSteps }
|
|
1217
1617
|
);
|
|
1218
|
-
const
|
|
1219
|
-
if (wantVision) {
|
|
1220
|
-
content.push({ type: "image", data: screenshot.toString("base64"), mimeType: "image/png" });
|
|
1221
|
-
}
|
|
1222
|
-
content.push({ type: "text", text: userText });
|
|
1223
|
-
const messages = [
|
|
1224
|
-
{ role: "system", content: systemPrompt },
|
|
1225
|
-
{ role: "user", content }
|
|
1226
|
-
];
|
|
1618
|
+
const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
|
|
1227
1619
|
this.logger.debug("Calling model", { step, url: pageInfo.url, vision: wantVision });
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
responseFormat: "json"
|
|
1235
|
-
});
|
|
1236
|
-
} catch (e) {
|
|
1237
|
-
failureCount++;
|
|
1238
|
-
this.logger.warn("Model call failed", { error: e?.message, failureCount });
|
|
1239
|
-
if (failureCount > this.maxFailures) {
|
|
1240
|
-
return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
1241
|
-
result: `Model call failed ${failureCount} times: ${e?.message}`,
|
|
1242
|
-
success: false
|
|
1243
|
-
});
|
|
1620
|
+
const { response, modelUsed } = await this.callModelWithFallback(messages, opts?.apiKey);
|
|
1621
|
+
if (!response) {
|
|
1622
|
+
consecutiveFailures++;
|
|
1623
|
+
actionHistory.push(`(model call failed \u2014 retrying, ${consecutiveFailures}/${this.maxFailures})`);
|
|
1624
|
+
if (consecutiveFailures > this.maxFailures) {
|
|
1625
|
+
return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
|
|
1244
1626
|
}
|
|
1245
|
-
actionHistory.push(`(model call failed: ${e?.message ?? "unknown"} \u2014 retrying)`);
|
|
1246
1627
|
continue;
|
|
1247
1628
|
}
|
|
1248
1629
|
if (this.costTracker && response.usage) {
|
|
1249
1630
|
this.costTracker.track({
|
|
1250
1631
|
runId: sessionId,
|
|
1251
1632
|
agentName: this.name,
|
|
1252
|
-
modelId:
|
|
1633
|
+
modelId: modelUsed.modelId,
|
|
1253
1634
|
usage: response.usage,
|
|
1254
1635
|
sessionId,
|
|
1255
1636
|
userId
|
|
1256
1637
|
});
|
|
1257
1638
|
}
|
|
1258
1639
|
const raw = typeof response.message.content === "string" ? response.message.content : "";
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
if (failureCount > this.maxFailures) {
|
|
1266
|
-
return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
1267
|
-
result: `Model returned invalid JSON ${failureCount} times. Last response: ${raw.slice(0, 200)}`,
|
|
1268
|
-
success: false
|
|
1269
|
-
});
|
|
1640
|
+
const envelope = this.parseEnvelope(raw);
|
|
1641
|
+
if (!envelope) {
|
|
1642
|
+
consecutiveFailures++;
|
|
1643
|
+
this.logger.warn("Failed to parse model response", { raw: raw.slice(0, 200), consecutiveFailures });
|
|
1644
|
+
if (consecutiveFailures > this.maxFailures) {
|
|
1645
|
+
return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "parse");
|
|
1270
1646
|
}
|
|
1271
1647
|
actionHistory.push("(invalid JSON response \u2014 retrying)");
|
|
1272
1648
|
continue;
|
|
1273
1649
|
}
|
|
1274
|
-
|
|
1650
|
+
consecutiveFailures = 0;
|
|
1651
|
+
historyTurns.push({ userText, hasScreenshot: wantVision, envelope });
|
|
1652
|
+
if (this.historyWindow > 0 && historyTurns.length > this.historyWindow) {
|
|
1653
|
+
historyTurns.splice(0, historyTurns.length - this.historyWindow);
|
|
1654
|
+
}
|
|
1655
|
+
const actions = (Array.isArray(envelope.action) ? envelope.action : [envelope.action]).slice(
|
|
1275
1656
|
0,
|
|
1276
1657
|
this.maxActionsPerStep
|
|
1277
1658
|
);
|
|
@@ -1282,24 +1663,12 @@ ${memoryContext}` : memoryContext;
|
|
|
1282
1663
|
const action = actions[ai];
|
|
1283
1664
|
let summary = summarizeAction(action);
|
|
1284
1665
|
if (this.credentials) summary = this.credentials.mask(summary);
|
|
1285
|
-
const
|
|
1286
|
-
if (
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
lastActionKey = actionKey;
|
|
1290
|
-
repeatCount = 1;
|
|
1291
|
-
}
|
|
1292
|
-
if (repeatCount > this.maxRepeats && action.action !== "done" && action.action !== "fail") {
|
|
1293
|
-
this.logger.warn("Stuck in a loop \u2014 same action repeated", {
|
|
1294
|
-
action: action.action,
|
|
1295
|
-
repeats: repeatCount,
|
|
1296
|
-
maxRepeats: this.maxRepeats
|
|
1297
|
-
});
|
|
1298
|
-
actionHistory.push(
|
|
1299
|
-
`\u26A0 LOOP DETECTED: "${summary}" repeated ${repeatCount} times. Auto-stopping.`
|
|
1300
|
-
);
|
|
1666
|
+
const advice = loop.recordAction(action);
|
|
1667
|
+
if (advice.severity === "abort" && action.action !== "done" && action.action !== "fail") {
|
|
1668
|
+
this.logger.warn("Loop detector aborting run", { advice });
|
|
1669
|
+
actionHistory.push(`\u26A0 ${advice.message ?? "loop detected \u2014 auto-stopping"}`);
|
|
1301
1670
|
didTerminate = {
|
|
1302
|
-
result:
|
|
1671
|
+
result: advice.message ?? "Stuck in a loop \u2014 auto-stopped.",
|
|
1303
1672
|
success: false
|
|
1304
1673
|
};
|
|
1305
1674
|
break;
|
|
@@ -1322,21 +1691,20 @@ ${memoryContext}` : memoryContext;
|
|
|
1322
1691
|
try {
|
|
1323
1692
|
const exec = await this.executeAction(browser, action, actionHistory, extractedContent);
|
|
1324
1693
|
stepOutput = exec?.output;
|
|
1325
|
-
if (exec?.didNavigate)
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
}
|
|
1329
|
-
if (action.action === "screenshot") {
|
|
1330
|
-
lastActionWasScreenshot = true;
|
|
1694
|
+
if (exec?.didNavigate) {
|
|
1695
|
+
didNavigate = true;
|
|
1696
|
+
await this.navigationHealthCheck(browser, pageInfo.url);
|
|
1331
1697
|
}
|
|
1698
|
+
if (action.action === "extract" && exec?.output) lastExtractResult = exec.output;
|
|
1699
|
+
if (action.action === "screenshot") lastActionWasScreenshot = true;
|
|
1332
1700
|
} catch (e) {
|
|
1333
1701
|
stepOk = false;
|
|
1334
|
-
|
|
1335
|
-
this.logger.warn("Action failed", { action: action.action, error: e?.message,
|
|
1702
|
+
consecutiveFailures++;
|
|
1703
|
+
this.logger.warn("Action failed", { action: action.action, error: e?.message, consecutiveFailures });
|
|
1336
1704
|
actionHistory.push(`(error: ${e?.message ?? "unknown"})`);
|
|
1337
|
-
if (
|
|
1705
|
+
if (consecutiveFailures > this.maxFailures) {
|
|
1338
1706
|
didTerminate = {
|
|
1339
|
-
result: `Action ${action.action} failed ${
|
|
1707
|
+
result: `Action ${action.action} failed ${consecutiveFailures} times. Last error: ${e?.message}`,
|
|
1340
1708
|
success: false
|
|
1341
1709
|
};
|
|
1342
1710
|
break;
|
|
@@ -1351,7 +1719,11 @@ ${memoryContext}` : memoryContext;
|
|
|
1351
1719
|
timestamp: /* @__PURE__ */ new Date(),
|
|
1352
1720
|
dom: domSnapshot,
|
|
1353
1721
|
output: stepOutput,
|
|
1354
|
-
ok: stepOk
|
|
1722
|
+
ok: stepOk,
|
|
1723
|
+
thinking: envelope.thinking,
|
|
1724
|
+
evaluationPreviousGoal: envelope.evaluationPreviousGoal,
|
|
1725
|
+
memory: envelope.memory,
|
|
1726
|
+
nextGoal: envelope.nextGoal
|
|
1355
1727
|
});
|
|
1356
1728
|
this.eventBus.emit("browser.step", {
|
|
1357
1729
|
index: steps.length - 1,
|
|
@@ -1366,11 +1738,8 @@ ${memoryContext}` : memoryContext;
|
|
|
1366
1738
|
return await this.finalize(browser, steps, startTime, opts, extractedContent, didTerminate);
|
|
1367
1739
|
}
|
|
1368
1740
|
}
|
|
1369
|
-
this.logger.warn("Max steps reached without
|
|
1370
|
-
return await this.
|
|
1371
|
-
result: `Task not completed within ${maxSteps} steps. Last actions: ${actionHistory.slice(-3).join("; ")}`,
|
|
1372
|
-
success: false
|
|
1373
|
-
});
|
|
1741
|
+
this.logger.warn("Max steps reached without explicit done", { maxSteps });
|
|
1742
|
+
return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "max_steps");
|
|
1374
1743
|
} catch (error) {
|
|
1375
1744
|
this.logger.error("Browser agent error", { error: error.message });
|
|
1376
1745
|
this.eventBus.emit("browser.error", { error });
|
|
@@ -1432,6 +1801,163 @@ ${memoryContext}` : memoryContext;
|
|
|
1432
1801
|
throw new Error(`Navigation blocked: ${host} is in prohibitedDomains`);
|
|
1433
1802
|
}
|
|
1434
1803
|
}
|
|
1804
|
+
/**
|
|
1805
|
+
* Build the message array sent to the model: system prompt + a compact
|
|
1806
|
+
* summary of older turns (if any) + the most recent `historyWindow`
|
|
1807
|
+
* turns verbatim + the current step's user message.
|
|
1808
|
+
*
|
|
1809
|
+
* Inspired by browser-use's history compaction. Keeps tokens bounded
|
|
1810
|
+
* while giving the model meaningful context about what it already
|
|
1811
|
+
* tried.
|
|
1812
|
+
*/
|
|
1813
|
+
buildMessages(systemPrompt, historyTurns, currentUserText, currentScreenshot) {
|
|
1814
|
+
const msgs = [{ role: "system", content: systemPrompt }];
|
|
1815
|
+
if (this.historyWindow > 0 && historyTurns.length > 0) {
|
|
1816
|
+
for (const turn of historyTurns) {
|
|
1817
|
+
const env = turn.envelope;
|
|
1818
|
+
const replay = [];
|
|
1819
|
+
if (env.thinking) replay.push(`thinking: ${env.thinking}`);
|
|
1820
|
+
if (env.evaluationPreviousGoal) replay.push(`evaluation: ${env.evaluationPreviousGoal}`);
|
|
1821
|
+
if (env.memory) replay.push(`memory: ${env.memory}`);
|
|
1822
|
+
if (env.nextGoal) replay.push(`next_goal: ${env.nextGoal}`);
|
|
1823
|
+
replay.push(`action: ${JSON.stringify(env.action)}`);
|
|
1824
|
+
msgs.push({ role: "assistant", content: replay.join("\n") });
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
const content = [];
|
|
1828
|
+
if (currentScreenshot && currentScreenshot.length > 0) {
|
|
1829
|
+
content.push({
|
|
1830
|
+
type: "image",
|
|
1831
|
+
data: currentScreenshot.toString("base64"),
|
|
1832
|
+
mimeType: "image/png"
|
|
1833
|
+
});
|
|
1834
|
+
}
|
|
1835
|
+
content.push({ type: "text", text: currentUserText });
|
|
1836
|
+
msgs.push({ role: "user", content });
|
|
1837
|
+
return msgs;
|
|
1838
|
+
}
|
|
1839
|
+
/**
|
|
1840
|
+
* Call the primary model, retrying once with `fallbackModel` on
|
|
1841
|
+
* transient errors (5xx, 429, network). Returns the response or
|
|
1842
|
+
* `null` if both models failed.
|
|
1843
|
+
*/
|
|
1844
|
+
async callModelWithFallback(messages, apiKey) {
|
|
1845
|
+
const reqOpts = { temperature: 0.1, maxTokens: 1024, apiKey, responseFormat: "json" };
|
|
1846
|
+
try {
|
|
1847
|
+
const r = await this.model.generate(messages, reqOpts);
|
|
1848
|
+
return { response: r, modelUsed: this.model };
|
|
1849
|
+
} catch (e) {
|
|
1850
|
+
if (this.fallbackModel && this.isTransientError(e)) {
|
|
1851
|
+
this.logger.warn("Primary model failed; trying fallbackModel", {
|
|
1852
|
+
error: e?.message,
|
|
1853
|
+
primary: this.model.modelId,
|
|
1854
|
+
fallback: this.fallbackModel.modelId
|
|
1855
|
+
});
|
|
1856
|
+
try {
|
|
1857
|
+
const r = await this.fallbackModel.generate(messages, reqOpts);
|
|
1858
|
+
return { response: r, modelUsed: this.fallbackModel };
|
|
1859
|
+
} catch (e2) {
|
|
1860
|
+
this.logger.warn("Fallback model also failed", { error: e2?.message });
|
|
1861
|
+
return { response: null, modelUsed: this.model };
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1864
|
+
this.logger.warn("Model call failed (no fallback configured)", { error: e?.message });
|
|
1865
|
+
return { response: null, modelUsed: this.model };
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
isTransientError(e) {
|
|
1869
|
+
const msg = String(e?.message ?? e).toLowerCase();
|
|
1870
|
+
if (msg.includes("rate limit") || msg.includes("429")) return true;
|
|
1871
|
+
if (msg.includes("timeout") || msg.includes("etimedout")) return true;
|
|
1872
|
+
if (msg.includes("econnreset") || msg.includes("network")) return true;
|
|
1873
|
+
if (/\b5\d\d\b/.test(msg)) return true;
|
|
1874
|
+
if (msg.includes("401") || msg.includes("402") || msg.includes("auth")) return true;
|
|
1875
|
+
return false;
|
|
1876
|
+
}
|
|
1877
|
+
/**
|
|
1878
|
+
* Parse the model's raw response into an `AgentOutput`. Tolerant to
|
|
1879
|
+
* three shapes:
|
|
1880
|
+
* - Full envelope: { thinking, evaluation_previous_goal, action, ... }
|
|
1881
|
+
* - Raw action object (legacy / `useThinking: false`)
|
|
1882
|
+
* - Raw action array
|
|
1883
|
+
*
|
|
1884
|
+
* Also strips ```json fences the model occasionally adds.
|
|
1885
|
+
*/
|
|
1886
|
+
parseEnvelope(raw) {
|
|
1887
|
+
if (!raw) return null;
|
|
1888
|
+
let s = raw.trim();
|
|
1889
|
+
const fence = s.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
|
|
1890
|
+
if (fence) s = fence[1].trim();
|
|
1891
|
+
let json;
|
|
1892
|
+
try {
|
|
1893
|
+
json = JSON.parse(s);
|
|
1894
|
+
} catch {
|
|
1895
|
+
const m = s.match(/[\[{][\s\S]*[\]}]/);
|
|
1896
|
+
if (!m) return null;
|
|
1897
|
+
try {
|
|
1898
|
+
json = JSON.parse(m[0]);
|
|
1899
|
+
} catch {
|
|
1900
|
+
return null;
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
if (Array.isArray(json)) return { action: json };
|
|
1904
|
+
if (json && typeof json === "object" && typeof json.action === "string") {
|
|
1905
|
+
return { action: json };
|
|
1906
|
+
}
|
|
1907
|
+
if (json && typeof json === "object" && "action" in json) {
|
|
1908
|
+
const out = { action: json.action };
|
|
1909
|
+
out.thinking = json.thinking ?? json.reasoning ?? void 0;
|
|
1910
|
+
out.evaluationPreviousGoal = json.evaluationPreviousGoal ?? json.evaluation_previous_goal ?? void 0;
|
|
1911
|
+
out.memory = json.memory ?? void 0;
|
|
1912
|
+
out.nextGoal = json.nextGoal ?? json.next_goal ?? void 0;
|
|
1913
|
+
return out;
|
|
1914
|
+
}
|
|
1915
|
+
return null;
|
|
1916
|
+
}
|
|
1917
|
+
/**
|
|
1918
|
+
* Quick post-navigation health check. If the page came back blank
|
|
1919
|
+
* (no body text and no interactive elements), reload once and wait
|
|
1920
|
+
* for stable. This catches the "FreightOS half-loaded font test" class
|
|
1921
|
+
* of failure before the LLM ever sees it.
|
|
1922
|
+
*/
|
|
1923
|
+
async navigationHealthCheck(browser, url) {
|
|
1924
|
+
try {
|
|
1925
|
+
const ok = await browser.pageText({ maxChars: 200 }).catch(() => "");
|
|
1926
|
+
if (ok && ok.trim().length > 5) return;
|
|
1927
|
+
this.logger.warn("Navigation produced an empty page; reloading once", { url });
|
|
1928
|
+
await this.sleep(800);
|
|
1929
|
+
try {
|
|
1930
|
+
await browser.navigate(url);
|
|
1931
|
+
} catch {
|
|
1932
|
+
}
|
|
1933
|
+
} catch {
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
/**
|
|
1937
|
+
* Force-finalize the run with whatever partial data the agent has.
|
|
1938
|
+
* Called when:
|
|
1939
|
+
* - `maxSteps` is exhausted without an explicit `done`,
|
|
1940
|
+
* - `maxFailures` is exceeded,
|
|
1941
|
+
* - the model can't produce parseable JSON enough times to make
|
|
1942
|
+
* forward progress.
|
|
1943
|
+
* The result is composed from `extractedContent` + the last few
|
|
1944
|
+
* action summaries so the caller gets something useful instead of
|
|
1945
|
+
* just a one-line error.
|
|
1946
|
+
*/
|
|
1947
|
+
async forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, reason) {
|
|
1948
|
+
const reasonLine = reason === "model" ? "Model call failed too many consecutive times." : reason === "parse" ? "Model produced invalid JSON too many times." : "Max step budget exhausted before an explicit `done`.";
|
|
1949
|
+
const tail = actionHistory.slice(-5).join(" \u2192 ");
|
|
1950
|
+
const extracts = extractedContent.length > 0 ? `
|
|
1951
|
+
|
|
1952
|
+
Extracts so far:
|
|
1953
|
+
${extractedContent.join("\n---\n")}` : "";
|
|
1954
|
+
const result = `${reasonLine}
|
|
1955
|
+
Last actions: ${tail || "(none)"}${extracts}`;
|
|
1956
|
+
return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
1957
|
+
result,
|
|
1958
|
+
success: false
|
|
1959
|
+
});
|
|
1960
|
+
}
|
|
1435
1961
|
async finalize(browser, steps, startTime, opts, extractedContent, outcome) {
|
|
1436
1962
|
let finalScreenshot;
|
|
1437
1963
|
let finalInfo;
|