bwb-browser 4.0.1 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +31 -17
- package/README.md +76 -32
- package/lib/act.mjs +474 -215
- package/lib/browser.mjs +145 -30
- package/lib/config.mjs +180 -0
- package/lib/diagnose.mjs +13 -11
- package/lib/fetch.mjs +187 -75
- package/lib/fingerprint.mjs +73 -28
- package/lib/helpers.mjs +86 -39
- package/lib/session.mjs +34 -7
- package/lib/setup.mjs +94 -13
- package/lib/tabs.mjs +127 -15
- package/lib/urlpolicy.mjs +137 -0
- package/lib/vigil.mjs +44 -21
- package/package.json +15 -7
- package/server.mjs +418 -211
package/lib/act.mjs
CHANGED
|
@@ -6,139 +6,295 @@
|
|
|
6
6
|
* form filling, content extraction, and scrolling.
|
|
7
7
|
*
|
|
8
8
|
* The goal: ONE tool call does what would normally take 5-10 tool calls.
|
|
9
|
-
*
|
|
9
|
+
*
|
|
10
|
+
* Ground rules (learned the hard way — see CHANGELOG 4.1.0):
|
|
11
|
+
* 1. Captured text is NEVER lowercased. Matching is case-insensitive, but a
|
|
12
|
+
* password typed as `mys3cretpass` is silently wrong.
|
|
13
|
+
* 2. Elements are located, measured and tagged inside ONE Runtime.evaluate.
|
|
14
|
+
* A "found element" object is never round-tripped through a CSS selector —
|
|
15
|
+
* `a` matches every link, so `querySelector('a')` returns the first one.
|
|
16
|
+
* 3. When matching is ambiguous, return candidates instead of guessing.
|
|
17
|
+
* Clicking "Buy now" because of a heuristic is the worst failure mode here.
|
|
10
18
|
*/
|
|
11
19
|
|
|
20
|
+
import { assertNavigable, UrlPolicyError } from "./urlpolicy.mjs";
|
|
21
|
+
|
|
22
|
+
const MARK_ATTR = "data-bwb-target";
|
|
23
|
+
const PREVIEW_CHARS = 500;
|
|
24
|
+
|
|
25
|
+
// Text that should never be clicked without an explicit confirmation.
|
|
26
|
+
const DESTRUCTIVE_RE =
|
|
27
|
+
/\b(delete|remove|drop|trash|erase|wipe|buy|purchase|pay|checkout|order|confirm|submit|send|publish|transfer|withdraw|cancel|close account|deactivate|unsubscribe|revoke|terminate|uninstall)\b/i;
|
|
28
|
+
|
|
29
|
+
/** Result text for a filled field, with secrets redacted (never echoed back). */
|
|
30
|
+
export function redactIfSecret(target, text) {
|
|
31
|
+
if (/pass|secret|token|otp|pin|cvv|card|ssn|security.?code/i.test(String(target || ""))) {
|
|
32
|
+
return { length: [...String(text)].length, redacted: true };
|
|
33
|
+
}
|
|
34
|
+
return { text, redacted: false };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// ─── In-page helpers (injected into the page expression) ─────────────────────
|
|
38
|
+
// Kept as strings so a single Runtime.evaluate can locate AND measure.
|
|
39
|
+
|
|
12
40
|
// ─── Smart Input Finder ──────────────────────────────────────────────────────
|
|
13
41
|
|
|
14
42
|
async function findInput(Runtime, labelText) {
|
|
43
|
+
const query = String(labelText || "").trim().toLowerCase();
|
|
15
44
|
const { result } = await Runtime.evaluate({
|
|
16
45
|
expression: `(() => {
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
const name = (el.name || '').toLowerCase();
|
|
25
|
-
const id = (el.id || '').toLowerCase();
|
|
26
|
-
const type = (el.type || '').toLowerCase();
|
|
46
|
+
const query = ${JSON.stringify(query)};
|
|
47
|
+
const inputs = Array.from(document.querySelectorAll(
|
|
48
|
+
'input:not([type=hidden]):not([type=submit]):not([type=button]), textarea, [contenteditable=true], select'
|
|
49
|
+
));
|
|
50
|
+
if (!inputs.length) return { found: false, candidates: [] };
|
|
51
|
+
|
|
52
|
+
const describe = (el) => {
|
|
27
53
|
let label = '';
|
|
28
54
|
if (el.id) {
|
|
29
|
-
const lbl = document.querySelector('label[for="' + el.id + '"]');
|
|
30
|
-
if (lbl) label = lbl.textContent
|
|
55
|
+
const lbl = document.querySelector('label[for="' + CSS.escape(el.id) + '"]');
|
|
56
|
+
if (lbl) label = lbl.textContent || '';
|
|
31
57
|
}
|
|
32
|
-
const parent = el.closest('label');
|
|
33
|
-
if (parent) label += ' ' + parent.textContent
|
|
58
|
+
const parent = el.closest && el.closest('label');
|
|
59
|
+
if (parent) label += ' ' + (parent.textContent || '');
|
|
60
|
+
return {
|
|
61
|
+
tag: el.tagName.toLowerCase(),
|
|
62
|
+
type: (el.type || '').toLowerCase(),
|
|
63
|
+
id: el.id || '',
|
|
64
|
+
name: el.name || '',
|
|
65
|
+
placeholder: el.placeholder || el.getAttribute('placeholder') || '',
|
|
66
|
+
aria: el.getAttribute('aria-label') || '',
|
|
67
|
+
label: label.trim().slice(0, 120),
|
|
68
|
+
visible: isVisible(el),
|
|
69
|
+
};
|
|
70
|
+
};
|
|
34
71
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
72
|
+
function isVisible(el) {
|
|
73
|
+
const r = el.getBoundingClientRect();
|
|
74
|
+
if (r.width > 0 && r.height > 0) return true;
|
|
75
|
+
const cs = window.getComputedStyle ? window.getComputedStyle(el) : null;
|
|
76
|
+
return !!(cs && cs.display !== 'none' && cs.visibility !== 'hidden');
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const meta = inputs.map(describe);
|
|
80
|
+
if (!query) {
|
|
81
|
+
// No target named: only an unambiguous single input is safe to pick.
|
|
82
|
+
const visibleIdx = meta.map((m, i) => (m.visible ? i : -1)).filter((i) => i >= 0);
|
|
83
|
+
if (visibleIdx.length === 1) {
|
|
84
|
+
return { found: true, index: visibleIdx[0], tag: meta[visibleIdx[0]].tag, type: meta[visibleIdx[0]].type };
|
|
38
85
|
}
|
|
86
|
+
return {
|
|
87
|
+
found: false,
|
|
88
|
+
candidates: meta.map((m) => ({
|
|
89
|
+
tag: m.tag, type: m.type, id: m.id, name: m.name, placeholder: m.placeholder,
|
|
90
|
+
})),
|
|
91
|
+
};
|
|
39
92
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
93
|
+
|
|
94
|
+
// Rank candidates: an input whose own metadata matches the requested
|
|
95
|
+
// field wins; a generic fallback (first input on the page) is never used.
|
|
96
|
+
let best = -1, bestScore = 0;
|
|
97
|
+
meta.forEach((m, i) => {
|
|
98
|
+
let score = 0;
|
|
99
|
+
const name = m.name.toLowerCase(), id = m.id.toLowerCase();
|
|
100
|
+
const ph = m.placeholder.toLowerCase(), aria = m.aria.toLowerCase();
|
|
101
|
+
const lbl = m.label.toLowerCase(), type = m.type;
|
|
102
|
+
if (name === query) score = 100;
|
|
103
|
+
else if (id === query) score = 95;
|
|
104
|
+
else if (ph === query) score = 90;
|
|
105
|
+
else if (aria === query) score = 85;
|
|
106
|
+
else if (lbl === query) score = 85;
|
|
107
|
+
else if (name.includes(query)) score = 70;
|
|
108
|
+
else if (id.includes(query)) score = 65;
|
|
109
|
+
else if (aria.includes(query)) score = 60;
|
|
110
|
+
else if (ph.includes(query)) score = 60;
|
|
111
|
+
else if (lbl.includes(query)) score = 55;
|
|
112
|
+
else if (type === query) score = 40;
|
|
113
|
+
if (score && !m.visible) score -= 5;
|
|
114
|
+
if (score > bestScore) { bestScore = score; best = i; }
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
if (best < 0) {
|
|
118
|
+
return { found: false, candidates: meta.map((m) => ({
|
|
119
|
+
tag: m.tag, type: m.type, id: m.id, name: m.name,
|
|
120
|
+
placeholder: m.placeholder, label: m.label,
|
|
121
|
+
})) };
|
|
50
122
|
}
|
|
123
|
+
return { found: true, index: best, tag: meta[best].tag, type: meta[best].type };
|
|
51
124
|
})()`,
|
|
52
125
|
returnByValue: true,
|
|
53
126
|
});
|
|
54
127
|
return result?.value || { found: false };
|
|
55
128
|
}
|
|
56
129
|
|
|
57
|
-
// ───
|
|
130
|
+
// ─── Element Finder by Text ──────────────────────────────────────────────────
|
|
131
|
+
// One page-side routine for every "find something by its text" case. Scoring,
|
|
132
|
+
// filtering, scrolling, measuring and tagging all happen inside ONE
|
|
133
|
+
// Runtime.evaluate against ONE element. A previous version returned a CSS
|
|
134
|
+
// selector and re-queried it, so `a` (every link) resolved to the first link:
|
|
135
|
+
// "click the Pricing link" clicked "Home".
|
|
58
136
|
|
|
59
|
-
|
|
137
|
+
const CLICKABLE_SELECTOR =
|
|
138
|
+
'a[href], button, [role=button], input[type=submit], input[type=button], ' +
|
|
139
|
+
'[onclick], summary, label, [role=link], [role=menuitem]';
|
|
140
|
+
|
|
141
|
+
// Containers worth extracting text from — deliberately excludes html/body so a
|
|
142
|
+
// fuzzy match can never return the entire document.
|
|
143
|
+
const TEXT_CONTAINER_SELECTOR =
|
|
144
|
+
'h1, h2, h3, h4, h5, h6, p, li, td, th, dt, dd, blockquote, pre, code, ' +
|
|
145
|
+
'section, article, main, div, span, a, summary, label, table, ul, ol, form, figcaption';
|
|
146
|
+
|
|
147
|
+
async function findByText(Runtime, text, selector) {
|
|
148
|
+
const query = String(text || "").trim().toLowerCase();
|
|
60
149
|
const { result } = await Runtime.evaluate({
|
|
61
150
|
expression: `(() => {
|
|
62
|
-
const
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
const
|
|
88
|
-
let
|
|
89
|
-
for (const
|
|
90
|
-
|
|
91
|
-
if (
|
|
151
|
+
const query = ${JSON.stringify(query)};
|
|
152
|
+
const selector = ${JSON.stringify(selector)};
|
|
153
|
+
const MARK = ${JSON.stringify(MARK_ATTR)};
|
|
154
|
+
const SKIP = new Set(['HTML','BODY','SCRIPT','STYLE','HEAD','META','LINK','NOSCRIPT','TEMPLATE']);
|
|
155
|
+
|
|
156
|
+
const visible = (el) => {
|
|
157
|
+
if (SKIP.has(el.tagName)) return false;
|
|
158
|
+
const r = el.getBoundingClientRect();
|
|
159
|
+
if (r.width > 0 && r.height > 0) return true;
|
|
160
|
+
const cs = window.getComputedStyle ? window.getComputedStyle(el) : null;
|
|
161
|
+
return !!(cs && cs.display !== 'none' && cs.visibility !== 'hidden');
|
|
162
|
+
};
|
|
163
|
+
const ownText = (el) => {
|
|
164
|
+
let t = '';
|
|
165
|
+
for (const n of el.childNodes) if (n.nodeType === 3) t += n.nodeValue;
|
|
166
|
+
return t.trim();
|
|
167
|
+
};
|
|
168
|
+
const signals = (el) => [
|
|
169
|
+
ownText(el),
|
|
170
|
+
el.getAttribute && el.getAttribute('aria-label'),
|
|
171
|
+
el.value, el.title, el.alt,
|
|
172
|
+
(el.textContent || '').trim(),
|
|
173
|
+
].filter((s) => s !== undefined && s !== null).map((s) => String(s).trim().toLowerCase());
|
|
174
|
+
|
|
175
|
+
function score(el) {
|
|
176
|
+
const full = (el.textContent || '').trim();
|
|
177
|
+
let best = 0;
|
|
178
|
+
for (const s of signals(el)) {
|
|
179
|
+
let v = 0;
|
|
180
|
+
if (s === query) v = 100; // exact
|
|
181
|
+
else if (s.startsWith(query)) v = 84; // starts with
|
|
182
|
+
else if (query.length >= 3 && s.includes(query)) v = 66; // contains
|
|
183
|
+
if (v > best) best = v;
|
|
184
|
+
}
|
|
185
|
+
if (best === 0 && query.length >= 3) {
|
|
186
|
+
const words = full.toLowerCase().split(/\\s+/);
|
|
187
|
+
const qw = query.split(/\\s+/).filter((x) => x.length >= 3);
|
|
188
|
+
let s = 0;
|
|
189
|
+
for (const x of qw) if (words.some((w) => w.includes(x))) s += 8;
|
|
190
|
+
best = s;
|
|
92
191
|
}
|
|
93
|
-
if (
|
|
94
|
-
|
|
95
|
-
|
|
192
|
+
if (best > 0 && best < 100) {
|
|
193
|
+
// Prefer the SMALLEST, most specific element holding the text:
|
|
194
|
+
// a page-sized container must never outrank the paragraph with the
|
|
195
|
+
// answer, and a leaf beats an identical wrapper.
|
|
196
|
+
best += el.querySelector('*') ? 0 : 4;
|
|
197
|
+
best -= Math.min(24, Math.max(0, full.length - query.length) / 40 | 0);
|
|
96
198
|
}
|
|
199
|
+
return best;
|
|
97
200
|
}
|
|
98
201
|
|
|
99
|
-
|
|
100
|
-
|
|
202
|
+
const scored = [];
|
|
203
|
+
for (const el of document.querySelectorAll(selector)) {
|
|
204
|
+
if (!visible(el)) continue;
|
|
205
|
+
const s = score(el);
|
|
206
|
+
if (s > 0) scored.push({ el, s });
|
|
207
|
+
}
|
|
208
|
+
if (!scored.length) return { found: false, candidates: [] };
|
|
209
|
+
scored.sort((a, b) => b.s - a.s);
|
|
101
210
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
text: (el.textContent || '').trim().slice(0, 200),
|
|
115
|
-
score,
|
|
116
|
-
href: el.href || '',
|
|
117
|
-
};
|
|
211
|
+
const candidates = scored.slice(0, 5).map(({ el, s }) => ({
|
|
212
|
+
tag: el.tagName.toLowerCase(),
|
|
213
|
+
text: String(el.innerText || el.value || el.textContent || '').trim().slice(0, 80),
|
|
214
|
+
href: el.href || '',
|
|
215
|
+
score: s,
|
|
216
|
+
}));
|
|
217
|
+
|
|
218
|
+
const top = scored[0];
|
|
219
|
+
// Too close to call: hand back the list instead of guessing. Clicking
|
|
220
|
+
// "Buy now" because of a heuristic is the worst failure mode here.
|
|
221
|
+
if (scored.length > 1 && scored[1].s >= top.s - 2 && top.s < 100) {
|
|
222
|
+
return { found: false, ambiguous: true, candidates };
|
|
118
223
|
}
|
|
224
|
+
|
|
225
|
+
const best = top.el;
|
|
226
|
+
try { best.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
|
|
227
|
+
const id = 'bwb' + Math.random().toString(36).slice(2, 9);
|
|
228
|
+
try { best.setAttribute(MARK, id); } catch {}
|
|
229
|
+
const r = best.getBoundingClientRect();
|
|
230
|
+
const cx = Math.round(r.x + r.width / 2);
|
|
231
|
+
const cy = Math.round(r.y + r.height / 2);
|
|
232
|
+
return {
|
|
233
|
+
found: true,
|
|
234
|
+
attr: MARK, id,
|
|
235
|
+
x: cx, y: cy,
|
|
236
|
+
width: r.width, height: r.height,
|
|
237
|
+
tag: best.tagName.toLowerCase(),
|
|
238
|
+
text: String(best.innerText || best.value || best.textContent || '').trim().slice(0, 200),
|
|
239
|
+
href: best.href || '',
|
|
240
|
+
score: top.s,
|
|
241
|
+
hit: (() => {
|
|
242
|
+
if (r.width <= 0 || r.height <= 0) return false;
|
|
243
|
+
const at = document.elementFromPoint(cx, cy);
|
|
244
|
+
return !!(at && (at === best || best.contains(at) || at.contains(best)));
|
|
245
|
+
})(),
|
|
246
|
+
};
|
|
119
247
|
})()`,
|
|
120
248
|
returnByValue: true,
|
|
121
249
|
});
|
|
122
250
|
return result?.value || { found: false };
|
|
123
251
|
}
|
|
124
252
|
|
|
253
|
+
/** Remove the temporary marker attribute from a located element. */
|
|
254
|
+
async function untag(Runtime, info) {
|
|
255
|
+
if (!info?.attr || !info?.id) return;
|
|
256
|
+
try {
|
|
257
|
+
await Runtime.evaluate({
|
|
258
|
+
expression: `document.querySelector('[${info.attr}="${info.id}"]')?.removeAttribute(${JSON.stringify(info.attr)})`,
|
|
259
|
+
});
|
|
260
|
+
} catch {}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Recover a capture group from the ORIGINAL instruction text, case intact.
|
|
264
|
+
* `match[0].length - match[1].length` arithmetic is wrong the moment a pattern
|
|
265
|
+
* has anything after the capture ("click the Pricing link" yielded "ng link").
|
|
266
|
+
* Uses the /d flag's match indices when available.
|
|
267
|
+
*/
|
|
268
|
+
function capture(orig, match, group = 1) {
|
|
269
|
+
const idx = match.indices?.[group];
|
|
270
|
+
if (idx) return orig.slice(match.index + idx[0], match.index + idx[1]);
|
|
271
|
+
return orig.slice(orig.length - match[group].length);
|
|
272
|
+
}
|
|
273
|
+
|
|
125
274
|
// ─── URL Normalizer ──────────────────────────────────────────────────────────
|
|
126
275
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
276
|
+
/**
|
|
277
|
+
* Keep the scheme the user gave, keep `www.`, keep path case, and accept
|
|
278
|
+
* localhost / IPs / ports. Returns null when the token is not a URL at all.
|
|
279
|
+
*/
|
|
280
|
+
export function normalizeUrl(text) {
|
|
281
|
+
let url = String(text || "").trim().replace(/[.,;:!?]+$/, "");
|
|
282
|
+
if (!url || /\s/.test(url)) return null;
|
|
283
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(url)) {
|
|
284
|
+
try {
|
|
285
|
+
const u = new URL(url);
|
|
286
|
+
return ["http:", "https:"].includes(u.protocol) ? u.href : null;
|
|
287
|
+
} catch {
|
|
138
288
|
return null;
|
|
139
289
|
}
|
|
140
290
|
}
|
|
141
|
-
|
|
291
|
+
url = url.replace(/[)\]}>.,;:!?'"]+$/, "");
|
|
292
|
+
const looksLikeHost =
|
|
293
|
+
/^[\w.-]+\.[a-z]{2,}(?:[/:?#][^\s]*)?$/i.test(url) ||
|
|
294
|
+
/^localhost(?::\d+)?(?:[/?#].*)?$/i.test(url) ||
|
|
295
|
+
/^\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?(?:[/?#].*)?$/.test(url);
|
|
296
|
+
if (!looksLikeHost) return null;
|
|
297
|
+
return `https://${url}`;
|
|
142
298
|
}
|
|
143
299
|
|
|
144
300
|
// ─── Execute Instruction ─────────────────────────────────────────────────────
|
|
@@ -154,15 +310,23 @@ function normalizeUrl(text) {
|
|
|
154
310
|
* "type X in Y" → Type X into input Y
|
|
155
311
|
* "get me X" / "extract X" → Find content by text
|
|
156
312
|
* "scroll down" / "scroll up" → Scroll page
|
|
157
|
-
* "what
|
|
313
|
+
* "what's on this page" → Page summary
|
|
158
314
|
*
|
|
159
|
-
* @param {
|
|
160
|
-
* @param {string} instruction
|
|
161
|
-
* @
|
|
315
|
+
* @param {object} protocol CDP protocol for the active tab
|
|
316
|
+
* @param {string} instruction
|
|
317
|
+
* @param {object} [opts] { navigate, confirmDestructive, force }
|
|
318
|
+
* @returns {Promise<object>}
|
|
162
319
|
*/
|
|
163
|
-
export async function executeInstruction(protocol, instruction) {
|
|
320
|
+
export async function executeInstruction(protocol, instruction, opts = {}) {
|
|
164
321
|
const { Runtime, Page, Input } = protocol;
|
|
165
|
-
const
|
|
322
|
+
const orig = String(instruction || "").trim();
|
|
323
|
+
// Matching is case-insensitive; the ORIGINAL text is what we type/navigate.
|
|
324
|
+
const lower = orig.toLowerCase();
|
|
325
|
+
const navigate = opts.navigate || defaultNavigate(Page, opts);
|
|
326
|
+
const confirmDestructive = opts.confirmDestructive !== false;
|
|
327
|
+
const force = opts.force === true;
|
|
328
|
+
// Settle delay after a click / search / navigation before reading the page.
|
|
329
|
+
const settle = (ms) => new Promise((r) => setTimeout(r, opts.settleMs ?? ms));
|
|
166
330
|
|
|
167
331
|
// ─── Pattern: "go to URL" / "navigate to URL" / "open URL" ──────────────
|
|
168
332
|
// Capture only the URL token — stop at natural instruction boundaries so
|
|
@@ -170,160 +334,154 @@ export async function executeInstruction(protocol, instruction) {
|
|
|
170
334
|
// the whole sentence into the URL.
|
|
171
335
|
const navMatch = lower.match(/^(?:go to|navigate to|open|visit|take me to)\s+([^\s,]+(?:\.[^\s,]+)*)(?=\s|$)/);
|
|
172
336
|
if (navMatch) {
|
|
173
|
-
|
|
337
|
+
// The URL keeps its case: `GitHub.com/Krish/Repo` must not become `krish/repo`.
|
|
338
|
+
const rawUrl = orig.match(/^(?:go to|navigate to|open|visit|take me to)\s+(\S+)/i)?.[1] || navMatch[1];
|
|
339
|
+
// An explicit non-http scheme is a refusal, never a silent fall-through to
|
|
340
|
+
// "treat it as a search query".
|
|
341
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {
|
|
342
|
+
return {
|
|
343
|
+
action: "navigate_error",
|
|
344
|
+
url: rawUrl,
|
|
345
|
+
error: `Refusing to navigate to ${rawUrl.split(":")[0]}: — only http and https are allowed.`,
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
const url = normalizeUrl(rawUrl);
|
|
174
349
|
if (url) {
|
|
175
|
-
|
|
176
|
-
|
|
350
|
+
try {
|
|
351
|
+
assertNavigable(url);
|
|
352
|
+
} catch (err) {
|
|
353
|
+
return { action: "navigate_error", url, error: err.message };
|
|
354
|
+
}
|
|
355
|
+
const nav = await navigate(url);
|
|
177
356
|
const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
|
|
178
|
-
const { result: textResult } = await Runtime.evaluate({
|
|
357
|
+
const { result: textResult } = await Runtime.evaluate({
|
|
358
|
+
expression: `String(document.body?.innerText || '').trim().slice(0, ${PREVIEW_CHARS * 2})`,
|
|
359
|
+
});
|
|
179
360
|
return {
|
|
180
361
|
action: "navigate",
|
|
181
|
-
url,
|
|
362
|
+
url: nav?.url || url,
|
|
182
363
|
title: titleResult?.value || "",
|
|
183
|
-
preview: (textResult?.value || "").slice(0,
|
|
364
|
+
preview: (textResult?.value || "").slice(0, PREVIEW_CHARS),
|
|
365
|
+
...(nav?.timedOut ? { timedOut: true } : {}),
|
|
184
366
|
};
|
|
185
367
|
}
|
|
186
368
|
// If not a URL, treat as search
|
|
187
369
|
}
|
|
188
370
|
|
|
189
|
-
// ─── Pattern: "go to URL" that's a search fallback ──────────────────────
|
|
190
|
-
// If navMatch but URL was invalid (normalizeUrl returned null), fall through to search
|
|
191
|
-
|
|
192
371
|
// ─── Pattern: "search for X" / "search X" ───────────────────────────────
|
|
193
372
|
// Anchored to string start: unanchored, "fill search with X" would be
|
|
194
373
|
// hijacked by this pattern before fill could handle it.
|
|
195
|
-
const searchMatch = lower.match(/^search\s+(?:for\s+)?(.+)/);
|
|
374
|
+
const searchMatch = lower.match(/^search\s+(?:for\s+)?(.+)/d);
|
|
196
375
|
if (searchMatch) {
|
|
197
|
-
const query = searchMatch
|
|
376
|
+
const query = capture(orig, searchMatch);
|
|
198
377
|
let inputInfo = await findInput(Runtime, "search");
|
|
199
|
-
if (!inputInfo.found
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
const el = document.querySelector('input[type=search], input[name=q], input[name=search], [role=search] input, form[role=search] input');
|
|
207
|
-
if (el) return JSON.stringify({ found: true, selector: 'input[type=search]' });
|
|
208
|
-
return JSON.stringify({ found: false });
|
|
209
|
-
})()`,
|
|
210
|
-
returnByValue: true,
|
|
211
|
-
});
|
|
212
|
-
inputInfo = JSON.parse(result?.value || "{}");
|
|
213
|
-
}
|
|
214
|
-
if (inputInfo.found && inputInfo.selector) {
|
|
215
|
-
await Runtime.evaluate({
|
|
216
|
-
expression: `document.querySelector(${JSON.stringify(inputInfo.selector)})?.focus()`,
|
|
217
|
-
});
|
|
218
|
-
await Input.insertText({ text: query });
|
|
219
|
-
await new Promise(r => setTimeout(r, 300));
|
|
220
|
-
// Try Enter key
|
|
221
|
-
await Input.dispatchKeyEvent({ type: "keyDown", key: "Enter", windowsVirtualKeyCode: 13 });
|
|
222
|
-
await Input.dispatchKeyEvent({ type: "keyUp", key: "Enter", windowsVirtualKeyCode: 13 });
|
|
223
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
378
|
+
if (!inputInfo.found) inputInfo = await findInput(Runtime, "query");
|
|
379
|
+
if (inputInfo.found) {
|
|
380
|
+
const cleared = await focusInputAt(Runtime, inputInfo.index);
|
|
381
|
+
if (cleared) {
|
|
382
|
+
await Input.insertText({ text: query });
|
|
383
|
+
await pressEnter(Input);
|
|
384
|
+
await settle(2000);
|
|
224
385
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
386
|
+
const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
|
|
387
|
+
const { result: urlResult } = await Runtime.evaluate({ expression: "location.href" });
|
|
388
|
+
const { result: textResult } = await Runtime.evaluate({
|
|
389
|
+
expression: `String(document.body?.innerText || '').trim().slice(0, ${PREVIEW_CHARS * 4})`,
|
|
390
|
+
});
|
|
391
|
+
return {
|
|
392
|
+
action: "search",
|
|
393
|
+
query,
|
|
394
|
+
title: titleResult?.value || "",
|
|
395
|
+
url: urlResult?.value || "",
|
|
396
|
+
preview: (textResult?.value || "").slice(0, PREVIEW_CHARS),
|
|
397
|
+
};
|
|
398
|
+
}
|
|
235
399
|
}
|
|
236
|
-
|
|
400
|
+
// Never type into an unrelated field: report what was on the page instead.
|
|
401
|
+
return {
|
|
402
|
+
action: "search_error",
|
|
403
|
+
query,
|
|
404
|
+
error: "No search input found on this page",
|
|
405
|
+
candidates: inputInfo.candidates || [],
|
|
406
|
+
};
|
|
237
407
|
}
|
|
238
408
|
|
|
239
409
|
// ─── Pattern: "click X" / "click the X button" / "click on X" ──────────
|
|
240
|
-
const clickMatch = lower.match(/click\s+(?:the\s+|on\s+)?(.+?)(?:\s+button|\s+link|\s+element)?$/);
|
|
410
|
+
const clickMatch = lower.match(/click\s+(?:the\s+|on\s+)?(.+?)(?:\s+button|\s+link|\s+element)?$/d);
|
|
241
411
|
if (clickMatch) {
|
|
242
|
-
const target = clickMatch
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
})()`,
|
|
256
|
-
});
|
|
257
|
-
const pos = result?.value ? JSON.parse(result.value) : null;
|
|
258
|
-
if (pos) {
|
|
259
|
-
await Input.dispatchMouseEvent({ type: "mousePressed", x: pos.x, y: pos.y, button: "left", clickCount: 1 });
|
|
260
|
-
await Input.dispatchMouseEvent({ type: "mouseReleased", x: pos.x, y: pos.y, button: "left", clickCount: 1 });
|
|
261
|
-
await new Promise(r => setTimeout(r, 1500));
|
|
262
|
-
|
|
263
|
-
const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
|
|
264
|
-
return { action: "click", target, clicked: el.text, title: titleResult?.value };
|
|
412
|
+
const target = capture(orig, clickMatch).replace(/^\s*(?:the|on)\s+/i, "");
|
|
413
|
+
const el = await findByText(Runtime, target, CLICKABLE_SELECTOR);
|
|
414
|
+
if (el.found) {
|
|
415
|
+
const destructive = confirmDestructive && DESTRUCTIVE_RE.test(target);
|
|
416
|
+
if (destructive && !force) {
|
|
417
|
+
return {
|
|
418
|
+
action: "needs_confirmation",
|
|
419
|
+
target,
|
|
420
|
+
matchedText: el.text,
|
|
421
|
+
tag: el.tag,
|
|
422
|
+
href: el.href || "",
|
|
423
|
+
error: `Refusing to click "${el.text.slice(0, 60)}" without confirmation — it looks destructive. Re-run with force:true if that is intended.`,
|
|
424
|
+
};
|
|
265
425
|
}
|
|
426
|
+
const { x, y } = el;
|
|
427
|
+
await Input.dispatchMouseEvent({ type: "mouseMoved", x, y, button: "none" });
|
|
428
|
+
await Input.dispatchMouseEvent({ type: "mousePressed", x, y, button: "left", clickCount: 1 });
|
|
429
|
+
await Input.dispatchMouseEvent({ type: "mouseReleased", x, y, button: "left", clickCount: 1 });
|
|
430
|
+
await settle(1500);
|
|
431
|
+
await untag(Runtime, el);
|
|
432
|
+
|
|
433
|
+
const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
|
|
434
|
+
const { result: urlResult } = await Runtime.evaluate({ expression: "location.href" });
|
|
435
|
+
return {
|
|
436
|
+
action: "click",
|
|
437
|
+
target,
|
|
438
|
+
clicked: el.text,
|
|
439
|
+
tag: el.tag,
|
|
440
|
+
...(el.hit === false ? { warning: "Element was off-screen or covered; the click may have missed." } : {}),
|
|
441
|
+
title: titleResult?.value || "",
|
|
442
|
+
url: urlResult?.value || "",
|
|
443
|
+
};
|
|
266
444
|
}
|
|
267
|
-
return {
|
|
445
|
+
return {
|
|
446
|
+
action: "click_error",
|
|
447
|
+
target,
|
|
448
|
+
error: el.ambiguous
|
|
449
|
+
? `Ambiguous — several elements match "${target}"`
|
|
450
|
+
: `Could not find element matching "${target}"`,
|
|
451
|
+
...(el.candidates?.length ? { candidates: el.candidates } : {}),
|
|
452
|
+
};
|
|
268
453
|
}
|
|
269
454
|
|
|
270
455
|
// ─── Pattern: "fill X with Y" / "type Y in X" / "enter Y into X" ──────
|
|
271
456
|
// NOTE: "fill X with Y" puts the VALUE in X ("fill email with foo@bar.com"),
|
|
272
457
|
// while "type Y in X" puts the TEXT in X ("type hello in search").
|
|
273
|
-
const fillWithMatch =
|
|
458
|
+
const fillWithMatch = orig.match(/fill\s+(.+?)\s+with\s+(.+)$/is);
|
|
274
459
|
if (fillWithMatch) {
|
|
275
|
-
|
|
276
|
-
const text = fillWithMatch[2];
|
|
277
|
-
const inputInfo = await findInput(Runtime, target);
|
|
278
|
-
if (inputInfo.found && inputInfo.selector) {
|
|
279
|
-
await Runtime.evaluate({
|
|
280
|
-
expression: `document.querySelector(${JSON.stringify(inputInfo.selector)})?.focus()`,
|
|
281
|
-
});
|
|
282
|
-
await Input.insertText({ text });
|
|
283
|
-
await new Promise(r => setTimeout(r, 200));
|
|
284
|
-
return { action: "fill", target, text, inputTag: inputInfo.tag };
|
|
285
|
-
}
|
|
286
|
-
return { action: "fill_error", target, error: `Could not find input matching "${target}"` };
|
|
460
|
+
return fillField(Runtime, Input, fillWithMatch[1], fillWithMatch[2], opts);
|
|
287
461
|
}
|
|
288
462
|
|
|
289
|
-
const fillMatch =
|
|
463
|
+
const fillMatch = orig.match(/(?:type|enter)\s+(.+?)\s+(?:in|into)\s+(.+)/is);
|
|
290
464
|
if (fillMatch) {
|
|
291
|
-
|
|
292
|
-
const target = fillMatch[2];
|
|
293
|
-
const inputInfo = await findInput(Runtime, target);
|
|
294
|
-
if (inputInfo.found && inputInfo.selector) {
|
|
295
|
-
await Runtime.evaluate({
|
|
296
|
-
expression: `document.querySelector(${JSON.stringify(inputInfo.selector)})?.focus()`,
|
|
297
|
-
});
|
|
298
|
-
await Input.insertText({ text });
|
|
299
|
-
await new Promise(r => setTimeout(r, 200));
|
|
300
|
-
return { action: "fill", target, text, inputTag: inputInfo.tag };
|
|
301
|
-
}
|
|
302
|
-
return { action: "fill_error", target, error: `Could not find input matching "${target}"` };
|
|
465
|
+
return fillField(Runtime, Input, fillMatch[2], fillMatch[1], opts);
|
|
303
466
|
}
|
|
304
467
|
|
|
305
|
-
// ─── Pattern: "get me X" / "extract X" / "find X" / "what
|
|
306
|
-
const extractMatch = lower.match(
|
|
468
|
+
// ─── Pattern: "get me X" / "extract X" / "find X" / "what's on this page" ──
|
|
469
|
+
const extractMatch = lower.match(
|
|
470
|
+
/^(?:get me|extract|find|show me|tell me about|what(?:'s| is| are))\s+(.+)/d);
|
|
307
471
|
if (extractMatch) {
|
|
308
|
-
const target = extractMatch
|
|
309
|
-
|
|
472
|
+
const target = capture(orig, extractMatch);
|
|
473
|
+
// "what's on this page" / "what is here" is a page summary, not a search.
|
|
474
|
+
if (/^(on\s+)?(this|the)\s+(page|site|website)$/i.test(target.trim()) || /^here$/i.test(target.trim())) {
|
|
475
|
+
return pageSummary(Runtime, target);
|
|
476
|
+
}
|
|
477
|
+
const el = await findByText(Runtime, target, TEXT_CONTAINER_SELECTOR);
|
|
310
478
|
if (el.found && el.text) {
|
|
311
479
|
return { action: "extract", target, content: el.text };
|
|
312
480
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
returnByValue: true,
|
|
318
|
-
});
|
|
319
|
-
const body = JSON.parse(bodyResult?.value || "{}");
|
|
320
|
-
return {
|
|
321
|
-
action: "extract",
|
|
322
|
-
target,
|
|
323
|
-
pageTitle: titleResult?.value || "",
|
|
324
|
-
headings: body.headings || [],
|
|
325
|
-
preview: (body.text || "").slice(0, 500),
|
|
326
|
-
};
|
|
481
|
+
if (el.ambiguous) {
|
|
482
|
+
return { action: "extract_error", target, error: `Ambiguous — several elements match "${target}"`, candidates: el.candidates };
|
|
483
|
+
}
|
|
484
|
+
return pageSummary(Runtime, target);
|
|
327
485
|
}
|
|
328
486
|
|
|
329
487
|
// ─── Pattern: "scroll down" / "scroll up" ──────────────────────────────
|
|
@@ -337,6 +495,99 @@ export async function executeInstruction(protocol, instruction) {
|
|
|
337
495
|
}
|
|
338
496
|
|
|
339
497
|
// ─── Fallback: return page info + help ─────────────────────────────────
|
|
498
|
+
return unknown(protocol, orig);
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// ─── Instruction steps ───────────────────────────────────────────────────────
|
|
502
|
+
|
|
503
|
+
function inputSelectorExpression() {
|
|
504
|
+
return JSON.stringify(
|
|
505
|
+
'input:not([type=hidden]):not([type=submit]):not([type=button]), textarea, [contenteditable=true], select'
|
|
506
|
+
);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
function defaultNavigate(Page, opts) {
|
|
510
|
+
return async (url) => {
|
|
511
|
+
await Page.navigate({ url });
|
|
512
|
+
await new Promise((r) => setTimeout(r, Math.min(opts.settleMs ?? 2000, 5000)));
|
|
513
|
+
return { url };
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
async function pressEnter(Input) {
|
|
518
|
+
const base = { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13 };
|
|
519
|
+
await Input.dispatchKeyEvent({ type: "keyDown", ...base, text: "\r" });
|
|
520
|
+
await Input.dispatchKeyEvent({ type: "keyUp", ...base });
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* Focus the input at `index` in the candidate list AND clear it. Focus alone
|
|
525
|
+
* appends to whatever was already there; `Ctrl+A` via raw key events without
|
|
526
|
+
* `modifiers` does not select-all either.
|
|
527
|
+
*/
|
|
528
|
+
async function focusInputAt(Runtime, index) {
|
|
529
|
+
const { result } = await Runtime.evaluate({
|
|
530
|
+
expression: `(() => {
|
|
531
|
+
const el = document.querySelectorAll(${inputSelectorExpression()})[${index}];
|
|
532
|
+
if (!el) return false;
|
|
533
|
+
el.focus();
|
|
534
|
+
try {
|
|
535
|
+
if (el.isContentEditable) { document.execCommand('selectAll'); }
|
|
536
|
+
else if (typeof el.select === 'function') { el.select(); }
|
|
537
|
+
else if ('value' in el) { el.value = ''; }
|
|
538
|
+
} catch {}
|
|
539
|
+
return true;
|
|
540
|
+
})()`,
|
|
541
|
+
returnByValue: true,
|
|
542
|
+
});
|
|
543
|
+
return result?.value === true;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
async function fillField(Runtime, Input, target, text, opts = {}) {
|
|
547
|
+
const inputInfo = await findInput(Runtime, target);
|
|
548
|
+
if (!inputInfo.found) {
|
|
549
|
+
return {
|
|
550
|
+
action: "fill_error",
|
|
551
|
+
target,
|
|
552
|
+
error: `Could not find input matching "${target}"`,
|
|
553
|
+
...(inputInfo.candidates?.length ? { candidates: inputInfo.candidates } : {}),
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
// Clear first: focus alone APPENDS to whatever was already there.
|
|
557
|
+
const cleared = await focusInputAt(Runtime, inputInfo.index);
|
|
558
|
+
if (!cleared) {
|
|
559
|
+
return { action: "fill_error", target, error: `Could not focus input matching "${target}"` };
|
|
560
|
+
}
|
|
561
|
+
await new Promise((r) => setTimeout(r, opts.settleMs === 0 ? 0 : 100));
|
|
562
|
+
await Input.insertText({ text });
|
|
563
|
+
await new Promise((r) => setTimeout(r, opts.settleMs === 0 ? 0 : 200));
|
|
564
|
+
const echo = redactIfSecret(target, text);
|
|
565
|
+
return { action: "fill", target, inputTag: inputInfo.tag, ...echo };
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
async function pageSummary(Runtime, target) {
|
|
569
|
+
const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
|
|
570
|
+
const { result } = await Runtime.evaluate({
|
|
571
|
+
expression: `JSON.stringify({
|
|
572
|
+
text: String(document.body?.innerText || '').trim().slice(0, ${PREVIEW_CHARS * 4}),
|
|
573
|
+
headings: Array.from(document.querySelectorAll('h1,h2,h3')).map(h => h.textContent.trim()).slice(0, 10),
|
|
574
|
+
links: Array.from(document.querySelectorAll('a[href]')).slice(0, 15).map(a => ({ text: a.textContent.trim().slice(0, 60), href: a.href })),
|
|
575
|
+
})`,
|
|
576
|
+
returnByValue: true,
|
|
577
|
+
});
|
|
578
|
+
const body = parseJson(result?.value) || {};
|
|
579
|
+
return {
|
|
580
|
+
action: "extract",
|
|
581
|
+
target,
|
|
582
|
+
pageTitle: titleResult?.value || "",
|
|
583
|
+
headings: body.headings || [],
|
|
584
|
+
links: body.links || [],
|
|
585
|
+
preview: (body.text || "").slice(0, PREVIEW_CHARS),
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
async function unknown(protocol, instruction) {
|
|
590
|
+
const { Runtime } = protocol;
|
|
340
591
|
const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
|
|
341
592
|
const { result: urlResult } = await Runtime.evaluate({ expression: "location.href" });
|
|
342
593
|
return {
|
|
@@ -347,3 +598,11 @@ export async function executeInstruction(protocol, instruction) {
|
|
|
347
598
|
tip: "Try: 'search for laptops', 'click the login button', 'go to google.com', 'fill email with test@example.com', 'extract prices', 'scroll down'",
|
|
348
599
|
};
|
|
349
600
|
}
|
|
601
|
+
|
|
602
|
+
function parseJson(value) {
|
|
603
|
+
if (!value) return null;
|
|
604
|
+
if (typeof value === "object") return value;
|
|
605
|
+
try { return JSON.parse(value); } catch { return null; }
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
export { UrlPolicyError };
|