bwb-browser 4.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/act.mjs CHANGED
@@ -6,139 +6,295 @@
6
6
  * form filling, content extraction, and scrolling.
7
7
  *
8
8
  * The goal: ONE tool call does what would normally take 5-10 tool calls.
9
- * This is the "Computer Use for MCP" moment.
9
+ *
10
+ * Ground rules (learned the hard way — see CHANGELOG 4.1.0):
11
+ * 1. Captured text is NEVER lowercased. Matching is case-insensitive, but a
12
+ * password typed as `mys3cretpass` is silently wrong.
13
+ * 2. Elements are located, measured and tagged inside ONE Runtime.evaluate.
14
+ * A "found element" object is never round-tripped through a CSS selector —
15
+ * `a` matches every link, so `querySelector('a')` returns the first one.
16
+ * 3. When matching is ambiguous, return candidates instead of guessing.
17
+ * Clicking "Buy now" because of a heuristic is the worst failure mode here.
10
18
  */
11
19
 
20
+ import { assertNavigable, UrlPolicyError } from "./urlpolicy.mjs";
21
+
22
+ const MARK_ATTR = "data-bwb-target";
23
+ const PREVIEW_CHARS = 500;
24
+
25
+ // Text that should never be clicked without an explicit confirmation.
26
+ const DESTRUCTIVE_RE =
27
+ /\b(delete|remove|drop|trash|erase|wipe|buy|purchase|pay|checkout|order|confirm|submit|send|publish|transfer|withdraw|cancel|close account|deactivate|unsubscribe|revoke|terminate|uninstall)\b/i;
28
+
29
+ /** Result text for a filled field, with secrets redacted (never echoed back). */
30
+ export function redactIfSecret(target, text) {
31
+ if (/pass|secret|token|otp|pin|cvv|card|ssn|security.?code/i.test(String(target || ""))) {
32
+ return { length: [...String(text)].length, redacted: true };
33
+ }
34
+ return { text, redacted: false };
35
+ }
36
+
37
+ // ─── In-page helpers (injected into the page expression) ─────────────────────
38
+ // Kept as strings so a single Runtime.evaluate can locate AND measure.
39
+
12
40
  // ─── Smart Input Finder ──────────────────────────────────────────────────────
13
41
 
14
42
  async function findInput(Runtime, labelText) {
43
+ const query = String(labelText || "").trim().toLowerCase();
15
44
  const { result } = await Runtime.evaluate({
16
45
  expression: `(() => {
17
- const inputs = document.querySelectorAll('input:not([type=hidden]):not([type=submit]):not([type=button]), textarea, [contenteditable=true]');
18
- const query = ${JSON.stringify((labelText || '').toLowerCase())};
19
- if (!query) return { found: true, selector: inputs[0] ? buildSelector(inputs[0]) : null };
20
-
21
- for (const el of inputs) {
22
- const p = (el.placeholder || '').toLowerCase();
23
- const aria = (el.getAttribute('aria-label') || '').toLowerCase();
24
- const name = (el.name || '').toLowerCase();
25
- const id = (el.id || '').toLowerCase();
26
- const type = (el.type || '').toLowerCase();
46
+ const query = ${JSON.stringify(query)};
47
+ const inputs = Array.from(document.querySelectorAll(
48
+ 'input:not([type=hidden]):not([type=submit]):not([type=button]), textarea, [contenteditable=true], select'
49
+ ));
50
+ if (!inputs.length) return { found: false, candidates: [] };
51
+
52
+ const describe = (el) => {
27
53
  let label = '';
28
54
  if (el.id) {
29
- const lbl = document.querySelector('label[for="' + el.id + '"]');
30
- if (lbl) label = lbl.textContent.toLowerCase();
55
+ const lbl = document.querySelector('label[for="' + CSS.escape(el.id) + '"]');
56
+ if (lbl) label = lbl.textContent || '';
31
57
  }
32
- const parent = el.closest('label');
33
- if (parent) label += ' ' + parent.textContent.toLowerCase();
58
+ const parent = el.closest && el.closest('label');
59
+ if (parent) label += ' ' + (parent.textContent || '');
60
+ return {
61
+ tag: el.tagName.toLowerCase(),
62
+ type: (el.type || '').toLowerCase(),
63
+ id: el.id || '',
64
+ name: el.name || '',
65
+ placeholder: el.placeholder || el.getAttribute('placeholder') || '',
66
+ aria: el.getAttribute('aria-label') || '',
67
+ label: label.trim().slice(0, 120),
68
+ visible: isVisible(el),
69
+ };
70
+ };
34
71
 
35
- const searchText = p + ' ' + aria + ' ' + label + ' ' + name + ' ' + id + ' ' + type;
36
- if (searchText.includes(query)) {
37
- return { found: true, selector: buildSelector(el), tag: el.tagName, type: el.type };
72
+ function isVisible(el) {
73
+ const r = el.getBoundingClientRect();
74
+ if (r.width > 0 && r.height > 0) return true;
75
+ const cs = window.getComputedStyle ? window.getComputedStyle(el) : null;
76
+ return !!(cs && cs.display !== 'none' && cs.visibility !== 'hidden');
77
+ }
78
+
79
+ const meta = inputs.map(describe);
80
+ if (!query) {
81
+ // No target named: only an unambiguous single input is safe to pick.
82
+ const visibleIdx = meta.map((m, i) => (m.visible ? i : -1)).filter((i) => i >= 0);
83
+ if (visibleIdx.length === 1) {
84
+ return { found: true, index: visibleIdx[0], tag: meta[visibleIdx[0]].tag, type: meta[visibleIdx[0]].type };
38
85
  }
86
+ return {
87
+ found: false,
88
+ candidates: meta.map((m) => ({
89
+ tag: m.tag, type: m.type, id: m.id, name: m.name, placeholder: m.placeholder,
90
+ })),
91
+ };
39
92
  }
40
- // Fallback: return first input if nothing matches
41
- if (inputs[0]) return { found: true, selector: buildSelector(inputs[0]), tag: inputs[0].tagName };
42
- return { found: false };
43
-
44
- function buildSelector(el) {
45
- if (el.id) return '#' + CSS.escape(el.id);
46
- if (el.name) return el.tagName.toLowerCase() + '[name=' + JSON.stringify(el.name) + ']';
47
- if (el.placeholder) return el.tagName.toLowerCase() + '[placeholder=' + JSON.stringify(el.placeholder) + ']';
48
- if (el.getAttribute('aria-label')) return el.tagName.toLowerCase() + '[aria-label=' + JSON.stringify(el.getAttribute('aria-label')) + ']';
49
- return el.tagName.toLowerCase();
93
+
94
+ // Rank candidates: an input whose own metadata matches the requested
95
+ // field wins; a generic fallback (first input on the page) is never used.
96
+ let best = -1, bestScore = 0;
97
+ meta.forEach((m, i) => {
98
+ let score = 0;
99
+ const name = m.name.toLowerCase(), id = m.id.toLowerCase();
100
+ const ph = m.placeholder.toLowerCase(), aria = m.aria.toLowerCase();
101
+ const lbl = m.label.toLowerCase(), type = m.type;
102
+ if (name === query) score = 100;
103
+ else if (id === query) score = 95;
104
+ else if (ph === query) score = 90;
105
+ else if (aria === query) score = 85;
106
+ else if (lbl === query) score = 85;
107
+ else if (name.includes(query)) score = 70;
108
+ else if (id.includes(query)) score = 65;
109
+ else if (aria.includes(query)) score = 60;
110
+ else if (ph.includes(query)) score = 60;
111
+ else if (lbl.includes(query)) score = 55;
112
+ else if (type === query) score = 40;
113
+ if (score && !m.visible) score -= 5;
114
+ if (score > bestScore) { bestScore = score; best = i; }
115
+ });
116
+
117
+ if (best < 0) {
118
+ return { found: false, candidates: meta.map((m) => ({
119
+ tag: m.tag, type: m.type, id: m.id, name: m.name,
120
+ placeholder: m.placeholder, label: m.label,
121
+ })) };
50
122
  }
123
+ return { found: true, index: best, tag: meta[best].tag, type: meta[best].type };
51
124
  })()`,
52
125
  returnByValue: true,
53
126
  });
54
127
  return result?.value || { found: false };
55
128
  }
56
129
 
57
- // ─── Smart Element Finder by Text ────────────────────────────────────────────
130
+ // ─── Element Finder by Text ──────────────────────────────────────────────────
131
+ // One page-side routine for every "find something by its text" case. Scoring,
132
+ // filtering, scrolling, measuring and tagging all happen inside ONE
133
+ // Runtime.evaluate against ONE element. A previous version returned a CSS
134
+ // selector and re-queried it, so `a` (every link) resolved to the first link:
135
+ // "click the Pricing link" clicked "Home".
58
136
 
59
- async function findElementByText(Runtime, text, tag = "*") {
137
+ const CLICKABLE_SELECTOR =
138
+ 'a[href], button, [role=button], input[type=submit], input[type=button], ' +
139
+ '[onclick], summary, label, [role=link], [role=menuitem]';
140
+
141
+ // Containers worth extracting text from — deliberately excludes html/body so a
142
+ // fuzzy match can never return the entire document.
143
+ const TEXT_CONTAINER_SELECTOR =
144
+ 'h1, h2, h3, h4, h5, h6, p, li, td, th, dt, dd, blockquote, pre, code, ' +
145
+ 'section, article, main, div, span, a, summary, label, table, ul, ol, form, figcaption';
146
+
147
+ async function findByText(Runtime, text, selector) {
148
+ const query = String(text || "").trim().toLowerCase();
60
149
  const { result } = await Runtime.evaluate({
61
150
  expression: `(() => {
62
- const elements = document.querySelectorAll('${tag}');
63
- const query = ${JSON.stringify(text.toLowerCase())};
64
- let best = null, bestScore = 0;
65
-
66
- for (const el of elements) {
67
- const t = (el.textContent || '').trim().toLowerCase();
68
- const aria = (el.getAttribute('aria-label') || '').toLowerCase();
69
- const val = (el.getAttribute('value') || '').toLowerCase();
70
- const title = (el.title || '').toLowerCase();
71
-
72
- // Exact match is best
73
- if (t === query) return makeResult(el, 100);
74
- // Contains match
75
- if (t.includes(query) && query.length > 2) return makeResult(el, 90);
76
- // Starts with
77
- if (t.startsWith(query)) return makeResult(el, 80);
78
- // aria-label match
79
- if (aria.includes(query)) return makeResult(el, 70);
80
- // title match
81
- if (title.includes(query)) return makeResult(el, 60);
82
- // value match
83
- if (val.includes(query)) return makeResult(el, 50);
84
-
85
- // Fuzzy: word-in-text match
86
- const words = t.split(/\\s+/);
87
- const qWords = query.split(/\\s+/);
88
- let score = 0;
89
- for (const qw of qWords) {
90
- if (qw.length < 3) continue;
91
- if (words.some(w => w.includes(qw))) score += 10;
151
+ const query = ${JSON.stringify(query)};
152
+ const selector = ${JSON.stringify(selector)};
153
+ const MARK = ${JSON.stringify(MARK_ATTR)};
154
+ const SKIP = new Set(['HTML','BODY','SCRIPT','STYLE','HEAD','META','LINK','NOSCRIPT','TEMPLATE']);
155
+
156
+ const visible = (el) => {
157
+ if (SKIP.has(el.tagName)) return false;
158
+ const r = el.getBoundingClientRect();
159
+ if (r.width > 0 && r.height > 0) return true;
160
+ const cs = window.getComputedStyle ? window.getComputedStyle(el) : null;
161
+ return !!(cs && cs.display !== 'none' && cs.visibility !== 'hidden');
162
+ };
163
+ const ownText = (el) => {
164
+ let t = '';
165
+ for (const n of el.childNodes) if (n.nodeType === 3) t += n.nodeValue;
166
+ return t.trim();
167
+ };
168
+ const signals = (el) => [
169
+ ownText(el),
170
+ el.getAttribute && el.getAttribute('aria-label'),
171
+ el.value, el.title, el.alt,
172
+ (el.textContent || '').trim(),
173
+ ].filter((s) => s !== undefined && s !== null).map((s) => String(s).trim().toLowerCase());
174
+
175
+ function score(el) {
176
+ const full = (el.textContent || '').trim();
177
+ let best = 0;
178
+ for (const s of signals(el)) {
179
+ let v = 0;
180
+ if (s === query) v = 100; // exact
181
+ else if (s.startsWith(query)) v = 84; // starts with
182
+ else if (query.length >= 3 && s.includes(query)) v = 66; // contains
183
+ if (v > best) best = v;
184
+ }
185
+ if (best === 0 && query.length >= 3) {
186
+ const words = full.toLowerCase().split(/\\s+/);
187
+ const qw = query.split(/\\s+/).filter((x) => x.length >= 3);
188
+ let s = 0;
189
+ for (const x of qw) if (words.some((w) => w.includes(x))) s += 8;
190
+ best = s;
92
191
  }
93
- if (score > bestScore) {
94
- bestScore = score;
95
- best = el;
192
+ if (best > 0 && best < 100) {
193
+ // Prefer the SMALLEST, most specific element holding the text:
194
+ // a page-sized container must never outrank the paragraph with the
195
+ // answer, and a leaf beats an identical wrapper.
196
+ best += el.querySelector('*') ? 0 : 4;
197
+ best -= Math.min(24, Math.max(0, full.length - query.length) / 40 | 0);
96
198
  }
199
+ return best;
97
200
  }
98
201
 
99
- if (best) return makeResult(best, bestScore);
100
- return { found: false };
202
+ const scored = [];
203
+ for (const el of document.querySelectorAll(selector)) {
204
+ if (!visible(el)) continue;
205
+ const s = score(el);
206
+ if (s > 0) scored.push({ el, s });
207
+ }
208
+ if (!scored.length) return { found: false, candidates: [] };
209
+ scored.sort((a, b) => b.s - a.s);
101
210
 
102
- function makeResult(el, score) {
103
- const tag = el.tagName.toLowerCase();
104
- let selector = tag;
105
- if (el.id) selector = '#' + CSS.escape(el.id);
106
- else if (el.getAttribute('data-testid')) selector = '[' + JSON.stringify(el.getAttribute('data-testid')) + ']';
107
- else if (el.className && typeof el.className === 'string') {
108
- selector = tag + '.' + el.className.split(' ').filter(Boolean).map(c => CSS.escape(c)).join('.');
109
- }
110
- return {
111
- found: true,
112
- selector,
113
- tag,
114
- text: (el.textContent || '').trim().slice(0, 200),
115
- score,
116
- href: el.href || '',
117
- };
211
+ const candidates = scored.slice(0, 5).map(({ el, s }) => ({
212
+ tag: el.tagName.toLowerCase(),
213
+ text: String(el.innerText || el.value || el.textContent || '').trim().slice(0, 80),
214
+ href: el.href || '',
215
+ score: s,
216
+ }));
217
+
218
+ const top = scored[0];
219
+ // Too close to call: hand back the list instead of guessing. Clicking
220
+ // "Buy now" because of a heuristic is the worst failure mode here.
221
+ if (scored.length > 1 && scored[1].s >= top.s - 2 && top.s < 100) {
222
+ return { found: false, ambiguous: true, candidates };
118
223
  }
224
+
225
+ const best = top.el;
226
+ try { best.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
227
+ const id = 'bwb' + Math.random().toString(36).slice(2, 9);
228
+ try { best.setAttribute(MARK, id); } catch {}
229
+ const r = best.getBoundingClientRect();
230
+ const cx = Math.round(r.x + r.width / 2);
231
+ const cy = Math.round(r.y + r.height / 2);
232
+ return {
233
+ found: true,
234
+ attr: MARK, id,
235
+ x: cx, y: cy,
236
+ width: r.width, height: r.height,
237
+ tag: best.tagName.toLowerCase(),
238
+ text: String(best.innerText || best.value || best.textContent || '').trim().slice(0, 200),
239
+ href: best.href || '',
240
+ score: top.s,
241
+ hit: (() => {
242
+ if (r.width <= 0 || r.height <= 0) return false;
243
+ const at = document.elementFromPoint(cx, cy);
244
+ return !!(at && (at === best || best.contains(at) || at.contains(best)));
245
+ })(),
246
+ };
119
247
  })()`,
120
248
  returnByValue: true,
121
249
  });
122
250
  return result?.value || { found: false };
123
251
  }
124
252
 
253
+ /** Remove the temporary marker attribute from a located element. */
254
+ async function untag(Runtime, info) {
255
+ if (!info?.attr || !info?.id) return;
256
+ try {
257
+ await Runtime.evaluate({
258
+ expression: `document.querySelector('[${info.attr}="${info.id}"]')?.removeAttribute(${JSON.stringify(info.attr)})`,
259
+ });
260
+ } catch {}
261
+ }
262
+ /**
263
+ * Recover a capture group from the ORIGINAL instruction text, case intact.
264
+ * `match[0].length - match[1].length` arithmetic is wrong the moment a pattern
265
+ * has anything after the capture ("click the Pricing link" yielded "ng link").
266
+ * Uses the /d flag's match indices when available.
267
+ */
268
+ function capture(orig, match, group = 1) {
269
+ const idx = match.indices?.[group];
270
+ if (idx) return orig.slice(match.index + idx[0], match.index + idx[1]);
271
+ return orig.slice(orig.length - match[group].length);
272
+ }
273
+
125
274
  // ─── URL Normalizer ──────────────────────────────────────────────────────────
126
275
 
127
- function normalizeUrl(text) {
128
- let url = text.trim();
129
- // Remove common prefixes from natural language
130
- url = url.replace(/^(https?:\/\/)?(www\.)?/i, "");
131
- // Handle "google.com" type inputs
132
- if (!url.startsWith("http://") && !url.startsWith("https://")) {
133
- // Check if it looks like a domain (contains a dot)
134
- if (url.includes(".") && !url.includes(" ")) {
135
- url = "https://" + url;
136
- } else {
137
- // Treat as search query
276
+ /**
277
+ * Keep the scheme the user gave, keep `www.`, keep path case, and accept
278
+ * localhost / IPs / ports. Returns null when the token is not a URL at all.
279
+ */
280
+ export function normalizeUrl(text) {
281
+ let url = String(text || "").trim().replace(/[.,;:!?]+$/, "");
282
+ if (!url || /\s/.test(url)) return null;
283
+ if (/^[a-z][a-z0-9+.-]*:/i.test(url)) {
284
+ try {
285
+ const u = new URL(url);
286
+ return ["http:", "https:"].includes(u.protocol) ? u.href : null;
287
+ } catch {
138
288
  return null;
139
289
  }
140
290
  }
141
- return url;
291
+ url = url.replace(/[)\]}>.,;:!?'"]+$/, "");
292
+ const looksLikeHost =
293
+ /^[\w.-]+\.[a-z]{2,}(?:[/:?#][^\s]*)?$/i.test(url) ||
294
+ /^localhost(?::\d+)?(?:[/?#].*)?$/i.test(url) ||
295
+ /^\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?(?:[/?#].*)?$/.test(url);
296
+ if (!looksLikeHost) return null;
297
+ return `https://${url}`;
142
298
  }
143
299
 
144
300
  // ─── Execute Instruction ─────────────────────────────────────────────────────
@@ -154,15 +310,23 @@ function normalizeUrl(text) {
154
310
  * "type X in Y" → Type X into input Y
155
311
  * "get me X" / "extract X" → Find content by text
156
312
  * "scroll down" / "scroll up" → Scroll page
157
- * "what is X" → Return page content about X
313
+ * "what's on this page" → Page summary
158
314
  *
159
- * @param {import('chrome-remote-interface').Protocol} protocol
160
- * @param {string} instruction - Natural language instruction
161
- * @returns {Promise<object>} Result of the action
315
+ * @param {object} protocol CDP protocol for the active tab
316
+ * @param {string} instruction
317
+ * @param {object} [opts] { navigate, confirmDestructive, force }
318
+ * @returns {Promise<object>}
162
319
  */
163
- export async function executeInstruction(protocol, instruction) {
320
+ export async function executeInstruction(protocol, instruction, opts = {}) {
164
321
  const { Runtime, Page, Input } = protocol;
165
- const lower = instruction.trim().toLowerCase();
322
+ const orig = String(instruction || "").trim();
323
+ // Matching is case-insensitive; the ORIGINAL text is what we type/navigate.
324
+ const lower = orig.toLowerCase();
325
+ const navigate = opts.navigate || defaultNavigate(Page, opts);
326
+ const confirmDestructive = opts.confirmDestructive !== false;
327
+ const force = opts.force === true;
328
+ // Settle delay after a click / search / navigation before reading the page.
329
+ const settle = (ms) => new Promise((r) => setTimeout(r, opts.settleMs ?? ms));
166
330
 
167
331
  // ─── Pattern: "go to URL" / "navigate to URL" / "open URL" ──────────────
168
332
  // Capture only the URL token — stop at natural instruction boundaries so
@@ -170,160 +334,154 @@ export async function executeInstruction(protocol, instruction) {
170
334
  // the whole sentence into the URL.
171
335
  const navMatch = lower.match(/^(?:go to|navigate to|open|visit|take me to)\s+([^\s,]+(?:\.[^\s,]+)*)(?=\s|$)/);
172
336
  if (navMatch) {
173
- const url = normalizeUrl(navMatch[1]);
337
+ // The URL keeps its case: `GitHub.com/Krish/Repo` must not become `krish/repo`.
338
+ const rawUrl = orig.match(/^(?:go to|navigate to|open|visit|take me to)\s+(\S+)/i)?.[1] || navMatch[1];
339
+ // An explicit non-http scheme is a refusal, never a silent fall-through to
340
+ // "treat it as a search query".
341
+ if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {
342
+ return {
343
+ action: "navigate_error",
344
+ url: rawUrl,
345
+ error: `Refusing to navigate to ${rawUrl.split(":")[0]}: — only http and https are allowed.`,
346
+ };
347
+ }
348
+ const url = normalizeUrl(rawUrl);
174
349
  if (url) {
175
- await Page.navigate({ url });
176
- await new Promise(r => setTimeout(r, 2000));
350
+ try {
351
+ assertNavigable(url);
352
+ } catch (err) {
353
+ return { action: "navigate_error", url, error: err.message };
354
+ }
355
+ const nav = await navigate(url);
177
356
  const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
178
- const { result: textResult } = await Runtime.evaluate({ expression: "(document.body?.textContent || '').trim().slice(0, 1000)" });
357
+ const { result: textResult } = await Runtime.evaluate({
358
+ expression: `String(document.body?.innerText || '').trim().slice(0, ${PREVIEW_CHARS * 2})`,
359
+ });
179
360
  return {
180
361
  action: "navigate",
181
- url,
362
+ url: nav?.url || url,
182
363
  title: titleResult?.value || "",
183
- preview: (textResult?.value || "").slice(0, 300),
364
+ preview: (textResult?.value || "").slice(0, PREVIEW_CHARS),
365
+ ...(nav?.timedOut ? { timedOut: true } : {}),
184
366
  };
185
367
  }
186
368
  // If not a URL, treat as search
187
369
  }
188
370
 
189
- // ─── Pattern: "go to URL" that's a search fallback ──────────────────────
190
- // If navMatch but URL was invalid (normalizeUrl returned null), fall through to search
191
-
192
371
  // ─── Pattern: "search for X" / "search X" ───────────────────────────────
193
372
  // Anchored to string start: unanchored, "fill search with X" would be
194
373
  // hijacked by this pattern before fill could handle it.
195
- const searchMatch = lower.match(/^search\s+(?:for\s+)?(.+)/);
374
+ const searchMatch = lower.match(/^search\s+(?:for\s+)?(.+)/d);
196
375
  if (searchMatch) {
197
- const query = searchMatch[1];
376
+ const query = capture(orig, searchMatch);
198
377
  let inputInfo = await findInput(Runtime, "search");
199
- if (!inputInfo.found || !inputInfo.selector) {
200
- inputInfo = await findInput(Runtime, "query");
201
- }
202
- if (!inputInfo.found || !inputInfo.selector) {
203
- // Try common search selectors
204
- const { result } = await Runtime.evaluate({
205
- expression: `(() => {
206
- const el = document.querySelector('input[type=search], input[name=q], input[name=search], [role=search] input, form[role=search] input');
207
- if (el) return JSON.stringify({ found: true, selector: 'input[type=search]' });
208
- return JSON.stringify({ found: false });
209
- })()`,
210
- returnByValue: true,
211
- });
212
- inputInfo = JSON.parse(result?.value || "{}");
213
- }
214
- if (inputInfo.found && inputInfo.selector) {
215
- await Runtime.evaluate({
216
- expression: `document.querySelector(${JSON.stringify(inputInfo.selector)})?.focus()`,
217
- });
218
- await Input.insertText({ text: query });
219
- await new Promise(r => setTimeout(r, 300));
220
- // Try Enter key
221
- await Input.dispatchKeyEvent({ type: "keyDown", key: "Enter", windowsVirtualKeyCode: 13 });
222
- await Input.dispatchKeyEvent({ type: "keyUp", key: "Enter", windowsVirtualKeyCode: 13 });
223
- await new Promise(r => setTimeout(r, 2000));
378
+ if (!inputInfo.found) inputInfo = await findInput(Runtime, "query");
379
+ if (inputInfo.found) {
380
+ const cleared = await focusInputAt(Runtime, inputInfo.index);
381
+ if (cleared) {
382
+ await Input.insertText({ text: query });
383
+ await pressEnter(Input);
384
+ await settle(2000);
224
385
 
225
- const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
226
- const { result: textResult } = await Runtime.evaluate({
227
- expression: "(document.body?.textContent || '').trim().slice(0, 2000)",
228
- });
229
- return {
230
- action: "search",
231
- query,
232
- title: titleResult?.value || "",
233
- preview: (textResult?.value || "").slice(0, 500),
234
- };
386
+ const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
387
+ const { result: urlResult } = await Runtime.evaluate({ expression: "location.href" });
388
+ const { result: textResult } = await Runtime.evaluate({
389
+ expression: `String(document.body?.innerText || '').trim().slice(0, ${PREVIEW_CHARS * 4})`,
390
+ });
391
+ return {
392
+ action: "search",
393
+ query,
394
+ title: titleResult?.value || "",
395
+ url: urlResult?.value || "",
396
+ preview: (textResult?.value || "").slice(0, PREVIEW_CHARS),
397
+ };
398
+ }
235
399
  }
236
- return { action: "search_error", query, error: "Could not find a search input on this page" };
400
+ // Never type into an unrelated field: report what was on the page instead.
401
+ return {
402
+ action: "search_error",
403
+ query,
404
+ error: "No search input found on this page",
405
+ candidates: inputInfo.candidates || [],
406
+ };
237
407
  }
238
408
 
239
409
  // ─── Pattern: "click X" / "click the X button" / "click on X" ──────────
240
- const clickMatch = lower.match(/click\s+(?:the\s+|on\s+)?(.+?)(?:\s+button|\s+link|\s+element)?$/);
410
+ const clickMatch = lower.match(/click\s+(?:the\s+|on\s+)?(.+?)(?:\s+button|\s+link|\s+element)?$/d);
241
411
  if (clickMatch) {
242
- const target = clickMatch[1];
243
- // Try buttons and clickable elements first
244
- let el = await findElementByText(Runtime, target, "button, [role=button], input[type=submit], input[type=button], a");
245
- if (!el.found) el = await findElementByText(Runtime, target, "a");
246
- if (!el.found) el = await findElementByText(Runtime, target);
247
- if (el.found && el.selector) {
248
- // Get bounding rect via JS for CDP click
249
- const { result } = await Runtime.evaluate({
250
- expression: `(() => {
251
- const el = document.querySelector(${JSON.stringify(el.selector)});
252
- if (!el) return null;
253
- const rect = el.getBoundingClientRect();
254
- return JSON.stringify({ x: Math.round(rect.x + rect.width/2), y: Math.round(rect.y + rect.height/2) });
255
- })()`,
256
- });
257
- const pos = result?.value ? JSON.parse(result.value) : null;
258
- if (pos) {
259
- await Input.dispatchMouseEvent({ type: "mousePressed", x: pos.x, y: pos.y, button: "left", clickCount: 1 });
260
- await Input.dispatchMouseEvent({ type: "mouseReleased", x: pos.x, y: pos.y, button: "left", clickCount: 1 });
261
- await new Promise(r => setTimeout(r, 1500));
262
-
263
- const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
264
- return { action: "click", target, clicked: el.text, title: titleResult?.value };
412
+ const target = capture(orig, clickMatch).replace(/^\s*(?:the|on)\s+/i, "");
413
+ const el = await findByText(Runtime, target, CLICKABLE_SELECTOR);
414
+ if (el.found) {
415
+ const destructive = confirmDestructive && DESTRUCTIVE_RE.test(target);
416
+ if (destructive && !force) {
417
+ return {
418
+ action: "needs_confirmation",
419
+ target,
420
+ matchedText: el.text,
421
+ tag: el.tag,
422
+ href: el.href || "",
423
+ error: `Refusing to click "${el.text.slice(0, 60)}" without confirmation — it looks destructive. Re-run with force:true if that is intended.`,
424
+ };
265
425
  }
426
+ const { x, y } = el;
427
+ await Input.dispatchMouseEvent({ type: "mouseMoved", x, y, button: "none" });
428
+ await Input.dispatchMouseEvent({ type: "mousePressed", x, y, button: "left", clickCount: 1 });
429
+ await Input.dispatchMouseEvent({ type: "mouseReleased", x, y, button: "left", clickCount: 1 });
430
+ await settle(1500);
431
+ await untag(Runtime, el);
432
+
433
+ const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
434
+ const { result: urlResult } = await Runtime.evaluate({ expression: "location.href" });
435
+ return {
436
+ action: "click",
437
+ target,
438
+ clicked: el.text,
439
+ tag: el.tag,
440
+ ...(el.hit === false ? { warning: "Element was off-screen or covered; the click may have missed." } : {}),
441
+ title: titleResult?.value || "",
442
+ url: urlResult?.value || "",
443
+ };
266
444
  }
267
- return { action: "click_error", target, error: `Could not find element matching "${target}"` };
445
+ return {
446
+ action: "click_error",
447
+ target,
448
+ error: el.ambiguous
449
+ ? `Ambiguous — several elements match "${target}"`
450
+ : `Could not find element matching "${target}"`,
451
+ ...(el.candidates?.length ? { candidates: el.candidates } : {}),
452
+ };
268
453
  }
269
454
 
270
455
  // ─── Pattern: "fill X with Y" / "type Y in X" / "enter Y into X" ──────
271
456
  // NOTE: "fill X with Y" puts the VALUE in X ("fill email with foo@bar.com"),
272
457
  // while "type Y in X" puts the TEXT in X ("type hello in search").
273
- const fillWithMatch = lower.match(/fill\s+(.+?)\s+with\s+(.+)/);
458
+ const fillWithMatch = orig.match(/fill\s+(.+?)\s+with\s+(.+)$/is);
274
459
  if (fillWithMatch) {
275
- const target = fillWithMatch[1];
276
- const text = fillWithMatch[2];
277
- const inputInfo = await findInput(Runtime, target);
278
- if (inputInfo.found && inputInfo.selector) {
279
- await Runtime.evaluate({
280
- expression: `document.querySelector(${JSON.stringify(inputInfo.selector)})?.focus()`,
281
- });
282
- await Input.insertText({ text });
283
- await new Promise(r => setTimeout(r, 200));
284
- return { action: "fill", target, text, inputTag: inputInfo.tag };
285
- }
286
- return { action: "fill_error", target, error: `Could not find input matching "${target}"` };
460
+ return fillField(Runtime, Input, fillWithMatch[1], fillWithMatch[2], opts);
287
461
  }
288
462
 
289
- const fillMatch = lower.match(/(?:type|enter)\s+(.+?)\s+(?:in|into)\s+(.+)/);
463
+ const fillMatch = orig.match(/(?:type|enter)\s+(.+?)\s+(?:in|into)\s+(.+)/is);
290
464
  if (fillMatch) {
291
- const text = fillMatch[1];
292
- const target = fillMatch[2];
293
- const inputInfo = await findInput(Runtime, target);
294
- if (inputInfo.found && inputInfo.selector) {
295
- await Runtime.evaluate({
296
- expression: `document.querySelector(${JSON.stringify(inputInfo.selector)})?.focus()`,
297
- });
298
- await Input.insertText({ text });
299
- await new Promise(r => setTimeout(r, 200));
300
- return { action: "fill", target, text, inputTag: inputInfo.tag };
301
- }
302
- return { action: "fill_error", target, error: `Could not find input matching "${target}"` };
465
+ return fillField(Runtime, Input, fillMatch[2], fillMatch[1], opts);
303
466
  }
304
467
 
305
- // ─── Pattern: "get me X" / "extract X" / "find X" / "what is X" ──────
306
- const extractMatch = lower.match(/^(?:get me|extract|find|what is|show me|tell me about)\s+(.+)/);
468
+ // ─── Pattern: "get me X" / "extract X" / "find X" / "what's on this page" ──
469
+ const extractMatch = lower.match(
470
+ /^(?:get me|extract|find|show me|tell me about|what(?:'s| is| are))\s+(.+)/d);
307
471
  if (extractMatch) {
308
- const target = extractMatch[1];
309
- const el = await findElementByText(Runtime, target);
472
+ const target = capture(orig, extractMatch);
473
+ // "what's on this page" / "what is here" is a page summary, not a search.
474
+ if (/^(on\s+)?(this|the)\s+(page|site|website)$/i.test(target.trim()) || /^here$/i.test(target.trim())) {
475
+ return pageSummary(Runtime, target);
476
+ }
477
+ const el = await findByText(Runtime, target, TEXT_CONTAINER_SELECTOR);
310
478
  if (el.found && el.text) {
311
479
  return { action: "extract", target, content: el.text };
312
480
  }
313
- // Return page summary
314
- const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
315
- const { result: bodyResult } = await Runtime.evaluate({
316
- expression: "JSON.stringify({ text: (document.body?.textContent || '').trim().slice(0, 2000), headings: Array.from(document.querySelectorAll('h1,h2,h3')).map(h => h.textContent.trim()).slice(0, 10) })",
317
- returnByValue: true,
318
- });
319
- const body = JSON.parse(bodyResult?.value || "{}");
320
- return {
321
- action: "extract",
322
- target,
323
- pageTitle: titleResult?.value || "",
324
- headings: body.headings || [],
325
- preview: (body.text || "").slice(0, 500),
326
- };
481
+ if (el.ambiguous) {
482
+ return { action: "extract_error", target, error: `Ambiguous — several elements match "${target}"`, candidates: el.candidates };
483
+ }
484
+ return pageSummary(Runtime, target);
327
485
  }
328
486
 
329
487
  // ─── Pattern: "scroll down" / "scroll up" ──────────────────────────────
@@ -337,6 +495,99 @@ export async function executeInstruction(protocol, instruction) {
337
495
  }
338
496
 
339
497
  // ─── Fallback: return page info + help ─────────────────────────────────
498
+ return unknown(protocol, orig);
499
+ }
500
+
501
+ // ─── Instruction steps ───────────────────────────────────────────────────────
502
+
503
+ function inputSelectorExpression() {
504
+ return JSON.stringify(
505
+ 'input:not([type=hidden]):not([type=submit]):not([type=button]), textarea, [contenteditable=true], select'
506
+ );
507
+ }
508
+
509
+ function defaultNavigate(Page, opts) {
510
+ return async (url) => {
511
+ await Page.navigate({ url });
512
+ await new Promise((r) => setTimeout(r, Math.min(opts.settleMs ?? 2000, 5000)));
513
+ return { url };
514
+ };
515
+ }
516
+
517
+ async function pressEnter(Input) {
518
+ const base = { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13 };
519
+ await Input.dispatchKeyEvent({ type: "keyDown", ...base, text: "\r" });
520
+ await Input.dispatchKeyEvent({ type: "keyUp", ...base });
521
+ }
522
+
523
+ /**
524
+ * Focus the input at `index` in the candidate list AND clear it. Focus alone
525
+ * appends to whatever was already there; `Ctrl+A` via raw key events without
526
+ * `modifiers` does not select-all either.
527
+ */
528
+ async function focusInputAt(Runtime, index) {
529
+ const { result } = await Runtime.evaluate({
530
+ expression: `(() => {
531
+ const el = document.querySelectorAll(${inputSelectorExpression()})[${index}];
532
+ if (!el) return false;
533
+ el.focus();
534
+ try {
535
+ if (el.isContentEditable) { document.execCommand('selectAll'); }
536
+ else if (typeof el.select === 'function') { el.select(); }
537
+ else if ('value' in el) { el.value = ''; }
538
+ } catch {}
539
+ return true;
540
+ })()`,
541
+ returnByValue: true,
542
+ });
543
+ return result?.value === true;
544
+ }
545
+
546
+ async function fillField(Runtime, Input, target, text, opts = {}) {
547
+ const inputInfo = await findInput(Runtime, target);
548
+ if (!inputInfo.found) {
549
+ return {
550
+ action: "fill_error",
551
+ target,
552
+ error: `Could not find input matching "${target}"`,
553
+ ...(inputInfo.candidates?.length ? { candidates: inputInfo.candidates } : {}),
554
+ };
555
+ }
556
+ // Clear first: focus alone APPENDS to whatever was already there.
557
+ const cleared = await focusInputAt(Runtime, inputInfo.index);
558
+ if (!cleared) {
559
+ return { action: "fill_error", target, error: `Could not focus input matching "${target}"` };
560
+ }
561
+ await new Promise((r) => setTimeout(r, opts.settleMs === 0 ? 0 : 100));
562
+ await Input.insertText({ text });
563
+ await new Promise((r) => setTimeout(r, opts.settleMs === 0 ? 0 : 200));
564
+ const echo = redactIfSecret(target, text);
565
+ return { action: "fill", target, inputTag: inputInfo.tag, ...echo };
566
+ }
567
+
568
+ async function pageSummary(Runtime, target) {
569
+ const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
570
+ const { result } = await Runtime.evaluate({
571
+ expression: `JSON.stringify({
572
+ text: String(document.body?.innerText || '').trim().slice(0, ${PREVIEW_CHARS * 4}),
573
+ headings: Array.from(document.querySelectorAll('h1,h2,h3')).map(h => h.textContent.trim()).slice(0, 10),
574
+ links: Array.from(document.querySelectorAll('a[href]')).slice(0, 15).map(a => ({ text: a.textContent.trim().slice(0, 60), href: a.href })),
575
+ })`,
576
+ returnByValue: true,
577
+ });
578
+ const body = parseJson(result?.value) || {};
579
+ return {
580
+ action: "extract",
581
+ target,
582
+ pageTitle: titleResult?.value || "",
583
+ headings: body.headings || [],
584
+ links: body.links || [],
585
+ preview: (body.text || "").slice(0, PREVIEW_CHARS),
586
+ };
587
+ }
588
+
589
+ async function unknown(protocol, instruction) {
590
+ const { Runtime } = protocol;
340
591
  const { result: titleResult } = await Runtime.evaluate({ expression: "document.title" });
341
592
  const { result: urlResult } = await Runtime.evaluate({ expression: "location.href" });
342
593
  return {
@@ -347,3 +598,11 @@ export async function executeInstruction(protocol, instruction) {
347
598
  tip: "Try: 'search for laptops', 'click the login button', 'go to google.com', 'fill email with test@example.com', 'extract prices', 'scroll down'",
348
599
  };
349
600
  }
601
+
602
+ function parseJson(value) {
603
+ if (!value) return null;
604
+ if (typeof value === "object") return value;
605
+ try { return JSON.parse(value); } catch { return null; }
606
+ }
607
+
608
+ export { UrlPolicyError };