dsh-inner-browser-use 0.34.28 → 0.34.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -18
- package/README.zh.md +26 -14
- package/client.js +223 -23
- package/docs/ARCHITECTURE.md +15 -12
- package/docs/ARTIFACTS.md +1 -1
- package/docs/BENCH.md +209 -34
- package/docs/FLASH.md +84 -0
- package/docs/ROADMAP.md +21 -15
- package/docs/VERIFY.md +20 -10
- package/index.js +186 -24
- package/package.json +17 -3
- package/read.js +633 -29
- package/runtime/flash-app/main.js +232 -0
- package/runtime/flash-app/package.json +1 -0
- package/runtime/flash-setup.mjs +151 -0
- package/runtime/swf.mjs +233 -0
- package/scripts/check-flash-local.mjs +104 -0
- package/scripts/check-flash-mirror.mjs +73 -0
- package/scripts/check-flash-tools.mjs +74 -0
- package/scripts/check-flash-window.mjs +58 -0
- package/scripts/doctor.mjs +3 -1
- package/sidecar.js +1098 -96
- package/tools.js +1068 -144
package/tools.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Two costs decide whether this is usable: the tokens a read spends, and the
|
|
6
6
|
* round trips a task spends. The tool surface here is shaped around both — a
|
|
7
7
|
* read can be narrowed (`mode`, `max_chars`) and diffed (`changed_only`), and
|
|
8
|
-
* `
|
|
8
|
+
* `browser_do` runs a run of steps in one call with waits that know when a page
|
|
9
9
|
* has settled.
|
|
10
10
|
*/
|
|
11
11
|
|
|
@@ -15,7 +15,7 @@ import { homedir } from 'node:os'
|
|
|
15
15
|
import { basename, join } from 'node:path'
|
|
16
16
|
import { mkdir, readdir, stat, unlink, writeFile } from 'node:fs/promises'
|
|
17
17
|
import { VIEW } from './sidecar.js?v=1'
|
|
18
|
-
import { readPage, renderPage, locate, covering, contextFor, identityOf, sameThing, differs, labelSpot, selectSpot, bounds, pictures, REF_GONE } from './read.js?v=1'
|
|
18
|
+
import { readPage, renderPage, readArticle, readRelevant, findOnPage, locate, covering, contextFor, identityOf, sameThing, differs, labelSpot, selectSpot, bounds, pictures, clickInDom, writeInDom, REF_GONE } from './read.js?v=1'
|
|
19
19
|
import { brief as askBrief, recordRound, lastRoundNote, lastRound, markRound, MARK_STATUS } from './review.js?v=1'
|
|
20
20
|
|
|
21
21
|
/** Which chat the running call belongs to, so each gets its own tab. */
|
|
@@ -26,6 +26,25 @@ function currentSession() {
|
|
|
26
26
|
return String(sessionScope.getStore() ?? '')
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
/** How a call asked to act: real mouse and keys, or the page's own DOM. */
|
|
30
|
+
const WAYS = new Set(['human', 'dom'])
|
|
31
|
+
|
|
32
|
+
/** The way each chat said to act, until it says otherwise. */
|
|
33
|
+
const chatWays = new Map()
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The way this call should act: what it asked for, else what its chat last set,
|
|
37
|
+
* else a real mouse and real keys, which no page can tell from a person.
|
|
38
|
+
* @param {any} args - the call's arguments.
|
|
39
|
+
* @returns {'human'|'dom'} which way.
|
|
40
|
+
*/
|
|
41
|
+
function wayOf(args) {
|
|
42
|
+
const said = String(args?.mode ?? '')
|
|
43
|
+
if (WAYS.has(said)) return /** @type {'human'|'dom'} */ (said)
|
|
44
|
+
const chat = currentSession()
|
|
45
|
+
return chatWays.get(chat) === 'dom' ? 'dom' : 'human'
|
|
46
|
+
}
|
|
47
|
+
|
|
29
48
|
/** Which project the running call belongs to, so its files stay together. */
|
|
30
49
|
export const workScope = new AsyncLocalStorage()
|
|
31
50
|
|
|
@@ -121,7 +140,7 @@ export async function pruneArtifacts(dir, keep) {
|
|
|
121
140
|
const follows = { count: 0 }
|
|
122
141
|
|
|
123
142
|
/** Shown when the sidecar has never been started in this run. */
|
|
124
|
-
export const SIDECAR_OFF = 'The sidecar browser is not running. Call
|
|
143
|
+
export const SIDECAR_OFF = 'The sidecar browser is not running. Call browser_open with the address first.'
|
|
125
144
|
|
|
126
145
|
/** Output schema every tool returns. */
|
|
127
146
|
/** The picture half of a screenshot result: shown only when the route can see it. */
|
|
@@ -193,6 +212,23 @@ async function brief(page) {
|
|
|
193
212
|
if (dialog !== null && dialog !== undefined) {
|
|
194
213
|
return `${String(page.url ?? '')} · dialog ${String(dialog.type ?? '')}: ${String(dialog.message ?? '')}`
|
|
195
214
|
}
|
|
215
|
+
/* On a Flash page the DOM around the plugin says nothing about what is on
|
|
216
|
+
screen, and reading it right after a click waits for the main thread the
|
|
217
|
+
plugin is holding. The bridge answers with the address and title instead. */
|
|
218
|
+
if (page.flashBridge !== null && page.flashBridge !== undefined) {
|
|
219
|
+
try {
|
|
220
|
+
const reply = await fetch(`http://127.0.0.1:${page.flashBridge}/info`, { signal: AbortSignal.timeout(600) })
|
|
221
|
+
if (reply.ok === true) {
|
|
222
|
+
const info = await reply.json()
|
|
223
|
+
const bits = [String(info?.url ?? page.url ?? '')]
|
|
224
|
+
if (String(info?.title ?? '') !== '') bits.push(String(info.title))
|
|
225
|
+
return bits.join(' · ')
|
|
226
|
+
}
|
|
227
|
+
} catch {
|
|
228
|
+
/* the cached address below is still a true answer */
|
|
229
|
+
}
|
|
230
|
+
return String(page.url ?? '')
|
|
231
|
+
}
|
|
196
232
|
const info = await evaluate(page, BRIEF_JS)
|
|
197
233
|
const bits = [String(info?.url ?? '')]
|
|
198
234
|
if (String(info?.title ?? '') !== '') bits.push(String(info.title))
|
|
@@ -208,9 +244,117 @@ async function brief(page) {
|
|
|
208
244
|
* @param {string} label - what was done.
|
|
209
245
|
* @returns {Promise<string>} reply text.
|
|
210
246
|
*/
|
|
247
|
+
/** The address each page last reported to the model, so it is said once. */
|
|
248
|
+
const saidUrls = new WeakMap()
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* The address is worth one line the first time and every time it changes; the
|
|
252
|
+
* same address on every step is a cost with nothing to teach — the model read
|
|
253
|
+
* it a step ago and still has it. Titles and focus still come through.
|
|
254
|
+
* @param {any} page - sidecar page.
|
|
255
|
+
* @param {string} briefText - address, title and focus as `brief` built them.
|
|
256
|
+
* @returns {string} the line to send.
|
|
257
|
+
*/
|
|
258
|
+
/**
|
|
259
|
+
* A PDF is a document, not a blank page: say so, and put its bytes somewhere a
|
|
260
|
+
* reader can open them. The mirror cannot draw a PDF, and a model looking at an
|
|
261
|
+
* empty screen would otherwise think the address had nothing on it.
|
|
262
|
+
* @param {any} page - the tab.
|
|
263
|
+
* @returns {Promise<string>} a line to append to the read, or ''.
|
|
264
|
+
*/
|
|
265
|
+
async function pdfNote(page) {
|
|
266
|
+
const found = page?.pdf
|
|
267
|
+
if (found === null || found === undefined) return ''
|
|
268
|
+
let file = { path: '', bytes: 0 }
|
|
269
|
+
try {
|
|
270
|
+
file = await page.documentFile()
|
|
271
|
+
} catch (error) {
|
|
272
|
+
file = { path: '', bytes: 0 }
|
|
273
|
+
}
|
|
274
|
+
const size = file.bytes > 0 ? ` (${String(Math.max(1, Math.round(file.bytes / 1024)))} KB)` : ''
|
|
275
|
+
return file.path === ''
|
|
276
|
+
? `\nthis address is a PDF${size}: the mirror cannot draw it, and its bytes could not be saved. browser_downloads lists what the browser has downloaded.`
|
|
277
|
+
: `\nthis address is a PDF${size}: the mirror cannot draw it. Saved to ${file.path} — envoy_read can read that file, or another document reader.`
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* The numbered controls that were not on the page at the read before this one.
|
|
282
|
+
* A model that just acted cares about what its action brought in, and saying so
|
|
283
|
+
* costs a few numbers instead of a whole second read.
|
|
284
|
+
* @param {any} before - the read before this one, or undefined.
|
|
285
|
+
* @param {any} after - the read now.
|
|
286
|
+
* @returns {number[]} refs in `after` that `before` did not have.
|
|
287
|
+
*/
|
|
288
|
+
export function freshRefs(before, after) {
|
|
289
|
+
if (before === undefined || before === null) return []
|
|
290
|
+
if (after === undefined || after === null) return []
|
|
291
|
+
const had = new Set((Array.isArray(before.controls) ? before.controls : []).map((control) => Number(control?.ref ?? -1)))
|
|
292
|
+
const fresh = []
|
|
293
|
+
for (const control of Array.isArray(after.controls) ? after.controls : []) {
|
|
294
|
+
const ref = Number(control?.ref ?? -1)
|
|
295
|
+
if (ref < 0 || had.has(ref)) continue
|
|
296
|
+
if (!fresh.includes(ref)) fresh.push(ref)
|
|
297
|
+
}
|
|
298
|
+
return fresh
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/** What an action did, in one line: where it went, whether the page moved at all. */
|
|
302
|
+
export async function outcome(page, before) {
|
|
303
|
+
if (typeof before !== 'string' || before === '') return ''
|
|
304
|
+
const after = await pageMark(page)
|
|
305
|
+
if (after === '') return ''
|
|
306
|
+
/* A movie has no DOM to compare: its frame counter is the whole answer. */
|
|
307
|
+
if (before.startsWith('flash:') && after.startsWith('flash:')) {
|
|
308
|
+
return before === after ? ' · no frame change' : ' · screen updated'
|
|
309
|
+
}
|
|
310
|
+
const [wasTitle, wasUrl, wasText, wasNodes] = before.split('|')
|
|
311
|
+
const [nowTitle, nowUrl, nowText, nowNodes] = after.split('|')
|
|
312
|
+
if (after === before) return ' · no visible change'
|
|
313
|
+
if (nowUrl !== wasUrl) return ` · navigated to ${String(nowUrl).slice(0, 300)}`
|
|
314
|
+
const words = Math.abs(Number(nowText ?? 0) - Number(wasText ?? 0))
|
|
315
|
+
const nodes = Math.abs(Number(nowNodes ?? 0) - Number(wasNodes ?? 0))
|
|
316
|
+
if (nowTitle !== wasTitle) return ' · page moved on'
|
|
317
|
+
if (words >= 20 || nodes >= 3) return ' · content changed'
|
|
318
|
+
return ' · no visible change'
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** Read results this chat already paid for, keyed by what the page looks like. */
|
|
322
|
+
const readSeen = new WeakMap()
|
|
323
|
+
|
|
324
|
+
export function oncePerPage(page, briefText) {
|
|
325
|
+
const parts = String(briefText).split(' · ')
|
|
326
|
+
const url = String(parts[0] ?? '')
|
|
327
|
+
const known = page === null || page === undefined ? undefined : saidUrls.get(page)
|
|
328
|
+
if (url !== '' && known !== undefined) {
|
|
329
|
+
saidUrls.set(page, url)
|
|
330
|
+
if (url === known && parts.length > 1) return parts.slice(1).join(' · ')
|
|
331
|
+
return briefText
|
|
332
|
+
}
|
|
333
|
+
if (url !== '' && page !== null && page !== undefined) saidUrls.set(page, url)
|
|
334
|
+
return briefText
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* A read that starts with an address the model was already given opens with the
|
|
339
|
+
* page itself. The address is how a page says which page it is, not something
|
|
340
|
+
* to repeat every read; a page that moved says the new address in full.
|
|
341
|
+
* @param {any} page - sidecar page.
|
|
342
|
+
* @param {string} url - the address this read is from.
|
|
343
|
+
* @param {string} text - the rendered read.
|
|
344
|
+
* @returns {string} the read, without a leading address already said.
|
|
345
|
+
*/
|
|
346
|
+
export function withoutSeenAddress(page, url, text) {
|
|
347
|
+
const body = String(text ?? '')
|
|
348
|
+
const address = String(url ?? '')
|
|
349
|
+
if (address === '' || page === null || page === undefined) return body
|
|
350
|
+
const first = body.split('\n')[0]
|
|
351
|
+
if (first !== address || saidUrls.get(page) !== address) return body
|
|
352
|
+
return body.slice(first.length + 1)
|
|
353
|
+
}
|
|
354
|
+
|
|
211
355
|
async function acted(page, args, look, label) {
|
|
212
356
|
if (args?.read === true) return await look()
|
|
213
|
-
return `ok ${label} · ${await brief(page)}`
|
|
357
|
+
return `ok ${label} · ${oncePerPage(page, await brief(page))}`
|
|
214
358
|
}
|
|
215
359
|
|
|
216
360
|
/**
|
|
@@ -231,7 +375,7 @@ async function acted(page, args, look, label) {
|
|
|
231
375
|
/** Titles a site shows while it asks the browser to prove a person is there. */
|
|
232
376
|
const CHECK_TITLES = ['请稍候', 'just a moment', 'checking your browser', 'verifying you are human', 'attention required']
|
|
233
377
|
|
|
234
|
-
const CHECK_NOTE = 'That is the site asking a browser to prove a person is there, and a windowless one never gets past it.
|
|
378
|
+
const CHECK_NOTE = 'That is the site asking a browser to prove a person is there, and a windowless one never gets past it. browser_open with headless false opens a window on the same profile, with the same logins.'
|
|
235
379
|
|
|
236
380
|
/**
|
|
237
381
|
* A check page: the site has not refused, it is asking for proof. Reading it
|
|
@@ -293,27 +437,36 @@ export async function unreachable(page) {
|
|
|
293
437
|
if (!isErrorPage(page)) return ''
|
|
294
438
|
/* The code is the part a model can act on: ERR_NAME_NOT_RESOLVED is a typo,
|
|
295
439
|
ERR_PROXY_CONNECTION_FAILED is this machine's proxy, ERR_TIMED_OUT is slow. */
|
|
296
|
-
let detail =
|
|
440
|
+
let detail = ''
|
|
297
441
|
try {
|
|
298
442
|
const said = await evaluate(page, `(() => {
|
|
299
443
|
const text = String(document.body ? document.body.innerText : '')
|
|
300
444
|
const code = (String(text).match(/ERR_[A-Z0-9_]+/) || [''])[0]
|
|
301
|
-
return code
|
|
445
|
+
return code
|
|
302
446
|
})()`)
|
|
303
|
-
if (typeof said === 'string'
|
|
447
|
+
if (typeof said === 'string') detail = said.trim()
|
|
304
448
|
} catch {
|
|
305
449
|
/* The error page itself refused to be read; the address still says enough. */
|
|
306
450
|
}
|
|
307
|
-
|
|
451
|
+
/* Chrome often writes the address it could not reach where the code would
|
|
452
|
+
be: that is still the one thing the person can check. */
|
|
453
|
+
if (detail === '') {
|
|
454
|
+
const title = String(page.title ?? '').trim()
|
|
455
|
+
if (title !== '' && !/\s/.test(title)) detail = title
|
|
456
|
+
}
|
|
457
|
+
if (detail === '') detail = String(page.wantUrl ?? page.lastUrl ?? '').replace(/^https?:\/\//, '').replace(/\/.*$/, '')
|
|
458
|
+
return `the network could not reach that address — the browser stopped on its own error page (${detail === '' ? 'no details' : detail}), which is not the site. Nothing on it is page content. Use browser_nav with another address, or fix the connection this machine needs.`
|
|
308
459
|
}
|
|
309
460
|
|
|
310
|
-
/** Key names this driver can press
|
|
461
|
+
/** Key names this driver can press: DOM `key` names for the named keys, and
|
|
462
|
+
the character itself for a printable one (Space also answers to `Space`). */
|
|
311
463
|
export const KEYS = {
|
|
312
464
|
Enter: { key: 'Enter', code: 'Enter', keyCode: 13, text: '\r' },
|
|
313
465
|
Tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
|
|
314
466
|
Escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
|
|
315
467
|
Backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
|
|
316
468
|
Delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
|
|
469
|
+
Insert: { key: 'Insert', code: 'Insert', keyCode: 45 },
|
|
317
470
|
ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
|
|
318
471
|
ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
|
|
319
472
|
ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
|
|
@@ -323,6 +476,20 @@ export const KEYS = {
|
|
|
323
476
|
PageUp: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
|
|
324
477
|
PageDown: { key: 'PageDown', code: 'PageDown', keyCode: 34 },
|
|
325
478
|
Space: { key: ' ', code: 'Space', keyCode: 32, text: ' ' },
|
|
479
|
+
ContextMenu: { key: 'ContextMenu', code: 'ContextMenu', keyCode: 93 },
|
|
480
|
+
CapsLock: { key: 'CapsLock', code: 'CapsLock', keyCode: 20 },
|
|
481
|
+
NumLock: { key: 'NumLock', code: 'NumLock', keyCode: 144 },
|
|
482
|
+
ScrollLock: { key: 'ScrollLock', code: 'ScrollLock', keyCode: 145 },
|
|
483
|
+
PrintScreen: { key: 'PrintScreen', code: 'PrintScreen', keyCode: 44 },
|
|
484
|
+
Pause: { key: 'Pause', code: 'Pause', keyCode: 19 },
|
|
485
|
+
NumpadEnter: { key: 'Enter', code: 'NumpadEnter', keyCode: 13, text: '\r' },
|
|
486
|
+
AudioVolumeMute: { key: 'AudioVolumeMute', code: 'AudioVolumeMute', keyCode: 173 },
|
|
487
|
+
AudioVolumeDown: { key: 'AudioVolumeDown', code: 'AudioVolumeDown', keyCode: 174 },
|
|
488
|
+
AudioVolumeUp: { key: 'AudioVolumeUp', code: 'AudioVolumeUp', keyCode: 175 },
|
|
489
|
+
MediaPlayPause: { key: 'MediaPlayPause', code: 'MediaPlayPause', keyCode: 179 },
|
|
490
|
+
MediaTrackNext: { key: 'MediaTrackNext', code: 'MediaTrackNext', keyCode: 176 },
|
|
491
|
+
MediaTrackPrevious: { key: 'MediaTrackPrevious', code: 'MediaTrackPrevious', keyCode: 177 },
|
|
492
|
+
MediaStop: { key: 'MediaStop', code: 'MediaStop', keyCode: 178 },
|
|
326
493
|
}
|
|
327
494
|
|
|
328
495
|
/* A game moves while a key is held, and the keys a game listens for are letters
|
|
@@ -331,12 +498,188 @@ export const KEYS = {
|
|
|
331
498
|
engines still look at. */
|
|
332
499
|
for (const letter of 'abcdefghijklmnopqrstuvwxyz') {
|
|
333
500
|
KEYS[letter] = { key: letter, code: `Key${letter.toUpperCase()}`, keyCode: letter.toUpperCase().charCodeAt(0), text: letter }
|
|
334
|
-
|
|
501
|
+
/* A capital is its own key event: the page reads `A` with shift held, and a
|
|
502
|
+
form inserts `A`, not `a`. */
|
|
503
|
+
KEYS[letter.toUpperCase()] = {
|
|
504
|
+
key: letter.toUpperCase(),
|
|
505
|
+
code: `Key${letter.toUpperCase()}`,
|
|
506
|
+
keyCode: letter.toUpperCase().charCodeAt(0),
|
|
507
|
+
text: letter.toUpperCase(),
|
|
508
|
+
shift: true,
|
|
509
|
+
}
|
|
335
510
|
}
|
|
336
511
|
for (const digit of '0123456789') {
|
|
337
512
|
KEYS[digit] = { key: digit, code: `Digit${digit}`, keyCode: digit.charCodeAt(0), text: digit }
|
|
338
513
|
}
|
|
339
514
|
|
|
515
|
+
/* The symbols above the digits and around the alphabet, both halves of every
|
|
516
|
+
key: a page reads `?` as Slash with shift down, and a field inserts `?`.
|
|
517
|
+
`keyCode` is the legacy VK number Windows and old pages compare against. */
|
|
518
|
+
const PUNCTUATION = [
|
|
519
|
+
['Backquote', '`', 192], ['Minus', '-', 189], ['Equal', '=', 187],
|
|
520
|
+
['BracketLeft', '[', 219], ['BracketRight', ']', 221], ['Backslash', '\\', 220],
|
|
521
|
+
['Semicolon', ';', 186], ['Quote', "'", 222], ['Comma', ',', 188],
|
|
522
|
+
['Period', '.', 190], ['Slash', '/', 191],
|
|
523
|
+
]
|
|
524
|
+
const SHIFTED_SYMBOLS = [
|
|
525
|
+
['~', 'Backquote', 192], ['!', 'Digit1', 49], ['@', 'Digit2', 50], ['#', 'Digit3', 51],
|
|
526
|
+
['$', 'Digit4', 52], ['%', 'Digit5', 53], ['^', 'Digit6', 54], ['&', 'Digit7', 55],
|
|
527
|
+
['*', 'Digit8', 56], ['(', 'Digit9', 57], [')', 'Digit0', 48], ['_', 'Minus', 189],
|
|
528
|
+
['+', 'Equal', 187], ['{', 'BracketLeft', 219], ['}', 'BracketRight', 221], ['|', 'Backslash', 220],
|
|
529
|
+
[':', 'Semicolon', 186], ['"', 'Quote', 222], ['<', 'Comma', 188], ['>', 'Period', 190],
|
|
530
|
+
['?', 'Slash', 191],
|
|
531
|
+
]
|
|
532
|
+
for (const [code, char, keyCode] of PUNCTUATION) {
|
|
533
|
+
KEYS[char] = { key: char, code, keyCode, text: char }
|
|
534
|
+
}
|
|
535
|
+
for (const [char, code, keyCode] of SHIFTED_SYMBOLS) {
|
|
536
|
+
KEYS[char] = { key: char, code, keyCode, text: char, shift: true }
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/* Function keys, the numeric keypad a page can listen to, and the keys that do
|
|
540
|
+
nothing in a text field but everything in an app: a canvas game, a document
|
|
541
|
+
editor, a video player. */
|
|
542
|
+
for (let n = 1; n <= 12; n += 1) {
|
|
543
|
+
KEYS[`F${n}`] = { key: `F${n}`, code: `F${n}`, keyCode: 111 + n }
|
|
544
|
+
}
|
|
545
|
+
for (const digit of '0123456789') {
|
|
546
|
+
KEYS[`Numpad${digit}`] = { key: digit, code: `Numpad${digit}`, keyCode: 96 + Number(digit), text: digit }
|
|
547
|
+
}
|
|
548
|
+
KEYS.NumpadAdd = { key: '+', code: 'NumpadAdd', keyCode: 107, text: '+' }
|
|
549
|
+
KEYS.NumpadSubtract = { key: '-', code: 'NumpadSubtract', keyCode: 109, text: '-' }
|
|
550
|
+
KEYS.NumpadMultiply = { key: '*', code: 'NumpadMultiply', keyCode: 106, text: '*' }
|
|
551
|
+
KEYS.NumpadDivide = { key: '/', code: 'NumpadDivide', keyCode: 111, text: '/' }
|
|
552
|
+
KEYS.NumpadDecimal = { key: '.', code: 'NumpadDecimal', keyCode: 110, text: '.' }
|
|
553
|
+
|
|
554
|
+
/* The names people actually type, and the spelling a model guesses. */
|
|
555
|
+
const KEY_ALIASES = {
|
|
556
|
+
esc: 'Escape', return: 'Enter', del: 'Delete', ins: 'Insert',
|
|
557
|
+
up: 'ArrowUp', down: 'ArrowDown', left: 'ArrowLeft', right: 'ArrowRight',
|
|
558
|
+
pageup: 'PageUp', pagedown: 'PageDown', spacebar: 'Space', space: 'Space',
|
|
559
|
+
menu: 'ContextMenu', prtsc: 'PrintScreen', printscreen: 'PrintScreen',
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* The event a browser would report for a named key.
|
|
564
|
+
* @param {string} name - a key in {@link KEYS}, or something close to one.
|
|
565
|
+
* @returns {{key: string, code: string, keyCode: number, text?: string, shift?: boolean}} the event fields.
|
|
566
|
+
* @throws {Error} when the key does not exist.
|
|
567
|
+
*/
|
|
568
|
+
export function keySpec(name) {
|
|
569
|
+
const raw = String(name ?? '')
|
|
570
|
+
const exact = KEYS[raw]
|
|
571
|
+
if (exact !== undefined) return exact
|
|
572
|
+
if (raw === ' ') return KEYS.Space
|
|
573
|
+
const alias = KEY_ALIASES[raw.toLowerCase()]
|
|
574
|
+
if (alias !== undefined) return KEYS[alias]
|
|
575
|
+
const capital = raw.charAt(0).toUpperCase() + raw.slice(1)
|
|
576
|
+
if (KEYS[capital] !== undefined) return KEYS[capital]
|
|
577
|
+
const fn = /^f([1-9]|1[0-2])$/i.exec(raw)
|
|
578
|
+
if (fn !== null) return KEYS[`F${fn[1]}`]
|
|
579
|
+
throw new Error(`sidecar: no such key ${raw}`) // ui-copy-ignore
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* Turn `Ctrl+Shift` into the CDP modifier bits. `Mod` is the platform's own
|
|
584
|
+
* command key: Command on macOS, Control on Windows and Linux — one spelling
|
|
585
|
+
* for the shortcut a page treats as copy, undo, select-all.
|
|
586
|
+
* @param {string} held - modifiers joined by +, any of Alt/Option, Ctrl/Control,
|
|
587
|
+
* Meta/Cmd/Command/Super/Win, Shift, Mod.
|
|
588
|
+
* @param {string} [platform] - defaults to this process's platform.
|
|
589
|
+
* @returns {number} CDP modifier bits: Alt 1, Ctrl 2, Meta 4, Shift 8.
|
|
590
|
+
*/
|
|
591
|
+
export function modifierBits(held, platform = process.platform) {
|
|
592
|
+
let bits = 0
|
|
593
|
+
for (const raw of String(held ?? '').split('+')) {
|
|
594
|
+
const one = raw.trim().toLowerCase()
|
|
595
|
+
if (one === 'alt' || one === 'option' || one === 'opt' || one === '⌥') bits |= 1
|
|
596
|
+
else if (one === 'ctrl' || one === 'control' || one === '^') bits |= 2
|
|
597
|
+
else if (one === 'meta' || one === 'cmd' || one === 'command' || one === 'super' || one === 'win' || one === '⌘') bits |= 4
|
|
598
|
+
else if (one === 'shift' || one === '⇧') bits |= 8
|
|
599
|
+
else if (one === 'mod') bits |= platform === 'darwin' ? 4 : 2
|
|
600
|
+
}
|
|
601
|
+
return bits
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* Turn what the mirror's client posted into the event the page should see. A
|
|
606
|
+
* capital letter carries its shift, a shortcut becomes the browser command it
|
|
607
|
+
* names, and a release carries no text so the field does not type twice.
|
|
608
|
+
* @param {any} body - `{ key, code?, keyCode?, modifiers?, text?, down?, repeat? }`.
|
|
609
|
+
* @returns {{name: string, down: boolean, repeat: boolean, spec: any, command: string|null}} the event.
|
|
610
|
+
*/
|
|
611
|
+
export function keyEvent(body) {
|
|
612
|
+
const name = String(body?.key ?? '')
|
|
613
|
+
let known
|
|
614
|
+
try {
|
|
615
|
+
known = keySpec(name)
|
|
616
|
+
} catch {
|
|
617
|
+
known = null
|
|
618
|
+
}
|
|
619
|
+
const held = Number(body?.modifiers ?? 0)
|
|
620
|
+
const down = body?.down !== false
|
|
621
|
+
const repeat = body?.repeat === true
|
|
622
|
+
const capital = name.length === 1 && name !== name.toLowerCase()
|
|
623
|
+
const modifiers = capital ? held | 8 : held
|
|
624
|
+
const command = down === true && repeat === false ? commandOf(name, modifiers) : null
|
|
625
|
+
const text = command !== null ? undefined : body?.text === undefined ? known?.text : String(body.text)
|
|
626
|
+
return {
|
|
627
|
+
name,
|
|
628
|
+
down,
|
|
629
|
+
repeat,
|
|
630
|
+
spec: {
|
|
631
|
+
key: name,
|
|
632
|
+
code: String(body?.code ?? known?.code ?? name),
|
|
633
|
+
keyCode: Number(body?.keyCode ?? 0) || known?.keyCode || 0,
|
|
634
|
+
text,
|
|
635
|
+
modifiers,
|
|
636
|
+
},
|
|
637
|
+
command,
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* Read a chord the way a shortcut is written: `Ctrl+Shift+Tab` is the held
|
|
643
|
+
* modifiers Ctrl and Shift around one Tab, and `Mod+a` is Command+a on macOS.
|
|
644
|
+
* @param {string} chord - keys joined by +, modifiers may be in it.
|
|
645
|
+
* @param {number} [modifiers] - bits already held.
|
|
646
|
+
* @returns {{names: string[], modifiers: number}} the keys to press and the bits.
|
|
647
|
+
*/
|
|
648
|
+
export function splitChord(chord, modifiers = 0) {
|
|
649
|
+
const bits = { alt: 1, ctrl: 2, control: 2, meta: 4, cmd: 4, command: 4, super: 4, win: 4, shift: 8, option: 1, opt: 1, mod: process.platform === 'darwin' ? 4 : 2 }
|
|
650
|
+
const names = []
|
|
651
|
+
let held = modifiers
|
|
652
|
+
for (const raw of String(chord ?? '').split('+')) {
|
|
653
|
+
const one = raw.trim()
|
|
654
|
+
if (one === '') continue
|
|
655
|
+
const bit = bits[one.toLowerCase()]
|
|
656
|
+
if (bit === undefined) names.push(one)
|
|
657
|
+
else held |= bit
|
|
658
|
+
}
|
|
659
|
+
return { names, modifiers: held }
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/**
|
|
663
|
+
* The editing command a browser runs for a shortcut, if any: copy and paste are
|
|
664
|
+
* commands, not key events, and a key event alone leaves the clipboard alone.
|
|
665
|
+
* @param {string} name - the key name.
|
|
666
|
+
* @param {number} modifiers - CDP modifier bits.
|
|
667
|
+
* @returns {string|null} a CDP command name, or null.
|
|
668
|
+
*/
|
|
669
|
+
export function commandOf(name, modifiers) {
|
|
670
|
+
/* Either command key counts: a page listens for the one its platform uses,
|
|
671
|
+
and the mirror can be driven from a machine with the other one. */
|
|
672
|
+
if ((modifiers & 6) === 0) return null
|
|
673
|
+
const one = String(name ?? '').toLowerCase()
|
|
674
|
+
if (one === 'v') return 'paste'
|
|
675
|
+
if (one === 'c') return 'copy'
|
|
676
|
+
if (one === 'x') return 'cut'
|
|
677
|
+
if (one === 'a') return 'selectAll'
|
|
678
|
+
if (one === 'z') return (modifiers & 8) !== 0 ? 'redo' : 'undo'
|
|
679
|
+
if (one === 'y') return 'redo'
|
|
680
|
+
return null
|
|
681
|
+
}
|
|
682
|
+
|
|
340
683
|
/**
|
|
341
684
|
* Press a key: keyDown, then keyUp, with the character when the key types one.
|
|
342
685
|
* @param {any} page - sidecar page.
|
|
@@ -345,10 +688,15 @@ for (const digit of '0123456789') {
|
|
|
345
688
|
* @returns {Promise<void>} done.
|
|
346
689
|
*/
|
|
347
690
|
export async function pressKey(page, name, modifiers = 0, holdMs = 0) {
|
|
348
|
-
const key =
|
|
349
|
-
|
|
350
|
-
const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers }
|
|
351
|
-
|
|
691
|
+
const key = keySpec(name)
|
|
692
|
+
const withShift = (key.shift === true ? modifiers | 8 : modifiers)
|
|
693
|
+
const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers: withShift }
|
|
694
|
+
/* Copy, paste, undo and select-all are editor commands: a key event alone
|
|
695
|
+
leaves the clipboard and the field where they were. */
|
|
696
|
+
const command = commandOf(key.text ?? key.key, withShift)
|
|
697
|
+
await page.key('keyDown', command === null
|
|
698
|
+
? (key.text === undefined ? base : { ...base, text: key.text })
|
|
699
|
+
: { ...base, text: undefined, commands: [command] })
|
|
352
700
|
/* A key held down is how a game moves: down, wait, up — not a tap. */
|
|
353
701
|
if (holdMs > 0) await new Promise((settle) => setTimeout(settle, holdMs))
|
|
354
702
|
await page.key('keyUp', base)
|
|
@@ -366,9 +714,8 @@ export async function pressKey(page, name, modifiers = 0, holdMs = 0) {
|
|
|
366
714
|
*/
|
|
367
715
|
export async function pressChord(page, names, modifiers = 0, holdMs = 0) {
|
|
368
716
|
const keys = names.map((name) => {
|
|
369
|
-
const key =
|
|
370
|
-
|
|
371
|
-
return { key, base: { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers } }
|
|
717
|
+
const key = keySpec(name)
|
|
718
|
+
return { key, base: { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers: key.shift === true ? modifiers | 8 : modifiers } }
|
|
372
719
|
})
|
|
373
720
|
for (const one of keys) await page.key('keyDown', one.key.text === undefined ? one.base : { ...one.base, text: one.key.text })
|
|
374
721
|
if (holdMs > 0) await new Promise((settle) => setTimeout(settle, holdMs))
|
|
@@ -434,7 +781,7 @@ export function personNote(sidecar) {
|
|
|
434
781
|
export function holdText(sidecar) {
|
|
435
782
|
const left = Number(sidecar?.holding?.() ?? 0)
|
|
436
783
|
if (left <= 0) return ''
|
|
437
|
-
return `the person is using the mirror (${String(Math.ceil(left / 1000))} s more) — nothing was sent. Let them finish, then repeat this call;
|
|
784
|
+
return `the person is using the mirror (${String(Math.ceil(left / 1000))} s more) — nothing was sent. Let them finish, then repeat this call; browser_resume takes the page back now.`
|
|
438
785
|
}
|
|
439
786
|
|
|
440
787
|
/**
|
|
@@ -468,11 +815,15 @@ async function coverProbe(page, ref) {
|
|
|
468
815
|
* @param {ReturnType<typeof coverProbe>} probe - what the page said is on top.
|
|
469
816
|
* @returns {string} a sentence, or an empty string.
|
|
470
817
|
*/
|
|
471
|
-
function coverNote(probe) {
|
|
818
|
+
function coverNote(probe, moved) {
|
|
472
819
|
if (probe === null || probe === undefined) return ''
|
|
473
820
|
const what = probe.tag === 'div' || probe.tag === 'section' ? 'an overlay' : `a ${probe.tag}`
|
|
474
|
-
|
|
475
|
-
${what} covers it${probe.name === '' ? '' : `: ${probe.name}`}
|
|
821
|
+
const head = `
|
|
822
|
+
${what} covers it${probe.name === '' ? '' : `: ${probe.name}`}`
|
|
823
|
+
/* A banner that is about to be cleared is not a warning: the line below it
|
|
824
|
+
already says what was clicked on the way, so do not claim the click will
|
|
825
|
+
land under something that is leaving. */
|
|
826
|
+
return moved ? head : `${head} — the click would land beneath it`
|
|
476
827
|
}
|
|
477
828
|
|
|
478
829
|
/**
|
|
@@ -487,14 +838,19 @@ ${what} covers it${probe.name === '' ? '' : `: ${probe.name}`} — the click lan
|
|
|
487
838
|
async function clearCover(page, ref) {
|
|
488
839
|
const probe = await coverProbe(page, ref)
|
|
489
840
|
if (probe === null) return { note: '', cleared: true, name: '' }
|
|
490
|
-
const note = coverNote(probe)
|
|
841
|
+
const note = coverNote(probe, probe.dismiss !== null)
|
|
491
842
|
const away = probe.dismiss
|
|
492
843
|
if (away === null) return { note, cleared: false, name: '' }
|
|
493
844
|
await page.mouse('mouseMoved', { x: away.x, y: away.y })
|
|
494
845
|
await page.mouse('mousePressed', { x: away.x, y: away.y, clickCount: 1 })
|
|
495
846
|
await page.mouse('mouseReleased', { x: away.x, y: away.y, clickCount: 1 })
|
|
496
|
-
|
|
497
|
-
|
|
847
|
+
/* Prove the banner left instead of guessing a paint delay: poll the same
|
|
848
|
+
question the refusal would ask, and stop the moment it answers clear. */
|
|
849
|
+
let again = await coverProbe(page, ref)
|
|
850
|
+
for (let look = 0; look < 8 && again !== null; look += 1) {
|
|
851
|
+
await wait(20)
|
|
852
|
+
again = await coverProbe(page, ref)
|
|
853
|
+
}
|
|
498
854
|
if (again !== null) return { note: `${note}
|
|
499
855
|
- clicking ${away.name} on it did not clear it`, cleared: false, name: away.name } // ui-copy-ignore
|
|
500
856
|
return { note: `${note}
|
|
@@ -524,12 +880,19 @@ function wait(ms) {
|
|
|
524
880
|
* @param {{contextId?: number, sessionId?: string}|null} [route] - frame to run in.
|
|
525
881
|
* @returns {Promise<any>} the value.
|
|
526
882
|
*/
|
|
527
|
-
async function evaluate(page, expression, route) {
|
|
883
|
+
async function evaluate(page, expression, route, raceMs = 2000) {
|
|
528
884
|
const params = { expression, returnByValue: true, awaitPromise: true }
|
|
529
885
|
if (route?.contextId !== undefined) params.contextId = route.contextId
|
|
886
|
+
/* A page that navigates mid-call destroys the context this ran in; the
|
|
887
|
+
rejection arrives after the race is already lost. Handled, so it cannot
|
|
888
|
+
take the whole process down with it. */
|
|
530
889
|
const call = page.cdp.send('Runtime.evaluate', params, route?.sessionId)
|
|
531
|
-
|
|
890
|
+
.catch((error) => ({ threw: String(error?.message ?? error) }))
|
|
891
|
+
let timer = null
|
|
892
|
+
const raced = await Promise.race([call, new Promise((done) => { timer = setTimeout(() => done('timeout'), raceMs) })])
|
|
893
|
+
clearTimeout(timer)
|
|
532
894
|
if (raced === 'timeout') return undefined
|
|
895
|
+
if (raced?.threw !== undefined) return undefined
|
|
533
896
|
const { result, exceptionDetails } = raced
|
|
534
897
|
if (exceptionDetails !== undefined) {
|
|
535
898
|
return { error: String(exceptionDetails.exception?.description ?? exceptionDetails.text ?? 'evaluate failed') }
|
|
@@ -537,6 +900,36 @@ async function evaluate(page, expression, route) {
|
|
|
537
900
|
return result?.value
|
|
538
901
|
}
|
|
539
902
|
|
|
903
|
+
/**
|
|
904
|
+
* What the page looks like in one cheap string: address, title, how many
|
|
905
|
+
* elements and how much text. Two of these equal says the click did nothing
|
|
906
|
+
* visible — worth saying, so the model does not click a dead control twice.
|
|
907
|
+
* A Flash page has no DOM to ask, so its frame counter answers instead.
|
|
908
|
+
* @param {any} page - sidecar page.
|
|
909
|
+
* @returns {Promise<string>} a fingerprint, or '' when the page would not say.
|
|
910
|
+
*/
|
|
911
|
+
export async function pageMark(page) {
|
|
912
|
+
if (page?.flashBridge !== null && page?.flashBridge !== undefined) {
|
|
913
|
+
return `flash:${Number(page.seq ?? 0)}:${Number(page.lastFrameAt ?? 0)}`
|
|
914
|
+
}
|
|
915
|
+
const said = await evaluate(page, `(() => { const body = document.body; return String(document.title || '').slice(0, 80) + '|' + String(location.href).slice(0, 200) + '|' + (body ? body.innerText.length : 0) + '|' + document.querySelectorAll('*').length })()`).catch(() => undefined)
|
|
916
|
+
return typeof said === 'string' ? said : ''
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
/**
|
|
920
|
+
* The page's answer, as an object.
|
|
921
|
+
* @param {any} said - what evaluate returned.
|
|
922
|
+
* @returns {any} the answer, or null when the page did not answer.
|
|
923
|
+
*/
|
|
924
|
+
function asSpot(said) {
|
|
925
|
+
if (typeof said !== 'string') return null
|
|
926
|
+
try {
|
|
927
|
+
return JSON.parse(said)
|
|
928
|
+
} catch {
|
|
929
|
+
return null
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
|
|
540
933
|
/**
|
|
541
934
|
* Wait until the page has settled: loading finished and the address stopped
|
|
542
935
|
* changing. Acting on a page mid-navigation is how a driver clicks the old page.
|
|
@@ -544,21 +937,62 @@ async function evaluate(page, expression, route) {
|
|
|
544
937
|
* @param {number} [timeout] - ceiling in milliseconds.
|
|
545
938
|
* @returns {Promise<string>} the address it settled on.
|
|
546
939
|
*/
|
|
547
|
-
export async function settle(page, timeout = 5000) {
|
|
940
|
+
export async function settle(page, timeout = 5000, since = 0) {
|
|
941
|
+
/* A Flash page has no load events and no DOM of its own to poll: the next
|
|
942
|
+
picture is the only sign the click changed anything, and the bridge paints
|
|
943
|
+
every pump, so this ends at the frame instead of at the timeout. */
|
|
944
|
+
if (page.flashBridge !== null && page.flashBridge !== undefined) {
|
|
945
|
+
const until = Date.now() + Math.min(timeout, 700)
|
|
946
|
+
const before = Number(page.seq ?? 0)
|
|
947
|
+
while (Date.now() < until) {
|
|
948
|
+
if (Number(page.seq ?? 0) > before || Number(page.lastFrameAt ?? 0) > since) break
|
|
949
|
+
await new Promise((resolve) => setTimeout(resolve, 20))
|
|
950
|
+
}
|
|
951
|
+
return String(page.url ?? '')
|
|
952
|
+
}
|
|
548
953
|
/* A click may have opened a tab. The mirror follows it, so every tool that
|
|
549
954
|
waited on a page ends up on the page the person would be looking at. */
|
|
550
955
|
const opened = typeof page.adoptOpened === 'function' ? await page.adoptOpened().catch(() => '') : ''
|
|
551
956
|
const deadline = Date.now() + timeout
|
|
957
|
+
const started = Date.now()
|
|
552
958
|
let last = ''
|
|
553
959
|
let steady = 0
|
|
554
960
|
while (Date.now() < deadline) {
|
|
555
961
|
if (page.dialog !== null && page.dialog !== undefined) return String(page.url ?? '')
|
|
556
|
-
|
|
557
|
-
|
|
962
|
+
/* The browser already said the load it began has finished. That is the same
|
|
963
|
+
answer a readyState poll gives, and it stays true while a page's own
|
|
964
|
+
scripts hold the main thread, which is exactly when the poll stalls. */
|
|
965
|
+
const began = Number(page.navStarted ?? 0)
|
|
966
|
+
const fired = Number(page.loadFired ?? 0)
|
|
967
|
+
/* Two event-only answers: nothing began loading since the action, or the
|
|
968
|
+
navigation this action started has already finished. Neither needs the
|
|
969
|
+
renderer, whose answer can take seconds on a page that is busy. */
|
|
970
|
+
if (since > 0 && began <= since) return String(page.url ?? '')
|
|
971
|
+
if (fired > 0 && fired >= began && (since === 0 || fired >= since)) return String(page.url ?? '')
|
|
972
|
+
/* A click opened a page whose own requests hang: the browser began loading
|
|
973
|
+
and nothing has finished since. A person presses Esc and uses what
|
|
974
|
+
arrived, and waiting the whole timeout only to give up helps no one. */
|
|
975
|
+
if (since > 0 && began > since && Date.now() - began > 900) {
|
|
976
|
+
await page.cdp.send('Page.stopLoading', {}).catch(() => {})
|
|
977
|
+
return String(page.url ?? '')
|
|
978
|
+
}
|
|
979
|
+
const state = await evaluate(page, '({ ready: document.readyState, href: location.href, n: document.getElementsByTagName("*").length, c: (document.body?.textContent ?? "").length })', null, 400)
|
|
980
|
+
/* Nothing began loading since the action, and the page is complete: the
|
|
981
|
+
click changed state without navigating, so there is nothing to wait for. */
|
|
982
|
+
if (since > 0 && state?.ready === 'complete' && Number(page.navStarted ?? 0) <= since) return String(state?.href ?? '')
|
|
983
|
+
const key = `${String(state?.ready)}|${String(state?.href)}|${String(state?.n)}|${String(state?.c)}`
|
|
558
984
|
steady = key === last ? steady + 1 : 0
|
|
559
985
|
last = key
|
|
986
|
+
/* A page that already says complete needs one steady poll, not two: the
|
|
987
|
+
second look is what a redirect needs, and a click usually has none. */
|
|
988
|
+
if (state?.ready === 'complete' && steady >= 1) return String(state?.href ?? '')
|
|
560
989
|
if (state?.ready !== 'loading' && steady >= 2) return String(state?.href ?? '')
|
|
561
|
-
|
|
990
|
+
/* A loaded page can change on the next paint; a loading one is either
|
|
991
|
+
navigating or about to be. Right after an action that is usually the
|
|
992
|
+
page the action opened, which lands in tens of milliseconds, so the
|
|
993
|
+
first quarter second is watched closely and only then slows down. */
|
|
994
|
+
const fresh = Date.now() - started < 250
|
|
995
|
+
await wait(state?.ready === 'complete' ? (fresh ? 12 : 25) : (fresh ? 12 : 60))
|
|
562
996
|
}
|
|
563
997
|
return ''
|
|
564
998
|
}
|
|
@@ -588,7 +1022,29 @@ async function until(page, expression, timeout) {
|
|
|
588
1022
|
* @returns {Promise<string>} a compact report.
|
|
589
1023
|
*/
|
|
590
1024
|
/** The first control whose words match, scrolled into view. Text can beat a number: it costs no read. */
|
|
591
|
-
async function findWords(page, words) {
|
|
1025
|
+
export async function findWords(page, words, known) {
|
|
1026
|
+
/* The words a caller used usually came from a read that already happened, so
|
|
1027
|
+
look among those numbers first: one position check is a single evaluate,
|
|
1028
|
+
while a fresh read walks the whole page. A number the page no longer has
|
|
1029
|
+
falls through to the read, so a redraw is not a missed click. */
|
|
1030
|
+
const squeezeOf = (text) => String(text).toLowerCase().replace(/\s+/g, '')
|
|
1031
|
+
const wantedFirst = squeezeOf(words)
|
|
1032
|
+
const remembered = Array.isArray(known) ? known.filter((control) => control.view !== false && squeezeOf(control.name ?? '').includes(wantedFirst)) : []
|
|
1033
|
+
remembered.sort((a, b) => Number(squeezeOf(a.name ?? '') === wantedFirst) - Number(squeezeOf(b.name ?? '') === wantedFirst))
|
|
1034
|
+
/* A page that repeats one word down a list ("Add to cart" on every card) has
|
|
1035
|
+
one obvious answer: the first one a person would reach. The last-first
|
|
1036
|
+
order that keeps a scrolled page fast also picked whichever duplicate sat
|
|
1037
|
+
lowest, so visible matches are tried in reading order first and the near
|
|
1038
|
+
end is only the fallback. */
|
|
1039
|
+
const firstInOrder = remembered.filter((control) => control.view !== false).sort((a, b) => Number(a.ref) - Number(b.ref))
|
|
1040
|
+
const nearEnd = remembered.slice(-8).reverse()
|
|
1041
|
+
for (const control of [...firstInOrder.slice(0, 4), ...nearEnd]) {
|
|
1042
|
+
const spot = await locate(page, Number(control.ref))
|
|
1043
|
+
if (spot.gone === true) continue
|
|
1044
|
+
const name = String(spot.name ?? control.name ?? '')
|
|
1045
|
+
if (!squeezeOf(name).includes(wantedFirst)) continue
|
|
1046
|
+
return { ...control, ref: Number(control.ref), name, x: Number(spot.x), y: Number(spot.y) }
|
|
1047
|
+
}
|
|
592
1048
|
const read = await readPage(page, { mode: 'text', maxChars: 1200, max: 200 })
|
|
593
1049
|
const shown = read.controls.filter((control) => control.view !== false)
|
|
594
1050
|
/* The words may sit in separate inline nodes: <span>Sign</span><span>in</span>
|
|
@@ -626,21 +1082,90 @@ async function fieldSpot(page, words) {
|
|
|
626
1082
|
* Where the field called these words is, clearing a banner over it first.
|
|
627
1083
|
* @param {any} page - sidecar page.
|
|
628
1084
|
* @param {string} words - what the field is called on screen.
|
|
629
|
-
* @returns {Promise<{x: number, y: number, name: string, lead: string}|null>} the click.
|
|
1085
|
+
* @returns {Promise<{x: number, y: number, name: string, lead: string, ref?: number}|null>} the click.
|
|
630
1086
|
*/
|
|
631
|
-
async function fieldByName(page, words) {
|
|
632
|
-
const hit = await findWords(page, words)
|
|
1087
|
+
async function fieldByName(page, words, known) {
|
|
1088
|
+
const hit = await findWords(page, words, known)
|
|
633
1089
|
const lead = hit === null ? '' : await coverLead(page, hit.ref)
|
|
634
1090
|
if (hit !== null && EDITABLE_KINDS.includes(String(hit.kind))) {
|
|
635
1091
|
const spot = await locate(page, hit.ref)
|
|
636
|
-
if (spot.gone !== true) return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead }
|
|
1092
|
+
if (spot.gone !== true) return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead, ref: Number(hit.ref) }
|
|
637
1093
|
}
|
|
638
1094
|
const field = await fieldSpot(page, words)
|
|
639
1095
|
if (field !== null) return { ...field, lead }
|
|
640
1096
|
if (hit === null) return null
|
|
641
1097
|
const spot = await locate(page, hit.ref)
|
|
642
1098
|
if (spot.gone === true) return null
|
|
643
|
-
return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead }
|
|
1099
|
+
return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead, ref: Number(hit.ref) }
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
/**
|
|
1103
|
+
* A date-like field ignores human typing: Chrome's date input wants its own
|
|
1104
|
+
* segments, so keystrokes leave the value empty while the call reports success.
|
|
1105
|
+
* Set it the way a form would, and say the value the page now holds.
|
|
1106
|
+
* @param {any} page - sidecar page.
|
|
1107
|
+
* @param {string} text - what the caller wanted in it.
|
|
1108
|
+
* @returns {Promise<string>} a clause about the field, or '' when it is not one.
|
|
1109
|
+
*/
|
|
1110
|
+
/* The input kinds a person cannot type into: Chrome gives each of them its own
|
|
1111
|
+
segment editor, and keys typed at it leave the value empty. */
|
|
1112
|
+
const DATE_KINDS = ['date', 'time', 'month', 'week', 'datetime-local']
|
|
1113
|
+
|
|
1114
|
+
/**
|
|
1115
|
+
* The date-like kind the focus is on, if it is on one.
|
|
1116
|
+
* @param {any} page - sidecar page.
|
|
1117
|
+
* @returns {Promise<string>} date, time… or '' when it is an ordinary field.
|
|
1118
|
+
*/
|
|
1119
|
+
async function dateLikeFocus(page) {
|
|
1120
|
+
const said = await evaluate(page, `(() => {
|
|
1121
|
+
const el = document.activeElement
|
|
1122
|
+
if (!el || String(el.tagName).toUpperCase() !== 'INPUT') return ''
|
|
1123
|
+
const kind = String(el.type)
|
|
1124
|
+
if (!${JSON.stringify(DATE_KINDS)}.includes(kind)) return ''
|
|
1125
|
+
if (String(el.value) !== '') return ''
|
|
1126
|
+
return JSON.stringify({ kind })
|
|
1127
|
+
})()`).catch(() => undefined)
|
|
1128
|
+
return String(asSpot(said)?.kind ?? '')
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
async function fillDateLike(page, text) {
|
|
1132
|
+
const kind = await dateLikeFocus(page)
|
|
1133
|
+
if (kind === '') return ''
|
|
1134
|
+
const empty = { kind }
|
|
1135
|
+
const want = String(text).trim()
|
|
1136
|
+
const after = await evaluate(page, `(() => {
|
|
1137
|
+
const el = document.activeElement
|
|
1138
|
+
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set
|
|
1139
|
+
setter.call(el, ${JSON.stringify(want)})
|
|
1140
|
+
el.dispatchEvent(new Event('input', { bubbles: true }))
|
|
1141
|
+
el.dispatchEvent(new Event('change', { bubbles: true }))
|
|
1142
|
+
return String(el.value)
|
|
1143
|
+
})()`).catch(() => undefined)
|
|
1144
|
+
if (typeof after !== 'string' || after === '') {
|
|
1145
|
+
return `that ${empty.kind} field would not take "${want}" — it wants ${empty.kind === 'time' ? '09:30' : '2026-03-04'}`
|
|
1146
|
+
}
|
|
1147
|
+
return `set ${after} in the ${empty.kind} field`
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
/**
|
|
1151
|
+
* Put text in the focused box the cheapest way that still works there.
|
|
1152
|
+
* A box that reacts while you type (suggestion lists, live search) needs the
|
|
1153
|
+
* keys in order; anything else takes one paste event, and the box itself is
|
|
1154
|
+
* the judge of whether it took it.
|
|
1155
|
+
* @param {any} page - sidecar page.
|
|
1156
|
+
* @param {string} text - what to type.
|
|
1157
|
+
* @param {string} [hint] - the field's name, to spot a live-search box.
|
|
1158
|
+
* @returns {Promise<void>} resolves when the box holds the text.
|
|
1159
|
+
*/
|
|
1160
|
+
export async function typeInto(page, text, hint = '') {
|
|
1161
|
+
const words = String(text)
|
|
1162
|
+
if (words === '') return
|
|
1163
|
+
const live = /search|query|lookup|filter|搜索|筛选|查询/i.test(hint)
|
|
1164
|
+
if (!live && (await page.quickType(words)) === true) {
|
|
1165
|
+
await wait(30)
|
|
1166
|
+
return
|
|
1167
|
+
}
|
|
1168
|
+
await page.humanType(words)
|
|
644
1169
|
}
|
|
645
1170
|
|
|
646
1171
|
/**
|
|
@@ -654,20 +1179,28 @@ async function writeAt(page, point, text, options = {}) {
|
|
|
654
1179
|
/* Approach the control the way a hand does: a couple of small moves that
|
|
655
1180
|
land in different places, not a teleport followed by a click. */
|
|
656
1181
|
const drift = (by) => ({ x: point.x + by, y: point.y + Math.round(by / 2) })
|
|
1182
|
+
/* The shape of the approach is what a page can see; the gaps between its
|
|
1183
|
+
points only buy latency. Three moves stay, the pauses are frames. */
|
|
657
1184
|
await page.mouse('mouseMoved', drift(-14 - Math.round(Math.random() * 10)))
|
|
658
|
-
await wait(
|
|
1185
|
+
await wait(5 + Math.round(Math.random() * 8))
|
|
659
1186
|
await page.mouse('mouseMoved', drift(-4 - Math.round(Math.random() * 4)))
|
|
660
|
-
await wait(
|
|
1187
|
+
await wait(4 + Math.round(Math.random() * 6))
|
|
661
1188
|
await page.mouse('mouseMoved', point)
|
|
662
|
-
await wait(
|
|
1189
|
+
await wait(5 + Math.round(Math.random() * 8))
|
|
663
1190
|
await page.mouse('mousePressed', { ...point, clickCount: 1 })
|
|
664
|
-
await wait(
|
|
1191
|
+
await wait(5 + Math.round(Math.random() * 9))
|
|
665
1192
|
await page.mouse('mouseReleased', { ...point, clickCount: 1 })
|
|
666
|
-
await wait(
|
|
1193
|
+
await wait(14 + Math.round(Math.random() * 18))
|
|
667
1194
|
if (options.clear === true) await clearFocus(page)
|
|
1195
|
+
/* A date or time input never takes keystrokes: noticing that before typing
|
|
1196
|
+
spends one round trip instead of fourteen keys and a failed readback. */
|
|
1197
|
+
if (await dateLikeFocus(page) !== '') return await fillDateLike(page, text)
|
|
668
1198
|
if (options.human === false) await page.insertText(text)
|
|
669
|
-
else await page.
|
|
1199
|
+
else await typeInto(page, text, String(options.hint ?? ''))
|
|
1200
|
+
const fixed = await fillDateLike(page, text)
|
|
1201
|
+
if (fixed !== '') return fixed
|
|
670
1202
|
if (options.submit === true) await pressKey(page, 'Enter')
|
|
1203
|
+
return ''
|
|
671
1204
|
}
|
|
672
1205
|
|
|
673
1206
|
/**
|
|
@@ -745,7 +1278,7 @@ async function runStep(page, step, known) {
|
|
|
745
1278
|
}
|
|
746
1279
|
}
|
|
747
1280
|
if (step.tap !== undefined && !/^\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*$/.test(String(step.tap))) {
|
|
748
|
-
const hit = await findWords(page, String(step.tap))
|
|
1281
|
+
const hit = await findWords(page, String(step.tap), known)
|
|
749
1282
|
if (hit === null) return stuck(`no control says ${JSON.stringify(String(step.tap))}`)
|
|
750
1283
|
const lead = await coverLead(page, hit.ref)
|
|
751
1284
|
const spot = await locate(page, hit.ref)
|
|
@@ -759,9 +1292,10 @@ async function runStep(page, step, known) {
|
|
|
759
1292
|
if (step.field !== undefined) {
|
|
760
1293
|
/* The words may sit on the field or in a div above a bare input. Only a
|
|
761
1294
|
click that lands on the field itself can be trusted with the typing. */
|
|
762
|
-
const target = await fieldByName(page, String(step.field))
|
|
1295
|
+
const target = await fieldByName(page, String(step.field), known)
|
|
763
1296
|
if (target === null) return stuck(`${String(step.field)}: ${REF_GONE}`)
|
|
764
|
-
await writeAt(page, { x: target.x, y: target.y }, String(step.text ?? ''), { clear: step.clear, submit: step.submit })
|
|
1297
|
+
const dated = await writeAt(page, { x: target.x, y: target.y }, String(step.text ?? ''), { clear: step.clear, submit: step.submit })
|
|
1298
|
+
if (dated !== '') return dated
|
|
765
1299
|
return `${target.lead}typed into "${target.name.slice(0, 60)}"`
|
|
766
1300
|
}
|
|
767
1301
|
if (step.tap !== undefined) {
|
|
@@ -830,8 +1364,36 @@ async function runStep(page, step, known) {
|
|
|
830
1364
|
* @param {{mode?: string, maxChars?: number}} options - narrowing.
|
|
831
1365
|
* @returns {Promise<any>} the read.
|
|
832
1366
|
*/
|
|
1367
|
+
/** A read with nothing in it often means the document is still being built:
|
|
1368
|
+
* samples of real pages showed zero controls when the read landed before
|
|
1369
|
+
* hydration, and the model then calls the page broken. */
|
|
1370
|
+
function looksEmpty(read) {
|
|
1371
|
+
return (read?.controls?.length ?? 0) === 0 && String(read?.text ?? '').trim().length < 40
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
/** @param {string} url - what the page claims to be. @returns {boolean} true when waiting can still help. */
|
|
1375
|
+
function canFill(url) {
|
|
1376
|
+
const said = String(url ?? '')
|
|
1377
|
+
return said.startsWith('http://') || said.startsWith('https://')
|
|
1378
|
+
}
|
|
1379
|
+
|
|
833
1380
|
async function remember(page, refs, options) {
|
|
834
|
-
|
|
1381
|
+
let read = await readPage(page, options)
|
|
1382
|
+
if (looksEmpty(read) && canFill(read.url)) {
|
|
1383
|
+
/* Hydration and client-side rendering finish a few hundred milliseconds
|
|
1384
|
+
after the address does. Read again until the page has something, but
|
|
1385
|
+
never longer than a person would wait for a page that stays empty. */
|
|
1386
|
+
const deadline = Date.now() + 1500
|
|
1387
|
+
while (looksEmpty(read) && canFill(read.url) && Date.now() < deadline) {
|
|
1388
|
+
await wait(120)
|
|
1389
|
+
const again = await readPage(page, options)
|
|
1390
|
+
if (!looksEmpty(again)) {
|
|
1391
|
+
read = again
|
|
1392
|
+
break
|
|
1393
|
+
}
|
|
1394
|
+
read = again
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
835
1397
|
refs.refs = read.controls
|
|
836
1398
|
refs.at = Date.now()
|
|
837
1399
|
return read
|
|
@@ -853,6 +1415,34 @@ export function sameRead(before, after) {
|
|
|
853
1415
|
return true
|
|
854
1416
|
}
|
|
855
1417
|
|
|
1418
|
+
/**
|
|
1419
|
+
* Where two readings of a page differ, said in one line. A counter moving, one
|
|
1420
|
+
* row appearing, a word edited: the page is mostly the same, and sending all of
|
|
1421
|
+
* it again is what makes a long session expensive.
|
|
1422
|
+
* @param {string} before - the text earlier.
|
|
1423
|
+
* @param {string} after - the text now.
|
|
1424
|
+
* @returns {string} a short line, or '' when the two have nothing in common.
|
|
1425
|
+
*/
|
|
1426
|
+
export function textChange(before, after) {
|
|
1427
|
+
const was = String(before ?? '')
|
|
1428
|
+
const now = String(after ?? '')
|
|
1429
|
+
let head = 0
|
|
1430
|
+
const shortest = Math.min(was.length, now.length)
|
|
1431
|
+
while (head < shortest && was[head] === now[head]) head += 1
|
|
1432
|
+
let tail = 0
|
|
1433
|
+
while (tail < shortest - head && was[was.length - 1 - tail] === now[now.length - 1 - tail]) tail += 1
|
|
1434
|
+
const gone = was.slice(head, was.length - tail)
|
|
1435
|
+
const came = now.slice(head, now.length - tail)
|
|
1436
|
+
/* A page that changed in one place is worth a line; one that shares almost no
|
|
1437
|
+
text is a different page, and a diff of it would be as long as the page. */
|
|
1438
|
+
if (head + tail < 12) return ''
|
|
1439
|
+
const clip = (text) => (text.length > 160 ? `${text.slice(0, 157)}…` : text)
|
|
1440
|
+
const near = clip(was.slice(Math.max(0, head - 24), head))
|
|
1441
|
+
const far = clip(now.slice(now.length - tail, now.length - tail + 24))
|
|
1442
|
+
const line = `${near === '' ? '' : `…${near} `}${gone === '' ? '' : `[${clip(gone)}] `}→ ${came === '' ? '' : `[${clip(came)}] `}${far === '' ? '' : `${far}…`}`
|
|
1443
|
+
return line.trim()
|
|
1444
|
+
}
|
|
1445
|
+
|
|
856
1446
|
/**
|
|
857
1447
|
* Say only what changed since the previous read.
|
|
858
1448
|
* @param {any} previous - the read before.
|
|
@@ -864,8 +1454,7 @@ export function diffRead(previous, next, options = {}) {
|
|
|
864
1454
|
const maxChars = Number(options.maxChars ?? 2000)
|
|
865
1455
|
const before = new Map(previous.controls.map((control) => [control.ref, control]))
|
|
866
1456
|
const after = new Map(next.controls.map((control) => [control.ref, control]))
|
|
867
|
-
const lines = [next.url]
|
|
868
|
-
if (previous.url !== next.url) lines.push(`address changed to ${next.url}`)
|
|
1457
|
+
const lines = [previous.url === next.url ? next.url : `address ${next.url}`]
|
|
869
1458
|
const added = next.controls.filter((control) => !before.has(control.ref))
|
|
870
1459
|
const removed = previous.controls.filter((control) => !after.has(control.ref))
|
|
871
1460
|
const changed = next.controls.filter((control) => {
|
|
@@ -881,9 +1470,14 @@ export function diffRead(previous, next, options = {}) {
|
|
|
881
1470
|
}
|
|
882
1471
|
if (added.length + removed.length + changed.length === 0) lines.push('no control changed')
|
|
883
1472
|
if (previous.text !== next.text) {
|
|
884
|
-
const
|
|
885
|
-
|
|
886
|
-
if (
|
|
1473
|
+
const said = textChange(previous.text, next.text)
|
|
1474
|
+
const room = Math.max(200, Math.min(1200, maxChars - 200))
|
|
1475
|
+
if (said !== '') lines.push(`text: ${said}`)
|
|
1476
|
+
else {
|
|
1477
|
+
const head = next.text.slice(0, room)
|
|
1478
|
+
lines.push('', 'text now:', head)
|
|
1479
|
+
if (next.text.length > head.length) lines.push(`… ${next.text.length - head.length} more characters`)
|
|
1480
|
+
}
|
|
887
1481
|
}
|
|
888
1482
|
if (next.focused !== null && previous.focused?.ref !== next.focused.ref) lines.push(`focus ${next.focused.ref} ${next.focused.name}`)
|
|
889
1483
|
return lines.join('\n')
|
|
@@ -938,7 +1532,7 @@ export async function extractRows(page, args) {
|
|
|
938
1532
|
const out = await page.cdp.send('Runtime.evaluate', { expression, returnByValue: true }).catch((error) => ({ error: String(error?.message ?? error) }))
|
|
939
1533
|
if (out?.error !== undefined && out.error !== null && typeof out.error === 'string') return `the page did not answer: ${out.error}`
|
|
940
1534
|
const value = out?.result?.value
|
|
941
|
-
if (value === undefined || value === null || value === '') return 'the page did not answer; try again, or read the page with
|
|
1535
|
+
if (value === undefined || value === null || value === '') return 'the page did not answer; try again, or read the page with browser_view.'
|
|
942
1536
|
if (value.gone === true) return `ref ${ref} is gone. Read the page again and use a new number.`
|
|
943
1537
|
if (value.bad === true) return `${select} is not a selector this browser understands.`
|
|
944
1538
|
const rows = Array.isArray(value.rows) ? value.rows : []
|
|
@@ -1006,7 +1600,7 @@ if (args?.ref !== undefined) {
|
|
|
1006
1600
|
}
|
|
1007
1601
|
return best === null ? null : { x: Math.round(best.x), y: Math.round(best.y), name: best.name }
|
|
1008
1602
|
})()`)
|
|
1009
|
-
if (at === null || at === undefined) return 'No box to type in on this page. Call
|
|
1603
|
+
if (at === null || at === undefined) return 'No box to type in on this page. Call browser_view to see what is there.'
|
|
1010
1604
|
point = { x: Number(at.x), y: Number(at.y) }
|
|
1011
1605
|
}
|
|
1012
1606
|
await page.mouse('mouseMoved', point)
|
|
@@ -1049,7 +1643,7 @@ const took = Date.now() - started
|
|
|
1049
1643
|
const echo = String(said.slice(0, 24))
|
|
1050
1644
|
if (fresh !== '' && echo.length >= 8 && !fresh.includes(echo)) {
|
|
1051
1645
|
const asking = /(sign in|log in|登录|注册)/i.test(fresh)
|
|
1052
|
-
return `the page changed but does not show the message: it did not take what was sent.${asking ? ' It is asking for a sign-in — sign in in the column, then ask again.' : ' Try
|
|
1646
|
+
return `the page changed but does not show the message: it did not take what was sent.${asking ? ' It is asking for a sign-in — sign in in the column, then ask again.' : ' Try browser_view to see what is in the way.'}`
|
|
1053
1647
|
}
|
|
1054
1648
|
/* A box that still holds the words is the page saying it did not take them:
|
|
1055
1649
|
a signed-out page leaves them there, and the old wording sent the reader
|
|
@@ -1095,21 +1689,40 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1095
1689
|
}
|
|
1096
1690
|
/** Looking is still useful while the person has the page. */
|
|
1097
1691
|
const withRead = async (work) => {
|
|
1692
|
+
/* A call starting means the last click is over: a tab that opens now is not
|
|
1693
|
+
going to be followed under this call's feet. */
|
|
1694
|
+
try {
|
|
1695
|
+
sidecar.page?.disarmFollow?.()
|
|
1696
|
+
} catch {
|
|
1697
|
+
/* the page is already gone */
|
|
1698
|
+
}
|
|
1098
1699
|
/* A plugin reload makes a new driver while the browser it started keeps
|
|
1099
1700
|
running. Refusing every read until someone opens a page again would throw
|
|
1100
1701
|
away a browser that is still there: attach to it instead. */
|
|
1101
1702
|
if (!sidecar.alive) {
|
|
1703
|
+
/* A renderer crash can take the whole browser with it, and the first
|
|
1704
|
+
relaunch sometimes loses the race with the old process exiting. One
|
|
1705
|
+
more try turns that into a fresh tab instead of a refusal. */
|
|
1102
1706
|
try {
|
|
1103
1707
|
await sidecar.ensure({ session: currentSession() })
|
|
1104
1708
|
} catch {
|
|
1105
|
-
|
|
1709
|
+
await wait(300)
|
|
1710
|
+
try {
|
|
1711
|
+
await sidecar.ensure({ session: currentSession() })
|
|
1712
|
+
} catch {
|
|
1713
|
+
return { text: SIDECAR_OFF }
|
|
1714
|
+
}
|
|
1106
1715
|
}
|
|
1107
1716
|
}
|
|
1108
1717
|
const page = await sidecar.ensure({ session: currentSession() })
|
|
1109
1718
|
const dead = await unreachable(page)
|
|
1110
1719
|
if (dead !== '') return { text: dead + personNote(sidecar) }
|
|
1111
1720
|
const text = await work(page)
|
|
1112
|
-
|
|
1721
|
+
/* Why a page stopped working is usually in its own log, and looking it up
|
|
1722
|
+
costs another call. This is the one place every read passes through, so
|
|
1723
|
+
say what went wrong since the previous one, once. */
|
|
1724
|
+
const trouble = typeof page.troubleOf === 'function' ? page.troubleOf() : ''
|
|
1725
|
+
return { text: text + (trouble === '' ? '' : `\n${trouble}`) + personNote(sidecar) }
|
|
1113
1726
|
}
|
|
1114
1727
|
const withPage = async (work) => {
|
|
1115
1728
|
if (!sidecar.alive) return { text: SIDECAR_OFF }
|
|
@@ -1119,6 +1732,8 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1119
1732
|
}
|
|
1120
1733
|
const look = async (page, options) => {
|
|
1121
1734
|
const read = await remember(page, refsFor(), options)
|
|
1735
|
+
/* A read already showed the address: the next action does not repeat it. */
|
|
1736
|
+
if (String(read.url ?? '') !== '') saidUrls.set(page, String(read.url))
|
|
1122
1737
|
const said = renderPage(read, options)
|
|
1123
1738
|
return isCheckPage(read) ? `${said}\n${CHECK_NOTE}` : said
|
|
1124
1739
|
}
|
|
@@ -1160,13 +1775,17 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1160
1775
|
|
|
1161
1776
|
return [
|
|
1162
1777
|
{
|
|
1163
|
-
name: '
|
|
1164
|
-
description: 'Start (or reuse) the
|
|
1778
|
+
name: 'browser_open', // ui-copy-ignore
|
|
1779
|
+
description: 'Start (or reuse) the background browser — a real Chrome with its own persistent profile — and open a URL in it. It is not the host\'s own browser tab: this Chrome runs on its own and only its picture is mirrored into this chat\'s right column, so it keeps working while the person looks elsewhere. Logins survive restarts. Use it only when the person explicitly asked for a browser this turn; reading a URL, a document or checking whether a site answers is `web_fetch`\'s job, not this one.',
|
|
1165
1780
|
parameters: {
|
|
1166
1781
|
type: 'object', // ui-copy-ignore
|
|
1167
1782
|
properties: {
|
|
1168
1783
|
url: { type: 'string', description: 'Address to open. Omit to reuse the current page.' },
|
|
1169
1784
|
headless: { type: 'boolean', description: 'Run without a window. Default true; turn off only if a site refuses a windowless browser.' },
|
|
1785
|
+
flash: {
|
|
1786
|
+
type: 'boolean',
|
|
1787
|
+
description: 'Play the page in the Flash-era browser instead of Chrome — for a site whose game is a Flash movie that never got an HTML5 version. Default off; switching either way restarts the browser.',
|
|
1788
|
+
},
|
|
1170
1789
|
},
|
|
1171
1790
|
additionalProperties: false,
|
|
1172
1791
|
},
|
|
@@ -1176,12 +1795,14 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1176
1795
|
/** @param {any} args - arguments. @param {any} exec - tool run. @returns {Promise<{text: string}>} what the page is. */
|
|
1177
1796
|
execute: async (args, exec) => {
|
|
1178
1797
|
const who = String(exec?.agent?.id ?? '')
|
|
1179
|
-
const page = await sidecar.ensure({ url: args?.url, headless: args?.headless, session: who })
|
|
1798
|
+
const page = await sidecar.ensure({ url: args?.url, headless: args?.headless, flash: args?.flash, session: who })
|
|
1180
1799
|
if (args?.url === undefined) await page.readTitle()
|
|
1181
1800
|
/* A window on the person's screen is their business, not a surprise: say
|
|
1182
1801
|
it, so nobody wonders what popped up. A site that refuses a windowless
|
|
1183
1802
|
browser is the only reason this happens. */
|
|
1184
|
-
|
|
1803
|
+
/* The Flash runtime is windowless even when headless is false: its window
|
|
1804
|
+
is never shown, so nothing popped up to explain. */
|
|
1805
|
+
const windowed = sidecar.headless === false && sidecar.flash !== true ? 'The browser is showing a window this time — a site only opens for a windowed one. Same profile, same logins.\n' : ''
|
|
1185
1806
|
/* The tab appears in the calling chat and nowhere else: the person may
|
|
1186
1807
|
be reading a different chat, and a browser there would be a surprise. */
|
|
1187
1808
|
if (who !== '') host.want?.(who) // ui-copy-ignore
|
|
@@ -1189,19 +1810,254 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1189
1810
|
const dead = await unreachable(page)
|
|
1190
1811
|
if (dead !== '') return { text: dead }
|
|
1191
1812
|
/* The column may still be drawing nothing: say so here, where the
|
|
1192
|
-
person is looking for a picture, rather than only in
|
|
1813
|
+
person is looking for a picture, rather than only in browser_state. */
|
|
1193
1814
|
const unseen = who === '' ? '' : viewerNote(await watchForViewer(sidecar, page, 1200))
|
|
1194
|
-
return { text: windowed + (unseen === '' ? '' : `${unseen}\n`) + await look(page, { mode: 'text' }) }
|
|
1815
|
+
return { text: windowed + (unseen === '' ? '' : `${unseen}\n`) + await look(page, { mode: 'text' }) + (await pdfNote(page)) }
|
|
1816
|
+
},
|
|
1817
|
+
},
|
|
1818
|
+
{
|
|
1819
|
+
name: 'browser_mode', // ui-copy-ignore
|
|
1820
|
+
description: 'Choose how this chat acts from now on. "human" moves the real mouse and presses real keys, which no page can tell from a person — games and pages that watch pointer movement need this. "dom" clicks and fills the controls the page registered, which reaches one that an overlay covers and leaves key handlers alone. A call can also pass its own mode, which wins for that call only.',
|
|
1821
|
+
parameters: {
|
|
1822
|
+
type: 'object', // ui-copy-ignore
|
|
1823
|
+
properties: {
|
|
1824
|
+
mode: { type: 'string', enum: ['human', 'dom'], description: 'Which way to act.' },
|
|
1825
|
+
},
|
|
1826
|
+
required: ['mode'], // ui-copy-ignore
|
|
1827
|
+
additionalProperties: false,
|
|
1828
|
+
},
|
|
1829
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
1830
|
+
timeoutMs: 10_000,
|
|
1831
|
+
isConcurrencySafe: () => true,
|
|
1832
|
+
/** @param {any} args - arguments. @returns {Promise<{text: string}>} which way is set. */
|
|
1833
|
+
execute: async (args) => {
|
|
1834
|
+
const said = String(args?.mode ?? '')
|
|
1835
|
+
if (!WAYS.has(said)) return 'Pass "human" or "dom".'
|
|
1836
|
+
const chat = currentSession()
|
|
1837
|
+
if (said === 'human') chatWays.delete(chat)
|
|
1838
|
+
else chatWays.set(chat, said)
|
|
1839
|
+
return said === 'human'
|
|
1840
|
+
? 'Real mouse and real keys from now on: pages cannot tell it from a person.'
|
|
1841
|
+
: "The page's own controls from now on: clicks and typing go through its DOM, so an overlay cannot swallow them."
|
|
1195
1842
|
},
|
|
1196
1843
|
},
|
|
1197
1844
|
{
|
|
1198
|
-
name: '
|
|
1199
|
-
description: '
|
|
1845
|
+
name: 'browser_prepare', // ui-copy-ignore
|
|
1846
|
+
description: 'Get a game or a heavy app page ready to play, in one call: read what is on it, find the surface that takes input, say whether it needs a Flash player this browser does not have, measure its real frame rate, and test whether keys reach it. Ends with what to do next instead of a blank page and no explanation.',
|
|
1200
1847
|
parameters: {
|
|
1201
1848
|
type: 'object', // ui-copy-ignore
|
|
1202
1849
|
properties: {
|
|
1203
|
-
|
|
1204
|
-
|
|
1850
|
+
seconds: { type: 'number', description: 'How long to count frames for. Default 1.2, at most 5.' },
|
|
1851
|
+
focus: { type: 'boolean', description: 'Click the surface to put keyboard focus in it. Default true.' },
|
|
1852
|
+
},
|
|
1853
|
+
additionalProperties: false,
|
|
1854
|
+
},
|
|
1855
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
1856
|
+
timeoutMs: 45_000,
|
|
1857
|
+
isConcurrencySafe: () => false,
|
|
1858
|
+
/** @param {any} args - arguments. @param {any} exec - tool run. @returns {Promise<{text: string}>} what the page is ready for. */
|
|
1859
|
+
execute: async (args, exec) => {
|
|
1860
|
+
const who = String(exec?.agent?.id ?? '')
|
|
1861
|
+
const page = await sidecar.ensure({ session: who })
|
|
1862
|
+
await settle(page, 8000)
|
|
1863
|
+
const seconds = Math.min(5, Math.max(0.5, Number(args?.seconds) || 1.2))
|
|
1864
|
+
const ms = Math.round(seconds * 1000)
|
|
1865
|
+
/* A page that never arrived is not a game that is not ready: say which. */
|
|
1866
|
+
const dead = await unreachable(page)
|
|
1867
|
+
if (dead !== '') return { text: dead }
|
|
1868
|
+
const read = await readPage(page, { mode: 'text', maxChars: 1500 })
|
|
1869
|
+
const said = asSpot(await evaluate(page, `(() => {
|
|
1870
|
+
const box = (el) => { const r = el.getBoundingClientRect(); return { w: Math.round(r.width), h: Math.round(r.height), x: Math.round(r.x + r.width / 2), y: Math.round(r.y + r.height / 2) } }
|
|
1871
|
+
const shown = (el) => {
|
|
1872
|
+
const r = el.getBoundingClientRect()
|
|
1873
|
+
const st = getComputedStyle(el)
|
|
1874
|
+
return r.width >= 120 && r.height >= 80 && st.visibility !== 'hidden' && st.display !== 'none' && Number(st.opacity) > 0.05
|
|
1875
|
+
}
|
|
1876
|
+
const spot = (el, tag) => {
|
|
1877
|
+
const first = box(el)
|
|
1878
|
+
const seen = first.y > 0 && first.y < innerHeight && first.x > 0 && first.x < innerWidth
|
|
1879
|
+
if (!seen) {
|
|
1880
|
+
/* A surface below the fold is not where the person is looking, and
|
|
1881
|
+
a click at its coordinate would land somewhere else. */
|
|
1882
|
+
el.scrollIntoView({ block: 'center', inline: 'center' })
|
|
1883
|
+
return { tag, src: String(el.getAttribute('src') ?? ''), ...box(el), scrolled: true }
|
|
1884
|
+
}
|
|
1885
|
+
return { tag, src: String(el.getAttribute('src') ?? ''), ...first }
|
|
1886
|
+
}
|
|
1887
|
+
const canvases = Array.from(document.querySelectorAll('canvas')).filter(shown).map((el) => spot(el, 'canvas'))
|
|
1888
|
+
const frames = Array.from(document.querySelectorAll('iframe')).filter(shown).map((el) => spot(el, 'frame'))
|
|
1889
|
+
const movies = document.querySelectorAll('embed, object').length
|
|
1890
|
+
return JSON.stringify({ canvases, frames, movies })
|
|
1891
|
+
})()`))
|
|
1892
|
+
const surface = said?.canvases?.[0] ?? said?.frames?.[0] ?? null
|
|
1893
|
+
/* Give a game the click a person gives it: some of them only start
|
|
1894
|
+
listening for keys once the surface has been touched. */
|
|
1895
|
+
let clicked = ''
|
|
1896
|
+
if (args?.focus !== false && surface !== null) {
|
|
1897
|
+
await page.mouse('mouseMoved', { x: surface.x - 6, y: surface.y - 4 }).catch(() => {})
|
|
1898
|
+
await wait(40)
|
|
1899
|
+
await page.mouse('mousePressed', { x: surface.x, y: surface.y, clickCount: 1 }).catch(() => {})
|
|
1900
|
+
await wait(30)
|
|
1901
|
+
await page.mouse('mouseReleased', { x: surface.x, y: surface.y, clickCount: 1 }).catch(() => {})
|
|
1902
|
+
clicked = `clicked the ${surface.tag} at ${surface.x},${surface.y}`
|
|
1903
|
+
}
|
|
1904
|
+
/* Count the page's own frames and listen for a key, in the page and in
|
|
1905
|
+
the biggest frame, because a game is usually not in the page itself. */
|
|
1906
|
+
const frames = await page.frameList().catch(() => [])
|
|
1907
|
+
const routeOfFrame = (frame) => {
|
|
1908
|
+
if (frame === undefined) return null
|
|
1909
|
+
const sessionId = frame.contextId === undefined && typeof page.childSession === 'function' ? page.childSession(frame.frameId) : undefined
|
|
1910
|
+
if (frame.contextId === undefined && sessionId === undefined) return null
|
|
1911
|
+
return { contextId: frame.contextId, sessionId }
|
|
1912
|
+
}
|
|
1913
|
+
const countFrames = `new Promise((done) => {
|
|
1914
|
+
window.__dshKeys = 0
|
|
1915
|
+
if (window.__dshKeyUp === undefined) {
|
|
1916
|
+
window.__dshKeyUp = true
|
|
1917
|
+
addEventListener('keydown', () => { window.__dshKeys += 1 }, true)
|
|
1918
|
+
}
|
|
1919
|
+
let n = 0
|
|
1920
|
+
const t0 = performance.now()
|
|
1921
|
+
const tick = () => { n += 1; if (performance.now() - t0 < ${ms}) requestAnimationFrame(tick) }
|
|
1922
|
+
requestAnimationFrame(tick)
|
|
1923
|
+
setTimeout(() => done(JSON.stringify({ fps: Math.round((n * 1000) / Math.max(1, performance.now() - t0)), keys: window.__dshKeys })), ${ms + 60})
|
|
1924
|
+
})`
|
|
1925
|
+
const mainRate = asSpot(await evaluate(page, countFrames))
|
|
1926
|
+
let frameRate = null
|
|
1927
|
+
const childFrames = frames.slice(1).filter((frame) => String(frame.url ?? '') !== 'chrome-error://chromewebdata/')
|
|
1928
|
+
const biggest = childFrames[0]
|
|
1929
|
+
const route = routeOfFrame(biggest)
|
|
1930
|
+
if (route !== null) frameRate = asSpot(await evaluate(page, countFrames, route))
|
|
1931
|
+
let keys = route === null ? 'none arrived — there is no frame to test' : 'none arrived in the frame'
|
|
1932
|
+
const keyDown = { type: 'keyDown', key: 'Shift', code: 'ShiftLeft', windowsVirtualKeyCode: 16, nativeVirtualKeyCode: 16 }
|
|
1933
|
+
await page.cdp.send('Input.dispatchKeyEvent', keyDown, route?.sessionId).catch(() => {})
|
|
1934
|
+
await page.cdp.send('Input.dispatchKeyEvent', { ...keyDown, key: 'Shift', type: 'keyUp' }, route?.sessionId).catch(() => {})
|
|
1935
|
+
await wait(80)
|
|
1936
|
+
const after = asSpot(await evaluate(page, 'JSON.stringify({ keys: window.__dshKeys ?? 0 })', route))
|
|
1937
|
+
const mainAfter = asSpot(await evaluate(page, 'JSON.stringify({ keys: window.__dshKeys ?? 0 })'))
|
|
1938
|
+
const arrived = Number(after?.keys ?? 0) + Number(mainAfter?.keys ?? 0) - Number(frameRate?.keys ?? 0) - Number(mainRate?.keys ?? 0)
|
|
1939
|
+
if (arrived > 0) keys = `arrived in the frame (${arrived} keydown event${arrived === 1 ? '' : 's'} from a real key)`
|
|
1940
|
+
else if (route !== null) keys = 'not in the frame — click inside it once, or the game only listens after it starts'
|
|
1941
|
+
const lines = []
|
|
1942
|
+
lines.push(`page: ${read.title === '' ? read.url : read.title} — ${read.controls.length} control(s), ${read.textChars} character(s)`)
|
|
1943
|
+
lines.push(surface === null ? 'surface: no canvas or frame takes the screen — the page itself takes input' : `surface: ${surface.tag} ${surface.w}x${surface.h} at ${surface.x},${surface.y}${surface.scrolled === true ? ' (scrolled it into view)' : ''}${clicked === '' ? '' : ` (${clicked})`}`)
|
|
1944
|
+
if (read.blind?.flash > 0 && sidecar.flash !== true) lines.push(`flash: ${read.blind.flash} movie(s) — open the same page with flash:true and it runs in this plugin's own player, in the column, with no window on the desktop`) // ui-copy-ignore
|
|
1945
|
+
else if (read.blind?.flash > 0) lines.push(`flash: ${read.blind.flash} movie(s) running on this plugin's own player`)
|
|
1946
|
+
else lines.push('flash: none found')
|
|
1947
|
+
lines.push(`frames: ${Math.max(0, frames.length - 1)} (${childFrames.filter((frame) => String(frame.url ?? '') === 'about:blank').length} of them about:blank)`)
|
|
1948
|
+
const rates = []
|
|
1949
|
+
if (mainRate !== null) rates.push(`page ${mainRate.fps} fps`)
|
|
1950
|
+
if (frameRate !== null) rates.push(`frame ${frameRate.fps} fps`)
|
|
1951
|
+
lines.push(`frame rate: ${rates.length === 0 ? 'the page could not count' : rates.join(', ')} over ${ms} ms`)
|
|
1952
|
+
lines.push(`keys: ${keys}`)
|
|
1953
|
+
lines.push(`mode: human${surface === null ? '' : ' — a game watches the mouse and the keys, and only real ones arrive trusted'}`)
|
|
1954
|
+
const trouble = read.blind?.flash > 0 && sidecar.flash !== true
|
|
1955
|
+
? 'call open with flash:true to run it here'
|
|
1956
|
+
: read.blind?.flash > 0
|
|
1957
|
+
? 'the movie needs a click inside it before it starts'
|
|
1958
|
+
: surface === null && Number(mainRate?.fps ?? 0) === 0
|
|
1959
|
+
? 'nothing on this page is animating — it may be waiting for a click, or the game is behind a login'
|
|
1960
|
+
: surface === null
|
|
1961
|
+
? 'ready to read and act on, though there is no canvas or frame to watch'
|
|
1962
|
+
: `ready: ${surface.tag} is live at ${mainRate?.fps ?? frameRate?.fps ?? 0} fps`
|
|
1963
|
+
lines.push(trouble)
|
|
1964
|
+
return { text: lines.join('\n') }
|
|
1965
|
+
},
|
|
1966
|
+
},
|
|
1967
|
+
{
|
|
1968
|
+
name: 'browser_find', // ui-copy-ignore
|
|
1969
|
+
description: 'Find words on the page without reading it all: returns matching numbered controls (click the ref) and matching text (click x/y), each with whether it is on screen. A phone-sized page has hundreds of lines; this answers "where does it say X" in one compact result.',
|
|
1970
|
+
parameters: {
|
|
1971
|
+
type: 'object', // ui-copy-ignore
|
|
1972
|
+
properties: {
|
|
1973
|
+
query: { type: 'string', description: 'Words to look for; case does not matter.' },
|
|
1974
|
+
regex: { type: 'boolean', description: 'Treat query as a regular expression.' },
|
|
1975
|
+
limit: { type: 'integer', description: 'Matches to return. Default 12, at most 40.' },
|
|
1976
|
+
},
|
|
1977
|
+
required: ['query'],
|
|
1978
|
+
additionalProperties: false,
|
|
1979
|
+
},
|
|
1980
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
1981
|
+
timeoutMs: 30_000,
|
|
1982
|
+
isConcurrencySafe: () => false,
|
|
1983
|
+
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the matches. */
|
|
1984
|
+
execute: async (args) => withRead(async (page) => {
|
|
1985
|
+
const query = String(args?.query ?? '').trim()
|
|
1986
|
+
if (query === '') return 'Give the words to look for.'
|
|
1987
|
+
const found = await findOnPage(page, query, { regex: args?.regex === true, limit: Number(args?.limit ?? 12) })
|
|
1988
|
+
if (found.fault !== undefined) return `the page did not answer when searched — ${String(found.fault).split('\n')[0].slice(0, 200)}`
|
|
1989
|
+
const matches = Array.isArray(found.matches) ? found.matches : []
|
|
1990
|
+
if (matches.length === 0) return `nothing on this page says ${JSON.stringify(query)} — try other words, or call browser_view for the whole page.`
|
|
1991
|
+
const lines = matches.map((one) => `${one.ref === 0 ? `at ${one.x},${one.y}` : one.ref} [${one.tag}] ${one.name}${one.view === true ? '' : ' (off screen)'}`)
|
|
1992
|
+
return `${matches.length} match(es) for ${JSON.stringify(query)}:\n${lines.join('\n')}`
|
|
1993
|
+
}),
|
|
1994
|
+
},
|
|
1995
|
+
{
|
|
1996
|
+
name: 'browser_read', // ui-copy-ignore
|
|
1997
|
+
description: 'Read the page as a person does: one Markdown block of headings, paragraphs, lists and links, no controls and no markup. Far fewer tokens than browser_view on long pages (articles, docs, threads, issue trackers), and it keeps links so a follow-up can be opened by address. Use browser_view when you need numbers to click or type; use browser_read when you only need the words.',
|
|
1998
|
+
parameters: {
|
|
1999
|
+
type: 'object', // ui-copy-ignore
|
|
2000
|
+
properties: {
|
|
2001
|
+
max_chars: { type: 'integer', description: 'Character ceiling. Default 6000; the result says when it was cut.' },
|
|
2002
|
+
query: { type: 'string', description: 'What you want to know. With a query, only the passages that answer it come back, ranked — the cheapest way to read a long page. Without one, the whole page comes back.' },
|
|
2003
|
+
force: { type: 'boolean', description: 'Read again even if the page has not changed since this chat last read it.' },
|
|
2004
|
+
},
|
|
2005
|
+
additionalProperties: false,
|
|
2006
|
+
},
|
|
2007
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
2008
|
+
timeoutMs: 30_000,
|
|
2009
|
+
isConcurrencySafe: () => false,
|
|
2010
|
+
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the words, as Markdown. */
|
|
2011
|
+
execute: async (args) => withRead(async (page) => {
|
|
2012
|
+
const asked = String(args?.query ?? '').trim()
|
|
2013
|
+
const ceiling = Number(args?.max_chars ?? (asked === '' ? 6000 : 4000))
|
|
2014
|
+
const mark = await pageMark(page)
|
|
2015
|
+
const key = mark === '' ? '' : `${mark}|${asked}|${ceiling}`
|
|
2016
|
+
const seen = page === null || page === undefined ? undefined : readSeen.get(page)
|
|
2017
|
+
/* The text this chat already has is not worth sending twice: a page that
|
|
2018
|
+
looks the same answers with where it is, not the same four thousand
|
|
2019
|
+
characters. force is for a caller that lost the earlier answer. */
|
|
2020
|
+
if (args?.force !== true && key !== '' && seen !== undefined && seen.key === key) {
|
|
2021
|
+
return `${mark.split('|')[1] ?? ''} · unchanged since this chat last read it (${String(seen.text.length)} chars) — the text is the one above. Call browser_view with changed_only to see what moved, or pass force for it again.`
|
|
2022
|
+
}
|
|
2023
|
+
const keep = (text) => {
|
|
2024
|
+
if (key !== '' && page !== null && page !== undefined) readSeen.set(page, { key, text })
|
|
2025
|
+
return text
|
|
2026
|
+
}
|
|
2027
|
+
if (asked !== '') {
|
|
2028
|
+
const found = await readRelevant(page, asked, { maxChars: ceiling })
|
|
2029
|
+
if (found.fault !== undefined) return `the page did not answer when read — ${String(found.fault).split('\n')[0].slice(0, 200)}`
|
|
2030
|
+
if (found.text === '') {
|
|
2031
|
+
/* The question missed every block: the page is still the answer, so
|
|
2032
|
+
hand it over whole rather than make the caller guess again. */
|
|
2033
|
+
const whole = await readArticle(page, { maxChars: ceiling })
|
|
2034
|
+
if (whole.fault === undefined && whole.text !== '') {
|
|
2035
|
+
return keep(`${whole.url}\n# ${whole.title === '' ? '(untitled)' : whole.title}\n\nnothing here answers ${JSON.stringify(asked)} (${String(found.total)} blocks read), so the whole page follows:\n\n${whole.text}${whole.truncated ? `\n\n(cut at ${String(whole.chars)} characters; raise max_chars for more)` : ''}`)
|
|
2036
|
+
}
|
|
2037
|
+
return `${found.url}\n# ${found.title === '' ? '(untitled)' : found.title}\n\nnothing here answers ${JSON.stringify(asked)} (${String(found.total)} blocks read). Call browser_view for the whole page, or try other words.`
|
|
2038
|
+
}
|
|
2039
|
+
return keep(`${found.url}\n# ${found.title === '' ? '(untitled)' : found.title}\n\n${found.text}\n\n(${String(found.matched)} of ${String(found.total)} blocks, ranked for ${JSON.stringify(asked)})`)
|
|
2040
|
+
}
|
|
2041
|
+
const read = await readArticle(page, { maxChars: ceiling })
|
|
2042
|
+
if (read.fault !== undefined) return `the page did not answer when read — ${read.fault.split('\n')[0].slice(0, 200)}`
|
|
2043
|
+
if (read.text === '') {
|
|
2044
|
+
const pdf = await pdfNote(page)
|
|
2045
|
+
if (pdf !== '') return `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}${pdf}`
|
|
2046
|
+
return `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}\n\nnothing to read here: the page draws its content, or it is still loading. Call browser_view to see what is on screen.`
|
|
2047
|
+
}
|
|
2048
|
+
const head = `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}`
|
|
2049
|
+
const tail = read.truncated ? `\n\n(cut at ${String(read.chars)} characters; raise max_chars for more)` : ''
|
|
2050
|
+
return keep(`${head}\n\n${read.text}${tail}`)
|
|
2051
|
+
}),
|
|
2052
|
+
},
|
|
2053
|
+
{
|
|
2054
|
+
name: 'browser_view', // ui-copy-ignore
|
|
2055
|
+
description: 'Read the sidecar browser: numbered controls and the words on screen. Numbers stay valid across reads and actions until the page navigates. Narrow it with mode (text, links, forms, full, map) and max_chars; ask for changed_only after an action to spend fewer tokens. Use map first on a page too big to read.',
|
|
2056
|
+
parameters: {
|
|
2057
|
+
type: 'object', // ui-copy-ignore
|
|
2058
|
+
properties: {
|
|
2059
|
+
mode: { type: 'string', enum: ['text', 'links', 'forms', 'full', 'map'], description: 'What to include. Default text: controls plus the visible words. map: a landmark summary of a big page — how many controls of each kind, the distinct ones with their numbers, and how many words there are, without the words.' },
|
|
2060
|
+
max_chars: { type: 'integer', description: 'Character budget for the page text. Default 4600.' },
|
|
1205
2061
|
changed_only: { type: 'boolean', description: 'Only what changed since the last read.' },
|
|
1206
2062
|
},
|
|
1207
2063
|
additionalProperties: false,
|
|
@@ -1226,31 +2082,42 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1226
2082
|
state.lastShape = JSON.stringify(options)
|
|
1227
2083
|
return text
|
|
1228
2084
|
}
|
|
2085
|
+
/* The read before this one answers two questions: what is new on this
|
|
2086
|
+
one, and whether the page moved at all. Keep it before either answer
|
|
2087
|
+
is overwritten. */
|
|
1229
2088
|
const before = state.last
|
|
1230
2089
|
state.last = read
|
|
2090
|
+
const fresh = freshRefs(before, read)
|
|
2091
|
+
const freshLine = fresh.length === 0 || fresh.length > 12 ? '' : `\nnew since the read before: ${fresh.map(String).join(', ')}`
|
|
1231
2092
|
/* Same page, same shape, same words: the numbers above still work, and
|
|
1232
2093
|
repeating the page would only spend tokens. */
|
|
1233
2094
|
if (sameRead(before, read) && state.lastShape === JSON.stringify(options)) {
|
|
1234
2095
|
const moved = Number(before.scroll?.y ?? 0) !== Number(read.scroll?.y ?? 0) ? `, scrolled to ${Math.round(Number(read.scroll?.y ?? 0))}` : ''
|
|
1235
|
-
|
|
2096
|
+
/* The address was said with the read above; "here" is the whole of it. */
|
|
2097
|
+
const where = saidUrls.get(page) === String(read.url ?? '') ? 'here' : read.url
|
|
2098
|
+
saidUrls.set(page, String(read.url ?? ''))
|
|
2099
|
+
return `still ${where}${moved} — nothing changed since the last read; the numbers above still work${note}`
|
|
1236
2100
|
}
|
|
1237
2101
|
state.lastShape = JSON.stringify(options)
|
|
1238
|
-
|
|
2102
|
+
const said = withoutSeenAddress(page, read.url, renderPage(read, options))
|
|
2103
|
+
saidUrls.set(page, String(read.url ?? ''))
|
|
2104
|
+
return said + freshLine + note + (await pdfNote(page))
|
|
1239
2105
|
}),
|
|
1240
2106
|
},
|
|
1241
2107
|
{
|
|
1242
|
-
name: '
|
|
2108
|
+
name: 'browser_click', // ui-copy-ignore
|
|
1243
2109
|
description: 'Click a numbered control, a point, or the first control whose words contain `text` — the last needs no read first. The click is real mouse input and the mirror shows where it landed.',
|
|
1244
2110
|
parameters: {
|
|
1245
2111
|
type: 'object', // ui-copy-ignore
|
|
1246
2112
|
properties: {
|
|
1247
|
-
ref: { type: 'integer', description: 'Number from
|
|
2113
|
+
ref: { type: 'integer', description: 'Number from browser_view.' },
|
|
1248
2114
|
text: { type: 'string', description: 'Words on the control, when its number is not known.' },
|
|
1249
2115
|
x: { type: 'integer', description: 'Viewport x, when clicking a point.' },
|
|
1250
2116
|
y: { type: 'integer', description: 'Viewport y, when clicking a point.' },
|
|
1251
2117
|
double: { type: 'boolean', description: 'Double-click.' },
|
|
1252
2118
|
right: { type: 'boolean', description: 'Right-click, opening a context menu.' },
|
|
1253
2119
|
hold_ms: { type: 'integer', description: 'Keep the button down this long before releasing: a press-and-hold a game or a drag handle needs.' },
|
|
2120
|
+
mode: { type: 'string', enum: ['human', 'dom'], description: '"human" moves the real mouse and is what a page cannot tell from a person; "dom" fires the page\'s own click on the numbered control, which reaches one an overlay covers. Omit to use what this chat set with browser_mode.' },
|
|
1254
2121
|
read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
|
|
1255
2122
|
},
|
|
1256
2123
|
additionalProperties: false,
|
|
@@ -1269,12 +2136,22 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1269
2136
|
if (spot.followed === true) followed = `the page redrew, so "${String(spot.was).slice(0, 50)}" has a new number — used where it is now`
|
|
1270
2137
|
point = { x: Number(spot.x), y: Number(spot.y) }
|
|
1271
2138
|
} else if (String(args?.text ?? '') !== '') {
|
|
1272
|
-
const hit = await findWords(page, String(args.text))
|
|
1273
|
-
if (hit === null) return `No control says ${JSON.stringify(String(args.text))}. Call
|
|
2139
|
+
const hit = await findWords(page, String(args.text), refsFor().refs)
|
|
2140
|
+
if (hit === null) return `No control says ${JSON.stringify(String(args.text))}. Call browser_view to see what is there.`
|
|
1274
2141
|
point = { x: hit.x, y: hit.y }
|
|
1275
2142
|
label = `"${hit.name.slice(0, 60)}"`
|
|
1276
2143
|
}
|
|
1277
|
-
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) return 'Give a ref from
|
|
2144
|
+
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) return 'Give a ref from browser_view, or both x and y.'
|
|
2145
|
+
if (wayOf(args) === 'dom') {
|
|
2146
|
+
if (args?.ref === undefined) return 'Acting on the page itself needs a number from browser_view: pass ref, or leave mode out to click like a person.'
|
|
2147
|
+
const pressedAt = Date.now()
|
|
2148
|
+
const said = await evaluate(page, clickInDom(Number(args.ref), { double: args?.double === true, right: args?.right === true })).catch(() => undefined)
|
|
2149
|
+
const hit = asSpot(said)
|
|
2150
|
+
if (hit?.ok !== true) return REF_GONE
|
|
2151
|
+
await settle(page, 5000, pressedAt)
|
|
2152
|
+
const called = await acted(page, args, () => look(page, { mode: 'text', maxChars: 3000 }), `click ${label} on the page itself`)
|
|
2153
|
+
return called + (followed === '' ? '' : ` (${followed})`)
|
|
2154
|
+
}
|
|
1278
2155
|
const button = args?.right === true ? 'right' : 'left'
|
|
1279
2156
|
let cover = ''
|
|
1280
2157
|
if (args?.ref !== undefined && button === 'left') {
|
|
@@ -1283,25 +2160,27 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1283
2160
|
/* An overlay that would not go away leaves the control underneath:
|
|
1284
2161
|
the click would land on the overlay, so it is not sent. */
|
|
1285
2162
|
if (gate.cleared !== true) {
|
|
1286
|
-
return `${cover.trim()}\nNot clicked: the control is still covered. Call
|
|
2163
|
+
return `${cover.trim()}\nNot clicked: the control is still covered. Call browser_view and handle the overlay first.`
|
|
1287
2164
|
}
|
|
1288
2165
|
if (gate.cleared && cover !== '') {
|
|
1289
2166
|
const again = await locate(page, Number(args.ref))
|
|
1290
2167
|
if (again.gone !== true) point = { x: Number(again.x), y: Number(again.y) }
|
|
1291
2168
|
}
|
|
1292
2169
|
}
|
|
2170
|
+
const before = args?.read === true ? '' : await pageMark(page)
|
|
2171
|
+
const pressedAt = Date.now()
|
|
1293
2172
|
await page.mouse('mouseMoved', point)
|
|
1294
2173
|
await page.mouse('mousePressed', { ...point, clickCount: args?.double === true ? 2 : 1, button })
|
|
1295
2174
|
const hold = Math.max(0, Math.min(60_000, Number(args?.hold_ms ?? 0)))
|
|
1296
2175
|
if (hold > 0) await new Promise((settle) => setTimeout(settle, hold))
|
|
1297
2176
|
await page.mouse('mouseReleased', { ...point, clickCount: args?.double === true ? 2 : 1, button })
|
|
1298
|
-
await settle(page)
|
|
2177
|
+
await settle(page, 5000, pressedAt)
|
|
1299
2178
|
const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 3000 }), `click ${label}`)
|
|
1300
|
-
return said + cover + (followed === '' ? '' : ` (${followed})`)
|
|
2179
|
+
return said + cover + (followed === '' ? '' : ` (${followed})`) + (await outcome(page, before))
|
|
1301
2180
|
}),
|
|
1302
2181
|
},
|
|
1303
2182
|
{
|
|
1304
|
-
name: '
|
|
2183
|
+
name: 'browser_type', // ui-copy-ignore
|
|
1305
2184
|
description: 'Type text into the sidecar browser — Chinese included — optionally pressing Enter. Pass a ref to focus that control first. With clear, the field is emptied first.',
|
|
1306
2185
|
parameters: {
|
|
1307
2186
|
type: 'object', // ui-copy-ignore
|
|
@@ -1311,6 +2190,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1311
2190
|
field: { type: 'string', description: 'The field is called this on screen, when you have no number for it.' },
|
|
1312
2191
|
submit: { type: 'boolean', description: 'Press Enter after typing.' },
|
|
1313
2192
|
clear: { type: 'boolean', description: 'Select all and delete before typing.' },
|
|
2193
|
+
mode: { type: 'string', enum: ['human', 'dom'], description: '"human" presses real keys and is what a page cannot tell from a person; "dom" sets the field and fires its input events, which leaves key handlers untouched. Omit to use what this chat set with browser_mode.' },
|
|
1314
2194
|
read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
|
|
1315
2195
|
},
|
|
1316
2196
|
required: ['text'], // ui-copy-ignore
|
|
@@ -1323,11 +2203,29 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1323
2203
|
execute: async (args) => withPage(async (page) => {
|
|
1324
2204
|
let cover = ''
|
|
1325
2205
|
let followed = ''
|
|
2206
|
+
if (wayOf(args) === 'dom') {
|
|
2207
|
+
let target = Number.isInteger(args?.ref) ? Number(args.ref) : undefined
|
|
2208
|
+
if (target === undefined && args?.field !== undefined) {
|
|
2209
|
+
const found = await fieldByName(page, String(args.field), refsFor().refs)
|
|
2210
|
+
if (found === null) return `${String(args.field)}: ${REF_GONE}`
|
|
2211
|
+
if (!Number.isInteger(found.ref)) return `${String(args.field)}: read the page and pass its number, since a field found by its label has none.`
|
|
2212
|
+
target = Number(found.ref)
|
|
2213
|
+
cover = found.lead
|
|
2214
|
+
}
|
|
2215
|
+
if (target === undefined) return 'Acting on the page itself needs a number from browser_view: pass ref, or leave mode out to type like a person.'
|
|
2216
|
+
const said = await evaluate(page, writeInDom(target, String(args?.text ?? ''), { clear: args?.clear === true, submit: args?.submit === true })).catch(() => undefined)
|
|
2217
|
+
const wrote = asSpot(said)
|
|
2218
|
+
if (wrote?.ok !== true) return REF_GONE
|
|
2219
|
+
await settle(page)
|
|
2220
|
+
const called = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type on the page itself')
|
|
2221
|
+
return `${called} (field now "${String(wrote.value).slice(0, 40)}")` + cover
|
|
2222
|
+
}
|
|
1326
2223
|
if (args?.field !== undefined && args?.ref === undefined) {
|
|
1327
|
-
const target = await fieldByName(page, String(args.field))
|
|
2224
|
+
const target = await fieldByName(page, String(args.field), refsFor().refs)
|
|
1328
2225
|
if (target === null) return `${String(args.field)}: ${REF_GONE}`
|
|
1329
2226
|
cover = target.lead
|
|
1330
|
-
await writeAt(page, { x: target.x, y: target.y }, String(args?.text ?? ''), { clear: args?.clear, submit: args?.submit === true })
|
|
2227
|
+
const dated = await writeAt(page, { x: target.x, y: target.y }, String(args?.text ?? ''), { clear: args?.clear, submit: args?.submit === true, hint: target.name })
|
|
2228
|
+
if (dated !== '') return dated + cover
|
|
1331
2229
|
await settle(page)
|
|
1332
2230
|
const called = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type')
|
|
1333
2231
|
return called + cover + ` (typed into "${target.name.slice(0, 50)}")` // ui-copy-ignore
|
|
@@ -1336,7 +2234,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1336
2234
|
const gate = await clearCover(page, Number(args.ref))
|
|
1337
2235
|
cover = gate.note
|
|
1338
2236
|
if (gate.cleared !== true) {
|
|
1339
|
-
return `${cover.trim()}\nNot typed into: the field is still covered. Call
|
|
2237
|
+
return `${cover.trim()}\nNot typed into: the field is still covered. Call browser_view and handle the overlay first.`
|
|
1340
2238
|
}
|
|
1341
2239
|
const spot = await findAgain(page, Number(args.ref))
|
|
1342
2240
|
if (spot.gone === true) return spot.why ?? REF_GONE + cover
|
|
@@ -1348,21 +2246,26 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1348
2246
|
await wait(80)
|
|
1349
2247
|
}
|
|
1350
2248
|
if (args?.clear === true) await clearFocus(page)
|
|
1351
|
-
await page
|
|
2249
|
+
if (await dateLikeFocus(page) !== '') return (await fillDateLike(page, String(args?.text ?? ''))) + cover
|
|
2250
|
+
const words = String(args?.text ?? '')
|
|
2251
|
+
const before = args?.read === true ? '' : await pageMark(page)
|
|
2252
|
+
await typeInto(page, words, String(args?.field ?? ''))
|
|
2253
|
+
const dated = await fillDateLike(page, words)
|
|
2254
|
+
if (dated !== '') return dated + cover
|
|
1352
2255
|
if (args?.submit === true) await pressKey(page, 'Enter')
|
|
1353
2256
|
await settle(page)
|
|
1354
2257
|
const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type')
|
|
1355
|
-
return said + cover + (followed === '' ? '' : ` (${followed})`) // ui-copy-ignore
|
|
2258
|
+
return said + cover + (followed === '' ? '' : ` (${followed})`) + (await outcome(page, before)) // ui-copy-ignore
|
|
1356
2259
|
}),
|
|
1357
2260
|
},
|
|
1358
2261
|
{
|
|
1359
|
-
name: '
|
|
2262
|
+
name: 'browser_ask', // ui-copy-ignore
|
|
1360
2263
|
description: 'Say something to a chat that is open in the page — a model, a support window, a search box — and read its answer back in the same call. Types into the box, presses Enter, then waits until the answer stops growing. Use it to get a review or a picture prompt answered without a read between every step.',
|
|
1361
2264
|
parameters: {
|
|
1362
2265
|
type: 'object', // ui-copy-ignore
|
|
1363
2266
|
properties: {
|
|
1364
2267
|
text: { type: 'string', description: 'What to say.' },
|
|
1365
|
-
ref: { type: 'integer', description: 'The box to type in, from
|
|
2268
|
+
ref: { type: 'integer', description: 'The box to type in, from browser_view. Omit to use the page\u2019s own chat box.' },
|
|
1366
2269
|
submit: { type: 'boolean', description: 'Press Enter to send. Default true.' },
|
|
1367
2270
|
timeout_ms: { type: 'integer', description: 'How long to wait for the answer. Default 60000.' },
|
|
1368
2271
|
},
|
|
@@ -1376,8 +2279,8 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1376
2279
|
execute: async (args) => withPage((page) => askInPage(page, args)),
|
|
1377
2280
|
},
|
|
1378
2281
|
{
|
|
1379
|
-
name: '
|
|
1380
|
-
description: 'Ask the chat in the page to review a repository and take its review back as a list: builds the message, sends it, waits for the answer, and writes the round down so the next round knows what was already said. Use it for a review round, not for a one-off question (that is
|
|
2282
|
+
name: 'browser_review', // ui-copy-ignore
|
|
2283
|
+
description: 'Ask the chat in the page to review a repository and take its review back as a list: builds the message, sends it, waits for the answer, and writes the round down so the next round knows what was already said. Use it for a review round, not for a one-off question (that is browser_ask).',
|
|
1381
2284
|
parameters: {
|
|
1382
2285
|
type: 'object', // ui-copy-ignore
|
|
1383
2286
|
properties: {
|
|
@@ -1385,7 +2288,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1385
2288
|
commit: { type: 'string', description: 'The commit the review is about. Omit for the newest one.' },
|
|
1386
2289
|
focus: { type: 'string', description: 'One thing to look at especially.' },
|
|
1387
2290
|
diff: { type: 'string', description: 'The change to review, as a diff. Include it when the reviewer cannot open the repository itself.' },
|
|
1388
|
-
ref: { type: 'integer', description: 'The box to type in, from
|
|
2291
|
+
ref: { type: 'integer', description: 'The box to type in, from browser_view. Omit to use the page\u2019s own chat box.' },
|
|
1389
2292
|
timeout_ms: { type: 'integer', description: 'How long to wait for the review. Default 120000.' },
|
|
1390
2293
|
},
|
|
1391
2294
|
required: ['repo'], // ui-copy-ignore
|
|
@@ -1428,8 +2331,8 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1428
2331
|
}),
|
|
1429
2332
|
},
|
|
1430
2333
|
{
|
|
1431
|
-
name: '
|
|
1432
|
-
description: 'Record what happened to each point of the last review round, so the next round leaves settled points out: point numbers are the ones
|
|
2334
|
+
name: 'browser_review_verdict', // ui-copy-ignore
|
|
2335
|
+
description: 'Record what happened to each point of the last review round, so the next round leaves settled points out: point numbers are the ones browser_review printed, and each mark is fixed, verified (checked in the code and the point does not apply), invalid, or open.',
|
|
1433
2336
|
parameters: {
|
|
1434
2337
|
type: 'object', // ui-copy-ignore
|
|
1435
2338
|
properties: {
|
|
@@ -1439,7 +2342,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1439
2342
|
items: { // ui-copy-ignore
|
|
1440
2343
|
type: 'object', // ui-copy-ignore
|
|
1441
2344
|
properties: { // ui-copy-ignore
|
|
1442
|
-
point: { type: 'integer', description: 'The point number
|
|
2345
|
+
point: { type: 'integer', description: 'The point number browser_review printed, starting at 1.' },
|
|
1443
2346
|
status: { type: 'string', enum: MARK_STATUS, description: 'fixed, verified, invalid, or open.' },
|
|
1444
2347
|
evidence: { type: 'string', description: 'Where the code shows it: file, test, or commit.' },
|
|
1445
2348
|
},
|
|
@@ -1457,7 +2360,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1457
2360
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} what was recorded. */
|
|
1458
2361
|
execute: async (args) => {
|
|
1459
2362
|
const written = markRound({ marks: Array.isArray(args?.mark) ? args.mark : [] })
|
|
1460
|
-
if (written === null) return 'No review round to mark: play one with
|
|
2363
|
+
if (written === null) return 'No review round to mark: play one with browser_review first.'
|
|
1461
2364
|
const counts = new Map()
|
|
1462
2365
|
for (const mark of written.marks) counts.set(mark.status, (counts.get(mark.status) ?? 0) + 1)
|
|
1463
2366
|
const said = [...counts].map(([status, n]) => `${n} ${status}`).join(', ')
|
|
@@ -1465,13 +2368,13 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1465
2368
|
},
|
|
1466
2369
|
},
|
|
1467
2370
|
{
|
|
1468
|
-
name: '
|
|
1469
|
-
description: 'Press one key in the
|
|
2371
|
+
name: 'browser_key', // ui-copy-ignore
|
|
2372
|
+
description: 'Press one key in the background browser: Enter, Tab, Escape, Backspace, Delete, Insert, arrows, Home, End, PageUp, PageDown, Space, F1-F12, the numeric keypad (Numpad0-Numpad9, NumpadEnter, NumpadAdd, ...), printable characters and every symbol on a US keyboard (including shifted ones such as ? : @), plus the keys with a name (ContextMenu, CapsLock, PrintScreen, media keys). Join keys with + to hold them together (ArrowLeft+ArrowUp, Shift+Tab), and hold_ms to keep them down. Modifiers accept Ctrl, Alt, Shift, Meta/Cmd, or Mod for the platform command key; Ctrl+C/X/V/A/Z/Y run the real copy, cut, paste, select-all, undo and redo.',
|
|
1470
2373
|
parameters: {
|
|
1471
2374
|
type: 'object', // ui-copy-ignore
|
|
1472
2375
|
properties: {
|
|
1473
|
-
key: { type: 'string', description: 'Key name, or several joined by + to hold them together (ArrowLeft+ArrowUp, Shift+ArrowLeft). Letters and
|
|
1474
|
-
modifiers: { type: 'string', description: 'Held while pressing: Ctrl, Alt, Shift or
|
|
2376
|
+
key: { type: 'string', description: 'Key name, or several joined by + to hold them together (ArrowLeft+ArrowUp, Shift+ArrowLeft). Letters, digits, punctuation and shifted symbols work too.' },
|
|
2377
|
+
modifiers: { type: 'string', description: 'Held while pressing: Ctrl, Alt, Shift, Meta/Cmd, or Mod (Command on macOS, Control on Windows and Linux), joined by +.' },
|
|
1475
2378
|
hold_ms: { type: 'integer', description: 'Keep the key down this long before releasing. Games move while a key is held.' },
|
|
1476
2379
|
read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
|
|
1477
2380
|
},
|
|
@@ -1484,12 +2387,14 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1484
2387
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
|
|
1485
2388
|
execute: async (args) => withPage(async (page) => {
|
|
1486
2389
|
const name = String(args?.key ?? 'Enter')
|
|
1487
|
-
const
|
|
1488
|
-
const mods =
|
|
2390
|
+
const chord = splitChord(name, modifierBits(args?.modifiers))
|
|
2391
|
+
const mods = chord.modifiers
|
|
1489
2392
|
const hold = Math.max(0, Math.min(60_000, Number(args?.hold_ms ?? 0)))
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
2393
|
+
if (chord.names.length === 0) throw new Error(`sidecar: ${name} holds no key, only modifiers`) // ui-copy-ignore
|
|
2394
|
+
/* One key goes through the plain path so an editing shortcut is sent as
|
|
2395
|
+
the browser command it is; a chord holds every key down together. */
|
|
2396
|
+
if (chord.names.length > 1) await pressChord(page, chord.names, mods, hold)
|
|
2397
|
+
else await pressKey(page, chord.names[0], mods, hold)
|
|
1493
2398
|
/* A held key is game input: waiting for the page to fall quiet after one
|
|
1494
2399
|
costs more than the frame it was meant to drive. */
|
|
1495
2400
|
if (hold === 0) await settle(page, 3000)
|
|
@@ -1497,7 +2402,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1497
2402
|
}),
|
|
1498
2403
|
},
|
|
1499
2404
|
{
|
|
1500
|
-
name: '
|
|
2405
|
+
name: 'browser_scroll', // ui-copy-ignore
|
|
1501
2406
|
description: 'Scroll the sidecar browser by wheel at its centre (negative goes up), then return what it shows.',
|
|
1502
2407
|
parameters: {
|
|
1503
2408
|
type: 'object', // ui-copy-ignore
|
|
@@ -1515,18 +2420,21 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1515
2420
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
|
|
1516
2421
|
execute: async (args) => withPage(async (page) => {
|
|
1517
2422
|
await page.wheel({ x: Number(args?.x ?? VIEW.width / 2), y: Number(args?.y ?? VIEW.height / 2), dy: Number(args?.dy ?? 600) })
|
|
2423
|
+
/* A wheel is a request, not a result: a list that loads more rows on
|
|
2424
|
+
scroll has not answered yet when the wheel returns. Give the scroll
|
|
2425
|
+
handler its turn, then let the page fall quiet. */
|
|
1518
2426
|
await wait(200)
|
|
1519
2427
|
const delta = Number(args?.dy ?? 600)
|
|
1520
2428
|
return acted(page, args, () => look(page, { mode: 'text', maxChars: 4000 }), `scroll ${delta > 0 ? '+' : ''}${delta}`)
|
|
1521
2429
|
}),
|
|
1522
2430
|
},
|
|
1523
2431
|
{
|
|
1524
|
-
name: '
|
|
2432
|
+
name: 'browser_hover', // ui-copy-ignore
|
|
1525
2433
|
description: 'Move the pointer onto a numbered control, so hover-only menus and tooltips open, then return what the page shows.',
|
|
1526
2434
|
parameters: {
|
|
1527
2435
|
type: 'object', // ui-copy-ignore
|
|
1528
2436
|
properties: {
|
|
1529
|
-
ref: { type: 'integer', description: 'Number from
|
|
2437
|
+
ref: { type: 'integer', description: 'Number from browser_view.' },
|
|
1530
2438
|
read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
|
|
1531
2439
|
},
|
|
1532
2440
|
required: ['ref'], // ui-copy-ignore
|
|
@@ -1546,7 +2454,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1546
2454
|
}),
|
|
1547
2455
|
},
|
|
1548
2456
|
{
|
|
1549
|
-
name: '
|
|
2457
|
+
name: 'browser_nav', // ui-copy-ignore
|
|
1550
2458
|
description: 'Change the sidecar browser\'s address: open a url, go back, go forward, or reload. Use this instead of clicking the browser chrome.',
|
|
1551
2459
|
parameters: {
|
|
1552
2460
|
type: 'object', // ui-copy-ignore
|
|
@@ -1576,7 +2484,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1576
2484
|
}),
|
|
1577
2485
|
},
|
|
1578
2486
|
{
|
|
1579
|
-
name: '
|
|
2487
|
+
name: 'browser_do', // ui-copy-ignore
|
|
1580
2488
|
description: 'Run several steps in one call and report one compact result: the whole of "click search, type a query, press Enter, wait for results" is one call instead of five. Steps run in order and stop at the first that cannot proceed. Each step: {ref, tap | text, submit}, {tap: "x,y"}, {tap: "Sign in"}, {field: "Email", text}, {text}, {key}, {scroll}, {wait_text}, {wait_gone}, {wait_url}, {wait_idle}, {eval}, {open}, {sleep_ms}. Waits know when the page has settled. A step that names a control clears a cookie banner in its way first.',
|
|
1581
2489
|
parameters: {
|
|
1582
2490
|
type: 'object', // ui-copy-ignore
|
|
@@ -1622,13 +2530,14 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1622
2530
|
if (steps.length === 0) return 'Give at least one step.'
|
|
1623
2531
|
const report = []
|
|
1624
2532
|
for (const [index, step] of steps.entries()) {
|
|
2533
|
+
const stepAt = Date.now()
|
|
1625
2534
|
const said = await runStep(page, step ?? {}, refsFor().refs)
|
|
1626
2535
|
const text = typeof said === 'string' ? said : said.text
|
|
1627
2536
|
report.push(`${index + 1}. ${text}`)
|
|
1628
2537
|
/* A step that never landed makes every later step a guess, so the
|
|
1629
2538
|
batch ends here instead of typing into a page that is not there. */
|
|
1630
2539
|
if (typeof said !== 'string' && said.blocked === true) {
|
|
1631
|
-
return `${report.join('\n')}\nStopped after step ${index + 1}. Call
|
|
2540
|
+
return `${report.join('\n')}\nStopped after step ${index + 1}. Call browser_view and plan again from what the page shows now.`
|
|
1632
2541
|
}
|
|
1633
2542
|
/* A step can navigate straight onto Chrome's error page — a mistyped
|
|
1634
2543
|
address, or a site that does not answer. The rest of the batch
|
|
@@ -1636,7 +2545,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1636
2545
|
here and says what really happened. */
|
|
1637
2546
|
if (isErrorPage(page)) return `${report.join('\n')}\nStopped after step ${index + 1}: ${await unreachable(page)}`
|
|
1638
2547
|
if (step?.sleep_ms === undefined && step?.wait_text === undefined && step?.wait_gone === undefined && step?.wait_url === undefined && step?.wait_idle === undefined) {
|
|
1639
|
-
await settle(page, Number(step?.timeout ?? 5000))
|
|
2548
|
+
await settle(page, Number(step?.timeout ?? 5000), stepAt)
|
|
1640
2549
|
}
|
|
1641
2550
|
}
|
|
1642
2551
|
const mode = String(args?.read ?? 'text')
|
|
@@ -1654,13 +2563,13 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1654
2563
|
}),
|
|
1655
2564
|
},
|
|
1656
2565
|
{
|
|
1657
|
-
name: '
|
|
2566
|
+
name: 'browser_extract', // ui-copy-ignore
|
|
1658
2567
|
description: 'Read the items a selector matches as one compact line each — lists, search results, tables, menus — instead of the whole page. Each row keeps its link when it has one. Use it when a page has more rows than the read budget allows; it costs a fraction of a full read.',
|
|
1659
2568
|
parameters: {
|
|
1660
2569
|
type: 'object', // ui-copy-ignore
|
|
1661
2570
|
properties: {
|
|
1662
2571
|
select: { type: 'string', description: 'CSS selector for the items, for example "ul.results > li".' },
|
|
1663
|
-
ref: { type: 'integer', description: 'Only look inside this control, as numbered by the last
|
|
2572
|
+
ref: { type: 'integer', description: 'Only look inside this control, as numbered by the last browser_view.' },
|
|
1664
2573
|
limit: { type: 'integer', description: 'Most rows to return. Default 40, at most 200.' },
|
|
1665
2574
|
},
|
|
1666
2575
|
required: ['select'], // ui-copy-ignore
|
|
@@ -1673,7 +2582,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1673
2582
|
execute: async (args) => withRead((page) => extractRows(page, args)),
|
|
1674
2583
|
},
|
|
1675
2584
|
{
|
|
1676
|
-
name: '
|
|
2585
|
+
name: 'browser_eval', // ui-copy-ignore
|
|
1677
2586
|
description: 'Run JavaScript in the sidecar page and return its value. For what the numbered controls cannot express: reading a table, setting storage, filling a widget. The page must be opened first.',
|
|
1678
2587
|
parameters: {
|
|
1679
2588
|
type: 'object', // ui-copy-ignore
|
|
@@ -1706,12 +2615,12 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1706
2615
|
}),
|
|
1707
2616
|
},
|
|
1708
2617
|
{
|
|
1709
|
-
name: '
|
|
2618
|
+
name: 'browser_select', // ui-copy-ignore
|
|
1710
2619
|
description: 'Choose an option in a dropdown, by visible label or value, then return the page.',
|
|
1711
2620
|
parameters: {
|
|
1712
2621
|
type: 'object', // ui-copy-ignore
|
|
1713
2622
|
properties: {
|
|
1714
|
-
ref: { type: 'integer', description: 'Number of the select control from
|
|
2623
|
+
ref: { type: 'integer', description: 'Number of the select control from browser_view.' },
|
|
1715
2624
|
field: { type: 'string', description: 'The dropdown is called this on screen, when you have no number for it.' },
|
|
1716
2625
|
value: { type: 'string', description: 'Option label or value to choose.' },
|
|
1717
2626
|
},
|
|
@@ -1723,20 +2632,30 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1723
2632
|
isConcurrencySafe: () => false,
|
|
1724
2633
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
|
|
1725
2634
|
execute: async (args) => withPage(async (page) => {
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
2635
|
+
let use = args
|
|
2636
|
+
/* One dropdown on the page needs no naming: the value alone says which
|
|
2637
|
+
one, and asking costs the caller a whole read. */
|
|
2638
|
+
if (use?.ref === undefined && use?.field === undefined) {
|
|
2639
|
+
const read = await readPage(page, { mode: 'text', maxChars: 1200, max: 200 })
|
|
2640
|
+
const drops = read.controls.filter((control) => control.view !== false && ['select', 'combobox', 'listbox'].includes(String(control.kind)))
|
|
2641
|
+
if (drops.length === 1) use = { ...use, ref: Number(drops[0].ref) }
|
|
2642
|
+
else if (drops.length > 1) return `which dropdown: ${drops.slice(0, 5).map((one) => `${String(one.name).slice(0, 30)} (${String(one.ref)})`).join(' · ')}`
|
|
2643
|
+
else return 'the page has no dropdown to choose in.' // ui-copy-ignore
|
|
2644
|
+
}
|
|
2645
|
+
if (use?.ref === undefined) {
|
|
2646
|
+
const field = String(use.field)
|
|
2647
|
+
const chosen = await evaluate(page, selectSpot(field, String(use?.value ?? '')))
|
|
1729
2648
|
const said = typeof chosen === 'string' ? chosen : chosen?.error === undefined ? '' : String(chosen.error)
|
|
1730
2649
|
if (said === '' || said.startsWith('no dropdown')) return said === '' ? 'the page did not answer; read it and try again.' : said // ui-copy-ignore
|
|
1731
2650
|
await settle(page, 3000)
|
|
1732
|
-
if (
|
|
2651
|
+
if (use?.read === true) return `${said}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
|
|
1733
2652
|
return `${said} · ${await brief(page)}`
|
|
1734
2653
|
}
|
|
1735
|
-
const spot = await findAgain(page, Number(
|
|
2654
|
+
const spot = await findAgain(page, Number(use?.ref))
|
|
1736
2655
|
if (spot.gone === true) return spot.why ?? REF_GONE
|
|
1737
|
-
const wanted = JSON.stringify(String(
|
|
1738
|
-
const route = await contextFor(page, Number(
|
|
1739
|
-
const local = route?.local ?? Number(
|
|
2656
|
+
const wanted = JSON.stringify(String(use?.value ?? ''))
|
|
2657
|
+
const route = await contextFor(page, Number(use?.ref))
|
|
2658
|
+
const local = route?.local ?? Number(use?.ref)
|
|
1740
2659
|
const said = await evaluate(page, `(() => {
|
|
1741
2660
|
const reg = window.__dshRead
|
|
1742
2661
|
const hit = reg && reg.find(${local})
|
|
@@ -1751,17 +2670,17 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1751
2670
|
return 'chose ' + option.text.trim()
|
|
1752
2671
|
})()`, route)
|
|
1753
2672
|
await settle(page, 3000)
|
|
1754
|
-
if (
|
|
2673
|
+
if (use?.read === true) return `${String(said)}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
|
|
1755
2674
|
return `${String(said)} · ${await brief(page)}`
|
|
1756
2675
|
}),
|
|
1757
2676
|
},
|
|
1758
2677
|
{
|
|
1759
|
-
name: '
|
|
2678
|
+
name: 'browser_upload', // ui-copy-ignore
|
|
1760
2679
|
description: 'Put files into a file input on the page, by absolute path, then return the page. The file chooser never opens.',
|
|
1761
2680
|
parameters: {
|
|
1762
2681
|
type: 'object', // ui-copy-ignore
|
|
1763
2682
|
properties: {
|
|
1764
|
-
ref: { type: 'integer', description: 'Number of the file input from
|
|
2683
|
+
ref: { type: 'integer', description: 'Number of the file input from browser_view.' },
|
|
1765
2684
|
files: { type: 'array', items: { type: 'string' }, description: 'Absolute paths on this machine.' },
|
|
1766
2685
|
},
|
|
1767
2686
|
required: ['ref', 'files'], // ui-copy-ignore
|
|
@@ -1790,14 +2709,14 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1790
2709
|
}),
|
|
1791
2710
|
},
|
|
1792
2711
|
{
|
|
1793
|
-
name: '
|
|
2712
|
+
name: 'browser_shot', // ui-copy-ignore
|
|
1794
2713
|
description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path; when the current model accepts images the picture comes back in the same call, so a captcha, a chart or a broken layout can be looked at instead of guessed. Default is the visible screen; full takes the whole page, ref takes one control.',
|
|
1795
2714
|
parameters: {
|
|
1796
2715
|
type: 'object', // ui-copy-ignore
|
|
1797
2716
|
properties: {
|
|
1798
2717
|
name: { type: 'string', description: 'File name without extension.' },
|
|
1799
2718
|
full: { type: 'boolean', description: 'The whole page, not just the screen.' },
|
|
1800
|
-
ref: { type: 'integer', description: 'One control from
|
|
2719
|
+
ref: { type: 'integer', description: 'One control from browser_view.' },
|
|
1801
2720
|
scale: { type: 'integer', description: 'Pixel ratio. Default 1; 2 is sharper and heavier.' },
|
|
1802
2721
|
},
|
|
1803
2722
|
additionalProperties: false,
|
|
@@ -1850,12 +2769,12 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1850
2769
|
/* The route may not carry pictures at all: then the file path is all
|
|
1851
2770
|
this call can give, and saying so is better than a silent path the
|
|
1852
2771
|
next step assumes was looked at. */
|
|
1853
|
-
if (image === undefined) return { text: `${said.text}\nno picture reached this result —
|
|
2772
|
+
if (image === undefined) return { text: `${said.text}\nno picture reached this result — browser_view reads the words here, browser_grab returns a picture as a file` }
|
|
1854
2773
|
return { text: said.text, image }
|
|
1855
2774
|
},
|
|
1856
2775
|
},
|
|
1857
2776
|
{
|
|
1858
|
-
name: '
|
|
2777
|
+
name: 'browser_grab', // ui-copy-ignore
|
|
1859
2778
|
description: 'Save the picture on the page as a file: an image, a chart, a drawing surface. Picks the largest one, or the one inside a given control.',
|
|
1860
2779
|
parameters: {
|
|
1861
2780
|
type: 'object', // ui-copy-ignore
|
|
@@ -1908,7 +2827,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1908
2827
|
}),
|
|
1909
2828
|
},
|
|
1910
2829
|
{
|
|
1911
|
-
name: '
|
|
2830
|
+
name: 'browser_state', // ui-copy-ignore
|
|
1912
2831
|
description: 'Report the sidecar browser: whether it runs, its profile, address, title, how many people are watching its mirror, and how many times a call had to find a control again after the page redrew it.',
|
|
1913
2832
|
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
1914
2833
|
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
@@ -1929,6 +2848,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1929
2848
|
`tab list ${[...sidecar.pages.entries()].map(([key, each]) => `${key.slice(-8)}=${String(each.url)}=${each.cdp?.closed === true ? 'closed' : 'open'}`).join(' | ')}`, // ui-copy-ignore
|
|
1930
2849
|
`default tab ${String(sidecar.page?.url)}`, // ui-copy-ignore
|
|
1931
2850
|
state.headless === false ? 'showing a window' : 'no window',
|
|
2851
|
+
sidecar.flash ? 'Flash-era runtime' : 'Chrome runtime',
|
|
1932
2852
|
`profile ${String(state.profile)}`,
|
|
1933
2853
|
`url ${String(state.url)}`,
|
|
1934
2854
|
`title ${String(state.title)}`,
|
|
@@ -1945,7 +2865,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1945
2865
|
},
|
|
1946
2866
|
|
|
1947
2867
|
{
|
|
1948
|
-
name: '
|
|
2868
|
+
name: 'browser_resume', // ui-copy-ignore
|
|
1949
2869
|
description: 'Take the page back from the person at the mirror: clears the hold so the next action is sent at once, instead of waiting the few seconds out.',
|
|
1950
2870
|
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
1951
2871
|
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
@@ -1958,7 +2878,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1958
2878
|
},
|
|
1959
2879
|
},
|
|
1960
2880
|
{
|
|
1961
|
-
name: '
|
|
2881
|
+
name: 'browser_downloads', // ui-copy-ignore
|
|
1962
2882
|
description: 'List what this browser has downloaded, newest first, with the file name, its size and when it arrived.',
|
|
1963
2883
|
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
1964
2884
|
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
@@ -1973,7 +2893,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1973
2893
|
},
|
|
1974
2894
|
|
|
1975
2895
|
{
|
|
1976
|
-
name: '
|
|
2896
|
+
name: 'browser_drag', // ui-copy-ignore
|
|
1977
2897
|
description: 'Drag from one place to another with real mouse steps: from a ref or a point, to a point or offset. Use it for sliders, dashboards and drag-and-drop lists.',
|
|
1978
2898
|
parameters: {
|
|
1979
2899
|
type: 'object', // ui-copy-ignore
|
|
@@ -2016,12 +2936,12 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2016
2936
|
}),
|
|
2017
2937
|
},
|
|
2018
2938
|
{
|
|
2019
|
-
name: '
|
|
2939
|
+
name: 'browser_check', // ui-copy-ignore
|
|
2020
2940
|
description: 'Set a checkbox, radio or switch to a known state, and say whether it took. Clicking blindly flips a control that is already right; this one looks first.',
|
|
2021
2941
|
parameters: {
|
|
2022
2942
|
type: 'object', // ui-copy-ignore
|
|
2023
2943
|
properties: {
|
|
2024
|
-
ref: { type: 'integer', description: 'Control number from
|
|
2944
|
+
ref: { type: 'integer', description: 'Control number from browser_view.' },
|
|
2025
2945
|
checked: { type: 'boolean', description: 'Wanted state. Omit to flip it.' },
|
|
2026
2946
|
read: { type: 'boolean', description: 'Read the page after.' },
|
|
2027
2947
|
},
|
|
@@ -2057,7 +2977,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2057
2977
|
}),
|
|
2058
2978
|
},
|
|
2059
2979
|
{
|
|
2060
|
-
name: '
|
|
2980
|
+
name: 'browser_wait', // ui-copy-ignore
|
|
2061
2981
|
description: 'Wait until the page shows a piece of text, stops showing it, reaches an address, or goes quiet. Cheaper than reading in a loop.',
|
|
2062
2982
|
parameters: {
|
|
2063
2983
|
type: 'object', // ui-copy-ignore
|
|
@@ -2093,7 +3013,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2093
3013
|
}),
|
|
2094
3014
|
},
|
|
2095
3015
|
{
|
|
2096
|
-
name: '
|
|
3016
|
+
name: 'browser_dialog', // ui-copy-ignore
|
|
2097
3017
|
description: 'Answer the alert, confirm or prompt the page is showing, or read what it asks. A dialog blocks the page until it is answered, so nothing else works while one is open.',
|
|
2098
3018
|
parameters: {
|
|
2099
3019
|
type: 'object', // ui-copy-ignore
|
|
@@ -2121,7 +3041,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2121
3041
|
}),
|
|
2122
3042
|
},
|
|
2123
3043
|
{
|
|
2124
|
-
name: '
|
|
3044
|
+
name: 'browser_console', // ui-copy-ignore
|
|
2125
3045
|
description: 'The console lines and errors this page has produced, newest last. Use it when a click seemed to do nothing.',
|
|
2126
3046
|
parameters: {
|
|
2127
3047
|
type: 'object', // ui-copy-ignore
|
|
@@ -2139,7 +3059,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2139
3059
|
}),
|
|
2140
3060
|
},
|
|
2141
3061
|
{
|
|
2142
|
-
name: '
|
|
3062
|
+
name: 'browser_failed', // ui-copy-ignore
|
|
2143
3063
|
description: 'Requests this page tried and that failed, newest last. A blank area is often one of these.',
|
|
2144
3064
|
parameters: {
|
|
2145
3065
|
type: 'object', // ui-copy-ignore
|
|
@@ -2157,7 +3077,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2157
3077
|
}),
|
|
2158
3078
|
},
|
|
2159
3079
|
{
|
|
2160
|
-
name: '
|
|
3080
|
+
name: 'browser_cookie',
|
|
2161
3081
|
description: 'Read or change the cookies this browser keeps. Login state is a cookie, so this is how you check a sign-in and how you keep one.',
|
|
2162
3082
|
parameters: {
|
|
2163
3083
|
type: 'object', // ui-copy-ignore
|
|
@@ -2201,7 +3121,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2201
3121
|
}),
|
|
2202
3122
|
},
|
|
2203
3123
|
{
|
|
2204
|
-
name: '
|
|
3124
|
+
name: 'browser_clipboard', // ui-copy-ignore
|
|
2205
3125
|
description: 'Read the sidecar browser\'s clipboard, or put text on it and paste it with a key press. For a page that only copies through a button, click it then read here.',
|
|
2206
3126
|
parameters: {
|
|
2207
3127
|
type: 'object', // ui-copy-ignore
|
|
@@ -2224,7 +3144,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2224
3144
|
}),
|
|
2225
3145
|
},
|
|
2226
3146
|
{
|
|
2227
|
-
name: '
|
|
3147
|
+
name: 'browser_network', // ui-copy-ignore
|
|
2228
3148
|
description: 'List the responses the page has received, newest first: method, status, type and address. Use it when a page fills itself from the network and the screen does not say why.',
|
|
2229
3149
|
parameters: {
|
|
2230
3150
|
type: 'object', // ui-copy-ignore
|
|
@@ -2250,7 +3170,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2250
3170
|
},
|
|
2251
3171
|
|
|
2252
3172
|
{
|
|
2253
|
-
name: '
|
|
3173
|
+
name: 'browser_tabs', // ui-copy-ignore
|
|
2254
3174
|
description: 'List the pages open in the browser, switch the mirror to another one, or close one. Use it when a site opens a link in a tab of its own.',
|
|
2255
3175
|
parameters: {
|
|
2256
3176
|
type: 'object', // ui-copy-ignore
|
|
@@ -2277,15 +3197,19 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2277
3197
|
return `tab ${index} · ${String(next?.url ?? '')}`
|
|
2278
3198
|
}
|
|
2279
3199
|
if (action === 'close') {
|
|
3200
|
+
/* Say what was closed, not what is left: after the close the page
|
|
3201
|
+
that was at that row is gone and reads back as about:blank. */
|
|
3202
|
+
const before = await sidecar.tabs(currentSession())
|
|
3203
|
+
const gone = String(before[index]?.url ?? '')
|
|
2280
3204
|
await sidecar.closeTab(index, currentSession())
|
|
2281
|
-
return `closed tab ${index} · ${
|
|
3205
|
+
return `closed tab ${index} · ${gone}`
|
|
2282
3206
|
}
|
|
2283
3207
|
throw new Error(`sidecar: unknown action ${action}`)
|
|
2284
3208
|
}),
|
|
2285
3209
|
},
|
|
2286
3210
|
|
|
2287
3211
|
{
|
|
2288
|
-
name: '
|
|
3212
|
+
name: 'browser_pdf', // ui-copy-ignore
|
|
2289
3213
|
description: 'Print the page to a PDF and return its path. Useful when the person wants the document rather than a picture of it.',
|
|
2290
3214
|
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
2291
3215
|
output: TEXT_OUTPUT, // ui-copy-ignore
|