dsh-inner-browser-use 0.27.0 → 0.34.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/tools.js CHANGED
@@ -5,26 +5,179 @@
5
5
  * Two costs decide whether this is usable: the tokens a read spends, and the
6
6
  * round trips a task spends. The tool surface here is shaped around both — a
7
7
  * read can be narrowed (`mode`, `max_chars`) and diffed (`changed_only`), and
8
- * `sidecar_do` runs a run of steps in one call with waits that know when a page
8
+ * `browser_do` runs a run of steps in one call with waits that know when a page
9
9
  * has settled.
10
10
  */
11
11
 
12
+ import { AsyncLocalStorage } from 'node:async_hooks'
13
+ import { createHash } from 'node:crypto'
12
14
  import { homedir } from 'node:os'
13
- import { join } from 'node:path'
14
- import { mkdir, writeFile } from 'node:fs/promises'
15
+ import { basename, join } from 'node:path'
16
+ import { mkdir, readdir, stat, unlink, writeFile } from 'node:fs/promises'
15
17
  import { VIEW } from './sidecar.js?v=1'
16
- import { readPage, renderPage, locate, covering, contextFor, identityOf, sameThing, labelSpot, selectSpot, bounds, pictures, REF_GONE } from './read.js?v=1'
18
+ import { readPage, renderPage, readArticle, readRelevant, findOnPage, locate, covering, contextFor, identityOf, sameThing, differs, labelSpot, selectSpot, bounds, pictures, clickInDom, writeInDom, REF_GONE } from './read.js?v=1'
19
+ import { brief as askBrief, recordRound, lastRoundNote, lastRound, markRound, MARK_STATUS } from './review.js?v=1'
17
20
 
18
- /** Where a screenshot from `sidecar_shot` lands, under the user's home. */
19
- export const SHOT_DIR = join('.dsh-inner-browser-use', 'screenshots')
21
+ /** Which chat the running call belongs to, so each gets its own tab. */
22
+ export const sessionScope = new AsyncLocalStorage()
23
+
24
+ /** @returns {string} the chat id of the running call. */
25
+ function currentSession() {
26
+ return String(sessionScope.getStore() ?? '')
27
+ }
28
+
29
+ /** How a call asked to act: real mouse and keys, or the page's own DOM. */
30
+ const WAYS = new Set(['human', 'dom'])
31
+
32
+ /** The way each chat said to act, until it says otherwise. */
33
+ const chatWays = new Map()
34
+
35
+ /**
36
+ * The way this call should act: what it asked for, else what its chat last set,
37
+ * else a real mouse and real keys, which no page can tell from a person.
38
+ * @param {any} args - the call's arguments.
39
+ * @returns {'human'|'dom'} which way.
40
+ */
41
+ function wayOf(args) {
42
+ const said = String(args?.mode ?? '')
43
+ if (WAYS.has(said)) return /** @type {'human'|'dom'} */ (said)
44
+ const chat = currentSession()
45
+ return chatWays.get(chat) === 'dom' ? 'dom' : 'human'
46
+ }
47
+
48
+ /** Which project the running call belongs to, so its files stay together. */
49
+ export const workScope = new AsyncLocalStorage()
50
+
51
+ /** Where files a call produces land, under the user's home. */
52
+ export const ARTIFACT_ROOT = String(process.env.DSH_INNER_BROWSER_ARTIFACTS ?? '') !== ''
53
+ ? String(process.env.DSH_INNER_BROWSER_ARTIFACTS)
54
+ : join(homedir(), '.dsh-inner-browser-use', 'artifacts')
55
+
56
+ /** How many files one kind keeps per chat; older ones go. */
57
+ export const ARTIFACT_KEEP = 40
58
+
59
+ /** Project folder name: the folder's own name, plus a short hash of its path. */
60
+ function projectKey() {
61
+ const cwd = String(workScope.getStore() ?? '')
62
+ if (cwd === '') return 'no-project'
63
+ const named = basename(cwd).toLowerCase().replace(/[^a-z0-9._-]/g, '-').replace(/^-+|-+$/g, '')
64
+ return `${named === '' ? 'project' : named}-${createHash('sha1').update(cwd).digest('hex').slice(0, 6)}`
65
+ }
66
+
67
+ /** Chat folder name: the short id a person can match to the chat. */
68
+ function sessionKey() {
69
+ const id = currentSession().replace(/^session-/, '').replace(/[^a-z0-9-]/gi, '')
70
+ return id === '' ? 'no-chat' : id.slice(0, 8)
71
+ }
72
+
73
+ /**
74
+ * Where one kind of file for this project and this chat goes.
75
+ * `~/.dsh-inner-browser-use/artifacts/<project>/<chat>/<kind>/`
76
+ * @param {string} kind - shots, pictures, pdf.
77
+ * @returns {string} absolute directory; not created yet.
78
+ */
79
+ export function artifactDir(kind) {
80
+ return join(ARTIFACT_ROOT, projectKey(), sessionKey(), kind)
81
+ }
82
+
83
+ /**
84
+ * Write one file where the project and the chat expect it, and keep the folder
85
+ * from growing without a limit.
86
+ * @param {string} kind - shots, pictures, pdf.
87
+ * @param {string} body - base64 body.
88
+ * @param {{name?: string, ext?: string, prefix?: string}} [options] - file name.
89
+ * @returns {Promise<string>} absolute path.
90
+ */
91
+ export async function saveArtifact(kind, body, options = {}) {
92
+ const dir = artifactDir(kind)
93
+ await mkdir(dir, { recursive: true })
94
+ const ext = String(options.ext ?? '.png')
95
+ /* A name is a name: no separators, no runs of dots that could read as a path
96
+ climb, and no leading dot to hide the file. */
97
+ const wanted = String(options.name ?? '').replace(/[^\w.-]/g, '').replace(/\.(png|jpe?g|webp|pdf)$/i, '').replace(/\.{2,}/g, '.').replace(/^\.+/, '')
98
+ const stamp = new Date().toISOString().replace(/[-:]/g, '').replace('T', '-').slice(0, 15)
99
+ const prefix = String(options.prefix ?? kind)
100
+ const file = join(dir, `${wanted === '' ? `${prefix}-${stamp}` : wanted}${ext}`)
101
+ await writeFile(file, Buffer.from(body, 'base64'))
102
+ await pruneArtifacts(dir, ARTIFACT_KEEP)
103
+ return file
104
+ }
105
+
106
+ /**
107
+ * Keep the newest `keep` files in one folder: a browser session that runs all
108
+ * day does not leave a folder nobody can look through.
109
+ * @param {string} dir - folder to trim.
110
+ * @param {number} keep - how many files survive.
111
+ * @returns {Promise<void>} done.
112
+ */
113
+ export async function pruneArtifacts(dir, keep) {
114
+ let names
115
+ try {
116
+ names = await readdir(dir)
117
+ } catch {
118
+ return
119
+ }
120
+ const listed = []
121
+ for (const name of names) {
122
+ try {
123
+ listed.push({ name, at: (await stat(join(dir, name))).mtimeMs })
124
+ } catch {
125
+ /* a file that vanished between listing and reading is already gone */
126
+ }
127
+ }
128
+ if (listed.length <= keep) return
129
+ listed.sort((left, right) => right.at - left.at)
130
+ for (const spare of listed.slice(keep)) {
131
+ try {
132
+ await unlink(join(dir, spare.name))
133
+ } catch {
134
+ /* best effort: a file in use stays */
135
+ }
136
+ }
137
+ }
20
138
 
21
139
  /** How many times a call found a control again after the page redrew it. */
22
140
  const follows = { count: 0 }
23
141
 
24
142
  /** Shown when the sidecar has never been started in this run. */
25
- export const SIDECAR_OFF = 'The sidecar browser is not running. Call sidecar_open with the address first.'
143
+ export const SIDECAR_OFF = 'The sidecar browser is not running. Call browser_open with the address first.'
26
144
 
27
145
  /** Output schema every tool returns. */
146
+ /** The picture half of a screenshot result: shown only when the route can see it. */
147
+ const SHOT_IMAGE = {
148
+ type: 'object', // ui-copy-ignore
149
+ properties: {
150
+ attachmentId: { type: 'string' },
151
+ mediaType: { type: 'string' },
152
+ bytes: { type: 'integer' }, // ui-copy-ignore
153
+ width: { type: 'integer' }, // ui-copy-ignore
154
+ height: { type: 'integer' }, // ui-copy-ignore
155
+ name: { type: 'string' }, // ui-copy-ignore
156
+ },
157
+ required: ['attachmentId', 'mediaType', 'bytes', 'width', 'height'],
158
+ additionalProperties: false,
159
+ }
160
+
161
+ /** A screenshot's path, plus the picture when this model can look at it. */
162
+ export const SHOT_OUTPUT = {
163
+ schema: { // ui-copy-ignore
164
+ type: 'object', // ui-copy-ignore
165
+ properties: { text: { type: 'string' }, image: SHOT_IMAGE },
166
+ required: ['text'],
167
+ additionalProperties: false,
168
+ },
169
+ /**
170
+ * @param {unknown} _args - call arguments, unused.
171
+ * @param {any} value - tool value.
172
+ * @returns {any[]} the text, and the picture when there is one.
173
+ */
174
+ render(_args, value) {
175
+ const blocks = [{ type: 'text', text: String(value?.text ?? '') }]
176
+ if (value?.image !== undefined) blocks.push({ type: 'image', attachment: value.image })
177
+ return blocks
178
+ },
179
+ }
180
+
28
181
  const TEXT_OUTPUT = {
29
182
  schema: { type: 'object', properties: { text: { type: 'string' } }, required: ['text'], additionalProperties: false },
30
183
  /**
@@ -45,18 +198,38 @@ const TEXT_OUTPUT = {
45
198
  * @param {any} page - sidecar page.
46
199
  * @returns {Promise<string>} address, title, focus and any open dialog.
47
200
  */
201
+ export const BRIEF_JS = `(() => {
202
+ const a = document.activeElement
203
+ const secret = a !== null && typeof a === 'object' && (String(a.type ?? '') === 'password' || String(a.getAttribute === undefined ? '' : a.getAttribute('type') ?? '') === 'password')
204
+ const focus = a !== null && a !== document.body && a !== document.documentElement
205
+ ? { name: String(a.getAttribute('aria-label') || a.getAttribute('placeholder') || (a.labels && a.labels[0] && a.labels[0].innerText) || a.innerText || a.name || a.tagName).trim().slice(0, 60), value: secret ? '' : String(a.value === undefined ? '' : a.value).slice(0, 60) }
206
+ : null
207
+ return { url: location.href, title: document.title.slice(0, 80), focus }
208
+ })()`
209
+
48
210
  async function brief(page) {
49
211
  const dialog = page.dialog
50
212
  if (dialog !== null && dialog !== undefined) {
51
213
  return `${String(page.url ?? '')} · dialog ${String(dialog.type ?? '')}: ${String(dialog.message ?? '')}`
52
214
  }
53
- const info = await evaluate(page, `(() => {
54
- const a = document.activeElement
55
- const focus = a !== null && a !== document.body && a !== document.documentElement
56
- ? { name: String(a.getAttribute('aria-label') || a.getAttribute('placeholder') || (a.labels && a.labels[0] && a.labels[0].innerText) || a.innerText || a.name || a.tagName).trim().slice(0, 60), value: String(a.value === undefined ? '' : a.value).slice(0, 60) }
57
- : null
58
- return { url: location.href, title: document.title.slice(0, 80), focus }
59
- })()`)
215
+ /* On a Flash page the DOM around the plugin says nothing about what is on
216
+ screen, and reading it right after a click waits for the main thread the
217
+ plugin is holding. The bridge answers with the address and title instead. */
218
+ if (page.flashBridge !== null && page.flashBridge !== undefined) {
219
+ try {
220
+ const reply = await fetch(`http://127.0.0.1:${page.flashBridge}/info`, { signal: AbortSignal.timeout(600) })
221
+ if (reply.ok === true) {
222
+ const info = await reply.json()
223
+ const bits = [String(info?.url ?? page.url ?? '')]
224
+ if (String(info?.title ?? '') !== '') bits.push(String(info.title))
225
+ return bits.join(' · ')
226
+ }
227
+ } catch {
228
+ /* the cached address below is still a true answer */
229
+ }
230
+ return String(page.url ?? '')
231
+ }
232
+ const info = await evaluate(page, BRIEF_JS)
60
233
  const bits = [String(info?.url ?? '')]
61
234
  if (String(info?.title ?? '') !== '') bits.push(String(info.title))
62
235
  if (info?.focus) bits.push(`focus ${info.focus.name}${info.focus.value === '' ? '' : ` = ${info.focus.value}`}`)
@@ -71,9 +244,117 @@ async function brief(page) {
71
244
  * @param {string} label - what was done.
72
245
  * @returns {Promise<string>} reply text.
73
246
  */
247
+ /** The address each page last reported to the model, so it is said once. */
248
+ const saidUrls = new WeakMap()
249
+
250
+ /**
251
+ * The address is worth one line the first time and every time it changes; the
252
+ * same address on every step is a cost with nothing to teach — the model read
253
+ * it a step ago and still has it. Titles and focus still come through.
254
+ * @param {any} page - sidecar page.
255
+ * @param {string} briefText - address, title and focus as `brief` built them.
256
+ * @returns {string} the line to send.
257
+ */
258
+ /**
259
+ * A PDF is a document, not a blank page: say so, and put its bytes somewhere a
260
+ * reader can open them. The mirror cannot draw a PDF, and a model looking at an
261
+ * empty screen would otherwise think the address had nothing on it.
262
+ * @param {any} page - the tab.
263
+ * @returns {Promise<string>} a line to append to the read, or ''.
264
+ */
265
+ async function pdfNote(page) {
266
+ const found = page?.pdf
267
+ if (found === null || found === undefined) return ''
268
+ let file = { path: '', bytes: 0 }
269
+ try {
270
+ file = await page.documentFile()
271
+ } catch (error) {
272
+ file = { path: '', bytes: 0 }
273
+ }
274
+ const size = file.bytes > 0 ? ` (${String(Math.max(1, Math.round(file.bytes / 1024)))} KB)` : ''
275
+ return file.path === ''
276
+ ? `\nthis address is a PDF${size}: the mirror cannot draw it, and its bytes could not be saved. browser_downloads lists what the browser has downloaded.`
277
+ : `\nthis address is a PDF${size}: the mirror cannot draw it. Saved to ${file.path} — envoy_read can read that file, or another document reader.`
278
+ }
279
+
280
+ /**
281
+ * The numbered controls that were not on the page at the read before this one.
282
+ * A model that just acted cares about what its action brought in, and saying so
283
+ * costs a few numbers instead of a whole second read.
284
+ * @param {any} before - the read before this one, or undefined.
285
+ * @param {any} after - the read now.
286
+ * @returns {number[]} refs in `after` that `before` did not have.
287
+ */
288
+ export function freshRefs(before, after) {
289
+ if (before === undefined || before === null) return []
290
+ if (after === undefined || after === null) return []
291
+ const had = new Set((Array.isArray(before.controls) ? before.controls : []).map((control) => Number(control?.ref ?? -1)))
292
+ const fresh = []
293
+ for (const control of Array.isArray(after.controls) ? after.controls : []) {
294
+ const ref = Number(control?.ref ?? -1)
295
+ if (ref < 0 || had.has(ref)) continue
296
+ if (!fresh.includes(ref)) fresh.push(ref)
297
+ }
298
+ return fresh
299
+ }
300
+
301
+ /** What an action did, in one line: where it went, whether the page moved at all. */
302
+ export async function outcome(page, before) {
303
+ if (typeof before !== 'string' || before === '') return ''
304
+ const after = await pageMark(page)
305
+ if (after === '') return ''
306
+ /* A movie has no DOM to compare: its frame counter is the whole answer. */
307
+ if (before.startsWith('flash:') && after.startsWith('flash:')) {
308
+ return before === after ? ' · no frame change' : ' · screen updated'
309
+ }
310
+ const [wasTitle, wasUrl, wasText, wasNodes] = before.split('|')
311
+ const [nowTitle, nowUrl, nowText, nowNodes] = after.split('|')
312
+ if (after === before) return ' · no visible change'
313
+ if (nowUrl !== wasUrl) return ` · navigated to ${String(nowUrl).slice(0, 300)}`
314
+ const words = Math.abs(Number(nowText ?? 0) - Number(wasText ?? 0))
315
+ const nodes = Math.abs(Number(nowNodes ?? 0) - Number(wasNodes ?? 0))
316
+ if (nowTitle !== wasTitle) return ' · page moved on'
317
+ if (words >= 20 || nodes >= 3) return ' · content changed'
318
+ return ' · no visible change'
319
+ }
320
+
321
+ /** Read results this chat already paid for, keyed by what the page looks like. */
322
+ const readSeen = new WeakMap()
323
+
324
+ export function oncePerPage(page, briefText) {
325
+ const parts = String(briefText).split(' · ')
326
+ const url = String(parts[0] ?? '')
327
+ const known = page === null || page === undefined ? undefined : saidUrls.get(page)
328
+ if (url !== '' && known !== undefined) {
329
+ saidUrls.set(page, url)
330
+ if (url === known && parts.length > 1) return parts.slice(1).join(' · ')
331
+ return briefText
332
+ }
333
+ if (url !== '' && page !== null && page !== undefined) saidUrls.set(page, url)
334
+ return briefText
335
+ }
336
+
337
+ /**
338
+ * A read that starts with an address the model was already given opens with the
339
+ * page itself. The address is how a page says which page it is, not something
340
+ * to repeat every read; a page that moved says the new address in full.
341
+ * @param {any} page - sidecar page.
342
+ * @param {string} url - the address this read is from.
343
+ * @param {string} text - the rendered read.
344
+ * @returns {string} the read, without a leading address already said.
345
+ */
346
+ export function withoutSeenAddress(page, url, text) {
347
+ const body = String(text ?? '')
348
+ const address = String(url ?? '')
349
+ if (address === '' || page === null || page === undefined) return body
350
+ const first = body.split('\n')[0]
351
+ if (first !== address || saidUrls.get(page) !== address) return body
352
+ return body.slice(first.length + 1)
353
+ }
354
+
74
355
  async function acted(page, args, look, label) {
75
356
  if (args?.read === true) return await look()
76
- return `ok ${label} · ${await brief(page)}`
357
+ return `ok ${label} · ${oncePerPage(page, await brief(page))}`
77
358
  }
78
359
 
79
360
  /**
@@ -94,7 +375,7 @@ async function acted(page, args, look, label) {
94
375
  /** Titles a site shows while it asks the browser to prove a person is there. */
95
376
  const CHECK_TITLES = ['请稍候', 'just a moment', 'checking your browser', 'verifying you are human', 'attention required']
96
377
 
97
- const CHECK_NOTE = 'That is the site asking a browser to prove a person is there, and a windowless one never gets past it. sidecar_open with headless false opens a window on the same profile, with the same logins.'
378
+ const CHECK_NOTE = 'That is the site asking a browser to prove a person is there, and a windowless one never gets past it. browser_open with headless false opens a window on the same profile, with the same logins.'
98
379
 
99
380
  /**
100
381
  * A check page: the site has not refused, it is asking for proof. Reading it
@@ -156,27 +437,36 @@ export async function unreachable(page) {
156
437
  if (!isErrorPage(page)) return ''
157
438
  /* The code is the part a model can act on: ERR_NAME_NOT_RESOLVED is a typo,
158
439
  ERR_PROXY_CONNECTION_FAILED is this machine's proxy, ERR_TIMED_OUT is slow. */
159
- let detail = String(page.title ?? '')
440
+ let detail = ''
160
441
  try {
161
442
  const said = await evaluate(page, `(() => {
162
443
  const text = String(document.body ? document.body.innerText : '')
163
444
  const code = (String(text).match(/ERR_[A-Z0-9_]+/) || [''])[0]
164
- return code === '' ? String(document.title || '') : code
445
+ return code
165
446
  })()`)
166
- if (typeof said === 'string' && said.trim() !== '') detail = said.trim()
447
+ if (typeof said === 'string') detail = said.trim()
167
448
  } catch {
168
449
  /* The error page itself refused to be read; the address still says enough. */
169
450
  }
170
- return `the network could not reach that address — the browser stopped on its own error page (${detail === '' ? 'no details' : detail}), which is not the site. Nothing on it is page content. Use sidecar_nav with another address, or fix the connection this machine needs.`
451
+ /* Chrome often writes the address it could not reach where the code would
452
+ be: that is still the one thing the person can check. */
453
+ if (detail === '') {
454
+ const title = String(page.title ?? '').trim()
455
+ if (title !== '' && !/\s/.test(title)) detail = title
456
+ }
457
+ if (detail === '') detail = String(page.wantUrl ?? page.lastUrl ?? '').replace(/^https?:\/\//, '').replace(/\/.*$/, '')
458
+ return `the network could not reach that address — the browser stopped on its own error page (${detail === '' ? 'no details' : detail}), which is not the site. Nothing on it is page content. Use browser_nav with another address, or fix the connection this machine needs.`
171
459
  }
172
460
 
173
- /** Key names this driver can press, spelled as a model would. */
461
+ /** Key names this driver can press: DOM `key` names for the named keys, and
462
+ the character itself for a printable one (Space also answers to `Space`). */
174
463
  export const KEYS = {
175
464
  Enter: { key: 'Enter', code: 'Enter', keyCode: 13, text: '\r' },
176
465
  Tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
177
466
  Escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
178
467
  Backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
179
468
  Delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
469
+ Insert: { key: 'Insert', code: 'Insert', keyCode: 45 },
180
470
  ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
181
471
  ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
182
472
  ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
@@ -186,6 +476,208 @@ export const KEYS = {
186
476
  PageUp: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
187
477
  PageDown: { key: 'PageDown', code: 'PageDown', keyCode: 34 },
188
478
  Space: { key: ' ', code: 'Space', keyCode: 32, text: ' ' },
479
+ ContextMenu: { key: 'ContextMenu', code: 'ContextMenu', keyCode: 93 },
480
+ CapsLock: { key: 'CapsLock', code: 'CapsLock', keyCode: 20 },
481
+ NumLock: { key: 'NumLock', code: 'NumLock', keyCode: 144 },
482
+ ScrollLock: { key: 'ScrollLock', code: 'ScrollLock', keyCode: 145 },
483
+ PrintScreen: { key: 'PrintScreen', code: 'PrintScreen', keyCode: 44 },
484
+ Pause: { key: 'Pause', code: 'Pause', keyCode: 19 },
485
+ NumpadEnter: { key: 'Enter', code: 'NumpadEnter', keyCode: 13, text: '\r' },
486
+ AudioVolumeMute: { key: 'AudioVolumeMute', code: 'AudioVolumeMute', keyCode: 173 },
487
+ AudioVolumeDown: { key: 'AudioVolumeDown', code: 'AudioVolumeDown', keyCode: 174 },
488
+ AudioVolumeUp: { key: 'AudioVolumeUp', code: 'AudioVolumeUp', keyCode: 175 },
489
+ MediaPlayPause: { key: 'MediaPlayPause', code: 'MediaPlayPause', keyCode: 179 },
490
+ MediaTrackNext: { key: 'MediaTrackNext', code: 'MediaTrackNext', keyCode: 176 },
491
+ MediaTrackPrevious: { key: 'MediaTrackPrevious', code: 'MediaTrackPrevious', keyCode: 177 },
492
+ MediaStop: { key: 'MediaStop', code: 'MediaStop', keyCode: 178 },
493
+ }
494
+
495
+ /* A game moves while a key is held, and the keys a game listens for are letters
496
+ and digits, not the ones a form needs. Each entry carries what a browser
497
+ reports: the `key` value, the physical `code`, and the legacy number old
498
+ engines still look at. */
499
+ for (const letter of 'abcdefghijklmnopqrstuvwxyz') {
500
+ KEYS[letter] = { key: letter, code: `Key${letter.toUpperCase()}`, keyCode: letter.toUpperCase().charCodeAt(0), text: letter }
501
+ /* A capital is its own key event: the page reads `A` with shift held, and a
502
+ form inserts `A`, not `a`. */
503
+ KEYS[letter.toUpperCase()] = {
504
+ key: letter.toUpperCase(),
505
+ code: `Key${letter.toUpperCase()}`,
506
+ keyCode: letter.toUpperCase().charCodeAt(0),
507
+ text: letter.toUpperCase(),
508
+ shift: true,
509
+ }
510
+ }
511
+ for (const digit of '0123456789') {
512
+ KEYS[digit] = { key: digit, code: `Digit${digit}`, keyCode: digit.charCodeAt(0), text: digit }
513
+ }
514
+
515
+ /* The symbols above the digits and around the alphabet, both halves of every
516
+ key: a page reads `?` as Slash with shift down, and a field inserts `?`.
517
+ `keyCode` is the legacy VK number Windows and old pages compare against. */
518
+ const PUNCTUATION = [
519
+ ['Backquote', '`', 192], ['Minus', '-', 189], ['Equal', '=', 187],
520
+ ['BracketLeft', '[', 219], ['BracketRight', ']', 221], ['Backslash', '\\', 220],
521
+ ['Semicolon', ';', 186], ['Quote', "'", 222], ['Comma', ',', 188],
522
+ ['Period', '.', 190], ['Slash', '/', 191],
523
+ ]
524
+ const SHIFTED_SYMBOLS = [
525
+ ['~', 'Backquote', 192], ['!', 'Digit1', 49], ['@', 'Digit2', 50], ['#', 'Digit3', 51],
526
+ ['$', 'Digit4', 52], ['%', 'Digit5', 53], ['^', 'Digit6', 54], ['&', 'Digit7', 55],
527
+ ['*', 'Digit8', 56], ['(', 'Digit9', 57], [')', 'Digit0', 48], ['_', 'Minus', 189],
528
+ ['+', 'Equal', 187], ['{', 'BracketLeft', 219], ['}', 'BracketRight', 221], ['|', 'Backslash', 220],
529
+ [':', 'Semicolon', 186], ['"', 'Quote', 222], ['<', 'Comma', 188], ['>', 'Period', 190],
530
+ ['?', 'Slash', 191],
531
+ ]
532
+ for (const [code, char, keyCode] of PUNCTUATION) {
533
+ KEYS[char] = { key: char, code, keyCode, text: char }
534
+ }
535
+ for (const [char, code, keyCode] of SHIFTED_SYMBOLS) {
536
+ KEYS[char] = { key: char, code, keyCode, text: char, shift: true }
537
+ }
538
+
539
+ /* Function keys, the numeric keypad a page can listen to, and the keys that do
540
+ nothing in a text field but everything in an app: a canvas game, a document
541
+ editor, a video player. */
542
+ for (let n = 1; n <= 12; n += 1) {
543
+ KEYS[`F${n}`] = { key: `F${n}`, code: `F${n}`, keyCode: 111 + n }
544
+ }
545
+ for (const digit of '0123456789') {
546
+ KEYS[`Numpad${digit}`] = { key: digit, code: `Numpad${digit}`, keyCode: 96 + Number(digit), text: digit }
547
+ }
548
+ KEYS.NumpadAdd = { key: '+', code: 'NumpadAdd', keyCode: 107, text: '+' }
549
+ KEYS.NumpadSubtract = { key: '-', code: 'NumpadSubtract', keyCode: 109, text: '-' }
550
+ KEYS.NumpadMultiply = { key: '*', code: 'NumpadMultiply', keyCode: 106, text: '*' }
551
+ KEYS.NumpadDivide = { key: '/', code: 'NumpadDivide', keyCode: 111, text: '/' }
552
+ KEYS.NumpadDecimal = { key: '.', code: 'NumpadDecimal', keyCode: 110, text: '.' }
553
+
554
+ /* The names people actually type, and the spelling a model guesses. */
555
+ const KEY_ALIASES = {
556
+ esc: 'Escape', return: 'Enter', del: 'Delete', ins: 'Insert',
557
+ up: 'ArrowUp', down: 'ArrowDown', left: 'ArrowLeft', right: 'ArrowRight',
558
+ pageup: 'PageUp', pagedown: 'PageDown', spacebar: 'Space', space: 'Space',
559
+ menu: 'ContextMenu', prtsc: 'PrintScreen', printscreen: 'PrintScreen',
560
+ }
561
+
562
+ /**
563
+ * The event a browser would report for a named key.
564
+ * @param {string} name - a key in {@link KEYS}, or something close to one.
565
+ * @returns {{key: string, code: string, keyCode: number, text?: string, shift?: boolean}} the event fields.
566
+ * @throws {Error} when the key does not exist.
567
+ */
568
+ export function keySpec(name) {
569
+ const raw = String(name ?? '')
570
+ const exact = KEYS[raw]
571
+ if (exact !== undefined) return exact
572
+ if (raw === ' ') return KEYS.Space
573
+ const alias = KEY_ALIASES[raw.toLowerCase()]
574
+ if (alias !== undefined) return KEYS[alias]
575
+ const capital = raw.charAt(0).toUpperCase() + raw.slice(1)
576
+ if (KEYS[capital] !== undefined) return KEYS[capital]
577
+ const fn = /^f([1-9]|1[0-2])$/i.exec(raw)
578
+ if (fn !== null) return KEYS[`F${fn[1]}`]
579
+ throw new Error(`sidecar: no such key ${raw}`) // ui-copy-ignore
580
+ }
581
+
582
+ /**
583
+ * Turn `Ctrl+Shift` into the CDP modifier bits. `Mod` is the platform's own
584
+ * command key: Command on macOS, Control on Windows and Linux — one spelling
585
+ * for the shortcut a page treats as copy, undo, select-all.
586
+ * @param {string} held - modifiers joined by +, any of Alt/Option, Ctrl/Control,
587
+ * Meta/Cmd/Command/Super/Win, Shift, Mod.
588
+ * @param {string} [platform] - defaults to this process's platform.
589
+ * @returns {number} CDP modifier bits: Alt 1, Ctrl 2, Meta 4, Shift 8.
590
+ */
591
+ export function modifierBits(held, platform = process.platform) {
592
+ let bits = 0
593
+ for (const raw of String(held ?? '').split('+')) {
594
+ const one = raw.trim().toLowerCase()
595
+ if (one === 'alt' || one === 'option' || one === 'opt' || one === '⌥') bits |= 1
596
+ else if (one === 'ctrl' || one === 'control' || one === '^') bits |= 2
597
+ else if (one === 'meta' || one === 'cmd' || one === 'command' || one === 'super' || one === 'win' || one === '⌘') bits |= 4
598
+ else if (one === 'shift' || one === '⇧') bits |= 8
599
+ else if (one === 'mod') bits |= platform === 'darwin' ? 4 : 2
600
+ }
601
+ return bits
602
+ }
603
+
604
+ /**
605
+ * Turn what the mirror's client posted into the event the page should see. A
606
+ * capital letter carries its shift, a shortcut becomes the browser command it
607
+ * names, and a release carries no text so the field does not type twice.
608
+ * @param {any} body - `{ key, code?, keyCode?, modifiers?, text?, down?, repeat? }`.
609
+ * @returns {{name: string, down: boolean, repeat: boolean, spec: any, command: string|null}} the event.
610
+ */
611
+ export function keyEvent(body) {
612
+ const name = String(body?.key ?? '')
613
+ let known
614
+ try {
615
+ known = keySpec(name)
616
+ } catch {
617
+ known = null
618
+ }
619
+ const held = Number(body?.modifiers ?? 0)
620
+ const down = body?.down !== false
621
+ const repeat = body?.repeat === true
622
+ const capital = name.length === 1 && name !== name.toLowerCase()
623
+ const modifiers = capital ? held | 8 : held
624
+ const command = down === true && repeat === false ? commandOf(name, modifiers) : null
625
+ const text = command !== null ? undefined : body?.text === undefined ? known?.text : String(body.text)
626
+ return {
627
+ name,
628
+ down,
629
+ repeat,
630
+ spec: {
631
+ key: name,
632
+ code: String(body?.code ?? known?.code ?? name),
633
+ keyCode: Number(body?.keyCode ?? 0) || known?.keyCode || 0,
634
+ text,
635
+ modifiers,
636
+ },
637
+ command,
638
+ }
639
+ }
640
+
641
+ /**
642
+ * Read a chord the way a shortcut is written: `Ctrl+Shift+Tab` is the held
643
+ * modifiers Ctrl and Shift around one Tab, and `Mod+a` is Command+a on macOS.
644
+ * @param {string} chord - keys joined by +, modifiers may be in it.
645
+ * @param {number} [modifiers] - bits already held.
646
+ * @returns {{names: string[], modifiers: number}} the keys to press and the bits.
647
+ */
648
+ export function splitChord(chord, modifiers = 0) {
649
+ const bits = { alt: 1, ctrl: 2, control: 2, meta: 4, cmd: 4, command: 4, super: 4, win: 4, shift: 8, option: 1, opt: 1, mod: process.platform === 'darwin' ? 4 : 2 }
650
+ const names = []
651
+ let held = modifiers
652
+ for (const raw of String(chord ?? '').split('+')) {
653
+ const one = raw.trim()
654
+ if (one === '') continue
655
+ const bit = bits[one.toLowerCase()]
656
+ if (bit === undefined) names.push(one)
657
+ else held |= bit
658
+ }
659
+ return { names, modifiers: held }
660
+ }
661
+
662
+ /**
663
+ * The editing command a browser runs for a shortcut, if any: copy and paste are
664
+ * commands, not key events, and a key event alone leaves the clipboard alone.
665
+ * @param {string} name - the key name.
666
+ * @param {number} modifiers - CDP modifier bits.
667
+ * @returns {string|null} a CDP command name, or null.
668
+ */
669
+ export function commandOf(name, modifiers) {
670
+ /* Either command key counts: a page listens for the one its platform uses,
671
+ and the mirror can be driven from a machine with the other one. */
672
+ if ((modifiers & 6) === 0) return null
673
+ const one = String(name ?? '').toLowerCase()
674
+ if (one === 'v') return 'paste'
675
+ if (one === 'c') return 'copy'
676
+ if (one === 'x') return 'cut'
677
+ if (one === 'a') return 'selectAll'
678
+ if (one === 'z') return (modifiers & 8) !== 0 ? 'redo' : 'undo'
679
+ if (one === 'y') return 'redo'
680
+ return null
189
681
  }
190
682
 
191
683
  /**
@@ -195,14 +687,41 @@ export const KEYS = {
195
687
  * @param {number} [modifiers] - CDP modifier bits: Alt 1, Ctrl 2, Meta 4, Shift 8.
196
688
  * @returns {Promise<void>} done.
197
689
  */
198
- export async function pressKey(page, name, modifiers = 0) {
199
- const key = KEYS[name] ?? KEYS[name.charAt(0).toUpperCase() + name.slice(1)]
200
- if (key === undefined) throw new Error(`sidecar: no such key ${name}`) // ui-copy-ignore
201
- const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers }
202
- await page.key('keyDown', key.text === undefined ? base : { ...base, text: key.text })
690
+ export async function pressKey(page, name, modifiers = 0, holdMs = 0) {
691
+ const key = keySpec(name)
692
+ const withShift = (key.shift === true ? modifiers | 8 : modifiers)
693
+ const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers: withShift }
694
+ /* Copy, paste, undo and select-all are editor commands: a key event alone
695
+ leaves the clipboard and the field where they were. */
696
+ const command = commandOf(key.text ?? key.key, withShift)
697
+ await page.key('keyDown', command === null
698
+ ? (key.text === undefined ? base : { ...base, text: key.text })
699
+ : { ...base, text: undefined, commands: [command] })
700
+ /* A key held down is how a game moves: down, wait, up — not a tap. */
701
+ if (holdMs > 0) await new Promise((settle) => setTimeout(settle, holdMs))
203
702
  await page.key('keyUp', base)
204
703
  }
205
704
 
705
+ /**
706
+ * Press several keys at once and hold them: the diagonal a game expects, and a
707
+ * chord such as Shift+ArrowLeft. Releasing in reverse order is what a pair of
708
+ * hands does, and some engines only notice the last key that went up.
709
+ * @param {any} page - sidecar page.
710
+ * @param {string[]} names - key names, for example ['ArrowLeft', 'ArrowUp'].
711
+ * @param {number} [modifiers] - CDP modifier bits.
712
+ * @param {number} [holdMs] - how long every key stays down.
713
+ * @returns {Promise<void>} done.
714
+ */
715
+ export async function pressChord(page, names, modifiers = 0, holdMs = 0) {
716
+ const keys = names.map((name) => {
717
+ const key = keySpec(name)
718
+ return { key, base: { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers: key.shift === true ? modifiers | 8 : modifiers } }
719
+ })
720
+ for (const one of keys) await page.key('keyDown', one.key.text === undefined ? one.base : { ...one.base, text: one.key.text })
721
+ if (holdMs > 0) await new Promise((settle) => setTimeout(settle, holdMs))
722
+ for (const one of [...keys].reverse()) await page.key('keyUp', one.base)
723
+ }
724
+
206
725
  /**
207
726
  * Empty the field that has focus, whatever this machine calls select-all:
208
727
  * Command on macOS, Control elsewhere. A page can swallow the shortcut, so any
@@ -262,7 +781,7 @@ export function personNote(sidecar) {
262
781
  export function holdText(sidecar) {
263
782
  const left = Number(sidecar?.holding?.() ?? 0)
264
783
  if (left <= 0) return ''
265
- return `the person is using the mirror (${String(Math.ceil(left / 1000))} s more) — nothing was sent. Let them finish, then repeat this call; sidecar_resume takes the page back now.`
784
+ return `the person is using the mirror (${String(Math.ceil(left / 1000))} s more) — nothing was sent. Let them finish, then repeat this call; browser_resume takes the page back now.`
266
785
  }
267
786
 
268
787
  /**
@@ -296,11 +815,15 @@ async function coverProbe(page, ref) {
296
815
  * @param {ReturnType<typeof coverProbe>} probe - what the page said is on top.
297
816
  * @returns {string} a sentence, or an empty string.
298
817
  */
299
- function coverNote(probe) {
818
+ function coverNote(probe, moved) {
300
819
  if (probe === null || probe === undefined) return ''
301
820
  const what = probe.tag === 'div' || probe.tag === 'section' ? 'an overlay' : `a ${probe.tag}`
302
- return `
303
- ${what} covers it${probe.name === '' ? '' : `: ${probe.name}`} — the click landed beneath it`
821
+ const head = `
822
+ ${what} covers it${probe.name === '' ? '' : `: ${probe.name}`}`
823
+ /* A banner that is about to be cleared is not a warning: the line below it
824
+ already says what was clicked on the way, so do not claim the click will
825
+ land under something that is leaving. */
826
+ return moved ? head : `${head} — the click would land beneath it`
304
827
  }
305
828
 
306
829
  /**
@@ -315,14 +838,19 @@ ${what} covers it${probe.name === '' ? '' : `: ${probe.name}`} — the click lan
315
838
  async function clearCover(page, ref) {
316
839
  const probe = await coverProbe(page, ref)
317
840
  if (probe === null) return { note: '', cleared: true, name: '' }
318
- const note = coverNote(probe)
841
+ const note = coverNote(probe, probe.dismiss !== null)
319
842
  const away = probe.dismiss
320
843
  if (away === null) return { note, cleared: false, name: '' }
321
844
  await page.mouse('mouseMoved', { x: away.x, y: away.y })
322
845
  await page.mouse('mousePressed', { x: away.x, y: away.y, clickCount: 1 })
323
846
  await page.mouse('mouseReleased', { x: away.x, y: away.y, clickCount: 1 })
324
- await wait(150)
325
- const again = await coverProbe(page, ref)
847
+ /* Prove the banner left instead of guessing a paint delay: poll the same
848
+ question the refusal would ask, and stop the moment it answers clear. */
849
+ let again = await coverProbe(page, ref)
850
+ for (let look = 0; look < 8 && again !== null; look += 1) {
851
+ await wait(20)
852
+ again = await coverProbe(page, ref)
853
+ }
326
854
  if (again !== null) return { note: `${note}
327
855
  - clicking ${away.name} on it did not clear it`, cleared: false, name: away.name } // ui-copy-ignore
328
856
  return { note: `${note}
@@ -352,12 +880,19 @@ function wait(ms) {
352
880
  * @param {{contextId?: number, sessionId?: string}|null} [route] - frame to run in.
353
881
  * @returns {Promise<any>} the value.
354
882
  */
355
- async function evaluate(page, expression, route) {
883
+ async function evaluate(page, expression, route, raceMs = 2000) {
356
884
  const params = { expression, returnByValue: true, awaitPromise: true }
357
885
  if (route?.contextId !== undefined) params.contextId = route.contextId
886
+ /* A page that navigates mid-call destroys the context this ran in; the
887
+ rejection arrives after the race is already lost. Handled, so it cannot
888
+ take the whole process down with it. */
358
889
  const call = page.cdp.send('Runtime.evaluate', params, route?.sessionId)
359
- const raced = await Promise.race([call, wait(2000).then(() => 'timeout')])
890
+ .catch((error) => ({ threw: String(error?.message ?? error) }))
891
+ let timer = null
892
+ const raced = await Promise.race([call, new Promise((done) => { timer = setTimeout(() => done('timeout'), raceMs) })])
893
+ clearTimeout(timer)
360
894
  if (raced === 'timeout') return undefined
895
+ if (raced?.threw !== undefined) return undefined
361
896
  const { result, exceptionDetails } = raced
362
897
  if (exceptionDetails !== undefined) {
363
898
  return { error: String(exceptionDetails.exception?.description ?? exceptionDetails.text ?? 'evaluate failed') }
@@ -365,6 +900,36 @@ async function evaluate(page, expression, route) {
365
900
  return result?.value
366
901
  }
367
902
 
903
+ /**
904
+ * What the page looks like in one cheap string: address, title, how many
905
+ * elements and how much text. Two of these equal says the click did nothing
906
+ * visible — worth saying, so the model does not click a dead control twice.
907
+ * A Flash page has no DOM to ask, so its frame counter answers instead.
908
+ * @param {any} page - sidecar page.
909
+ * @returns {Promise<string>} a fingerprint, or '' when the page would not say.
910
+ */
911
+ export async function pageMark(page) {
912
+ if (page?.flashBridge !== null && page?.flashBridge !== undefined) {
913
+ return `flash:${Number(page.seq ?? 0)}:${Number(page.lastFrameAt ?? 0)}`
914
+ }
915
+ const said = await evaluate(page, `(() => { const body = document.body; return String(document.title || '').slice(0, 80) + '|' + String(location.href).slice(0, 200) + '|' + (body ? body.innerText.length : 0) + '|' + document.querySelectorAll('*').length })()`).catch(() => undefined)
916
+ return typeof said === 'string' ? said : ''
917
+ }
918
+
919
+ /**
920
+ * The page's answer, as an object.
921
+ * @param {any} said - what evaluate returned.
922
+ * @returns {any} the answer, or null when the page did not answer.
923
+ */
924
+ function asSpot(said) {
925
+ if (typeof said !== 'string') return null
926
+ try {
927
+ return JSON.parse(said)
928
+ } catch {
929
+ return null
930
+ }
931
+ }
932
+
368
933
  /**
369
934
  * Wait until the page has settled: loading finished and the address stopped
370
935
  * changing. Acting on a page mid-navigation is how a driver clicks the old page.
@@ -372,21 +937,62 @@ async function evaluate(page, expression, route) {
372
937
  * @param {number} [timeout] - ceiling in milliseconds.
373
938
  * @returns {Promise<string>} the address it settled on.
374
939
  */
375
- export async function settle(page, timeout = 5000) {
940
+ export async function settle(page, timeout = 5000, since = 0) {
941
+ /* A Flash page has no load events and no DOM of its own to poll: the next
942
+ picture is the only sign the click changed anything, and the bridge paints
943
+ every pump, so this ends at the frame instead of at the timeout. */
944
+ if (page.flashBridge !== null && page.flashBridge !== undefined) {
945
+ const until = Date.now() + Math.min(timeout, 700)
946
+ const before = Number(page.seq ?? 0)
947
+ while (Date.now() < until) {
948
+ if (Number(page.seq ?? 0) > before || Number(page.lastFrameAt ?? 0) > since) break
949
+ await new Promise((resolve) => setTimeout(resolve, 20))
950
+ }
951
+ return String(page.url ?? '')
952
+ }
376
953
  /* A click may have opened a tab. The mirror follows it, so every tool that
377
954
  waited on a page ends up on the page the person would be looking at. */
378
955
  const opened = typeof page.adoptOpened === 'function' ? await page.adoptOpened().catch(() => '') : ''
379
956
  const deadline = Date.now() + timeout
957
+ const started = Date.now()
380
958
  let last = ''
381
959
  let steady = 0
382
960
  while (Date.now() < deadline) {
383
961
  if (page.dialog !== null && page.dialog !== undefined) return String(page.url ?? '')
384
- const state = await evaluate(page, '({ ready: document.readyState, href: location.href })')
385
- const key = `${String(state?.ready)}|${String(state?.href)}`
962
+ /* The browser already said the load it began has finished. That is the same
963
+ answer a readyState poll gives, and it stays true while a page's own
964
+ scripts hold the main thread, which is exactly when the poll stalls. */
965
+ const began = Number(page.navStarted ?? 0)
966
+ const fired = Number(page.loadFired ?? 0)
967
+ /* Two event-only answers: nothing began loading since the action, or the
968
+ navigation this action started has already finished. Neither needs the
969
+ renderer, whose answer can take seconds on a page that is busy. */
970
+ if (since > 0 && began <= since) return String(page.url ?? '')
971
+ if (fired > 0 && fired >= began && (since === 0 || fired >= since)) return String(page.url ?? '')
972
+ /* A click opened a page whose own requests hang: the browser began loading
973
+ and nothing has finished since. A person presses Esc and uses what
974
+ arrived, and waiting the whole timeout only to give up helps no one. */
975
+ if (since > 0 && began > since && Date.now() - began > 900) {
976
+ await page.cdp.send('Page.stopLoading', {}).catch(() => {})
977
+ return String(page.url ?? '')
978
+ }
979
+ const state = await evaluate(page, '({ ready: document.readyState, href: location.href, n: document.getElementsByTagName("*").length, c: (document.body?.textContent ?? "").length })', null, 400)
980
+ /* Nothing began loading since the action, and the page is complete: the
981
+ click changed state without navigating, so there is nothing to wait for. */
982
+ if (since > 0 && state?.ready === 'complete' && Number(page.navStarted ?? 0) <= since) return String(state?.href ?? '')
983
+ const key = `${String(state?.ready)}|${String(state?.href)}|${String(state?.n)}|${String(state?.c)}`
386
984
  steady = key === last ? steady + 1 : 0
387
985
  last = key
986
+ /* A page that already says complete needs one steady poll, not two: the
987
+ second look is what a redirect needs, and a click usually has none. */
988
+ if (state?.ready === 'complete' && steady >= 1) return String(state?.href ?? '')
388
989
  if (state?.ready !== 'loading' && steady >= 2) return String(state?.href ?? '')
389
- await wait(60)
990
+ /* A loaded page can change on the next paint; a loading one is either
991
+ navigating or about to be. Right after an action that is usually the
992
+ page the action opened, which lands in tens of milliseconds, so the
993
+ first quarter second is watched closely and only then slows down. */
994
+ const fresh = Date.now() - started < 250
995
+ await wait(state?.ready === 'complete' ? (fresh ? 12 : 25) : (fresh ? 12 : 60))
390
996
  }
391
997
  return ''
392
998
  }
@@ -416,7 +1022,29 @@ async function until(page, expression, timeout) {
416
1022
  * @returns {Promise<string>} a compact report.
417
1023
  */
418
1024
  /** The first control whose words match, scrolled into view. Text can beat a number: it costs no read. */
419
- async function findWords(page, words) {
1025
+ export async function findWords(page, words, known) {
1026
+ /* The words a caller used usually came from a read that already happened, so
1027
+ look among those numbers first: one position check is a single evaluate,
1028
+ while a fresh read walks the whole page. A number the page no longer has
1029
+ falls through to the read, so a redraw is not a missed click. */
1030
+ const squeezeOf = (text) => String(text).toLowerCase().replace(/\s+/g, '')
1031
+ const wantedFirst = squeezeOf(words)
1032
+ const remembered = Array.isArray(known) ? known.filter((control) => control.view !== false && squeezeOf(control.name ?? '').includes(wantedFirst)) : []
1033
+ remembered.sort((a, b) => Number(squeezeOf(a.name ?? '') === wantedFirst) - Number(squeezeOf(b.name ?? '') === wantedFirst))
1034
+ /* A page that repeats one word down a list ("Add to cart" on every card) has
1035
+ one obvious answer: the first one a person would reach. The last-first
1036
+ order that keeps a scrolled page fast also picked whichever duplicate sat
1037
+ lowest, so visible matches are tried in reading order first and the near
1038
+ end is only the fallback. */
1039
+ const firstInOrder = remembered.filter((control) => control.view !== false).sort((a, b) => Number(a.ref) - Number(b.ref))
1040
+ const nearEnd = remembered.slice(-8).reverse()
1041
+ for (const control of [...firstInOrder.slice(0, 4), ...nearEnd]) {
1042
+ const spot = await locate(page, Number(control.ref))
1043
+ if (spot.gone === true) continue
1044
+ const name = String(spot.name ?? control.name ?? '')
1045
+ if (!squeezeOf(name).includes(wantedFirst)) continue
1046
+ return { ...control, ref: Number(control.ref), name, x: Number(spot.x), y: Number(spot.y) }
1047
+ }
420
1048
  const read = await readPage(page, { mode: 'text', maxChars: 1200, max: 200 })
421
1049
  const shown = read.controls.filter((control) => control.view !== false)
422
1050
  /* The words may sit in separate inline nodes: <span>Sign</span><span>in</span>
@@ -454,21 +1082,90 @@ async function fieldSpot(page, words) {
454
1082
  * Where the field called these words is, clearing a banner over it first.
455
1083
  * @param {any} page - sidecar page.
456
1084
  * @param {string} words - what the field is called on screen.
457
- * @returns {Promise<{x: number, y: number, name: string, lead: string}|null>} the click.
1085
+ * @returns {Promise<{x: number, y: number, name: string, lead: string, ref?: number}|null>} the click.
458
1086
  */
459
- async function fieldByName(page, words) {
460
- const hit = await findWords(page, words)
1087
+ async function fieldByName(page, words, known) {
1088
+ const hit = await findWords(page, words, known)
461
1089
  const lead = hit === null ? '' : await coverLead(page, hit.ref)
462
1090
  if (hit !== null && EDITABLE_KINDS.includes(String(hit.kind))) {
463
1091
  const spot = await locate(page, hit.ref)
464
- if (spot.gone !== true) return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead }
1092
+ if (spot.gone !== true) return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead, ref: Number(hit.ref) }
465
1093
  }
466
1094
  const field = await fieldSpot(page, words)
467
1095
  if (field !== null) return { ...field, lead }
468
1096
  if (hit === null) return null
469
1097
  const spot = await locate(page, hit.ref)
470
1098
  if (spot.gone === true) return null
471
- return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead }
1099
+ return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead, ref: Number(hit.ref) }
1100
+ }
1101
+
1102
+ /**
1103
+ * A date-like field ignores human typing: Chrome's date input wants its own
1104
+ * segments, so keystrokes leave the value empty while the call reports success.
1105
+ * Set it the way a form would, and say the value the page now holds.
1106
+ * @param {any} page - sidecar page.
1107
+ * @param {string} text - what the caller wanted in it.
1108
+ * @returns {Promise<string>} a clause about the field, or '' when it is not one.
1109
+ */
1110
+ /* The input kinds a person cannot type into: Chrome gives each of them its own
1111
+ segment editor, and keys typed at it leave the value empty. */
1112
+ const DATE_KINDS = ['date', 'time', 'month', 'week', 'datetime-local']
1113
+
1114
+ /**
1115
+ * The date-like kind the focus is on, if it is on one.
1116
+ * @param {any} page - sidecar page.
1117
+ * @returns {Promise<string>} date, time… or '' when it is an ordinary field.
1118
+ */
1119
+ async function dateLikeFocus(page) {
1120
+ const said = await evaluate(page, `(() => {
1121
+ const el = document.activeElement
1122
+ if (!el || String(el.tagName).toUpperCase() !== 'INPUT') return ''
1123
+ const kind = String(el.type)
1124
+ if (!${JSON.stringify(DATE_KINDS)}.includes(kind)) return ''
1125
+ if (String(el.value) !== '') return ''
1126
+ return JSON.stringify({ kind })
1127
+ })()`).catch(() => undefined)
1128
+ return String(asSpot(said)?.kind ?? '')
1129
+ }
1130
+
1131
+ async function fillDateLike(page, text) {
1132
+ const kind = await dateLikeFocus(page)
1133
+ if (kind === '') return ''
1134
+ const empty = { kind }
1135
+ const want = String(text).trim()
1136
+ const after = await evaluate(page, `(() => {
1137
+ const el = document.activeElement
1138
+ const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set
1139
+ setter.call(el, ${JSON.stringify(want)})
1140
+ el.dispatchEvent(new Event('input', { bubbles: true }))
1141
+ el.dispatchEvent(new Event('change', { bubbles: true }))
1142
+ return String(el.value)
1143
+ })()`).catch(() => undefined)
1144
+ if (typeof after !== 'string' || after === '') {
1145
+ return `that ${empty.kind} field would not take "${want}" — it wants ${empty.kind === 'time' ? '09:30' : '2026-03-04'}`
1146
+ }
1147
+ return `set ${after} in the ${empty.kind} field`
1148
+ }
1149
+
1150
+ /**
1151
+ * Put text in the focused box the cheapest way that still works there.
1152
+ * A box that reacts while you type (suggestion lists, live search) needs the
1153
+ * keys in order; anything else takes one paste event, and the box itself is
1154
+ * the judge of whether it took it.
1155
+ * @param {any} page - sidecar page.
1156
+ * @param {string} text - what to type.
1157
+ * @param {string} [hint] - the field's name, to spot a live-search box.
1158
+ * @returns {Promise<void>} resolves when the box holds the text.
1159
+ */
1160
+ export async function typeInto(page, text, hint = '') {
1161
+ const words = String(text)
1162
+ if (words === '') return
1163
+ const live = /search|query|lookup|filter|搜索|筛选|查询/i.test(hint)
1164
+ if (!live && (await page.quickType(words)) === true) {
1165
+ await wait(30)
1166
+ return
1167
+ }
1168
+ await page.humanType(words)
472
1169
  }
473
1170
 
474
1171
  /**
@@ -479,13 +1176,31 @@ async function fieldByName(page, words) {
479
1176
  * @param {{clear?: boolean, submit?: boolean}} options - how.
480
1177
  */
481
1178
  async function writeAt(page, point, text, options = {}) {
1179
+ /* Approach the control the way a hand does: a couple of small moves that
1180
+ land in different places, not a teleport followed by a click. */
1181
+ const drift = (by) => ({ x: point.x + by, y: point.y + Math.round(by / 2) })
1182
+ /* The shape of the approach is what a page can see; the gaps between its
1183
+ points only buy latency. Three moves stay, the pauses are frames. */
1184
+ await page.mouse('mouseMoved', drift(-14 - Math.round(Math.random() * 10)))
1185
+ await wait(5 + Math.round(Math.random() * 8))
1186
+ await page.mouse('mouseMoved', drift(-4 - Math.round(Math.random() * 4)))
1187
+ await wait(4 + Math.round(Math.random() * 6))
482
1188
  await page.mouse('mouseMoved', point)
1189
+ await wait(5 + Math.round(Math.random() * 8))
483
1190
  await page.mouse('mousePressed', { ...point, clickCount: 1 })
1191
+ await wait(5 + Math.round(Math.random() * 9))
484
1192
  await page.mouse('mouseReleased', { ...point, clickCount: 1 })
485
- await wait(80)
1193
+ await wait(14 + Math.round(Math.random() * 18))
486
1194
  if (options.clear === true) await clearFocus(page)
487
- await page.insertText(text)
1195
+ /* A date or time input never takes keystrokes: noticing that before typing
1196
+ spends one round trip instead of fourteen keys and a failed readback. */
1197
+ if (await dateLikeFocus(page) !== '') return await fillDateLike(page, text)
1198
+ if (options.human === false) await page.insertText(text)
1199
+ else await typeInto(page, text, String(options.hint ?? ''))
1200
+ const fixed = await fillDateLike(page, text)
1201
+ if (fixed !== '') return fixed
488
1202
  if (options.submit === true) await pressKey(page, 'Enter')
1203
+ return ''
489
1204
  }
490
1205
 
491
1206
  /**
@@ -498,8 +1213,11 @@ async function writeAt(page, point, text, options = {}) {
498
1213
  */
499
1214
  async function followRef(page, ref, known) {
500
1215
  const spot = await locate(page, ref)
501
- if (spot.gone !== true) return spot
502
1216
  const was = Array.isArray(known) ? known.find((control) => Number(control.ref) === Number(ref)) : undefined
1217
+ if (spot.gone !== true) {
1218
+ const drift = differs(was, spot)
1219
+ return drift === '' ? spot : { gone: true, why: drift }
1220
+ }
503
1221
  if (was === undefined) return spot
504
1222
  const want = identityOf(was)
505
1223
  if (want.name === '') return spot
@@ -512,12 +1230,40 @@ async function followRef(page, ref, known) {
512
1230
  return { ...next, followed: true, was: want.name }
513
1231
  }
514
1232
 
1233
+ /**
1234
+ * One read state per chat, created the first time that chat reads.
1235
+ * @param {Map<string, any>} store - per-chat states.
1236
+ * @param {string} id - chat id.
1237
+ * @param {any} fallback - state for calls that name no chat.
1238
+ * @returns {any} the state of that chat.
1239
+ */
1240
+ export function readStateFor(store, id, fallback) {
1241
+ const key = String(id ?? '')
1242
+ const known = store.get(key)
1243
+ if (known !== undefined) return known
1244
+ /* The one object the plugin was started with belongs to calls that name no
1245
+ chat, so a chat that does name itself gets an empty one of its own. */
1246
+ const fresh = key === '' ? fallback : { refs: [], at: 0 }
1247
+ store.set(key, fresh)
1248
+ return fresh
1249
+ }
1250
+
1251
+ /**
1252
+ * A step that could not be carried out. The batch stops here: the next step was
1253
+ * written against a page that never arrived.
1254
+ * @param {string} text - what happened.
1255
+ * @returns {{text: string, blocked: boolean}} the step result.
1256
+ */
1257
+ function stuck(text) {
1258
+ return { text, blocked: true }
1259
+ }
1260
+
515
1261
  async function runStep(page, step, known) {
516
1262
  if (step.ref !== undefined) {
517
1263
  if (step.tap !== undefined || step.text !== undefined) {
518
1264
  const lead = await coverLead(page, Number(step.ref))
519
1265
  const spot = await followRef(page, Number(step.ref), known)
520
- if (spot.gone === true) return `ref ${String(step.ref)}: ${REF_GONE}`
1266
+ if (spot.gone === true) return stuck(`ref ${String(step.ref)}: ${REF_GONE}`)
521
1267
  const after = spot.followed === true ? ` (it was redrawn under a new number, used where it is now)` : ''
522
1268
  const point = { x: Number(spot.x), y: Number(spot.y) }
523
1269
  await page.mouse('mouseMoved', point)
@@ -526,17 +1272,17 @@ async function runStep(page, step, known) {
526
1272
  if (step.tap !== undefined) return `${lead}clicked ${String(step.ref)} ${String(spot.name ?? '')}`.trim() + after
527
1273
  await wait(80)
528
1274
  if (step.clear === true) await clearFocus(page)
529
- await page.insertText(String(step.text))
1275
+ await page.humanType(String(step.text))
530
1276
  if (step.submit === true) await pressKey(page, 'Enter')
531
1277
  return `${lead}typed into ${String(step.ref)} ${String(spot.name ?? '')}`.trim() + after
532
1278
  }
533
1279
  }
534
1280
  if (step.tap !== undefined && !/^\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*$/.test(String(step.tap))) {
535
- const hit = await findWords(page, String(step.tap))
536
- if (hit === null) return `no control says ${JSON.stringify(String(step.tap))}`
1281
+ const hit = await findWords(page, String(step.tap), known)
1282
+ if (hit === null) return stuck(`no control says ${JSON.stringify(String(step.tap))}`)
537
1283
  const lead = await coverLead(page, hit.ref)
538
1284
  const spot = await locate(page, hit.ref)
539
- if (spot.gone === true) return `ref ${String(hit.ref)}: ${REF_GONE}`
1285
+ if (spot.gone === true) return stuck(`ref ${String(hit.ref)}: ${REF_GONE}`)
540
1286
  const point = { x: Number(spot.x), y: Number(spot.y) }
541
1287
  await page.mouse('mouseMoved', point)
542
1288
  await page.mouse('mousePressed', { ...point, clickCount: 1 })
@@ -546,9 +1292,10 @@ async function runStep(page, step, known) {
546
1292
  if (step.field !== undefined) {
547
1293
  /* The words may sit on the field or in a div above a bare input. Only a
548
1294
  click that lands on the field itself can be trusted with the typing. */
549
- const target = await fieldByName(page, String(step.field))
550
- if (target === null) return `${String(step.field)}: ${REF_GONE}`
551
- await writeAt(page, { x: target.x, y: target.y }, String(step.text ?? ''), { clear: step.clear, submit: step.submit })
1295
+ const target = await fieldByName(page, String(step.field), known)
1296
+ if (target === null) return stuck(`${String(step.field)}: ${REF_GONE}`)
1297
+ const dated = await writeAt(page, { x: target.x, y: target.y }, String(step.text ?? ''), { clear: step.clear, submit: step.submit })
1298
+ if (dated !== '') return dated
552
1299
  return `${target.lead}typed into "${target.name.slice(0, 60)}"`
553
1300
  }
554
1301
  if (step.tap !== undefined) {
@@ -561,7 +1308,7 @@ async function runStep(page, step, known) {
561
1308
  }
562
1309
  if (step.text !== undefined) {
563
1310
  if (step.clear === true) await clearFocus(page)
564
- await page.insertText(String(step.text))
1311
+ await page.humanType(String(step.text))
565
1312
  if (step.submit === true) await pressKey(page, 'Enter')
566
1313
  return `typed ${String(step.text).slice(0, 60)}`
567
1314
  }
@@ -577,17 +1324,17 @@ async function runStep(page, step, known) {
577
1324
  if (step.wait_text !== undefined) {
578
1325
  const needle = JSON.stringify(String(step.wait_text))
579
1326
  const ok = await until(page, `document.body ? document.body.innerText.includes(${needle}) : false`, Number(step.timeout ?? 8000))
580
- return ok ? `saw ${String(step.wait_text)}` : `did not see ${String(step.wait_text)}`
1327
+ return ok ? `saw ${String(step.wait_text)}` : stuck(`did not see ${String(step.wait_text)}`)
581
1328
  }
582
1329
  if (step.wait_gone !== undefined) {
583
1330
  const needle = JSON.stringify(String(step.wait_gone))
584
1331
  const ok = await until(page, `document.body ? !document.body.innerText.includes(${needle}) : true`, Number(step.timeout ?? 8000))
585
- return ok ? `${String(step.wait_gone)} went away` : `${String(step.wait_gone)} stayed`
1332
+ return ok ? `${String(step.wait_gone)} went away` : stuck(`${String(step.wait_gone)} stayed`)
586
1333
  }
587
1334
  if (step.wait_url !== undefined) {
588
1335
  const needle = JSON.stringify(String(step.wait_url))
589
1336
  const ok = await until(page, `location.href.includes(${needle})`, Number(step.timeout ?? 8000))
590
- return ok ? `at ${String(step.wait_url)}` : `still at a different address`
1337
+ return ok ? `at ${String(step.wait_url)}` : stuck(`still at a different address`)
591
1338
  }
592
1339
  if (step.open !== undefined) {
593
1340
  const landed = await page.navigate(String(step.open))
@@ -617,13 +1364,85 @@ async function runStep(page, step, known) {
617
1364
  * @param {{mode?: string, maxChars?: number}} options - narrowing.
618
1365
  * @returns {Promise<any>} the read.
619
1366
  */
1367
+ /** A read with nothing in it often means the document is still being built:
1368
+ * samples of real pages showed zero controls when the read landed before
1369
+ * hydration, and the model then calls the page broken. */
1370
+ function looksEmpty(read) {
1371
+ return (read?.controls?.length ?? 0) === 0 && String(read?.text ?? '').trim().length < 40
1372
+ }
1373
+
1374
+ /** @param {string} url - what the page claims to be. @returns {boolean} true when waiting can still help. */
1375
+ function canFill(url) {
1376
+ const said = String(url ?? '')
1377
+ return said.startsWith('http://') || said.startsWith('https://')
1378
+ }
1379
+
620
1380
  async function remember(page, refs, options) {
621
- const read = await readPage(page, options)
1381
+ let read = await readPage(page, options)
1382
+ if (looksEmpty(read) && canFill(read.url)) {
1383
+ /* Hydration and client-side rendering finish a few hundred milliseconds
1384
+ after the address does. Read again until the page has something, but
1385
+ never longer than a person would wait for a page that stays empty. */
1386
+ const deadline = Date.now() + 1500
1387
+ while (looksEmpty(read) && canFill(read.url) && Date.now() < deadline) {
1388
+ await wait(120)
1389
+ const again = await readPage(page, options)
1390
+ if (!looksEmpty(again)) {
1391
+ read = again
1392
+ break
1393
+ }
1394
+ read = again
1395
+ }
1396
+ }
622
1397
  refs.refs = read.controls
623
1398
  refs.at = Date.now()
624
1399
  return read
625
1400
  }
626
1401
 
1402
+ /**
1403
+ * Two reads of the same page in the same shape say the same thing. Saying it
1404
+ * again costs tokens and teaches nothing, so the second read can be one line.
1405
+ * @param {any} before - the earlier read.
1406
+ * @param {any} after - the read now.
1407
+ * @returns {boolean} true when nothing a read would show has changed.
1408
+ */
1409
+ export function sameRead(before, after) {
1410
+ if (before === undefined || before === null || after === undefined || after === null) return false
1411
+ if (before.url !== after.url || before.title !== after.title) return false
1412
+ if (before.text !== after.text) return false
1413
+ if (JSON.stringify(before.controls) !== JSON.stringify(after.controls)) return false
1414
+ if ((before.focused?.ref ?? null) !== (after.focused?.ref ?? null)) return false
1415
+ return true
1416
+ }
1417
+
1418
+ /**
1419
+ * Where two readings of a page differ, said in one line. A counter moving, one
1420
+ * row appearing, a word edited: the page is mostly the same, and sending all of
1421
+ * it again is what makes a long session expensive.
1422
+ * @param {string} before - the text earlier.
1423
+ * @param {string} after - the text now.
1424
+ * @returns {string} a short line, or '' when the two have nothing in common.
1425
+ */
1426
+ export function textChange(before, after) {
1427
+ const was = String(before ?? '')
1428
+ const now = String(after ?? '')
1429
+ let head = 0
1430
+ const shortest = Math.min(was.length, now.length)
1431
+ while (head < shortest && was[head] === now[head]) head += 1
1432
+ let tail = 0
1433
+ while (tail < shortest - head && was[was.length - 1 - tail] === now[now.length - 1 - tail]) tail += 1
1434
+ const gone = was.slice(head, was.length - tail)
1435
+ const came = now.slice(head, now.length - tail)
1436
+ /* A page that changed in one place is worth a line; one that shares almost no
1437
+ text is a different page, and a diff of it would be as long as the page. */
1438
+ if (head + tail < 12) return ''
1439
+ const clip = (text) => (text.length > 160 ? `${text.slice(0, 157)}…` : text)
1440
+ const near = clip(was.slice(Math.max(0, head - 24), head))
1441
+ const far = clip(now.slice(now.length - tail, now.length - tail + 24))
1442
+ const line = `${near === '' ? '' : `…${near} `}${gone === '' ? '' : `[${clip(gone)}] `}→ ${came === '' ? '' : `[${clip(came)}] `}${far === '' ? '' : `${far}…`}`
1443
+ return line.trim()
1444
+ }
1445
+
627
1446
  /**
628
1447
  * Say only what changed since the previous read.
629
1448
  * @param {any} previous - the read before.
@@ -635,8 +1454,7 @@ export function diffRead(previous, next, options = {}) {
635
1454
  const maxChars = Number(options.maxChars ?? 2000)
636
1455
  const before = new Map(previous.controls.map((control) => [control.ref, control]))
637
1456
  const after = new Map(next.controls.map((control) => [control.ref, control]))
638
- const lines = [next.url]
639
- if (previous.url !== next.url) lines.push(`address changed to ${next.url}`)
1457
+ const lines = [previous.url === next.url ? next.url : `address ${next.url}`]
640
1458
  const added = next.controls.filter((control) => !before.has(control.ref))
641
1459
  const removed = previous.controls.filter((control) => !after.has(control.ref))
642
1460
  const changed = next.controls.filter((control) => {
@@ -652,9 +1470,14 @@ export function diffRead(previous, next, options = {}) {
652
1470
  }
653
1471
  if (added.length + removed.length + changed.length === 0) lines.push('no control changed')
654
1472
  if (previous.text !== next.text) {
655
- const head = next.text.slice(0, Math.max(200, Math.min(1200, maxChars - 800)))
656
- lines.push('', 'text now:', head)
657
- if (next.text.length > head.length) lines.push(`… ${next.text.length - head.length} more characters`)
1473
+ const said = textChange(previous.text, next.text)
1474
+ const room = Math.max(200, Math.min(1200, maxChars - 200))
1475
+ if (said !== '') lines.push(`text: ${said}`)
1476
+ else {
1477
+ const head = next.text.slice(0, room)
1478
+ lines.push('', 'text now:', head)
1479
+ if (next.text.length > head.length) lines.push(`… ${next.text.length - head.length} more characters`)
1480
+ }
658
1481
  }
659
1482
  if (next.focused !== null && previous.focused?.ref !== next.focused.ref) lines.push(`focus ${next.focused.ref} ${next.focused.name}`)
660
1483
  return lines.join('\n')
@@ -709,7 +1532,7 @@ export async function extractRows(page, args) {
709
1532
  const out = await page.cdp.send('Runtime.evaluate', { expression, returnByValue: true }).catch((error) => ({ error: String(error?.message ?? error) }))
710
1533
  if (out?.error !== undefined && out.error !== null && typeof out.error === 'string') return `the page did not answer: ${out.error}`
711
1534
  const value = out?.result?.value
712
- if (value === undefined || value === null || value === '') return 'the page did not answer; try again, or read the page with sidecar_view.'
1535
+ if (value === undefined || value === null || value === '') return 'the page did not answer; try again, or read the page with browser_view.'
713
1536
  if (value.gone === true) return `ref ${ref} is gone. Read the page again and use a new number.`
714
1537
  if (value.bad === true) return `${select} is not a selector this browser understands.`
715
1538
  const rows = Array.isArray(value.rows) ? value.rows : []
@@ -719,7 +1542,140 @@ export async function extractRows(page, args) {
719
1542
  return `${rows.length} of ${Number(value.total)} rows matching ${select}:\n${said.join('\n')}${more}`
720
1543
  }
721
1544
 
1545
+ /**
1546
+ * Say one thing to a box in the page and take the answer back. The wait ends
1547
+ * when the answer has stopped changing, so a streaming reply is not cut off and
1548
+ * a page that never answers does not hold the call open forever.
1549
+ * @param {any} page - sidecar page.
1550
+ * @param {{text?: string, ref?: number, submit?: boolean, timeout_ms?: number}} args - what to say.
1551
+ * @returns {Promise<string>} the answer, or why there is none.
1552
+ */
1553
+ /**
1554
+ * Whether a box on the page still holds the words that were typed.
1555
+ * A box that keeps them never sent them, which is what a signed-out page does.
1556
+ * @param {any} page - sidecar page.
1557
+ * @param {string} words - what was typed.
1558
+ * @returns {Promise<boolean>} true when the words are still in a box.
1559
+ */
1560
+ async function boxHolds(page, words) {
1561
+ const wanted = String(words).slice(0, 24)
1562
+ if (wanted === '') return false
1563
+ const held = await evaluate(page, `(() => {
1564
+ const wanted = ${JSON.stringify(wanted)}
1565
+ for (const el of document.querySelectorAll('textarea, [contenteditable=""], [contenteditable="true"], input[type="text"], input[type="search"], input:not([type])')) {
1566
+ if (String(el.value ?? el.innerText ?? '').trim().includes(wanted)) return true
1567
+ }
1568
+ return false
1569
+ })()`)
1570
+ return held === true
1571
+ }
1572
+
1573
+ /**
1574
+ * Sends a line to whatever chat the page holds and waits for the answer to stop
1575
+ * growing. The ref is followed the same way a click follows it, so a reply box
1576
+ * that was redrawn since the read still gets the text.
1577
+ * @param {any} page - sidecar page.
1578
+ * @param {any} args - text, optional ref, optional timeout.
1579
+ * @returns {Promise<string>} the answer.
1580
+ */
1581
+ export async function askInPage(page, args) {
1582
+ const said = String(args?.text ?? '').trim()
1583
+ if (said === '') return 'Give me something to say.'
1584
+ const budget = Math.max(3000, Math.min(180_000, Math.trunc(Number(args?.timeout_ms ?? 60_000)) || 60_000))
1585
+ const before = await readPage(page, { mode: 'text', maxChars: 4000 })
1586
+ let point = null
1587
+ if (args?.ref !== undefined) {
1588
+ const spot = await followRef(page, Number(args.ref))
1589
+ if (spot.gone === true) return spot.why ?? REF_GONE
1590
+ point = { x: Number(spot.x), y: Number(spot.y) }
1591
+ } else {
1592
+ const at = await evaluate(page, `(() => {
1593
+ let best = null
1594
+ for (const el of document.querySelectorAll('textarea, [contenteditable=""], [contenteditable="true"], input[type="text"], input[type="search"], input:not([type])')) {
1595
+ const b = el.getBoundingClientRect()
1596
+ const s = getComputedStyle(el)
1597
+ if (s.visibility === 'hidden' || s.display === 'none' || b.width < 80 || b.height < 18) continue
1598
+ if (b.bottom < 0 || b.top > innerHeight) continue
1599
+ if (best === null || b.width * b.height > best.area) best = { area: b.width * b.height, x: b.x + Math.min(24, b.width / 2), y: b.y + b.height / 2, name: String(el.getAttribute('placeholder') ?? el.getAttribute('aria-label') ?? '').slice(0, 40) }
1600
+ }
1601
+ return best === null ? null : { x: Math.round(best.x), y: Math.round(best.y), name: best.name }
1602
+ })()`)
1603
+ if (at === null || at === undefined) return 'No box to type in on this page. Call browser_view to see what is there.'
1604
+ point = { x: Number(at.x), y: Number(at.y) }
1605
+ }
1606
+ await page.mouse('mouseMoved', point)
1607
+ await page.mouse('mousePressed', { ...point, clickCount: 1 })
1608
+ await page.mouse('mouseReleased', { ...point, clickCount: 1 })
1609
+ await wait(60)
1610
+ await page.humanType(said)
1611
+ if (args?.submit !== false) await pressKey(page, 'Enter')
1612
+ await wait(500)
1613
+ if (args?.submit !== false && (await boxHolds(page, said))) {
1614
+ return `the words are still in the box 0.5s after pressing Enter: the page did not take the message. If it is waiting for a sign-in, sign in in the column, then ask again.`
1615
+ }
1616
+ const started = Date.now()
1617
+ const deadline = started + budget
1618
+ let last = before.text
1619
+ let fresh = ''
1620
+ let steady = 0
1621
+ let quiet = 0
1622
+ while (Date.now() < deadline) {
1623
+ await wait(500)
1624
+ const now = await readPage(page, { mode: 'text', maxChars: 4000 })
1625
+ if (now.text !== last) {
1626
+ last = now.text
1627
+ fresh = now.text
1628
+ steady = 0
1629
+ quiet = 0
1630
+ continue
1631
+ }
1632
+ if (fresh !== '') {
1633
+ steady += 1
1634
+ if (steady >= 3) break
1635
+ } else {
1636
+ quiet += 1
1637
+ if (quiet >= 4 && Date.now() - started > 4000) break
1638
+ }
1639
+ }
1640
+ const took = Date.now() - started
1641
+ /* A page that shows neither the words in a box nor the message itself never
1642
+ took it: the chips moved, the menu redrew, and none of that is an answer. */
1643
+ const echo = String(said.slice(0, 24))
1644
+ if (fresh !== '' && echo.length >= 8 && !fresh.includes(echo)) {
1645
+ const asking = /(sign in|log in|登录|注册)/i.test(fresh)
1646
+ return `the page changed but does not show the message: it did not take what was sent.${asking ? ' It is asking for a sign-in — sign in in the column, then ask again.' : ' Try browser_view to see what is in the way.'}`
1647
+ }
1648
+ /* A box that still holds the words is the page saying it did not take them:
1649
+ a signed-out page leaves them there, and the old wording sent the reader
1650
+ looking for an answer that was never asked for. */
1651
+ const stuck = fresh === '' && (await boxHolds(page, said))
1652
+ if (fresh === '') {
1653
+ if (stuck) return `sent it — no answer after ${Math.round(took / 100) / 10}s, and the words are still in the box: the page did not take the message. If it is waiting for a sign-in, sign in in the column, then ask again.`
1654
+ return `sent it — no answer after ${Math.round(took / 100) / 10}s; the page still says: ${before.text.slice(0, 400)}`
1655
+ }
1656
+ /* The answer may replace the words that were there, so cut at the last
1657
+ character the two reads share, not at the old length. */
1658
+ let shared = 0
1659
+ while (shared < fresh.length && shared < before.text.length && fresh[shared] === before.text[shared]) shared += 1
1660
+ const added = fresh.slice(shared)
1661
+ /* Lines are kept: a reviewer's list is one point per line, and flattening it
1662
+ would leave the next round with a paragraph instead of items. */
1663
+ const words = added.replace(/[ \t]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim().slice(0, 2500)
1664
+ return `the answer came back in ${Math.round(took / 100) / 10}s:\n${words}`
1665
+ }
1666
+
722
1667
  export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1668
+ /* A ref belongs to the page it was read from, and two chats read two pages.
1669
+ One shared read state meant chat B's read replaced chat A's baseline, so A's
1670
+ next "what changed" was a diff against a page it never saw and a number from
1671
+ A could be matched against B's control. Each chat now keeps its own. */
1672
+ const byChat = new Map()
1673
+ /**
1674
+ * The read state of the chat this call speaks for.
1675
+ * @param {string} [id] - chat id; the current one by default.
1676
+ * @returns {{refs: any[], at: number, last?: any, lastShape?: string}} its state.
1677
+ */
1678
+ const refsFor = (id = currentSession()) => readStateFor(byChat, id, refs)
723
1679
  /* Reading, clicking and typing on Chrome's own error page is not driving the
724
1680
  site: the text there is Chrome talking, the buttons reload. Those calls are
725
1681
  refused with the error code instead. The tools that report the browser
@@ -727,18 +1683,46 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
727
1683
  run, because they are how a stuck person finds out what happened. */
728
1684
  const withRawPage = async (work) => {
729
1685
  if (!sidecar.alive) return { text: SIDECAR_OFF }
730
- const page = await sidecar.ensure()
1686
+ const page = await sidecar.ensure({ session: currentSession() })
731
1687
  const text = await work(page)
732
1688
  return { text: text + personNote(sidecar) }
733
1689
  }
734
1690
  /** Looking is still useful while the person has the page. */
735
1691
  const withRead = async (work) => {
736
- if (!sidecar.alive) return { text: SIDECAR_OFF }
737
- const page = await sidecar.ensure()
1692
+ /* A call starting means the last click is over: a tab that opens now is not
1693
+ going to be followed under this call's feet. */
1694
+ try {
1695
+ sidecar.page?.disarmFollow?.()
1696
+ } catch {
1697
+ /* the page is already gone */
1698
+ }
1699
+ /* A plugin reload makes a new driver while the browser it started keeps
1700
+ running. Refusing every read until someone opens a page again would throw
1701
+ away a browser that is still there: attach to it instead. */
1702
+ if (!sidecar.alive) {
1703
+ /* A renderer crash can take the whole browser with it, and the first
1704
+ relaunch sometimes loses the race with the old process exiting. One
1705
+ more try turns that into a fresh tab instead of a refusal. */
1706
+ try {
1707
+ await sidecar.ensure({ session: currentSession() })
1708
+ } catch {
1709
+ await wait(300)
1710
+ try {
1711
+ await sidecar.ensure({ session: currentSession() })
1712
+ } catch {
1713
+ return { text: SIDECAR_OFF }
1714
+ }
1715
+ }
1716
+ }
1717
+ const page = await sidecar.ensure({ session: currentSession() })
738
1718
  const dead = await unreachable(page)
739
1719
  if (dead !== '') return { text: dead + personNote(sidecar) }
740
1720
  const text = await work(page)
741
- return { text: text + personNote(sidecar) }
1721
+ /* Why a page stopped working is usually in its own log, and looking it up
1722
+ costs another call. This is the one place every read passes through, so
1723
+ say what went wrong since the previous one, once. */
1724
+ const trouble = typeof page.troubleOf === 'function' ? page.troubleOf() : ''
1725
+ return { text: text + (trouble === '' ? '' : `\n${trouble}`) + personNote(sidecar) }
742
1726
  }
743
1727
  const withPage = async (work) => {
744
1728
  if (!sidecar.alive) return { text: SIDECAR_OFF }
@@ -747,7 +1731,9 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
747
1731
  return withRead(work)
748
1732
  }
749
1733
  const look = async (page, options) => {
750
- const read = await remember(page, refs, options)
1734
+ const read = await remember(page, refsFor(), options)
1735
+ /* A read already showed the address: the next action does not repeat it. */
1736
+ if (String(read.url ?? '') !== '') saidUrls.set(page, String(read.url))
751
1737
  const said = renderPage(read, options)
752
1738
  return isCheckPage(read) ? `${said}\n${CHECK_NOTE}` : said
753
1739
  }
@@ -768,8 +1754,11 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
768
1754
  */
769
1755
  const findAgain = async (page, ref) => {
770
1756
  const spot = await locate(page, ref)
771
- if (spot.gone !== true) return spot
772
- const known = (refs?.refs ?? []).find((control) => Number(control.ref) === Number(ref))
1757
+ const known = (refsFor().refs ?? []).find((control) => Number(control.ref) === Number(ref))
1758
+ if (spot.gone !== true) {
1759
+ const drift = differs(known, spot)
1760
+ return drift === '' ? spot : { gone: true, why: drift }
1761
+ }
773
1762
  if (known === undefined) return spot
774
1763
  const want = identityOf(known)
775
1764
  if (want.name === '') return spot
@@ -786,13 +1775,17 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
786
1775
 
787
1776
  return [
788
1777
  {
789
- name: 'sidecar_open', // ui-copy-ignore
790
- description: 'Start (or reuse) the sidecar browser — a real Chrome with its own persistent profile — and open a URL in it. It runs in the background and its screen appears in this chat\'s sidebar tab, so it keeps working while the person looks elsewhere. Logins survive restarts.',
1778
+ name: 'browser_open', // ui-copy-ignore
1779
+ description: 'Start (or reuse) the background browser — a real Chrome with its own persistent profile — and open a URL in it. It is not the host\'s own browser tab: this Chrome runs on its own and only its picture is mirrored into this chat\'s right column, so it keeps working while the person looks elsewhere. Logins survive restarts.',
791
1780
  parameters: {
792
1781
  type: 'object', // ui-copy-ignore
793
1782
  properties: {
794
1783
  url: { type: 'string', description: 'Address to open. Omit to reuse the current page.' },
795
1784
  headless: { type: 'boolean', description: 'Run without a window. Default true; turn off only if a site refuses a windowless browser.' },
1785
+ flash: {
1786
+ type: 'boolean',
1787
+ description: 'Play the page in the Flash-era browser instead of Chrome — for a site whose game is a Flash movie that never got an HTML5 version. Default off; switching either way restarts the browser.',
1788
+ },
796
1789
  },
797
1790
  additionalProperties: false,
798
1791
  },
@@ -801,33 +1794,270 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
801
1794
  isConcurrencySafe: () => false,
802
1795
  /** @param {any} args - arguments. @param {any} exec - tool run. @returns {Promise<{text: string}>} what the page is. */
803
1796
  execute: async (args, exec) => {
804
- const page = await sidecar.ensure({ url: args?.url, headless: args?.headless })
1797
+ const who = String(exec?.agent?.id ?? '')
1798
+ const page = await sidecar.ensure({ url: args?.url, headless: args?.headless, flash: args?.flash, session: who })
805
1799
  if (args?.url === undefined) await page.readTitle()
806
1800
  /* A window on the person's screen is their business, not a surprise: say
807
1801
  it, so nobody wonders what popped up. A site that refuses a windowless
808
1802
  browser is the only reason this happens. */
809
- const windowed = sidecar.headless === false ? 'The browser is showing a window this time — a site only opens for a windowed one. Same profile, same logins.\n' : ''
1803
+ /* The Flash runtime is windowless even when headless is false: its window
1804
+ is never shown, so nothing popped up to explain. */
1805
+ const windowed = sidecar.headless === false && sidecar.flash !== true ? 'The browser is showing a window this time — a site only opens for a windowed one. Same profile, same logins.\n' : ''
810
1806
  /* The tab appears in the calling chat and nowhere else: the person may
811
1807
  be reading a different chat, and a browser there would be a surprise. */
812
- const who = String(exec?.agent?.id ?? '')
813
1808
  if (who !== '') host.want?.(who) // ui-copy-ignore
814
1809
  await settle(page, 8000)
815
1810
  const dead = await unreachable(page)
816
1811
  if (dead !== '') return { text: dead }
817
1812
  /* The column may still be drawing nothing: say so here, where the
818
- person is looking for a picture, rather than only in sidecar_state. */
1813
+ person is looking for a picture, rather than only in browser_state. */
819
1814
  const unseen = who === '' ? '' : viewerNote(await watchForViewer(sidecar, page, 1200))
820
- return { text: windowed + (unseen === '' ? '' : `${unseen}\n`) + await look(page, { mode: 'text' }) }
1815
+ return { text: windowed + (unseen === '' ? '' : `${unseen}\n`) + await look(page, { mode: 'text' }) + (await pdfNote(page)) }
1816
+ },
1817
+ },
1818
+ {
1819
+ name: 'browser_mode', // ui-copy-ignore
1820
+ description: 'Choose how this chat acts from now on. "human" moves the real mouse and presses real keys, which no page can tell from a person — games and pages that watch pointer movement need this. "dom" clicks and fills the controls the page registered, which reaches one that an overlay covers and leaves key handlers alone. A call can also pass its own mode, which wins for that call only.',
1821
+ parameters: {
1822
+ type: 'object', // ui-copy-ignore
1823
+ properties: {
1824
+ mode: { type: 'string', enum: ['human', 'dom'], description: 'Which way to act.' },
1825
+ },
1826
+ required: ['mode'], // ui-copy-ignore
1827
+ additionalProperties: false,
1828
+ },
1829
+ output: TEXT_OUTPUT, // ui-copy-ignore
1830
+ timeoutMs: 10_000,
1831
+ isConcurrencySafe: () => true,
1832
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} which way is set. */
1833
+ execute: async (args) => {
1834
+ const said = String(args?.mode ?? '')
1835
+ if (!WAYS.has(said)) return 'Pass "human" or "dom".'
1836
+ const chat = currentSession()
1837
+ if (said === 'human') chatWays.delete(chat)
1838
+ else chatWays.set(chat, said)
1839
+ return said === 'human'
1840
+ ? 'Real mouse and real keys from now on: pages cannot tell it from a person.'
1841
+ : "The page's own controls from now on: clicks and typing go through its DOM, so an overlay cannot swallow them."
1842
+ },
1843
+ },
1844
+ {
1845
+ name: 'browser_prepare', // ui-copy-ignore
1846
+ description: 'Get a game or a heavy app page ready to play, in one call: read what is on it, find the surface that takes input, say whether it needs a Flash player this browser does not have, measure its real frame rate, and test whether keys reach it. Ends with what to do next instead of a blank page and no explanation.',
1847
+ parameters: {
1848
+ type: 'object', // ui-copy-ignore
1849
+ properties: {
1850
+ seconds: { type: 'number', description: 'How long to count frames for. Default 1.2, at most 5.' },
1851
+ focus: { type: 'boolean', description: 'Click the surface to put keyboard focus in it. Default true.' },
1852
+ },
1853
+ additionalProperties: false,
1854
+ },
1855
+ output: TEXT_OUTPUT, // ui-copy-ignore
1856
+ timeoutMs: 45_000,
1857
+ isConcurrencySafe: () => false,
1858
+ /** @param {any} args - arguments. @param {any} exec - tool run. @returns {Promise<{text: string}>} what the page is ready for. */
1859
+ execute: async (args, exec) => {
1860
+ const who = String(exec?.agent?.id ?? '')
1861
+ const page = await sidecar.ensure({ session: who })
1862
+ await settle(page, 8000)
1863
+ const seconds = Math.min(5, Math.max(0.5, Number(args?.seconds) || 1.2))
1864
+ const ms = Math.round(seconds * 1000)
1865
+ /* A page that never arrived is not a game that is not ready: say which. */
1866
+ const dead = await unreachable(page)
1867
+ if (dead !== '') return { text: dead }
1868
+ const read = await readPage(page, { mode: 'text', maxChars: 1500 })
1869
+ const said = asSpot(await evaluate(page, `(() => {
1870
+ const box = (el) => { const r = el.getBoundingClientRect(); return { w: Math.round(r.width), h: Math.round(r.height), x: Math.round(r.x + r.width / 2), y: Math.round(r.y + r.height / 2) } }
1871
+ const shown = (el) => {
1872
+ const r = el.getBoundingClientRect()
1873
+ const st = getComputedStyle(el)
1874
+ return r.width >= 120 && r.height >= 80 && st.visibility !== 'hidden' && st.display !== 'none' && Number(st.opacity) > 0.05
1875
+ }
1876
+ const spot = (el, tag) => {
1877
+ const first = box(el)
1878
+ const seen = first.y > 0 && first.y < innerHeight && first.x > 0 && first.x < innerWidth
1879
+ if (!seen) {
1880
+ /* A surface below the fold is not where the person is looking, and
1881
+ a click at its coordinate would land somewhere else. */
1882
+ el.scrollIntoView({ block: 'center', inline: 'center' })
1883
+ return { tag, src: String(el.getAttribute('src') ?? ''), ...box(el), scrolled: true }
1884
+ }
1885
+ return { tag, src: String(el.getAttribute('src') ?? ''), ...first }
1886
+ }
1887
+ const canvases = Array.from(document.querySelectorAll('canvas')).filter(shown).map((el) => spot(el, 'canvas'))
1888
+ const frames = Array.from(document.querySelectorAll('iframe')).filter(shown).map((el) => spot(el, 'frame'))
1889
+ const movies = document.querySelectorAll('embed, object').length
1890
+ return JSON.stringify({ canvases, frames, movies })
1891
+ })()`))
1892
+ const surface = said?.canvases?.[0] ?? said?.frames?.[0] ?? null
1893
+ /* Give a game the click a person gives it: some of them only start
1894
+ listening for keys once the surface has been touched. */
1895
+ let clicked = ''
1896
+ if (args?.focus !== false && surface !== null) {
1897
+ await page.mouse('mouseMoved', { x: surface.x - 6, y: surface.y - 4 }).catch(() => {})
1898
+ await wait(40)
1899
+ await page.mouse('mousePressed', { x: surface.x, y: surface.y, clickCount: 1 }).catch(() => {})
1900
+ await wait(30)
1901
+ await page.mouse('mouseReleased', { x: surface.x, y: surface.y, clickCount: 1 }).catch(() => {})
1902
+ clicked = `clicked the ${surface.tag} at ${surface.x},${surface.y}`
1903
+ }
1904
+ /* Count the page's own frames and listen for a key, in the page and in
1905
+ the biggest frame, because a game is usually not in the page itself. */
1906
+ const frames = await page.frameList().catch(() => [])
1907
+ const routeOfFrame = (frame) => {
1908
+ if (frame === undefined) return null
1909
+ const sessionId = frame.contextId === undefined && typeof page.childSession === 'function' ? page.childSession(frame.frameId) : undefined
1910
+ if (frame.contextId === undefined && sessionId === undefined) return null
1911
+ return { contextId: frame.contextId, sessionId }
1912
+ }
1913
+ const countFrames = `new Promise((done) => {
1914
+ window.__dshKeys = 0
1915
+ if (window.__dshKeyUp === undefined) {
1916
+ window.__dshKeyUp = true
1917
+ addEventListener('keydown', () => { window.__dshKeys += 1 }, true)
1918
+ }
1919
+ let n = 0
1920
+ const t0 = performance.now()
1921
+ const tick = () => { n += 1; if (performance.now() - t0 < ${ms}) requestAnimationFrame(tick) }
1922
+ requestAnimationFrame(tick)
1923
+ setTimeout(() => done(JSON.stringify({ fps: Math.round((n * 1000) / Math.max(1, performance.now() - t0)), keys: window.__dshKeys })), ${ms + 60})
1924
+ })`
1925
+ const mainRate = asSpot(await evaluate(page, countFrames))
1926
+ let frameRate = null
1927
+ const childFrames = frames.slice(1).filter((frame) => String(frame.url ?? '') !== 'chrome-error://chromewebdata/')
1928
+ const biggest = childFrames[0]
1929
+ const route = routeOfFrame(biggest)
1930
+ if (route !== null) frameRate = asSpot(await evaluate(page, countFrames, route))
1931
+ let keys = route === null ? 'none arrived — there is no frame to test' : 'none arrived in the frame'
1932
+ const keyDown = { type: 'keyDown', key: 'Shift', code: 'ShiftLeft', windowsVirtualKeyCode: 16, nativeVirtualKeyCode: 16 }
1933
+ await page.cdp.send('Input.dispatchKeyEvent', keyDown, route?.sessionId).catch(() => {})
1934
+ await page.cdp.send('Input.dispatchKeyEvent', { ...keyDown, key: 'Shift', type: 'keyUp' }, route?.sessionId).catch(() => {})
1935
+ await wait(80)
1936
+ const after = asSpot(await evaluate(page, 'JSON.stringify({ keys: window.__dshKeys ?? 0 })', route))
1937
+ const mainAfter = asSpot(await evaluate(page, 'JSON.stringify({ keys: window.__dshKeys ?? 0 })'))
1938
+ const arrived = Number(after?.keys ?? 0) + Number(mainAfter?.keys ?? 0) - Number(frameRate?.keys ?? 0) - Number(mainRate?.keys ?? 0)
1939
+ if (arrived > 0) keys = `arrived in the frame (${arrived} keydown event${arrived === 1 ? '' : 's'} from a real key)`
1940
+ else if (route !== null) keys = 'not in the frame — click inside it once, or the game only listens after it starts'
1941
+ const lines = []
1942
+ lines.push(`page: ${read.title === '' ? read.url : read.title} — ${read.controls.length} control(s), ${read.textChars} character(s)`)
1943
+ lines.push(surface === null ? 'surface: no canvas or frame takes the screen — the page itself takes input' : `surface: ${surface.tag} ${surface.w}x${surface.h} at ${surface.x},${surface.y}${surface.scrolled === true ? ' (scrolled it into view)' : ''}${clicked === '' ? '' : ` (${clicked})`}`)
1944
+ if (read.blind?.flash > 0 && sidecar.flash !== true) lines.push(`flash: ${read.blind.flash} movie(s) — open the same page with flash:true and it runs in this plugin's own player, in the column, with no window on the desktop`) // ui-copy-ignore
1945
+ else if (read.blind?.flash > 0) lines.push(`flash: ${read.blind.flash} movie(s) running on this plugin's own player`)
1946
+ else lines.push('flash: none found')
1947
+ lines.push(`frames: ${Math.max(0, frames.length - 1)} (${childFrames.filter((frame) => String(frame.url ?? '') === 'about:blank').length} of them about:blank)`)
1948
+ const rates = []
1949
+ if (mainRate !== null) rates.push(`page ${mainRate.fps} fps`)
1950
+ if (frameRate !== null) rates.push(`frame ${frameRate.fps} fps`)
1951
+ lines.push(`frame rate: ${rates.length === 0 ? 'the page could not count' : rates.join(', ')} over ${ms} ms`)
1952
+ lines.push(`keys: ${keys}`)
1953
+ lines.push(`mode: human${surface === null ? '' : ' — a game watches the mouse and the keys, and only real ones arrive trusted'}`)
1954
+ const trouble = read.blind?.flash > 0 && sidecar.flash !== true
1955
+ ? 'call open with flash:true to run it here'
1956
+ : read.blind?.flash > 0
1957
+ ? 'the movie needs a click inside it before it starts'
1958
+ : surface === null && Number(mainRate?.fps ?? 0) === 0
1959
+ ? 'nothing on this page is animating — it may be waiting for a click, or the game is behind a login'
1960
+ : surface === null
1961
+ ? 'ready to read and act on, though there is no canvas or frame to watch'
1962
+ : `ready: ${surface.tag} is live at ${mainRate?.fps ?? frameRate?.fps ?? 0} fps`
1963
+ lines.push(trouble)
1964
+ return { text: lines.join('\n') }
821
1965
  },
822
1966
  },
823
1967
  {
824
- name: 'sidecar_view', // ui-copy-ignore
825
- description: 'Read the sidecar browser: numbered controls and the words on screen. Numbers stay valid across reads and actions until the page navigates. Narrow it with mode (text, links, forms, full) and max_chars; ask for changed_only after an action to spend fewer tokens.',
1968
+ name: 'browser_find', // ui-copy-ignore
1969
+ description: 'Find words on the page without reading it all: returns matching numbered controls (click the ref) and matching text (click x/y), each with whether it is on screen. A phone-sized page has hundreds of lines; this answers "where does it say X" in one compact result.',
826
1970
  parameters: {
827
1971
  type: 'object', // ui-copy-ignore
828
1972
  properties: {
829
- mode: { type: 'string', enum: ['text', 'links', 'forms', 'full'], description: 'What to include. Default text: controls plus the visible words.' },
830
- max_chars: { type: 'integer', description: 'Character budget for the page text. Default 6000.' },
1973
+ query: { type: 'string', description: 'Words to look for; case does not matter.' },
1974
+ regex: { type: 'boolean', description: 'Treat query as a regular expression.' },
1975
+ limit: { type: 'integer', description: 'Matches to return. Default 12, at most 40.' },
1976
+ },
1977
+ required: ['query'],
1978
+ additionalProperties: false,
1979
+ },
1980
+ output: TEXT_OUTPUT, // ui-copy-ignore
1981
+ timeoutMs: 30_000,
1982
+ isConcurrencySafe: () => false,
1983
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the matches. */
1984
+ execute: async (args) => withRead(async (page) => {
1985
+ const query = String(args?.query ?? '').trim()
1986
+ if (query === '') return 'Give the words to look for.'
1987
+ const found = await findOnPage(page, query, { regex: args?.regex === true, limit: Number(args?.limit ?? 12) })
1988
+ if (found.fault !== undefined) return `the page did not answer when searched — ${String(found.fault).split('\n')[0].slice(0, 200)}`
1989
+ const matches = Array.isArray(found.matches) ? found.matches : []
1990
+ if (matches.length === 0) return `nothing on this page says ${JSON.stringify(query)} — try other words, or call browser_view for the whole page.`
1991
+ const lines = matches.map((one) => `${one.ref === 0 ? `at ${one.x},${one.y}` : one.ref} [${one.tag}] ${one.name}${one.view === true ? '' : ' (off screen)'}`)
1992
+ return `${matches.length} match(es) for ${JSON.stringify(query)}:\n${lines.join('\n')}`
1993
+ }),
1994
+ },
1995
+ {
1996
+ name: 'browser_read', // ui-copy-ignore
1997
+ description: 'Read the page as a person does: one Markdown block of headings, paragraphs, lists and links, no controls and no markup. Far fewer tokens than browser_view on long pages (articles, docs, threads, issue trackers), and it keeps links so a follow-up can be opened by address. Use browser_view when you need numbers to click or type; use browser_read when you only need the words.',
1998
+ parameters: {
1999
+ type: 'object', // ui-copy-ignore
2000
+ properties: {
2001
+ max_chars: { type: 'integer', description: 'Character ceiling. Default 6000; the result says when it was cut.' },
2002
+ query: { type: 'string', description: 'What you want to know. With a query, only the passages that answer it come back, ranked — the cheapest way to read a long page. Without one, the whole page comes back.' },
2003
+ force: { type: 'boolean', description: 'Read again even if the page has not changed since this chat last read it.' },
2004
+ },
2005
+ additionalProperties: false,
2006
+ },
2007
+ output: TEXT_OUTPUT, // ui-copy-ignore
2008
+ timeoutMs: 30_000,
2009
+ isConcurrencySafe: () => false,
2010
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the words, as Markdown. */
2011
+ execute: async (args) => withRead(async (page) => {
2012
+ const asked = String(args?.query ?? '').trim()
2013
+ const ceiling = Number(args?.max_chars ?? (asked === '' ? 6000 : 4000))
2014
+ const mark = await pageMark(page)
2015
+ const key = mark === '' ? '' : `${mark}|${asked}|${ceiling}`
2016
+ const seen = page === null || page === undefined ? undefined : readSeen.get(page)
2017
+ /* The text this chat already has is not worth sending twice: a page that
2018
+ looks the same answers with where it is, not the same four thousand
2019
+ characters. force is for a caller that lost the earlier answer. */
2020
+ if (args?.force !== true && key !== '' && seen !== undefined && seen.key === key) {
2021
+ return `${mark.split('|')[1] ?? ''} · unchanged since this chat last read it (${String(seen.text.length)} chars) — the text is the one above. Call browser_view with changed_only to see what moved, or pass force for it again.`
2022
+ }
2023
+ const keep = (text) => {
2024
+ if (key !== '' && page !== null && page !== undefined) readSeen.set(page, { key, text })
2025
+ return text
2026
+ }
2027
+ if (asked !== '') {
2028
+ const found = await readRelevant(page, asked, { maxChars: ceiling })
2029
+ if (found.fault !== undefined) return `the page did not answer when read — ${String(found.fault).split('\n')[0].slice(0, 200)}`
2030
+ if (found.text === '') {
2031
+ /* The question missed every block: the page is still the answer, so
2032
+ hand it over whole rather than make the caller guess again. */
2033
+ const whole = await readArticle(page, { maxChars: ceiling })
2034
+ if (whole.fault === undefined && whole.text !== '') {
2035
+ return keep(`${whole.url}\n# ${whole.title === '' ? '(untitled)' : whole.title}\n\nnothing here answers ${JSON.stringify(asked)} (${String(found.total)} blocks read), so the whole page follows:\n\n${whole.text}${whole.truncated ? `\n\n(cut at ${String(whole.chars)} characters; raise max_chars for more)` : ''}`)
2036
+ }
2037
+ return `${found.url}\n# ${found.title === '' ? '(untitled)' : found.title}\n\nnothing here answers ${JSON.stringify(asked)} (${String(found.total)} blocks read). Call browser_view for the whole page, or try other words.`
2038
+ }
2039
+ return keep(`${found.url}\n# ${found.title === '' ? '(untitled)' : found.title}\n\n${found.text}\n\n(${String(found.matched)} of ${String(found.total)} blocks, ranked for ${JSON.stringify(asked)})`)
2040
+ }
2041
+ const read = await readArticle(page, { maxChars: ceiling })
2042
+ if (read.fault !== undefined) return `the page did not answer when read — ${read.fault.split('\n')[0].slice(0, 200)}`
2043
+ if (read.text === '') {
2044
+ const pdf = await pdfNote(page)
2045
+ if (pdf !== '') return `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}${pdf}`
2046
+ return `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}\n\nnothing to read here: the page draws its content, or it is still loading. Call browser_view to see what is on screen.`
2047
+ }
2048
+ const head = `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}`
2049
+ const tail = read.truncated ? `\n\n(cut at ${String(read.chars)} characters; raise max_chars for more)` : ''
2050
+ return keep(`${head}\n\n${read.text}${tail}`)
2051
+ }),
2052
+ },
2053
+ {
2054
+ name: 'browser_view', // ui-copy-ignore
2055
+ description: 'Read the sidecar browser: numbered controls and the words on screen. Numbers stay valid across reads and actions until the page navigates. Narrow it with mode (text, links, forms, full, map) and max_chars; ask for changed_only after an action to spend fewer tokens. Use map first on a page too big to read.',
2056
+ parameters: {
2057
+ type: 'object', // ui-copy-ignore
2058
+ properties: {
2059
+ mode: { type: 'string', enum: ['text', 'links', 'forms', 'full', 'map'], description: 'What to include. Default text: controls plus the visible words. map: a landmark summary of a big page — how many controls of each kind, the distinct ones with their numbers, and how many words there are, without the words.' },
2060
+ max_chars: { type: 'integer', description: 'Character budget for the page text. Default 4600.' },
831
2061
  changed_only: { type: 'boolean', description: 'Only what changed since the last read.' },
832
2062
  },
833
2063
  additionalProperties: false,
@@ -838,25 +2068,56 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
838
2068
  /** @param {any} args - arguments. @returns {Promise<{text: string}>} the read. */
839
2069
  execute: async (args) => withRead(async (page) => {
840
2070
  const options = { mode: String(args?.mode ?? 'text'), maxChars: Number(args?.max_chars ?? 4600) }
841
- const read = await remember(page, refs, options)
2071
+ /* Each chat keeps its own read state: a shared one let another chat's
2072
+ read replace this chat's baseline, so its numbers and its "what
2073
+ changed" answered about a page it never saw. */
2074
+ const state = refsFor()
2075
+ const read = await remember(page, state, options)
842
2076
  const note = isCheckPage(read) ? `\n${CHECK_NOTE}` : ''
843
- if (args?.changed_only === true && refs.last !== undefined) return diffRead(refs.last, read, options) + note
844
- refs.last = read
845
- return renderPage(read, options) + note
2077
+ if (args?.changed_only === true && state.last !== undefined) {
2078
+ /* The diff becomes the new baseline: a page that moved zero -> one
2079
+ should not report that move again on the next read. */
2080
+ const text = diffRead(state.last, read, options) + note
2081
+ state.last = read
2082
+ state.lastShape = JSON.stringify(options)
2083
+ return text
2084
+ }
2085
+ /* The read before this one answers two questions: what is new on this
2086
+ one, and whether the page moved at all. Keep it before either answer
2087
+ is overwritten. */
2088
+ const before = state.last
2089
+ state.last = read
2090
+ const fresh = freshRefs(before, read)
2091
+ const freshLine = fresh.length === 0 || fresh.length > 12 ? '' : `\nnew since the read before: ${fresh.map(String).join(', ')}`
2092
+ /* Same page, same shape, same words: the numbers above still work, and
2093
+ repeating the page would only spend tokens. */
2094
+ if (sameRead(before, read) && state.lastShape === JSON.stringify(options)) {
2095
+ const moved = Number(before.scroll?.y ?? 0) !== Number(read.scroll?.y ?? 0) ? `, scrolled to ${Math.round(Number(read.scroll?.y ?? 0))}` : ''
2096
+ /* The address was said with the read above; "here" is the whole of it. */
2097
+ const where = saidUrls.get(page) === String(read.url ?? '') ? 'here' : read.url
2098
+ saidUrls.set(page, String(read.url ?? ''))
2099
+ return `still ${where}${moved} — nothing changed since the last read; the numbers above still work${note}`
2100
+ }
2101
+ state.lastShape = JSON.stringify(options)
2102
+ const said = withoutSeenAddress(page, read.url, renderPage(read, options))
2103
+ saidUrls.set(page, String(read.url ?? ''))
2104
+ return said + freshLine + note + (await pdfNote(page))
846
2105
  }),
847
2106
  },
848
2107
  {
849
- name: 'sidecar_click', // ui-copy-ignore
2108
+ name: 'browser_click', // ui-copy-ignore
850
2109
  description: 'Click a numbered control, a point, or the first control whose words contain `text` — the last needs no read first. The click is real mouse input and the mirror shows where it landed.',
851
2110
  parameters: {
852
2111
  type: 'object', // ui-copy-ignore
853
2112
  properties: {
854
- ref: { type: 'integer', description: 'Number from sidecar_view.' },
2113
+ ref: { type: 'integer', description: 'Number from browser_view.' },
855
2114
  text: { type: 'string', description: 'Words on the control, when its number is not known.' },
856
2115
  x: { type: 'integer', description: 'Viewport x, when clicking a point.' },
857
2116
  y: { type: 'integer', description: 'Viewport y, when clicking a point.' },
858
2117
  double: { type: 'boolean', description: 'Double-click.' },
859
2118
  right: { type: 'boolean', description: 'Right-click, opening a context menu.' },
2119
+ hold_ms: { type: 'integer', description: 'Keep the button down this long before releasing: a press-and-hold a game or a drag handle needs.' },
2120
+ mode: { type: 'string', enum: ['human', 'dom'], description: '"human" moves the real mouse and is what a page cannot tell from a person; "dom" fires the page\'s own click on the numbered control, which reaches one an overlay covers. Omit to use what this chat set with browser_mode.' },
860
2121
  read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
861
2122
  },
862
2123
  additionalProperties: false,
@@ -871,36 +2132,55 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
871
2132
  let followed = ''
872
2133
  if (args?.ref !== undefined) {
873
2134
  const spot = await findAgain(page, Number(args.ref))
874
- if (spot.gone === true) return REF_GONE
2135
+ if (spot.gone === true) return spot.why ?? REF_GONE
875
2136
  if (spot.followed === true) followed = `the page redrew, so "${String(spot.was).slice(0, 50)}" has a new number — used where it is now`
876
2137
  point = { x: Number(spot.x), y: Number(spot.y) }
877
2138
  } else if (String(args?.text ?? '') !== '') {
878
- const hit = await findWords(page, String(args.text))
879
- if (hit === null) return `No control says ${JSON.stringify(String(args.text))}. Call sidecar_view to see what is there.`
2139
+ const hit = await findWords(page, String(args.text), refsFor().refs)
2140
+ if (hit === null) return `No control says ${JSON.stringify(String(args.text))}. Call browser_view to see what is there.`
880
2141
  point = { x: hit.x, y: hit.y }
881
2142
  label = `"${hit.name.slice(0, 60)}"`
882
2143
  }
883
- if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) return 'Give a ref from sidecar_view, or both x and y.'
2144
+ if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) return 'Give a ref from browser_view, or both x and y.'
2145
+ if (wayOf(args) === 'dom') {
2146
+ if (args?.ref === undefined) return 'Acting on the page itself needs a number from browser_view: pass ref, or leave mode out to click like a person.'
2147
+ const pressedAt = Date.now()
2148
+ const said = await evaluate(page, clickInDom(Number(args.ref), { double: args?.double === true, right: args?.right === true })).catch(() => undefined)
2149
+ const hit = asSpot(said)
2150
+ if (hit?.ok !== true) return REF_GONE
2151
+ await settle(page, 5000, pressedAt)
2152
+ const called = await acted(page, args, () => look(page, { mode: 'text', maxChars: 3000 }), `click ${label} on the page itself`)
2153
+ return called + (followed === '' ? '' : ` (${followed})`)
2154
+ }
884
2155
  const button = args?.right === true ? 'right' : 'left'
885
2156
  let cover = ''
886
2157
  if (args?.ref !== undefined && button === 'left') {
887
2158
  const gate = await clearCover(page, Number(args.ref))
888
2159
  cover = gate.note
2160
+ /* An overlay that would not go away leaves the control underneath:
2161
+ the click would land on the overlay, so it is not sent. */
2162
+ if (gate.cleared !== true) {
2163
+ return `${cover.trim()}\nNot clicked: the control is still covered. Call browser_view and handle the overlay first.`
2164
+ }
889
2165
  if (gate.cleared && cover !== '') {
890
2166
  const again = await locate(page, Number(args.ref))
891
2167
  if (again.gone !== true) point = { x: Number(again.x), y: Number(again.y) }
892
2168
  }
893
2169
  }
2170
+ const before = args?.read === true ? '' : await pageMark(page)
2171
+ const pressedAt = Date.now()
894
2172
  await page.mouse('mouseMoved', point)
895
2173
  await page.mouse('mousePressed', { ...point, clickCount: args?.double === true ? 2 : 1, button })
2174
+ const hold = Math.max(0, Math.min(60_000, Number(args?.hold_ms ?? 0)))
2175
+ if (hold > 0) await new Promise((settle) => setTimeout(settle, hold))
896
2176
  await page.mouse('mouseReleased', { ...point, clickCount: args?.double === true ? 2 : 1, button })
897
- await settle(page)
2177
+ await settle(page, 5000, pressedAt)
898
2178
  const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 3000 }), `click ${label}`)
899
- return said + cover + (followed === '' ? '' : ` (${followed})`)
2179
+ return said + cover + (followed === '' ? '' : ` (${followed})`) + (await outcome(page, before))
900
2180
  }),
901
2181
  },
902
2182
  {
903
- name: 'sidecar_type', // ui-copy-ignore
2183
+ name: 'browser_type', // ui-copy-ignore
904
2184
  description: 'Type text into the sidecar browser — Chinese included — optionally pressing Enter. Pass a ref to focus that control first. With clear, the field is emptied first.',
905
2185
  parameters: {
906
2186
  type: 'object', // ui-copy-ignore
@@ -910,6 +2190,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
910
2190
  field: { type: 'string', description: 'The field is called this on screen, when you have no number for it.' },
911
2191
  submit: { type: 'boolean', description: 'Press Enter after typing.' },
912
2192
  clear: { type: 'boolean', description: 'Select all and delete before typing.' },
2193
+ mode: { type: 'string', enum: ['human', 'dom'], description: '"human" presses real keys and is what a page cannot tell from a person; "dom" sets the field and fires its input events, which leaves key handlers untouched. Omit to use what this chat set with browser_mode.' },
913
2194
  read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
914
2195
  },
915
2196
  required: ['text'], // ui-copy-ignore
@@ -922,11 +2203,29 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
922
2203
  execute: async (args) => withPage(async (page) => {
923
2204
  let cover = ''
924
2205
  let followed = ''
2206
+ if (wayOf(args) === 'dom') {
2207
+ let target = Number.isInteger(args?.ref) ? Number(args.ref) : undefined
2208
+ if (target === undefined && args?.field !== undefined) {
2209
+ const found = await fieldByName(page, String(args.field), refsFor().refs)
2210
+ if (found === null) return `${String(args.field)}: ${REF_GONE}`
2211
+ if (!Number.isInteger(found.ref)) return `${String(args.field)}: read the page and pass its number, since a field found by its label has none.`
2212
+ target = Number(found.ref)
2213
+ cover = found.lead
2214
+ }
2215
+ if (target === undefined) return 'Acting on the page itself needs a number from browser_view: pass ref, or leave mode out to type like a person.'
2216
+ const said = await evaluate(page, writeInDom(target, String(args?.text ?? ''), { clear: args?.clear === true, submit: args?.submit === true })).catch(() => undefined)
2217
+ const wrote = asSpot(said)
2218
+ if (wrote?.ok !== true) return REF_GONE
2219
+ await settle(page)
2220
+ const called = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type on the page itself')
2221
+ return `${called} (field now "${String(wrote.value).slice(0, 40)}")` + cover
2222
+ }
925
2223
  if (args?.field !== undefined && args?.ref === undefined) {
926
- const target = await fieldByName(page, String(args.field))
2224
+ const target = await fieldByName(page, String(args.field), refsFor().refs)
927
2225
  if (target === null) return `${String(args.field)}: ${REF_GONE}`
928
2226
  cover = target.lead
929
- await writeAt(page, { x: target.x, y: target.y }, String(args?.text ?? ''), { clear: args?.clear, submit: args?.submit === true })
2227
+ const dated = await writeAt(page, { x: target.x, y: target.y }, String(args?.text ?? ''), { clear: args?.clear, submit: args?.submit === true, hint: target.name })
2228
+ if (dated !== '') return dated + cover
930
2229
  await settle(page)
931
2230
  const called = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type')
932
2231
  return called + cover + ` (typed into "${target.name.slice(0, 50)}")` // ui-copy-ignore
@@ -934,8 +2233,11 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
934
2233
  if (args?.ref !== undefined) {
935
2234
  const gate = await clearCover(page, Number(args.ref))
936
2235
  cover = gate.note
2236
+ if (gate.cleared !== true) {
2237
+ return `${cover.trim()}\nNot typed into: the field is still covered. Call browser_view and handle the overlay first.`
2238
+ }
937
2239
  const spot = await findAgain(page, Number(args.ref))
938
- if (spot.gone === true) return REF_GONE + cover
2240
+ if (spot.gone === true) return spot.why ?? REF_GONE + cover
939
2241
  if (spot.followed === true) followed = `the page redrew, so "${String(spot.was).slice(0, 50)}" has a new number — typed where it is now`
940
2242
  const point = { x: Number(spot.x), y: Number(spot.y) }
941
2243
  await page.mouse('mouseMoved', point)
@@ -944,21 +2246,136 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
944
2246
  await wait(80)
945
2247
  }
946
2248
  if (args?.clear === true) await clearFocus(page)
947
- await page.insertText(String(args?.text ?? ''))
2249
+ if (await dateLikeFocus(page) !== '') return (await fillDateLike(page, String(args?.text ?? ''))) + cover
2250
+ const words = String(args?.text ?? '')
2251
+ const before = args?.read === true ? '' : await pageMark(page)
2252
+ await typeInto(page, words, String(args?.field ?? ''))
2253
+ const dated = await fillDateLike(page, words)
2254
+ if (dated !== '') return dated + cover
948
2255
  if (args?.submit === true) await pressKey(page, 'Enter')
949
2256
  await settle(page)
950
2257
  const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type')
951
- return said + cover + (followed === '' ? '' : ` (${followed})`) // ui-copy-ignore
2258
+ return said + cover + (followed === '' ? '' : ` (${followed})`) + (await outcome(page, before)) // ui-copy-ignore
952
2259
  }),
953
2260
  },
954
2261
  {
955
- name: 'sidecar_key', // ui-copy-ignore
956
- description: 'Press one key in the sidecar browser: Enter, Tab, Escape, Backspace, Delete, ArrowUp/ArrowDown/ArrowLeft/ArrowRight, Home, End, PageUp, PageDown, Space.',
2262
+ name: 'browser_ask', // ui-copy-ignore
2263
+ description: 'Say something to a chat that is open in the page — a model, a support window, a search box — and read its answer back in the same call. Types into the box, presses Enter, then waits until the answer stops growing. Use it to get a review or a picture prompt answered without a read between every step.',
957
2264
  parameters: {
958
2265
  type: 'object', // ui-copy-ignore
959
2266
  properties: {
960
- key: { type: 'string', description: 'Key name.' },
961
- modifiers: { type: 'string', description: 'Held while pressing: Ctrl, Alt, Shift or Meta, joined by +.' },
2267
+ text: { type: 'string', description: 'What to say.' },
2268
+ ref: { type: 'integer', description: 'The box to type in, from browser_view. Omit to use the page\u2019s own chat box.' },
2269
+ submit: { type: 'boolean', description: 'Press Enter to send. Default true.' },
2270
+ timeout_ms: { type: 'integer', description: 'How long to wait for the answer. Default 60000.' },
2271
+ },
2272
+ required: ['text'], // ui-copy-ignore
2273
+ additionalProperties: false,
2274
+ },
2275
+ output: TEXT_OUTPUT, // ui-copy-ignore
2276
+ timeoutMs: 180_000,
2277
+ isConcurrencySafe: () => false,
2278
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the answer. */
2279
+ execute: async (args) => withPage((page) => askInPage(page, args)),
2280
+ },
2281
+ {
2282
+ name: 'browser_review', // ui-copy-ignore
2283
+ description: 'Ask the chat in the page to review a repository and take its review back as a list: builds the message, sends it, waits for the answer, and writes the round down so the next round knows what was already said. Use it for a review round, not for a one-off question (that is browser_ask).',
2284
+ parameters: {
2285
+ type: 'object', // ui-copy-ignore
2286
+ properties: {
2287
+ repo: { type: 'string', description: 'Address of the repository to review.' },
2288
+ commit: { type: 'string', description: 'The commit the review is about. Omit for the newest one.' },
2289
+ focus: { type: 'string', description: 'One thing to look at especially.' },
2290
+ diff: { type: 'string', description: 'The change to review, as a diff. Include it when the reviewer cannot open the repository itself.' },
2291
+ ref: { type: 'integer', description: 'The box to type in, from browser_view. Omit to use the page\u2019s own chat box.' },
2292
+ timeout_ms: { type: 'integer', description: 'How long to wait for the review. Default 120000.' },
2293
+ },
2294
+ required: ['repo'], // ui-copy-ignore
2295
+ additionalProperties: false,
2296
+ },
2297
+ output: TEXT_OUTPUT, // ui-copy-ignore
2298
+ timeoutMs: 240_000,
2299
+ isConcurrencySafe: () => false,
2300
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the review. */
2301
+ execute: async (args) => withPage(async (page) => {
2302
+ const repo = String(args?.repo ?? '').trim()
2303
+ if (repo === '') return 'Which repository should the reviewer look at?'
2304
+ const commit = String(args?.commit ?? '').trim()
2305
+ /* What was settled last time travels with the question. */
2306
+ const before = lastRound()
2307
+ const already = Array.isArray(before?.marks) ? before.marks : []
2308
+ const asked = askBrief({ repo, commit, focus: String(args?.focus ?? ''), diff: String(args?.diff ?? ''), already })
2309
+ const started = Date.now()
2310
+ const told = await askInPage(page, { text: asked, ref: args?.ref, timeout_ms: args?.timeout_ms ?? 120_000 })
2311
+ const tookMs = Date.now() - started
2312
+ if (!String(told).startsWith('the answer came back in')) return told
2313
+ const answer = String(told).split('\n').slice(1).join('\n').trim()
2314
+ const { round } = recordRound({ commit, question: asked, answer, tookMs })
2315
+ /* A wall answers too, and its words are not advice: name it rather than
2316
+ call the login screen prose. */
2317
+ const wall = /(sign in|log in|登录|注册|verif|checking your browser)/i.test(answer)
2318
+ /* The question sitting there unanswered is not prose: say that instead. */
2319
+ const busy = asked.slice(0, 24)
2320
+ const waiting = round.points.length === 0 && busy.length >= 8 && answer.includes(busy)
2321
+ const list = round.points.length > 0
2322
+ ? round.points.map((point, at) => `${at + 1}. ${point}`).join('\n')
2323
+ : waiting
2324
+ ? `the page has the message but has not answered yet; it shows: ${answer.replace(/\s+/g, ' ').slice(0, 200)}`
2325
+ : wall
2326
+ ? `the page is asking to sign in first, so there is no review yet: ${answer.replace(/\s+/g, ' ').slice(0, 200)}`
2327
+ : `It answered in prose; no list to work from. It said: ${answer.replace(/\s+/g, ' ').slice(0, 200)}`
2328
+ const settled = already.filter((mark) => mark.status !== 'open').length
2329
+ const carried = settled === 0 ? '' : `\n(${settled} earlier point${settled === 1 ? '' : 's'} left out as already handled)`
2330
+ return `review round: ${commit === '' ? 'newest commit' : commit.slice(0, 12)} in ${Math.round(tookMs / 100) / 10}s\n${list}${carried}`
2331
+ }),
2332
+ },
2333
+ {
2334
+ name: 'browser_review_verdict', // ui-copy-ignore
2335
+ description: 'Record what happened to each point of the last review round, so the next round leaves settled points out: point numbers are the ones browser_review printed, and each mark is fixed, verified (checked in the code and the point does not apply), invalid, or open.',
2336
+ parameters: {
2337
+ type: 'object', // ui-copy-ignore
2338
+ properties: {
2339
+ mark: { // ui-copy-ignore
2340
+ type: 'array',
2341
+ description: 'One entry per point: which point, its status, and the evidence that backs it.',
2342
+ items: { // ui-copy-ignore
2343
+ type: 'object', // ui-copy-ignore
2344
+ properties: { // ui-copy-ignore
2345
+ point: { type: 'integer', description: 'The point number browser_review printed, starting at 1.' },
2346
+ status: { type: 'string', enum: MARK_STATUS, description: 'fixed, verified, invalid, or open.' },
2347
+ evidence: { type: 'string', description: 'Where the code shows it: file, test, or commit.' },
2348
+ },
2349
+ required: ['point', 'status'], // ui-copy-ignore
2350
+ additionalProperties: false,
2351
+ },
2352
+ },
2353
+ },
2354
+ required: ['mark'], // ui-copy-ignore
2355
+ additionalProperties: false,
2356
+ },
2357
+ output: TEXT_OUTPUT, // ui-copy-ignore
2358
+ timeoutMs: 10_000,
2359
+ isConcurrencySafe: () => false,
2360
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} what was recorded. */
2361
+ execute: async (args) => {
2362
+ const written = markRound({ marks: Array.isArray(args?.mark) ? args.mark : [] })
2363
+ if (written === null) return 'No review round to mark: play one with browser_review first.'
2364
+ const counts = new Map()
2365
+ for (const mark of written.marks) counts.set(mark.status, (counts.get(mark.status) ?? 0) + 1)
2366
+ const said = [...counts].map(([status, n]) => `${n} ${status}`).join(', ')
2367
+ return `review round ${String(written.round.commit || 'newest').slice(0, 12)}: recorded ${written.marks.length} of ${written.round.points.length} points (${said}).`
2368
+ },
2369
+ },
2370
+ {
2371
+ name: 'browser_key', // ui-copy-ignore
2372
+ description: 'Press one key in the background browser: Enter, Tab, Escape, Backspace, Delete, Insert, arrows, Home, End, PageUp, PageDown, Space, F1-F12, the numeric keypad (Numpad0-Numpad9, NumpadEnter, NumpadAdd, ...), printable characters and every symbol on a US keyboard (including shifted ones such as ? : @), plus the keys with a name (ContextMenu, CapsLock, PrintScreen, media keys). Join keys with + to hold them together (ArrowLeft+ArrowUp, Shift+Tab), and hold_ms to keep them down. Modifiers accept Ctrl, Alt, Shift, Meta/Cmd, or Mod for the platform command key; Ctrl+C/X/V/A/Z/Y run the real copy, cut, paste, select-all, undo and redo.',
2373
+ parameters: {
2374
+ type: 'object', // ui-copy-ignore
2375
+ properties: {
2376
+ key: { type: 'string', description: 'Key name, or several joined by + to hold them together (ArrowLeft+ArrowUp, Shift+ArrowLeft). Letters, digits, punctuation and shifted symbols work too.' },
2377
+ modifiers: { type: 'string', description: 'Held while pressing: Ctrl, Alt, Shift, Meta/Cmd, or Mod (Command on macOS, Control on Windows and Linux), joined by +.' },
2378
+ hold_ms: { type: 'integer', description: 'Keep the key down this long before releasing. Games move while a key is held.' },
962
2379
  read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
963
2380
  },
964
2381
  required: ['key'], // ui-copy-ignore
@@ -970,15 +2387,22 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
970
2387
  /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
971
2388
  execute: async (args) => withPage(async (page) => {
972
2389
  const name = String(args?.key ?? 'Enter')
973
- const held = String(args?.modifiers ?? '')
974
- const mods = (held.includes('Alt') ? 1 : 0) + (held.includes('Ctrl') ? 2 : 0) + (held.includes('Meta') ? 4 : 0) + (held.includes('Shift') ? 8 : 0)
975
- await pressKey(page, name, mods)
976
- await settle(page, 3000)
2390
+ const chord = splitChord(name, modifierBits(args?.modifiers))
2391
+ const mods = chord.modifiers
2392
+ const hold = Math.max(0, Math.min(60_000, Number(args?.hold_ms ?? 0)))
2393
+ if (chord.names.length === 0) throw new Error(`sidecar: ${name} holds no key, only modifiers`) // ui-copy-ignore
2394
+ /* One key goes through the plain path so an editing shortcut is sent as
2395
+ the browser command it is; a chord holds every key down together. */
2396
+ if (chord.names.length > 1) await pressChord(page, chord.names, mods, hold)
2397
+ else await pressKey(page, chord.names[0], mods, hold)
2398
+ /* A held key is game input: waiting for the page to fall quiet after one
2399
+ costs more than the frame it was meant to drive. */
2400
+ if (hold === 0) await settle(page, 3000)
977
2401
  return acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), `key ${name}`)
978
2402
  }),
979
2403
  },
980
2404
  {
981
- name: 'sidecar_scroll', // ui-copy-ignore
2405
+ name: 'browser_scroll', // ui-copy-ignore
982
2406
  description: 'Scroll the sidecar browser by wheel at its centre (negative goes up), then return what it shows.',
983
2407
  parameters: {
984
2408
  type: 'object', // ui-copy-ignore
@@ -996,18 +2420,21 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
996
2420
  /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
997
2421
  execute: async (args) => withPage(async (page) => {
998
2422
  await page.wheel({ x: Number(args?.x ?? VIEW.width / 2), y: Number(args?.y ?? VIEW.height / 2), dy: Number(args?.dy ?? 600) })
2423
+ /* A wheel is a request, not a result: a list that loads more rows on
2424
+ scroll has not answered yet when the wheel returns. Give the scroll
2425
+ handler its turn, then let the page fall quiet. */
999
2426
  await wait(200)
1000
2427
  const delta = Number(args?.dy ?? 600)
1001
2428
  return acted(page, args, () => look(page, { mode: 'text', maxChars: 4000 }), `scroll ${delta > 0 ? '+' : ''}${delta}`)
1002
2429
  }),
1003
2430
  },
1004
2431
  {
1005
- name: 'sidecar_hover', // ui-copy-ignore
2432
+ name: 'browser_hover', // ui-copy-ignore
1006
2433
  description: 'Move the pointer onto a numbered control, so hover-only menus and tooltips open, then return what the page shows.',
1007
2434
  parameters: {
1008
2435
  type: 'object', // ui-copy-ignore
1009
2436
  properties: {
1010
- ref: { type: 'integer', description: 'Number from sidecar_view.' },
2437
+ ref: { type: 'integer', description: 'Number from browser_view.' },
1011
2438
  read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
1012
2439
  },
1013
2440
  required: ['ref'], // ui-copy-ignore
@@ -1019,7 +2446,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1019
2446
  /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
1020
2447
  execute: async (args) => withPage(async (page) => {
1021
2448
  const spot = await findAgain(page, Number(args?.ref))
1022
- if (spot.gone === true) return REF_GONE
2449
+ if (spot.gone === true) return spot.why ?? REF_GONE
1023
2450
  const cursor = await page.hover({ x: Number(spot.x), y: Number(spot.y) })
1024
2451
  await wait(400)
1025
2452
  const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2500 }), `hover ${String(args?.ref)} · ${cursor}`)
@@ -1027,7 +2454,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1027
2454
  }),
1028
2455
  },
1029
2456
  {
1030
- name: 'sidecar_nav', // ui-copy-ignore
2457
+ name: 'browser_nav', // ui-copy-ignore
1031
2458
  description: 'Change the sidecar browser\'s address: open a url, go back, go forward, or reload. Use this instead of clicking the browser chrome.',
1032
2459
  parameters: {
1033
2460
  type: 'object', // ui-copy-ignore
@@ -1057,7 +2484,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1057
2484
  }),
1058
2485
  },
1059
2486
  {
1060
- name: 'sidecar_do', // ui-copy-ignore
2487
+ name: 'browser_do', // ui-copy-ignore
1061
2488
  description: 'Run several steps in one call and report one compact result: the whole of "click search, type a query, press Enter, wait for results" is one call instead of five. Steps run in order and stop at the first that cannot proceed. Each step: {ref, tap | text, submit}, {tap: "x,y"}, {tap: "Sign in"}, {field: "Email", text}, {text}, {key}, {scroll}, {wait_text}, {wait_gone}, {wait_url}, {wait_idle}, {eval}, {open}, {sleep_ms}. Waits know when the page has settled. A step that names a control clears a cookie banner in its way first.',
1062
2489
  parameters: {
1063
2490
  type: 'object', // ui-copy-ignore
@@ -1103,39 +2530,46 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1103
2530
  if (steps.length === 0) return 'Give at least one step.'
1104
2531
  const report = []
1105
2532
  for (const [index, step] of steps.entries()) {
1106
- const said = await runStep(page, step ?? {}, refs?.refs)
1107
- report.push(`${index + 1}. ${said}`)
1108
- if (/^ref \d+: That control is gone/.test(said)) return `${report.join('\n')}\nStopped: the page changed. Call sidecar_view and use the new numbers.`
2533
+ const stepAt = Date.now()
2534
+ const said = await runStep(page, step ?? {}, refsFor().refs)
2535
+ const text = typeof said === 'string' ? said : said.text
2536
+ report.push(`${index + 1}. ${text}`)
2537
+ /* A step that never landed makes every later step a guess, so the
2538
+ batch ends here instead of typing into a page that is not there. */
2539
+ if (typeof said !== 'string' && said.blocked === true) {
2540
+ return `${report.join('\n')}\nStopped after step ${index + 1}. Call browser_view and plan again from what the page shows now.`
2541
+ }
1109
2542
  /* A step can navigate straight onto Chrome's error page — a mistyped
1110
2543
  address, or a site that does not answer. The rest of the batch
1111
2544
  would then type into "This site can't be reached", so it stops
1112
2545
  here and says what really happened. */
1113
2546
  if (isErrorPage(page)) return `${report.join('\n')}\nStopped after step ${index + 1}: ${await unreachable(page)}`
1114
2547
  if (step?.sleep_ms === undefined && step?.wait_text === undefined && step?.wait_gone === undefined && step?.wait_url === undefined && step?.wait_idle === undefined) {
1115
- await settle(page, Number(step?.timeout ?? 5000))
2548
+ await settle(page, Number(step?.timeout ?? 5000), stepAt)
1116
2549
  }
1117
2550
  }
1118
2551
  const mode = String(args?.read ?? 'text')
1119
2552
  if (mode === 'none') return report.join('\n')
1120
2553
  const options = { mode: mode === 'changed' ? 'text' : mode, maxChars: Number(args?.max_chars ?? 3000) }
1121
- const read = await remember(page, refs, options)
1122
- if (mode === 'changed' && refs.last !== undefined) {
1123
- const diff = diffRead(refs.last, read, options)
1124
- refs.last = read
2554
+ const state = refsFor()
2555
+ const read = await remember(page, state, options)
2556
+ if (mode === 'changed' && state.last !== undefined) {
2557
+ const diff = diffRead(state.last, read, options)
2558
+ state.last = read
1125
2559
  return `${report.join('\n')}\n${diff}`
1126
2560
  }
1127
- refs.last = read
2561
+ state.last = read
1128
2562
  return `${report.join('\n')}\n\n${renderPage(read, options)}`
1129
2563
  }),
1130
2564
  },
1131
2565
  {
1132
- name: 'sidecar_extract', // ui-copy-ignore
2566
+ name: 'browser_extract', // ui-copy-ignore
1133
2567
  description: 'Read the items a selector matches as one compact line each — lists, search results, tables, menus — instead of the whole page. Each row keeps its link when it has one. Use it when a page has more rows than the read budget allows; it costs a fraction of a full read.',
1134
2568
  parameters: {
1135
2569
  type: 'object', // ui-copy-ignore
1136
2570
  properties: {
1137
2571
  select: { type: 'string', description: 'CSS selector for the items, for example "ul.results > li".' },
1138
- ref: { type: 'integer', description: 'Only look inside this control, as numbered by the last sidecar_view.' },
2572
+ ref: { type: 'integer', description: 'Only look inside this control, as numbered by the last browser_view.' },
1139
2573
  limit: { type: 'integer', description: 'Most rows to return. Default 40, at most 200.' },
1140
2574
  },
1141
2575
  required: ['select'], // ui-copy-ignore
@@ -1148,7 +2582,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1148
2582
  execute: async (args) => withRead((page) => extractRows(page, args)),
1149
2583
  },
1150
2584
  {
1151
- name: 'sidecar_eval', // ui-copy-ignore
2585
+ name: 'browser_eval', // ui-copy-ignore
1152
2586
  description: 'Run JavaScript in the sidecar page and return its value. For what the numbered controls cannot express: reading a table, setting storage, filling a widget. The page must be opened first.',
1153
2587
  parameters: {
1154
2588
  type: 'object', // ui-copy-ignore
@@ -1181,12 +2615,12 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1181
2615
  }),
1182
2616
  },
1183
2617
  {
1184
- name: 'sidecar_select', // ui-copy-ignore
2618
+ name: 'browser_select', // ui-copy-ignore
1185
2619
  description: 'Choose an option in a dropdown, by visible label or value, then return the page.',
1186
2620
  parameters: {
1187
2621
  type: 'object', // ui-copy-ignore
1188
2622
  properties: {
1189
- ref: { type: 'integer', description: 'Number of the select control from sidecar_view.' },
2623
+ ref: { type: 'integer', description: 'Number of the select control from browser_view.' },
1190
2624
  field: { type: 'string', description: 'The dropdown is called this on screen, when you have no number for it.' },
1191
2625
  value: { type: 'string', description: 'Option label or value to choose.' },
1192
2626
  },
@@ -1198,20 +2632,30 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1198
2632
  isConcurrencySafe: () => false,
1199
2633
  /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
1200
2634
  execute: async (args) => withPage(async (page) => {
1201
- if (args?.ref === undefined) {
1202
- if (args?.field === undefined) return 'name the dropdown: pass its number from sidecar_view, or the words on screen that name it.' // ui-copy-ignore
1203
- const chosen = await evaluate(page, selectSpot(String(args.field), String(args?.value ?? '')))
2635
+ let use = args
2636
+ /* One dropdown on the page needs no naming: the value alone says which
2637
+ one, and asking costs the caller a whole read. */
2638
+ if (use?.ref === undefined && use?.field === undefined) {
2639
+ const read = await readPage(page, { mode: 'text', maxChars: 1200, max: 200 })
2640
+ const drops = read.controls.filter((control) => control.view !== false && ['select', 'combobox', 'listbox'].includes(String(control.kind)))
2641
+ if (drops.length === 1) use = { ...use, ref: Number(drops[0].ref) }
2642
+ else if (drops.length > 1) return `which dropdown: ${drops.slice(0, 5).map((one) => `${String(one.name).slice(0, 30)} (${String(one.ref)})`).join(' · ')}`
2643
+ else return 'the page has no dropdown to choose in.' // ui-copy-ignore
2644
+ }
2645
+ if (use?.ref === undefined) {
2646
+ const field = String(use.field)
2647
+ const chosen = await evaluate(page, selectSpot(field, String(use?.value ?? '')))
1204
2648
  const said = typeof chosen === 'string' ? chosen : chosen?.error === undefined ? '' : String(chosen.error)
1205
2649
  if (said === '' || said.startsWith('no dropdown')) return said === '' ? 'the page did not answer; read it and try again.' : said // ui-copy-ignore
1206
2650
  await settle(page, 3000)
1207
- if (args?.read === true) return `${said}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
2651
+ if (use?.read === true) return `${said}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
1208
2652
  return `${said} · ${await brief(page)}`
1209
2653
  }
1210
- const spot = await findAgain(page, Number(args?.ref))
1211
- if (spot.gone === true) return REF_GONE
1212
- const wanted = JSON.stringify(String(args?.value ?? ''))
1213
- const route = await contextFor(page, Number(args?.ref))
1214
- const local = route?.local ?? Number(args?.ref)
2654
+ const spot = await findAgain(page, Number(use?.ref))
2655
+ if (spot.gone === true) return spot.why ?? REF_GONE
2656
+ const wanted = JSON.stringify(String(use?.value ?? ''))
2657
+ const route = await contextFor(page, Number(use?.ref))
2658
+ const local = route?.local ?? Number(use?.ref)
1215
2659
  const said = await evaluate(page, `(() => {
1216
2660
  const reg = window.__dshRead
1217
2661
  const hit = reg && reg.find(${local})
@@ -1226,17 +2670,17 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1226
2670
  return 'chose ' + option.text.trim()
1227
2671
  })()`, route)
1228
2672
  await settle(page, 3000)
1229
- if (args?.read === true) return `${String(said)}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
2673
+ if (use?.read === true) return `${String(said)}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
1230
2674
  return `${String(said)} · ${await brief(page)}`
1231
2675
  }),
1232
2676
  },
1233
2677
  {
1234
- name: 'sidecar_upload', // ui-copy-ignore
2678
+ name: 'browser_upload', // ui-copy-ignore
1235
2679
  description: 'Put files into a file input on the page, by absolute path, then return the page. The file chooser never opens.',
1236
2680
  parameters: {
1237
2681
  type: 'object', // ui-copy-ignore
1238
2682
  properties: {
1239
- ref: { type: 'integer', description: 'Number of the file input from sidecar_view.' },
2683
+ ref: { type: 'integer', description: 'Number of the file input from browser_view.' },
1240
2684
  files: { type: 'array', items: { type: 'string' }, description: 'Absolute paths on this machine.' },
1241
2685
  },
1242
2686
  required: ['ref', 'files'], // ui-copy-ignore
@@ -1255,7 +2699,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1255
2699
  if (route?.contextId !== undefined) uploadParams.contextId = route.contextId
1256
2700
  const handle = await page.cdp.send('Runtime.evaluate', uploadParams, route?.sessionId)
1257
2701
  const objectId = handle.result?.objectId
1258
- if (objectId === undefined) return REF_GONE
2702
+ if (objectId === undefined) return spot.why ?? REF_GONE
1259
2703
  const node = await page.cdp.send('DOM.describeNode', { objectId }, route?.sessionId)
1260
2704
  const backendNodeId = node?.node?.backendNodeId
1261
2705
  if (backendNodeId === undefined) return 'That control cannot take a file.'
@@ -1265,33 +2709,43 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1265
2709
  }),
1266
2710
  },
1267
2711
  {
1268
- name: 'sidecar_shot', // ui-copy-ignore
1269
- description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path. Default is the visible screen; full takes the whole page, ref takes one control.',
2712
+ name: 'browser_shot', // ui-copy-ignore
2713
+ description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path; when the current model accepts images the picture comes back in the same call, so a captcha, a chart or a broken layout can be looked at instead of guessed. Default is the visible screen; full takes the whole page, ref takes one control.',
1270
2714
  parameters: {
1271
2715
  type: 'object', // ui-copy-ignore
1272
2716
  properties: {
1273
2717
  name: { type: 'string', description: 'File name without extension.' },
1274
2718
  full: { type: 'boolean', description: 'The whole page, not just the screen.' },
1275
- ref: { type: 'integer', description: 'One control from sidecar_view.' },
2719
+ ref: { type: 'integer', description: 'One control from browser_view.' },
1276
2720
  scale: { type: 'integer', description: 'Pixel ratio. Default 1; 2 is sharper and heavier.' },
1277
2721
  },
1278
2722
  additionalProperties: false,
1279
2723
  },
1280
- output: TEXT_OUTPUT, // ui-copy-ignore
2724
+ output: SHOT_OUTPUT, // ui-copy-ignore
1281
2725
  timeoutMs: 40_000,
1282
2726
  isConcurrencySafe: () => false,
1283
- /** @param {any} args - arguments. @returns {Promise<{text: string}>} the saved path. */
1284
- execute: async (args) => withRead(async (page) => {
2727
+ /**
2728
+ * A picture nobody can look at is only a file path: when the route declares
2729
+ * image input, the same call carries the PNG as an attachment so the next
2730
+ * step can be planned from what is actually on screen instead of guessing.
2731
+ * @param {any} args - arguments.
2732
+ * @param {any} [exec] - call context, for the route and its attachments.
2733
+ * @returns {Promise<any>} the saved path, and the picture when available.
2734
+ */
2735
+ execute: async (args, exec) => {
2736
+ /** @type {{data: string, file: string}|undefined} */
2737
+ let taken
2738
+ const said = await withRead(async (page) => {
1285
2739
  const scale = Math.max(1, Math.min(3, Number(args?.scale ?? 1)))
1286
2740
  let clip = undefined
1287
2741
  if (args?.ref !== undefined) {
1288
2742
  const spot = await findAgain(page, Number(args.ref))
1289
- if (spot.gone === true) return REF_GONE
2743
+ if (spot.gone === true) return spot.why ?? REF_GONE
1290
2744
  /* The picture is of the control, not of a guess around its middle:
1291
2745
  a canvas can be most of the page, a checkbox a few pixels. */
1292
2746
  const box = await evaluate(page, bounds(Number(args.ref)))
1293
2747
  const rect = typeof box === 'string' ? JSON.parse(box) : box
1294
- if (rect?.gone === true) return REF_GONE
2748
+ if (rect?.gone === true) return spot.why ?? REF_GONE
1295
2749
  const pad = 8
1296
2750
  clip = {
1297
2751
  x: Math.max(0, Math.round(Number(rect.x) - pad)),
@@ -1301,16 +2755,26 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1301
2755
  }
1302
2756
  }
1303
2757
  const data = await page.shot({ full: args?.full === true, rect: clip, scale })
1304
- const dir = join(homedir(), SHOT_DIR)
1305
- await mkdir(dir, { recursive: true })
1306
- const name = String(args?.name ?? '').replace(/[^\w.-]/g, '') || `sidecar-${Date.now()}`
1307
- const file = join(dir, name.endsWith('.png') ? name : `${name}.png`)
1308
- await writeFile(file, Buffer.from(data, 'base64'))
2758
+ const file = await saveArtifact('shots', data, { name: args?.name, prefix: 'shot' })
2759
+ taken = { data, file }
1309
2760
  return file
1310
- }),
2761
+ })
2762
+ if (taken === undefined) return said
2763
+ let image
2764
+ try {
2765
+ image = await host.showShot?.(taken.data, taken.file.split('/').pop() ?? 'shot.png', exec) // ui-copy-ignore
2766
+ } catch {
2767
+ image = undefined
2768
+ }
2769
+ /* The route may not carry pictures at all: then the file path is all
2770
+ this call can give, and saying so is better than a silent path the
2771
+ next step assumes was looked at. */
2772
+ if (image === undefined) return { text: `${said.text}\nno picture reached this result — browser_view reads the words here, browser_grab returns a picture as a file` }
2773
+ return { text: said.text, image }
2774
+ },
1311
2775
  },
1312
2776
  {
1313
- name: 'sidecar_grab', // ui-copy-ignore
2777
+ name: 'browser_grab', // ui-copy-ignore
1314
2778
  description: 'Save the picture on the page as a file: an image, a chart, a drawing surface. Picks the largest one, or the one inside a given control.',
1315
2779
  parameters: {
1316
2780
  type: 'object', // ui-copy-ignore
@@ -1334,9 +2798,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1334
2798
  found = { pictures: [] }
1335
2799
  }
1336
2800
  const rows = []
1337
- const dir = join(homedir(), SHOT_DIR)
1338
- await mkdir(dir, { recursive: true })
1339
- const base = String(args?.name ?? '').replace(/[^\w.-]/g, '') || `sidecar-picture-${Date.now()}`
2801
+ const base = String(args?.name ?? '').replace(/[^\w.-]/g, '')
1340
2802
  const list = Array.isArray(found?.pictures) ? found.pictures : []
1341
2803
  for (let index = 0; index < list.length; index += 1) {
1342
2804
  const item = list[index]
@@ -1345,8 +2807,11 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1345
2807
  if (mark < 0) continue
1346
2808
  const type = String(item?.type ?? 'image/png')
1347
2809
  const ext = type.includes('jpeg') ? 'jpg' : type.includes('webp') ? 'webp' : type.includes('svg') ? 'svg' : 'png'
1348
- const file = join(dir, `${base}${list.length > 1 ? `-${index + 1}` : ''}.${ext}`)
1349
- await writeFile(file, Buffer.from(body.slice(mark + 1), 'base64'))
2810
+ const file = await saveArtifact('pictures', body.slice(mark + 1), {
2811
+ name: base === '' ? '' : `${base}${list.length > 1 ? `-${index + 1}` : ''}`,
2812
+ ext: `.${ext}`,
2813
+ prefix: 'picture',
2814
+ })
1350
2815
  const size = `${Math.round(Number(item.width) || 0)}x${Math.round(Number(item.height) || 0)}`
1351
2816
  /* Say what the picture looks like in words: a model that cannot open
1352
2817
  a file still needs to know a chart came back blank. */
@@ -1362,7 +2827,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1362
2827
  }),
1363
2828
  },
1364
2829
  {
1365
- name: 'sidecar_state', // ui-copy-ignore
2830
+ name: 'browser_state', // ui-copy-ignore
1366
2831
  description: 'Report the sidecar browser: whether it runs, its profile, address, title, how many people are watching its mirror, and how many times a call had to find a control again after the page redrew it.',
1367
2832
  parameters: { type: 'object', properties: {}, additionalProperties: false },
1368
2833
  output: TEXT_OUTPUT, // ui-copy-ignore
@@ -1372,13 +2837,24 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1372
2837
  execute: async () => {
1373
2838
  const state = await sidecar.state()
1374
2839
  const owner = String(host.owner?.() ?? '') // ui-copy-ignore
2840
+ const session = currentSession()
2841
+ const seats = String(host.seats?.() ?? '?') // ui-copy-ignore
2842
+ const mine = sidecar.pageFor(session)
1375
2843
  const rows = [
1376
2844
  state.alive ? 'running' : 'stopped',
2845
+ `chat ${session === '' ? '(no chat id)' : session}`,
2846
+ `this chat's tab ${mine === null ? 'none' : String(mine.url)}`, // ui-copy-ignore
2847
+ `tabs in the browser ${String(sidecar.pages.size)}`,
2848
+ `tab list ${[...sidecar.pages.entries()].map(([key, each]) => `${key.slice(-8)}=${String(each.url)}=${each.cdp?.closed === true ? 'closed' : 'open'}`).join(' | ')}`, // ui-copy-ignore
2849
+ `default tab ${String(sidecar.page?.url)}`, // ui-copy-ignore
1377
2850
  state.headless === false ? 'showing a window' : 'no window',
2851
+ sidecar.flash ? 'Flash-era runtime' : 'Chrome runtime',
1378
2852
  `profile ${String(state.profile)}`,
1379
2853
  `url ${String(state.url)}`,
1380
2854
  `title ${String(state.title)}`,
1381
2855
  `viewers ${String(state.viewers)}`,
2856
+ `listening chats ${seats}`, // ui-copy-ignore
2857
+ `mirror owner ${owner === '' ? 'none' : owner}`, // ui-copy-ignore
1382
2858
  owner === '' ? 'no owner' : `owner ${owner}`,
1383
2859
  `refs followed ${String(follows.count)}`,
1384
2860
  holdText(sidecar) === '' ? 'person free' : `person using it (${String(Math.ceil(Number(sidecar.holding()) / 1000))} s left)`,
@@ -1389,7 +2865,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1389
2865
  },
1390
2866
 
1391
2867
  {
1392
- name: 'sidecar_resume', // ui-copy-ignore
2868
+ name: 'browser_resume', // ui-copy-ignore
1393
2869
  description: 'Take the page back from the person at the mirror: clears the hold so the next action is sent at once, instead of waiting the few seconds out.',
1394
2870
  parameters: { type: 'object', properties: {}, additionalProperties: false },
1395
2871
  output: TEXT_OUTPUT, // ui-copy-ignore
@@ -1402,7 +2878,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1402
2878
  },
1403
2879
  },
1404
2880
  {
1405
- name: 'sidecar_downloads', // ui-copy-ignore
2881
+ name: 'browser_downloads', // ui-copy-ignore
1406
2882
  description: 'List what this browser has downloaded, newest first, with the file name, its size and when it arrived.',
1407
2883
  parameters: { type: 'object', properties: {}, additionalProperties: false },
1408
2884
  output: TEXT_OUTPUT, // ui-copy-ignore
@@ -1417,7 +2893,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1417
2893
  },
1418
2894
 
1419
2895
  {
1420
- name: 'sidecar_drag', // ui-copy-ignore
2896
+ name: 'browser_drag', // ui-copy-ignore
1421
2897
  description: 'Drag from one place to another with real mouse steps: from a ref or a point, to a point or offset. Use it for sliders, dashboards and drag-and-drop lists.',
1422
2898
  parameters: {
1423
2899
  type: 'object', // ui-copy-ignore
@@ -1440,7 +2916,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1440
2916
  let start = { x: Number(args?.x ?? NaN), y: Number(args?.y ?? NaN) }
1441
2917
  if (args?.from !== undefined) {
1442
2918
  const spot = await findAgain(page, Number(args.from))
1443
- if (spot.gone === true) return REF_GONE
2919
+ if (spot.gone === true) return spot.why ?? REF_GONE
1444
2920
  start = { x: Number(spot.x), y: Number(spot.y) }
1445
2921
  }
1446
2922
  const end = { x: Number(args?.to_x ?? NaN), y: Number(args?.to_y ?? NaN) }
@@ -1460,12 +2936,12 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1460
2936
  }),
1461
2937
  },
1462
2938
  {
1463
- name: 'sidecar_check', // ui-copy-ignore
2939
+ name: 'browser_check', // ui-copy-ignore
1464
2940
  description: 'Set a checkbox, radio or switch to a known state, and say whether it took. Clicking blindly flips a control that is already right; this one looks first.',
1465
2941
  parameters: {
1466
2942
  type: 'object', // ui-copy-ignore
1467
2943
  properties: {
1468
- ref: { type: 'integer', description: 'Control number from sidecar_view.' },
2944
+ ref: { type: 'integer', description: 'Control number from browser_view.' },
1469
2945
  checked: { type: 'boolean', description: 'Wanted state. Omit to flip it.' },
1470
2946
  read: { type: 'boolean', description: 'Read the page after.' },
1471
2947
  },
@@ -1485,11 +2961,11 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1485
2961
  if (checkRoute?.contextId !== undefined) checkParams.contextId = checkRoute.contextId
1486
2962
  const probe = await page.cdp.send('Runtime.evaluate', checkParams, checkRoute?.sessionId)
1487
2963
  const now = probe.result?.value
1488
- if (now === null || now === undefined) return REF_GONE
2964
+ if (now === null || now === undefined) return spot.why ?? REF_GONE
1489
2965
  const want = args?.checked === undefined ? !Boolean(now.checked) : args.checked === true
1490
2966
  if (Boolean(now.checked) !== want) {
1491
2967
  const spot = await findAgain(page, Number(args.ref))
1492
- if (spot.gone === true) return REF_GONE
2968
+ if (spot.gone === true) return spot.why ?? REF_GONE
1493
2969
  await page.mouse('mouseMoved', { x: Number(spot.x), y: Number(spot.y) })
1494
2970
  await page.mouse('mousePressed', { x: Number(spot.x), y: Number(spot.y) })
1495
2971
  await page.mouse('mouseReleased', { x: Number(spot.x), y: Number(spot.y) })
@@ -1501,7 +2977,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1501
2977
  }),
1502
2978
  },
1503
2979
  {
1504
- name: 'sidecar_wait', // ui-copy-ignore
2980
+ name: 'browser_wait', // ui-copy-ignore
1505
2981
  description: 'Wait until the page shows a piece of text, stops showing it, reaches an address, or goes quiet. Cheaper than reading in a loop.',
1506
2982
  parameters: {
1507
2983
  type: 'object', // ui-copy-ignore
@@ -1537,7 +3013,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1537
3013
  }),
1538
3014
  },
1539
3015
  {
1540
- name: 'sidecar_dialog', // ui-copy-ignore
3016
+ name: 'browser_dialog', // ui-copy-ignore
1541
3017
  description: 'Answer the alert, confirm or prompt the page is showing, or read what it asks. A dialog blocks the page until it is answered, so nothing else works while one is open.',
1542
3018
  parameters: {
1543
3019
  type: 'object', // ui-copy-ignore
@@ -1565,7 +3041,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1565
3041
  }),
1566
3042
  },
1567
3043
  {
1568
- name: 'sidecar_console', // ui-copy-ignore
3044
+ name: 'browser_console', // ui-copy-ignore
1569
3045
  description: 'The console lines and errors this page has produced, newest last. Use it when a click seemed to do nothing.',
1570
3046
  parameters: {
1571
3047
  type: 'object', // ui-copy-ignore
@@ -1583,7 +3059,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1583
3059
  }),
1584
3060
  },
1585
3061
  {
1586
- name: 'sidecar_failed', // ui-copy-ignore
3062
+ name: 'browser_failed', // ui-copy-ignore
1587
3063
  description: 'Requests this page tried and that failed, newest last. A blank area is often one of these.',
1588
3064
  parameters: {
1589
3065
  type: 'object', // ui-copy-ignore
@@ -1601,7 +3077,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1601
3077
  }),
1602
3078
  },
1603
3079
  {
1604
- name: 'sidecar_cookie',
3080
+ name: 'browser_cookie',
1605
3081
  description: 'Read or change the cookies this browser keeps. Login state is a cookie, so this is how you check a sign-in and how you keep one.',
1606
3082
  parameters: {
1607
3083
  type: 'object', // ui-copy-ignore
@@ -1645,7 +3121,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1645
3121
  }),
1646
3122
  },
1647
3123
  {
1648
- name: 'sidecar_clipboard', // ui-copy-ignore
3124
+ name: 'browser_clipboard', // ui-copy-ignore
1649
3125
  description: 'Read the sidecar browser\'s clipboard, or put text on it and paste it with a key press. For a page that only copies through a button, click it then read here.',
1650
3126
  parameters: {
1651
3127
  type: 'object', // ui-copy-ignore
@@ -1668,7 +3144,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1668
3144
  }),
1669
3145
  },
1670
3146
  {
1671
- name: 'sidecar_network', // ui-copy-ignore
3147
+ name: 'browser_network', // ui-copy-ignore
1672
3148
  description: 'List the responses the page has received, newest first: method, status, type and address. Use it when a page fills itself from the network and the screen does not say why.',
1673
3149
  parameters: {
1674
3150
  type: 'object', // ui-copy-ignore
@@ -1694,7 +3170,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1694
3170
  },
1695
3171
 
1696
3172
  {
1697
- name: 'sidecar_tabs', // ui-copy-ignore
3173
+ name: 'browser_tabs', // ui-copy-ignore
1698
3174
  description: 'List the pages open in the browser, switch the mirror to another one, or close one. Use it when a site opens a link in a tab of its own.',
1699
3175
  parameters: {
1700
3176
  type: 'object', // ui-copy-ignore
@@ -1711,25 +3187,29 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1711
3187
  execute: async (args) => withRawPage(async (page) => {
1712
3188
  const action = String(args?.action ?? 'list')
1713
3189
  if (action === 'list') {
1714
- const rows = await sidecar.tabs()
3190
+ const rows = await sidecar.tabs(currentSession())
1715
3191
  if (rows.length === 0) return 'no tabs'
1716
3192
  return rows.map((row, i) => `${i}${row.current ? ' *' : ''} ${row.title || row.url} · ${row.url}`).join('\n')
1717
3193
  }
1718
3194
  const index = Number(args?.index ?? 0)
1719
3195
  if (action === 'switch') {
1720
- const next = await sidecar.useTab(index)
3196
+ const next = await sidecar.useTab(index, currentSession())
1721
3197
  return `tab ${index} · ${String(next?.url ?? '')}`
1722
3198
  }
1723
3199
  if (action === 'close') {
1724
- await sidecar.closeTab(index)
1725
- return `closed tab ${index} · ${String(sidecar.page?.url ?? '')}`
3200
+ /* Say what was closed, not what is left: after the close the page
3201
+ that was at that row is gone and reads back as about:blank. */
3202
+ const before = await sidecar.tabs(currentSession())
3203
+ const gone = String(before[index]?.url ?? '')
3204
+ await sidecar.closeTab(index, currentSession())
3205
+ return `closed tab ${index} · ${gone}`
1726
3206
  }
1727
3207
  throw new Error(`sidecar: unknown action ${action}`)
1728
3208
  }),
1729
3209
  },
1730
3210
 
1731
3211
  {
1732
- name: 'sidecar_pdf', // ui-copy-ignore
3212
+ name: 'browser_pdf', // ui-copy-ignore
1733
3213
  description: 'Print the page to a PDF and return its path. Useful when the person wants the document rather than a picture of it.',
1734
3214
  parameters: { type: 'object', properties: {}, additionalProperties: false },
1735
3215
  output: TEXT_OUTPUT, // ui-copy-ignore
@@ -1738,11 +3218,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
1738
3218
  /** @returns {Promise<{text: string}>} where the PDF is. */
1739
3219
  execute: async () => withRawPage(async (page) => {
1740
3220
  const data = await page.pdf()
1741
- const dir = join(homedir(), SHOT_DIR)
1742
- await mkdir(dir, { recursive: true })
1743
- const file = join(dir, `sidecar-${Date.now()}.pdf`)
1744
- await writeFile(file, Buffer.from(data, 'base64'))
1745
- return file
3221
+ return saveArtifact('pdf', data, { ext: '.pdf', prefix: 'page' })
1746
3222
  }),
1747
3223
  },
1748
3224
  ]