dsh-inner-browser-use 0.13.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/tools.js ADDED
@@ -0,0 +1,1652 @@
1
+ /**
2
+ * The model-facing half: what a call can do to the sidecar browser, and what it
3
+ * says back.
4
+ *
5
+ * Two costs decide whether this is usable: the tokens a read spends, and the
6
+ * round trips a task spends. The tool surface here is shaped around both — a
7
+ * read can be narrowed (`mode`, `max_chars`) and diffed (`changed_only`), and
8
+ * `sidecar_do` runs a run of steps in one call with waits that know when a page
9
+ * has settled.
10
+ */
11
+
12
+ import { homedir } from 'node:os'
13
+ import { join } from 'node:path'
14
+ import { mkdir, writeFile } from 'node:fs/promises'
15
+ import { VIEW } from './sidecar.js?v=1'
16
+ import { readPage, renderPage, locate, covering, contextFor, identityOf, sameThing, labelSpot, REF_GONE } from './read.js?v=1'
17
+
18
+ /** Where a screenshot from `sidecar_shot` lands, under the user's home. */
19
+ export const SHOT_DIR = join('.dsh-inner-browser-use', 'screenshots')
20
+
21
+ /** How many times a call found a control again after the page redrew it. */
22
+ const follows = { count: 0 }
23
+
24
+ /** Shown when the sidecar has never been started in this run. */
25
+ export const SIDECAR_OFF = 'The sidecar browser is not running. Call sidecar_open with the address first.'
26
+
27
+ /** Output schema every tool returns. */
28
+ const TEXT_OUTPUT = {
29
+ schema: { type: 'object', properties: { text: { type: 'string' } }, required: ['text'], additionalProperties: false },
30
+ /**
31
+ * @param {unknown} _args - call arguments, unused.
32
+ * @param {unknown} value - tool value.
33
+ * @returns {Array<{type: string, text: string}>} blocks.
34
+ */
35
+ render(_args, value) {
36
+ return [{ type: 'text', text: String(value?.text ?? '') }]
37
+ },
38
+ }
39
+
40
+
41
+ /**
42
+ * One line about where the page now is. A step that worked says so in a few
43
+ * words; the model reads the page only when it needs to, which is most of the
44
+ * saving over reading after every action.
45
+ * @param {any} page - sidecar page.
46
+ * @returns {Promise<string>} address, title, focus and any open dialog.
47
+ */
48
+ async function brief(page) {
49
+ const dialog = page.dialog
50
+ if (dialog !== null && dialog !== undefined) {
51
+ return `${String(page.url ?? '')} · dialog ${String(dialog.type ?? '')}: ${String(dialog.message ?? '')}`
52
+ }
53
+ const info = await evaluate(page, `(() => {
54
+ const a = document.activeElement
55
+ const focus = a !== null && a !== document.body && a !== document.documentElement
56
+ ? { name: String(a.getAttribute('aria-label') || a.getAttribute('placeholder') || (a.labels && a.labels[0] && a.labels[0].innerText) || a.innerText || a.name || a.tagName).trim().slice(0, 60), value: String(a.value === undefined ? '' : a.value).slice(0, 60) }
57
+ : null
58
+ return { url: location.href, title: document.title.slice(0, 80), focus }
59
+ })()`)
60
+ const bits = [String(info?.url ?? '')]
61
+ if (String(info?.title ?? '') !== '') bits.push(String(info.title))
62
+ if (info?.focus) bits.push(`focus ${info.focus.name}${info.focus.value === '' ? '' : ` = ${info.focus.value}`}`)
63
+ return bits.join(' · ')
64
+ }
65
+
66
+ /**
67
+ * What an action returns: a short line, or the whole read when asked for one.
68
+ * @param {any} page - sidecar page.
69
+ * @param {any} args - tool arguments.
70
+ * @param {() => Promise<string>} look - full read.
71
+ * @param {string} label - what was done.
72
+ * @returns {Promise<string>} reply text.
73
+ */
74
+ async function acted(page, args, look, label) {
75
+ if (args?.read === true) return await look()
76
+ return `ok ${label} · ${await brief(page)}`
77
+ }
78
+
79
+ /**
80
+ * Say the network failed, instead of reading Chrome's own error page as if it
81
+ * were the site. Those pages mention proxies and firewalls, and they answer
82
+ * every question with something that looks like content — a read returns the
83
+ * text "This site can't be reached", a click lands on "Reload". A model that
84
+ * treats that as the page will report a broken site, or worse, act on it.
85
+ * @param {any} page - sidecar page.
86
+ * @returns {Promise<string>} a sentence, or an empty string when the page is real.
87
+ */
88
+ /**
89
+ * Whether the tab is sitting on one of Chrome's own error pages, without asking
90
+ * the page anything.
91
+ * @param {any} page - sidecar page.
92
+ * @returns {boolean} true while the browser is showing its own failure page.
93
+ */
94
+ /** Titles a site shows while it asks the browser to prove a person is there. */
95
+ const CHECK_TITLES = ['请稍候', 'just a moment', 'checking your browser', 'verifying you are human', 'attention required']
96
+
97
+ const CHECK_NOTE = 'That is the site asking a browser to prove a person is there, and a windowless one never gets past it. sidecar_open with headless false opens a window on the same profile, with the same logins.'
98
+
99
+ /**
100
+ * A check page: the site has not refused, it is asking for proof. Reading it
101
+ * again cannot help, and neither can a click: the answer is a window.
102
+ * @param {{url?: string, title?: string}} read - what the page says it is.
103
+ * @returns {boolean} whether this is a check page.
104
+ */
105
+ /**
106
+ * What to say when nothing is drawing the mirror.
107
+ *
108
+ * The picture lives in the column of the chat that asked for it, and that tab
109
+ * only exists once the page has loaded the plugin's drawing half. A page that
110
+ * was open before the plugin was installed draws nothing, so the browser runs
111
+ * unseen behind the window — which is exactly when a person says "I cannot see
112
+ * it". Naming it, with the one thing that fixes it, beats a bare "viewers 0".
113
+ */
114
+ export const NO_VIEWER = 'Nothing is drawing the mirror in the column yet — this page was loaded before the mirror was, so refresh it once. The window on screen is the same browser.'
115
+
116
+ /**
117
+ * The sentence for how many tabs are drawing the mirror.
118
+ * @param {number} viewers - tabs drawing frames.
119
+ * @returns {string} a line to append, or '' when something is watching.
120
+ */
121
+ export function viewerNote(viewers) {
122
+ return Number(viewers) > 0 ? '' : NO_VIEWER
123
+ }
124
+
125
+ /**
126
+ * How many tabs end up drawing the mirror, once the column has had a moment.
127
+ * The drawing half attaches while the page is still arriving, so asking once,
128
+ * too early, would always answer nothing.
129
+ * @param {any} sidecar - the browser driver.
130
+ * @param {any} page - sidecar page.
131
+ * @param {number} ms - how long to give it.
132
+ * @returns {Promise<number>} tabs drawing the mirror.
133
+ */
134
+ async function watchForViewer(sidecar, page, ms) {
135
+ const until = Date.now() + ms
136
+ for (;;) {
137
+ const seen = Number(page?.frames?.size ?? sidecar?.page?.frames?.size ?? 0)
138
+ if (seen > 0 || Date.now() >= until) return seen
139
+ await wait(80)
140
+ }
141
+ }
142
+
143
+ export function isCheckPage(read) {
144
+ const where = String(read?.url ?? '')
145
+ if (where.includes('__cf_chl')) return true
146
+ const title = String(read?.title ?? '').toLowerCase()
147
+ return CHECK_TITLES.some((word) => title.includes(word))
148
+ }
149
+
150
+ export function isErrorPage(page) {
151
+ const url = String(page?.url ?? '')
152
+ return url.startsWith('chrome-error://') || url.startsWith('edge-error://')
153
+ }
154
+
155
+ export async function unreachable(page) {
156
+ if (!isErrorPage(page)) return ''
157
+ /* The code is the part a model can act on: ERR_NAME_NOT_RESOLVED is a typo,
158
+ ERR_PROXY_CONNECTION_FAILED is this machine's proxy, ERR_TIMED_OUT is slow. */
159
+ let detail = String(page.title ?? '')
160
+ try {
161
+ const said = await evaluate(page, `(() => {
162
+ const text = String(document.body ? document.body.innerText : '')
163
+ const code = (String(text).match(/ERR_[A-Z0-9_]+/) || [''])[0]
164
+ return code === '' ? String(document.title || '') : code
165
+ })()`)
166
+ if (typeof said === 'string' && said.trim() !== '') detail = said.trim()
167
+ } catch {
168
+ /* The error page itself refused to be read; the address still says enough. */
169
+ }
170
+ return `the network could not reach that address — the browser stopped on its own error page (${detail === '' ? 'no details' : detail}), which is not the site. Nothing on it is page content. Use sidecar_nav with another address, or fix the connection this machine needs.`
171
+ }
172
+
173
+ /** Key names this driver can press, spelled as a model would. */
174
+ export const KEYS = {
175
+ Enter: { key: 'Enter', code: 'Enter', keyCode: 13, text: '\r' },
176
+ Tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
177
+ Escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
178
+ Backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
179
+ Delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
180
+ ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
181
+ ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
182
+ ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
183
+ ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 },
184
+ Home: { key: 'Home', code: 'Home', keyCode: 36 },
185
+ End: { key: 'End', code: 'End', keyCode: 35 },
186
+ PageUp: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
187
+ PageDown: { key: 'PageDown', code: 'PageDown', keyCode: 34 },
188
+ Space: { key: ' ', code: 'Space', keyCode: 32, text: ' ' },
189
+ }
190
+
191
+ /**
192
+ * Press a key: keyDown, then keyUp, with the character when the key types one.
193
+ * @param {any} page - sidecar page.
194
+ * @param {string} name - key name.
195
+ * @param {number} [modifiers] - CDP modifier bits: Alt 1, Ctrl 2, Meta 4, Shift 8.
196
+ * @returns {Promise<void>} done.
197
+ */
198
+ export async function pressKey(page, name, modifiers = 0) {
199
+ const key = KEYS[name] ?? KEYS[name.charAt(0).toUpperCase() + name.slice(1)]
200
+ if (key === undefined) throw new Error(`sidecar: no such key ${name}`) // ui-copy-ignore
201
+ const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers }
202
+ await page.key('keyDown', key.text === undefined ? base : { ...base, text: key.text })
203
+ await page.key('keyUp', base)
204
+ }
205
+
206
+ /**
207
+ * Empty the field that has focus, whatever this machine calls select-all:
208
+ * Command on macOS, Control elsewhere. A page can swallow the shortcut, so any
209
+ * remaining text is selected in the page instead, ready for the next insert.
210
+ * @param {any} page - sidecar page.
211
+ * @returns {Promise<void>} the field is empty, or its text is selected.
212
+ */
213
+ export async function clearFocus(page) {
214
+ const command = process.platform === 'darwin' ? 4 : 2 // ui-copy-ignore
215
+ const chord = { key: 'a', code: 'KeyA', windowsVirtualKeyCode: 65, nativeVirtualKeyCode: 65, modifiers: command }
216
+ await page.key('keyDown', chord)
217
+ await page.key('keyUp', chord)
218
+ await pressKey(page, 'Backspace')
219
+ const leftover = await page.cdp.send('Runtime.evaluate', {
220
+ expression: `(() => {
221
+ const field = document.activeElement
222
+ if (field === null) return false
223
+ const text = typeof field.value === 'string' ? field.value : (field.isContentEditable === true ? field.textContent : '')
224
+ if (String(text ?? '') === '') return false
225
+ if (typeof field.select === 'function') field.select()
226
+ else if (field.isContentEditable === true) document.execCommand('selectAll')
227
+ return true
228
+ })()`,
229
+ returnByValue: true,
230
+ }).catch(() => null)
231
+ if (leftover?.result?.value === true) return
232
+ }
233
+
234
+ /**
235
+ * Say the person touched the mirror themselves since the last tool call, once.
236
+ * The agent shares the page with them, so it should read what they left rather
237
+ * than assume the page is where its own last step put it.
238
+ * @param {any} sidecar - the browser driver.
239
+ * @returns {string} a sentence to append, or an empty string.
240
+ */
241
+ export function personNote(sidecar) {
242
+ const person = sidecar?.person
243
+ if (person === undefined || person === null || person.at === 0 || person.seen === true) return '' // ui-copy-ignore
244
+ person.seen = true
245
+ const age = Date.now() - person.at
246
+ if (age > 120_000) return ''
247
+ const when = age < 3000 ? 'just now' : `${Math.round(age / 1000)} s ago`
248
+ const at = person.kind === 'click' || person.kind === 'wheel' ? ` at ${Math.round(person.x)},${Math.round(person.y)}` : '' // ui-copy-ignore
249
+ return `\nthe person also used the page: ${person.kind}${at}, ${when}`
250
+ }
251
+
252
+ /**
253
+ * What an action says when the person is using the mirror right now.
254
+ *
255
+ * The pane is theirs as much as the agent's: someone filling a field by hand
256
+ * should not have the agent click somewhere else under their fingers. Hovering
257
+ * is not taking the page, only real input is, and the hold is short, so a stray
258
+ * click costs a few seconds rather than a run.
259
+ * @param {any} sidecar - the browser driver.
260
+ * @returns {string} a refusal, or '' when the agent may act.
261
+ */
262
+ export function holdText(sidecar) {
263
+ const left = Number(sidecar?.holding?.() ?? 0)
264
+ if (left <= 0) return ''
265
+ return `the person is using the mirror (${String(Math.ceil(left / 1000))} s more) — nothing was sent. Let them finish, then repeat this call; sidecar_resume takes the page back now.`
266
+ }
267
+
268
+ /**
269
+ * Ask the page what is sitting over a control, and which control would take
270
+ * that away.
271
+ * @param {any} page - sidecar page.
272
+ * @param {number} ref - ref that was about to be used.
273
+ * @returns {Promise<null|{tag: string, name: string, dismiss: null|{x: number, y: number, name: string}}>} the thing on top.
274
+ */
275
+ async function coverProbe(page, ref) {
276
+ if (!(Number(ref) > 0) || Number(ref) >= 900_000) return null
277
+ const said = await page.cdp.send('Runtime.evaluate', { expression: covering(Number(ref)), returnByValue: true }).catch(() => null)
278
+ const value = String(said?.result?.value ?? '')
279
+ if (value === '') return null
280
+ let top = null
281
+ try {
282
+ top = JSON.parse(value)
283
+ } catch (error) {
284
+ return null
285
+ }
286
+ if (top === null || typeof top !== 'object') return null
287
+ const dismiss = top.dismiss !== null && typeof top.dismiss === 'object' && Number.isFinite(Number(top.dismiss.x))
288
+ ? { x: Number(top.dismiss.x), y: Number(top.dismiss.y), name: String(top.dismiss.name ?? '') }
289
+ : null
290
+ return { tag: String(top.tag ?? ''), name: String(top.text ?? ''), dismiss } // ui-copy-ignore
291
+ }
292
+
293
+ /**
294
+ * Say when something else is sitting over the control, because a click that
295
+ * lands under a banner looks exactly like a click that did nothing.
296
+ * @param {ReturnType<typeof coverProbe>} probe - what the page said is on top.
297
+ * @returns {string} a sentence, or an empty string.
298
+ */
299
+ function coverNote(probe) {
300
+ if (probe === null || probe === undefined) return ''
301
+ const what = probe.tag === 'div' || probe.tag === 'section' ? 'an overlay' : `a ${probe.tag}`
302
+ return `
303
+ ${what} covers it${probe.name === '' ? '' : `: ${probe.name}`} — the click landed beneath it`
304
+ }
305
+
306
+ /**
307
+ * Clear a cookie banner or modal the way the person would, so the click the
308
+ * agent meant still happens. Only a small safe set of words counts as a way
309
+ * out — an overlay offering to pay, delete or sign out is reported, never
310
+ * accepted on the person's behalf.
311
+ * @param {any} page - sidecar page.
312
+ * @param {number} ref - ref the agent meant to use.
313
+ * @returns {Promise<{note: string, cleared: boolean, name: string}>} what happened.
314
+ */
315
+ async function clearCover(page, ref) {
316
+ const probe = await coverProbe(page, ref)
317
+ if (probe === null) return { note: '', cleared: true, name: '' }
318
+ const note = coverNote(probe)
319
+ const away = probe.dismiss
320
+ if (away === null) return { note, cleared: false, name: '' }
321
+ await page.mouse('mouseMoved', { x: away.x, y: away.y })
322
+ await page.mouse('mousePressed', { x: away.x, y: away.y, clickCount: 1 })
323
+ await page.mouse('mouseReleased', { x: away.x, y: away.y, clickCount: 1 })
324
+ await wait(150)
325
+ const again = await coverProbe(page, ref)
326
+ if (again !== null) return { note: `${note}
327
+ - clicking ${away.name} on it did not clear it`, cleared: false, name: away.name } // ui-copy-ignore
328
+ return { note: `${note}
329
+ - clicked ${away.name} on it first, then the control`, cleared: true, name: away.name } // ui-copy-ignore
330
+ }
331
+
332
+ /**
333
+ * The same clearing, said in one clause so a numbered step stays one line.
334
+ * @param {any} page - sidecar page.
335
+ * @param {number} ref - ref the step was about to use.
336
+ * @returns {Promise<string>} a prefix like `after Accept on a banner, ` or empty.
337
+ */
338
+ export async function coverLead(page, ref) {
339
+ const gate = await clearCover(page, ref)
340
+ return gate.name === '' ? '' : `after ${gate.name} on a banner, `
341
+ }
342
+
343
+ /** @param {number} ms - how long. */
344
+ function wait(ms) {
345
+ return new Promise((settle) => setTimeout(settle, ms))
346
+ }
347
+
348
+ /**
349
+ * Evaluate an expression and return its value, JSON-safe.
350
+ * @param {any} page - sidecar page.
351
+ * @param {string} expression - JavaScript.
352
+ * @param {{contextId?: number, sessionId?: string}|null} [route] - frame to run in.
353
+ * @returns {Promise<any>} the value.
354
+ */
355
+ async function evaluate(page, expression, route) {
356
+ const params = { expression, returnByValue: true, awaitPromise: true }
357
+ if (route?.contextId !== undefined) params.contextId = route.contextId
358
+ const call = page.cdp.send('Runtime.evaluate', params, route?.sessionId)
359
+ const raced = await Promise.race([call, wait(2000).then(() => 'timeout')])
360
+ if (raced === 'timeout') return undefined
361
+ const { result, exceptionDetails } = raced
362
+ if (exceptionDetails !== undefined) {
363
+ return { error: String(exceptionDetails.exception?.description ?? exceptionDetails.text ?? 'evaluate failed') }
364
+ }
365
+ return result?.value
366
+ }
367
+
368
+ /**
369
+ * Wait until the page has settled: loading finished and the address stopped
370
+ * changing. Acting on a page mid-navigation is how a driver clicks the old page.
371
+ * @param {any} page - sidecar page.
372
+ * @param {number} [timeout] - ceiling in milliseconds.
373
+ * @returns {Promise<string>} the address it settled on.
374
+ */
375
+ export async function settle(page, timeout = 5000) {
376
+ /* A click may have opened a tab. The mirror follows it, so every tool that
377
+ waited on a page ends up on the page the person would be looking at. */
378
+ const opened = typeof page.adoptOpened === 'function' ? await page.adoptOpened().catch(() => '') : ''
379
+ const deadline = Date.now() + timeout
380
+ let last = ''
381
+ let steady = 0
382
+ while (Date.now() < deadline) {
383
+ if (page.dialog !== null && page.dialog !== undefined) return String(page.url ?? '')
384
+ const state = await evaluate(page, '({ ready: document.readyState, href: location.href })')
385
+ const key = `${String(state?.ready)}|${String(state?.href)}`
386
+ steady = key === last ? steady + 1 : 0
387
+ last = key
388
+ if (state?.ready !== 'loading' && steady >= 2) return String(state?.href ?? '')
389
+ await wait(60)
390
+ }
391
+ return ''
392
+ }
393
+
394
+ /**
395
+ * Poll the page until a condition holds.
396
+ * @param {any} page - sidecar page.
397
+ * @param {string} expression - a JavaScript expression that is true when done.
398
+ * @param {number} timeout - ceiling in milliseconds.
399
+ * @returns {Promise<boolean>} whether it ever held.
400
+ */
401
+ async function until(page, expression, timeout) {
402
+ const deadline = Date.now() + timeout
403
+ while (Date.now() < deadline) {
404
+ const value = await evaluate(page, expression)
405
+ if (value === true) return true
406
+ await wait(100)
407
+ }
408
+ return false
409
+ }
410
+
411
+ /**
412
+ * One step of a batch. Kept small on purpose: every step is one thing a person
413
+ * would do with a mouse or a keyboard.
414
+ * @param {any} page - sidecar page.
415
+ * @param {any} step - the step.
416
+ * @returns {Promise<string>} a compact report.
417
+ */
418
+ /** The first control whose words match, scrolled into view. Text can beat a number: it costs no read. */
419
+ async function findWords(page, words) {
420
+ const read = await readPage(page, { mode: 'text', maxChars: 1200, max: 200 })
421
+ const shown = read.controls.filter((control) => control.view !== false)
422
+ const wanted = words.toLowerCase()
423
+ const hit = shown.find((control) => control.name.toLowerCase() === wanted) ?? shown.find((control) => control.name.toLowerCase().includes(wanted))
424
+ if (hit === undefined) return null
425
+ const spot = await locate(page, hit.ref)
426
+ if (spot.gone === true) return null
427
+ return { ...hit, x: Number(spot.x), y: Number(spot.y) }
428
+ }
429
+
430
+ /** Kinds that take typed text, so a click on one is a click on a field. */
431
+ const EDITABLE_KINDS = ['input', 'textarea', 'select', 'combobox', 'textbox', 'searchbox', 'password', 'search', 'email', 'number', 'tel', 'url', 'date']
432
+
433
+ /**
434
+ * The field the words belong to, when the words are not on the field itself.
435
+ * @param {any} page - sidecar page.
436
+ * @param {string} words - what the field is called on screen.
437
+ * @returns {Promise<{x: number, y: number, name: string}|null>} where to click.
438
+ */
439
+ async function fieldSpot(page, words) {
440
+ const said = await evaluate(page, labelSpot(words))
441
+ if (said.error !== undefined || said.result === undefined) return null
442
+ const spot = said.result
443
+ if (spot === undefined || spot === null || spot.name === undefined) return null
444
+ return { x: Number(spot.x), y: Number(spot.y), name: String(spot.name) }
445
+ }
446
+
447
+ /**
448
+ * Where the control behind a ref is now, following it across a re-render.
449
+ * The module-level sibling of findAgain, for runStep, which has no closure.
450
+ * @param {any} page - sidecar page.
451
+ * @param {number} ref - the number the caller used.
452
+ * @param {any[]} [known] - controls from the read that produced the ref.
453
+ * @returns {Promise<{x?: number, y?: number, gone?: boolean, followed?: boolean, was?: string}>} where it is.
454
+ */
455
+ async function followRef(page, ref, known) {
456
+ const spot = await locate(page, ref)
457
+ if (spot.gone !== true) return spot
458
+ const was = Array.isArray(known) ? known.find((control) => Number(control.ref) === Number(ref)) : undefined
459
+ if (was === undefined) return spot
460
+ const want = identityOf(was)
461
+ if (want.name === '') return spot
462
+ const read = await readPage(page, { mode: 'text', maxChars: 600, max: 400 })
463
+ const again = read.controls.filter((control) => sameThing(want, control))
464
+ if (again.length !== 1) return spot
465
+ const next = await locate(page, Number(again[0].ref))
466
+ if (next.gone === true) return spot
467
+ follows.count += 1
468
+ return { ...next, followed: true, was: want.name }
469
+ }
470
+
471
+ async function runStep(page, step, known) {
472
+ if (step.ref !== undefined) {
473
+ if (step.tap !== undefined || step.text !== undefined) {
474
+ const lead = await coverLead(page, Number(step.ref))
475
+ const spot = await followRef(page, Number(step.ref), known)
476
+ if (spot.gone === true) return `ref ${String(step.ref)}: ${REF_GONE}`
477
+ const after = spot.followed === true ? ` (it was redrawn under a new number, used where it is now)` : ''
478
+ const point = { x: Number(spot.x), y: Number(spot.y) }
479
+ await page.mouse('mouseMoved', point)
480
+ await page.mouse('mousePressed', { ...point, clickCount: 1 })
481
+ await page.mouse('mouseReleased', { ...point, clickCount: 1 })
482
+ if (step.tap !== undefined) return `${lead}clicked ${String(step.ref)} ${String(spot.name ?? '')}`.trim() + after
483
+ await wait(80)
484
+ if (step.clear === true) await clearFocus(page)
485
+ await page.insertText(String(step.text))
486
+ if (step.submit === true) await pressKey(page, 'Enter')
487
+ return `${lead}typed into ${String(step.ref)} ${String(spot.name ?? '')}`.trim() + after
488
+ }
489
+ }
490
+ if (step.tap !== undefined && !/^\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*$/.test(String(step.tap))) {
491
+ const hit = await findWords(page, String(step.tap))
492
+ if (hit === null) return `no control says ${JSON.stringify(String(step.tap))}`
493
+ const lead = await coverLead(page, hit.ref)
494
+ const spot = await locate(page, hit.ref)
495
+ if (spot.gone === true) return `ref ${String(hit.ref)}: ${REF_GONE}`
496
+ const point = { x: Number(spot.x), y: Number(spot.y) }
497
+ await page.mouse('mouseMoved', point)
498
+ await page.mouse('mousePressed', { ...point, clickCount: 1 })
499
+ await page.mouse('mouseReleased', { ...point, clickCount: 1 })
500
+ return `${lead}clicked "${hit.name.slice(0, 60)}"`
501
+ }
502
+ if (step.field !== undefined) {
503
+ /* The words may sit on the field or in a div above a bare input. Only a
504
+ click that lands on the field itself can be trusted with the typing. */
505
+ const hit = await findWords(page, String(step.field))
506
+ let lead = ''
507
+ let point = null
508
+ let called = ''
509
+ if (hit !== null && EDITABLE_KINDS.includes(String(hit.kind))) {
510
+ lead = await coverLead(page, hit.ref)
511
+ const spot = await locate(page, hit.ref)
512
+ if (spot.gone !== true) {
513
+ point = { x: Number(spot.x), y: Number(spot.y) }
514
+ called = String(hit.name)
515
+ }
516
+ }
517
+ if (point === null) {
518
+ const field = await fieldSpot(page, String(step.field))
519
+ if (field !== null) {
520
+ point = { x: field.x, y: field.y }
521
+ called = field.name
522
+ }
523
+ }
524
+ if (point === null) {
525
+ if (hit === null) return `no control says ${JSON.stringify(String(step.field))}`
526
+ const spot = await locate(page, hit.ref)
527
+ if (spot.gone === true) return `ref ${String(hit.ref)}: ${REF_GONE}`
528
+ point = { x: Number(spot.x), y: Number(spot.y) }
529
+ called = String(hit.name)
530
+ }
531
+ await page.mouse('mouseMoved', point)
532
+ await page.mouse('mousePressed', { ...point, clickCount: 1 })
533
+ await page.mouse('mouseReleased', { ...point, clickCount: 1 })
534
+ await wait(80)
535
+ if (step.clear === true) await clearFocus(page)
536
+ await page.insertText(String(step.text ?? ''))
537
+ if (step.submit === true) await pressKey(page, 'Enter')
538
+ return `${lead}typed into "${called.slice(0, 60)}"`
539
+ }
540
+ if (step.tap !== undefined) {
541
+ const [x, y] = String(step.tap).split(',').map(Number)
542
+ const point = { x, y }
543
+ await page.mouse('mouseMoved', point)
544
+ await page.mouse('mousePressed', { ...point, clickCount: 1 })
545
+ await page.mouse('mouseReleased', { ...point, clickCount: 1 })
546
+ return `clicked ${String(x)},${String(y)}`
547
+ }
548
+ if (step.text !== undefined) {
549
+ if (step.clear === true) await clearFocus(page)
550
+ await page.insertText(String(step.text))
551
+ if (step.submit === true) await pressKey(page, 'Enter')
552
+ return `typed ${String(step.text).slice(0, 60)}`
553
+ }
554
+ if (step.key !== undefined) {
555
+ await pressKey(page, String(step.key))
556
+ return `pressed ${String(step.key)}`
557
+ }
558
+ if (step.scroll !== undefined) {
559
+ const dy = Number(step.scroll)
560
+ await page.wheel({ x: VIEW.width / 2, y: VIEW.height / 2, dy })
561
+ return `scrolled ${dy > 0 ? 'down' : 'up'} ${Math.abs(dy)}`
562
+ }
563
+ if (step.wait_text !== undefined) {
564
+ const needle = JSON.stringify(String(step.wait_text))
565
+ const ok = await until(page, `document.body ? document.body.innerText.includes(${needle}) : false`, Number(step.timeout ?? 8000))
566
+ return ok ? `saw ${String(step.wait_text)}` : `did not see ${String(step.wait_text)}`
567
+ }
568
+ if (step.wait_gone !== undefined) {
569
+ const needle = JSON.stringify(String(step.wait_gone))
570
+ const ok = await until(page, `document.body ? !document.body.innerText.includes(${needle}) : true`, Number(step.timeout ?? 8000))
571
+ return ok ? `${String(step.wait_gone)} went away` : `${String(step.wait_gone)} stayed`
572
+ }
573
+ if (step.wait_url !== undefined) {
574
+ const needle = JSON.stringify(String(step.wait_url))
575
+ const ok = await until(page, `location.href.includes(${needle})`, Number(step.timeout ?? 8000))
576
+ return ok ? `at ${String(step.wait_url)}` : `still at a different address`
577
+ }
578
+ if (step.open !== undefined) {
579
+ const landed = await page.navigate(String(step.open))
580
+ return `opened ${landed?.url ?? String(step.open)}`
581
+ }
582
+ if (step.eval !== undefined) {
583
+ const value = await evaluate(page, String(step.eval))
584
+ const text = typeof value === 'string' ? value : JSON.stringify(value)
585
+ return `eval ${String(text).slice(0, 300)}`
586
+ }
587
+ if (step.sleep_ms !== undefined) {
588
+ await wait(Math.min(10_000, Number(step.sleep_ms)))
589
+ return `waited ${String(step.sleep_ms)} ms`
590
+ }
591
+ if (step.wait_idle !== undefined) {
592
+ await settle(page, Number(step.timeout ?? 5000))
593
+ return 'idle'
594
+ }
595
+ return 'nothing to do'
596
+ }
597
+
598
+ /**
599
+ * Read the page and keep the previous read, so `changed_only` has something to
600
+ * compare against.
601
+ * @param {any} page - sidecar page.
602
+ * @param {{refs: any[], at: number, last?: any}} refs - read state for this page.
603
+ * @param {{mode?: string, maxChars?: number}} options - narrowing.
604
+ * @returns {Promise<any>} the read.
605
+ */
606
+ async function remember(page, refs, options) {
607
+ const read = await readPage(page, options)
608
+ refs.refs = read.controls
609
+ refs.at = Date.now()
610
+ return read
611
+ }
612
+
613
+ /**
614
+ * Say only what changed since the previous read.
615
+ * @param {any} previous - the read before.
616
+ * @param {any} next - the read now.
617
+ * @param {{maxChars?: number}} options - narrowing.
618
+ * @returns {string} a diff, small by construction.
619
+ */
620
+ export function diffRead(previous, next, options = {}) {
621
+ const maxChars = Number(options.maxChars ?? 2000)
622
+ const before = new Map(previous.controls.map((control) => [control.ref, control]))
623
+ const after = new Map(next.controls.map((control) => [control.ref, control]))
624
+ const lines = [next.url]
625
+ if (previous.url !== next.url) lines.push(`address changed to ${next.url}`)
626
+ const added = next.controls.filter((control) => !before.has(control.ref))
627
+ const removed = previous.controls.filter((control) => !after.has(control.ref))
628
+ const changed = next.controls.filter((control) => {
629
+ const old = before.get(control.ref)
630
+ if (old === undefined) return false
631
+ return old.name !== control.name || old.value !== control.value || old.checked !== control.checked || old.disabled !== control.disabled
632
+ })
633
+ for (const control of added.slice(0, 40)) lines.push(`+ ${control.ref} [${control.kind}] ${control.name}`)
634
+ for (const control of removed.slice(0, 40)) lines.push(`- ${control.ref} ${control.name}`)
635
+ for (const control of changed.slice(0, 40)) {
636
+ const old = before.get(control.ref)
637
+ lines.push(`~ ${control.ref} ${control.name}${old?.value !== control.value ? `: ${old?.value ?? ''} → ${control.value}` : ''}${old?.checked !== control.checked ? (control.checked ? ' (now checked)' : ' (now unchecked)') : ''}`)
638
+ }
639
+ if (added.length + removed.length + changed.length === 0) lines.push('no control changed')
640
+ if (previous.text !== next.text) {
641
+ const head = next.text.slice(0, Math.max(200, Math.min(1200, maxChars - 800)))
642
+ lines.push('', 'text now:', head)
643
+ if (next.text.length > head.length) lines.push(`… ${next.text.length - head.length} more characters`)
644
+ }
645
+ if (next.focused !== null && previous.focused?.ref !== next.focused.ref) lines.push(`focus ${next.focused.ref} ${next.focused.name}`)
646
+ return lines.join('\n')
647
+ }
648
+
649
+ /**
650
+ * The tool surface.
651
+ * @param {any} sidecar - the sidecar browser.
652
+ * @param {{refs: any[], at: number, last?: any}} refs - read state.
653
+ * @param {{logger?: any}} [host] - host services. // ui-copy-ignore
654
+ * @returns {any[]} tool definitions.
655
+ */
656
+ /**
657
+ * Read what a selector matches as one line per item.
658
+ * A list read this way costs a fraction of the whole page and keeps the model on
659
+ * the items it came for: the text, plus the link when the item has one.
660
+ * @param {any} page - sidecar page.
661
+ * @param {any} args - select, ref, limit.
662
+ * @returns {Promise<string>} one line per row, and how many there were.
663
+ */
664
+ export async function extractRows(page, args) {
665
+ const select = typeof args?.select === 'string' ? args.select.trim() : ''
666
+ if (select === '') return 'Give a selector: what to read, for example "ul.results > li".'
667
+ const ref = Math.trunc(Number(args?.ref ?? 0)) || 0
668
+ const limit = Math.min(200, Math.max(1, Math.trunc(Number(args?.limit ?? 40)) || 40))
669
+ const expression = `(() => {
670
+ const reg = window.__dshRead
671
+ let scope = document
672
+ if (${ref} > 0) {
673
+ const hit = reg !== undefined && typeof reg.find === 'function' ? reg.find(${ref}) : null
674
+ if (hit === null || hit === undefined || hit.el === null || hit.el === undefined) return { gone: true }
675
+ scope = hit.el
676
+ }
677
+ let nodes
678
+ try { nodes = scope.querySelectorAll(${JSON.stringify(select)}) }
679
+ catch { return { bad: true } }
680
+ const rows = []
681
+ for (const node of nodes) {
682
+ if (rows.length >= ${limit}) break
683
+ const raw = typeof node.innerText === 'string' && node.innerText !== '' ? node.innerText : String(node.textContent ?? '')
684
+ const text = raw.replace(/\\s+/g, ' ').trim().slice(0, 200)
685
+ const own = node.matches !== undefined && node.matches('a[href]') === true ? node : null
686
+ const link = own ?? (typeof node.querySelector === 'function' ? node.querySelector('a[href]') : null)
687
+ const full = link !== null && link !== undefined && typeof link.href === 'string' ? link.href : ''
688
+ /* A row on this site says the same with the path alone; the origin is
689
+ already in the result and would be repeated thirty times. */
690
+ const href = full.startsWith(location.origin) ? full.slice(location.origin.length) : full
691
+ rows.push({ text, href })
692
+ }
693
+ return { rows, total: nodes.length }
694
+ })()`
695
+ const out = await page.cdp.send('Runtime.evaluate', { expression, returnByValue: true }).catch((error) => ({ error: String(error?.message ?? error) }))
696
+ if (out?.error !== undefined && out.error !== null && typeof out.error === 'string') return `the page did not answer: ${out.error}`
697
+ const value = out?.result?.value
698
+ if (value === undefined || value === null || value === '') return 'the page did not answer; try again, or read the page with sidecar_view.'
699
+ if (value.gone === true) return `ref ${ref} is gone. Read the page again and use a new number.`
700
+ if (value.bad === true) return `${select} is not a selector this browser understands.`
701
+ const rows = Array.isArray(value.rows) ? value.rows : []
702
+ if (rows.length === 0) return `nothing matched ${select} — the page may still be loading, or the selector is off.`
703
+ const said = rows.map((row, index) => `${index + 1}. ${row.text === '' ? '(empty)' : row.text}${row.href === '' ? '' : ` · ${row.href}`}`)
704
+ const more = Number(value.total) > rows.length ? `\n… ${Number(value.total) - rows.length} more rows` : ''
705
+ return `${rows.length} of ${Number(value.total)} rows matching ${select}:\n${said.join('\n')}${more}`
706
+ }
707
+
708
+ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
709
+ /* Reading, clicking and typing on Chrome's own error page is not driving the
710
+ site: the text there is Chrome talking, the buttons reload. Those calls are
711
+ refused with the error code instead. The tools that report the browser
712
+ itself — the tab list, the console, failed requests, the address — still
713
+ run, because they are how a stuck person finds out what happened. */
714
+ const withRawPage = async (work) => {
715
+ if (!sidecar.alive) return { text: SIDECAR_OFF }
716
+ const page = await sidecar.ensure()
717
+ const text = await work(page)
718
+ return { text: text + personNote(sidecar) }
719
+ }
720
+ /** Looking is still useful while the person has the page. */
721
+ const withRead = async (work) => {
722
+ if (!sidecar.alive) return { text: SIDECAR_OFF }
723
+ const page = await sidecar.ensure()
724
+ const dead = await unreachable(page)
725
+ if (dead !== '') return { text: dead + personNote(sidecar) }
726
+ const text = await work(page)
727
+ return { text: text + personNote(sidecar) }
728
+ }
729
+ const withPage = async (work) => {
730
+ if (!sidecar.alive) return { text: SIDECAR_OFF }
731
+ const hold = holdText(sidecar)
732
+ if (hold !== '') return { text: hold }
733
+ return withRead(work)
734
+ }
735
+ const look = async (page, options) => {
736
+ const read = await remember(page, refs, options)
737
+ const said = renderPage(read, options)
738
+ return isCheckPage(read) ? `${said}\n${CHECK_NOTE}` : said
739
+ }
740
+
741
+ /**
742
+ * Where the control behind a ref is now, following it across a re-render.
743
+ *
744
+ * A ref is a number for an element; a framework that redraws its list throws
745
+ * every element away and makes new ones with a fresh number, so the next call
746
+ * with the old number finds nothing. The page did not become unusable — the
747
+ * control is on screen, saying the same words, usually in the same place.
748
+ * Handing back REF_GONE costs the model a whole round: one read to find the
749
+ * number and one more call to use it. So look the control up by what it is,
750
+ * and only say it is gone when what it is, is gone too.
751
+ * @param {any} page - sidecar page.
752
+ * @param {number} ref - the number the caller used.
753
+ * @returns {Promise<{x?: number, y?: number, gone?: boolean, followed?: boolean, was?: string}>} where it is.
754
+ */
755
+ const findAgain = async (page, ref) => {
756
+ const spot = await locate(page, ref)
757
+ if (spot.gone !== true) return spot
758
+ const known = (refs?.refs ?? []).find((control) => Number(control.ref) === Number(ref))
759
+ if (known === undefined) return spot
760
+ const want = identityOf(known)
761
+ if (want.name === '') return spot
762
+ const read = await readPage(page, { mode: 'text', maxChars: 600, max: 400 })
763
+ const again = read.controls.filter((control) => sameThing(want, control))
764
+ /* Same words twice on one page is a choice only the person can make: two
765
+ "Delete" buttons in different rows are not interchangeable. */
766
+ if (again.length !== 1) return spot
767
+ const next = await locate(page, Number(again[0].ref))
768
+ if (next.gone === true) return spot
769
+ follows.count += 1
770
+ return { ...next, followed: true, was: want.name }
771
+ }
772
+
773
+ return [
774
+ {
775
+ name: 'sidecar_open', // ui-copy-ignore
776
+ description: 'Start (or reuse) the sidecar browser — a real Chrome with its own persistent profile — and open a URL in it. It runs in the background and its screen appears in this chat\'s sidebar tab, so it keeps working while the person looks elsewhere. Logins survive restarts.',
777
+ parameters: {
778
+ type: 'object', // ui-copy-ignore
779
+ properties: {
780
+ url: { type: 'string', description: 'Address to open. Omit to reuse the current page.' },
781
+ headless: { type: 'boolean', description: 'Run without a window. Default true; turn off only if a site refuses a windowless browser.' },
782
+ },
783
+ additionalProperties: false,
784
+ },
785
+ output: TEXT_OUTPUT, // ui-copy-ignore
786
+ timeoutMs: 60_000,
787
+ isConcurrencySafe: () => false,
788
+ /** @param {any} args - arguments. @param {any} exec - tool run. @returns {Promise<{text: string}>} what the page is. */
789
+ execute: async (args, exec) => {
790
+ const page = await sidecar.ensure({ url: args?.url, headless: args?.headless })
791
+ if (args?.url === undefined) await page.readTitle()
792
+ /* A window on the person's screen is their business, not a surprise: say
793
+ it, so nobody wonders what popped up. A site that refuses a windowless
794
+ browser is the only reason this happens. */
795
+ const windowed = sidecar.headless === false ? 'The browser is showing a window this time — a site only opens for a windowed one. Same profile, same logins.\n' : ''
796
+ /* The tab appears in the calling chat and nowhere else: the person may
797
+ be reading a different chat, and a browser there would be a surprise. */
798
+ const who = String(exec?.agent?.id ?? '')
799
+ if (who !== '') host.want?.(who) // ui-copy-ignore
800
+ await settle(page, 8000)
801
+ const dead = await unreachable(page)
802
+ if (dead !== '') return { text: dead }
803
+ /* The column may still be drawing nothing: say so here, where the
804
+ person is looking for a picture, rather than only in sidecar_state. */
805
+ const unseen = who === '' ? '' : viewerNote(await watchForViewer(sidecar, page, 1200))
806
+ return { text: windowed + (unseen === '' ? '' : `${unseen}\n`) + await look(page, { mode: 'text' }) }
807
+ },
808
+ },
809
+ {
810
+ name: 'sidecar_view', // ui-copy-ignore
811
+ description: 'Read the sidecar browser: numbered controls and the words on screen. Numbers stay valid across reads and actions until the page navigates. Narrow it with mode (text, links, forms, full) and max_chars; ask for changed_only after an action to spend fewer tokens.',
812
+ parameters: {
813
+ type: 'object', // ui-copy-ignore
814
+ properties: {
815
+ mode: { type: 'string', enum: ['text', 'links', 'forms', 'full'], description: 'What to include. Default text: controls plus the visible words.' },
816
+ max_chars: { type: 'integer', description: 'Character budget for the page text. Default 6000.' },
817
+ changed_only: { type: 'boolean', description: 'Only what changed since the last read.' },
818
+ },
819
+ additionalProperties: false,
820
+ },
821
+ output: TEXT_OUTPUT, // ui-copy-ignore
822
+ timeoutMs: 30_000,
823
+ isConcurrencySafe: () => false,
824
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the read. */
825
+ execute: async (args) => withRead(async (page) => {
826
+ const options = { mode: String(args?.mode ?? 'text'), maxChars: Number(args?.max_chars ?? 4600) }
827
+ const read = await remember(page, refs, options)
828
+ const note = isCheckPage(read) ? `\n${CHECK_NOTE}` : ''
829
+ if (args?.changed_only === true && refs.last !== undefined) return diffRead(refs.last, read, options) + note
830
+ refs.last = read
831
+ return renderPage(read, options) + note
832
+ }),
833
+ },
834
+ {
835
+ name: 'sidecar_click', // ui-copy-ignore
836
+ description: 'Click a numbered control, a point, or the first control whose words contain `text` — the last needs no read first. The click is real mouse input and the mirror shows where it landed.',
837
+ parameters: {
838
+ type: 'object', // ui-copy-ignore
839
+ properties: {
840
+ ref: { type: 'integer', description: 'Number from sidecar_view.' },
841
+ text: { type: 'string', description: 'Words on the control, when its number is not known.' },
842
+ x: { type: 'integer', description: 'Viewport x, when clicking a point.' },
843
+ y: { type: 'integer', description: 'Viewport y, when clicking a point.' },
844
+ double: { type: 'boolean', description: 'Double-click.' },
845
+ right: { type: 'boolean', description: 'Right-click, opening a context menu.' },
846
+ read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
847
+ },
848
+ additionalProperties: false,
849
+ },
850
+ output: TEXT_OUTPUT, // ui-copy-ignore
851
+ timeoutMs: 30_000,
852
+ isConcurrencySafe: () => false,
853
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
854
+ execute: async (args) => withPage(async (page) => {
855
+ let point = { x: Number(args?.x ?? NaN), y: Number(args?.y ?? NaN) }
856
+ let label = args?.ref === undefined ? `${point.x},${point.y}` : String(args.ref)
857
+ let followed = ''
858
+ if (args?.ref !== undefined) {
859
+ const spot = await findAgain(page, Number(args.ref))
860
+ if (spot.gone === true) return REF_GONE
861
+ if (spot.followed === true) followed = `the page redrew, so "${String(spot.was).slice(0, 50)}" has a new number — used where it is now`
862
+ point = { x: Number(spot.x), y: Number(spot.y) }
863
+ } else if (String(args?.text ?? '') !== '') {
864
+ const hit = await findWords(page, String(args.text))
865
+ if (hit === null) return `No control says ${JSON.stringify(String(args.text))}. Call sidecar_view to see what is there.`
866
+ point = { x: hit.x, y: hit.y }
867
+ label = `"${hit.name.slice(0, 60)}"`
868
+ }
869
+ if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) return 'Give a ref from sidecar_view, or both x and y.'
870
+ const button = args?.right === true ? 'right' : 'left'
871
+ let cover = ''
872
+ if (args?.ref !== undefined && button === 'left') {
873
+ const gate = await clearCover(page, Number(args.ref))
874
+ cover = gate.note
875
+ if (gate.cleared && cover !== '') {
876
+ const again = await locate(page, Number(args.ref))
877
+ if (again.gone !== true) point = { x: Number(again.x), y: Number(again.y) }
878
+ }
879
+ }
880
+ await page.mouse('mouseMoved', point)
881
+ await page.mouse('mousePressed', { ...point, clickCount: args?.double === true ? 2 : 1, button })
882
+ await page.mouse('mouseReleased', { ...point, clickCount: args?.double === true ? 2 : 1, button })
883
+ await settle(page)
884
+ const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 3000 }), `click ${label}`)
885
+ return said + cover + (followed === '' ? '' : ` (${followed})`)
886
+ }),
887
+ },
888
+ {
889
+ name: 'sidecar_type', // ui-copy-ignore
890
+ description: 'Type text into the sidecar browser — Chinese included — optionally pressing Enter. Pass a ref to focus that control first. With clear, the field is emptied first.',
891
+ parameters: {
892
+ type: 'object', // ui-copy-ignore
893
+ properties: {
894
+ text: { type: 'string', description: 'Text to type.' },
895
+ ref: { type: 'integer', description: 'Control to click before typing.' },
896
+ submit: { type: 'boolean', description: 'Press Enter after typing.' },
897
+ clear: { type: 'boolean', description: 'Select all and delete before typing.' },
898
+ read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
899
+ },
900
+ required: ['text'], // ui-copy-ignore
901
+ additionalProperties: false,
902
+ },
903
+ output: TEXT_OUTPUT, // ui-copy-ignore
904
+ timeoutMs: 40_000,
905
+ isConcurrencySafe: () => false,
906
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
907
+ execute: async (args) => withPage(async (page) => {
908
+ let cover = ''
909
+ let followed = ''
910
+ if (args?.ref !== undefined) {
911
+ const gate = await clearCover(page, Number(args.ref))
912
+ cover = gate.note
913
+ const spot = await findAgain(page, Number(args.ref))
914
+ if (spot.gone === true) return REF_GONE + cover
915
+ if (spot.followed === true) followed = `the page redrew, so "${String(spot.was).slice(0, 50)}" has a new number — typed where it is now`
916
+ const point = { x: Number(spot.x), y: Number(spot.y) }
917
+ await page.mouse('mouseMoved', point)
918
+ await page.mouse('mousePressed', { ...point, clickCount: 1 })
919
+ await page.mouse('mouseReleased', { ...point, clickCount: 1 })
920
+ await wait(80)
921
+ }
922
+ if (args?.clear === true) await clearFocus(page)
923
+ await page.insertText(String(args?.text ?? ''))
924
+ if (args?.submit === true) await pressKey(page, 'Enter')
925
+ await settle(page)
926
+ const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type')
927
+ return said + cover + (followed === '' ? '' : ` (${followed})`) // ui-copy-ignore
928
+ }),
929
+ },
930
+ {
931
+ name: 'sidecar_key', // ui-copy-ignore
932
+ description: 'Press one key in the sidecar browser: Enter, Tab, Escape, Backspace, Delete, ArrowUp/ArrowDown/ArrowLeft/ArrowRight, Home, End, PageUp, PageDown, Space.',
933
+ parameters: {
934
+ type: 'object', // ui-copy-ignore
935
+ properties: {
936
+ key: { type: 'string', description: 'Key name.' },
937
+ modifiers: { type: 'string', description: 'Held while pressing: Ctrl, Alt, Shift or Meta, joined by +.' },
938
+ read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
939
+ },
940
+ required: ['key'], // ui-copy-ignore
941
+ additionalProperties: false,
942
+ },
943
+ output: TEXT_OUTPUT, // ui-copy-ignore
944
+ timeoutMs: 20_000,
945
+ isConcurrencySafe: () => false,
946
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
947
+ execute: async (args) => withPage(async (page) => {
948
+ const name = String(args?.key ?? 'Enter')
949
+ const held = String(args?.modifiers ?? '')
950
+ const mods = (held.includes('Alt') ? 1 : 0) + (held.includes('Ctrl') ? 2 : 0) + (held.includes('Meta') ? 4 : 0) + (held.includes('Shift') ? 8 : 0)
951
+ await pressKey(page, name, mods)
952
+ await settle(page, 3000)
953
+ return acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), `key ${name}`)
954
+ }),
955
+ },
956
+ {
957
+ name: 'sidecar_scroll', // ui-copy-ignore
958
+ description: 'Scroll the sidecar browser by wheel at its centre (negative goes up), then return what it shows.',
959
+ parameters: {
960
+ type: 'object', // ui-copy-ignore
961
+ properties: {
962
+ dy: { type: 'integer', description: 'Wheel delta: 600 is about a screen down.' },
963
+ x: { type: 'integer', description: 'Viewport x for the wheel.' },
964
+ y: { type: 'integer', description: 'Viewport y for the wheel.' },
965
+ read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
966
+ },
967
+ additionalProperties: false,
968
+ },
969
+ output: TEXT_OUTPUT, // ui-copy-ignore
970
+ timeoutMs: 20_000,
971
+ isConcurrencySafe: () => false,
972
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
973
+ execute: async (args) => withPage(async (page) => {
974
+ await page.wheel({ x: Number(args?.x ?? VIEW.width / 2), y: Number(args?.y ?? VIEW.height / 2), dy: Number(args?.dy ?? 600) })
975
+ await wait(200)
976
+ const delta = Number(args?.dy ?? 600)
977
+ return acted(page, args, () => look(page, { mode: 'text', maxChars: 4000 }), `scroll ${delta > 0 ? '+' : ''}${delta}`)
978
+ }),
979
+ },
980
+ {
981
+ name: 'sidecar_hover', // ui-copy-ignore
982
+ description: 'Move the pointer onto a numbered control, so hover-only menus and tooltips open, then return what the page shows.',
983
+ parameters: {
984
+ type: 'object', // ui-copy-ignore
985
+ properties: {
986
+ ref: { type: 'integer', description: 'Number from sidecar_view.' },
987
+ read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
988
+ },
989
+ required: ['ref'], // ui-copy-ignore
990
+ additionalProperties: false,
991
+ },
992
+ output: TEXT_OUTPUT, // ui-copy-ignore
993
+ timeoutMs: 20_000,
994
+ isConcurrencySafe: () => false,
995
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
996
+ execute: async (args) => withPage(async (page) => {
997
+ const spot = await findAgain(page, Number(args?.ref))
998
+ if (spot.gone === true) return REF_GONE
999
+ const cursor = await page.hover({ x: Number(spot.x), y: Number(spot.y) })
1000
+ await wait(400)
1001
+ const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2500 }), `hover ${String(args?.ref)} · ${cursor}`)
1002
+ return said + (spot.followed === true ? ` (the page redrew, so "${String(spot.was).slice(0, 50)}" has a new number — hovered where it is now)` : '')
1003
+ }),
1004
+ },
1005
+ {
1006
+ name: 'sidecar_nav', // ui-copy-ignore
1007
+ description: 'Change the sidecar browser\'s address: open a url, go back, go forward, or reload. Use this instead of clicking the browser chrome.',
1008
+ parameters: {
1009
+ type: 'object', // ui-copy-ignore
1010
+ properties: {
1011
+ url: { type: 'string', description: 'Address to open.' },
1012
+ back: { type: 'boolean', description: 'Go back one history entry.' },
1013
+ forward: { type: 'boolean', description: 'Go forward one entry.' },
1014
+ reload: { type: 'boolean', description: 'Reload the current page.' },
1015
+ read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
1016
+ },
1017
+ additionalProperties: false,
1018
+ },
1019
+ output: TEXT_OUTPUT, // ui-copy-ignore
1020
+ timeoutMs: 60_000,
1021
+ isConcurrencySafe: () => false,
1022
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
1023
+ execute: async (args) => withRawPage(async (page) => {
1024
+ if (typeof args?.url === 'string' && args.url !== '') await page.navigate(args.url)
1025
+ else if (args?.back === true) await page.history(-1)
1026
+ else if (args?.forward === true) await page.history(1)
1027
+ else if (args?.reload === true) await page.reload()
1028
+ else return 'Give a url, or ask to go back, forward or reload.'
1029
+ await settle(page, 8000)
1030
+ const dead = await unreachable(page)
1031
+ if (dead !== '') return dead
1032
+ return acted(page, args, () => look(page, { mode: 'text' }), 'navigate') // ui-copy-ignore
1033
+ }),
1034
+ },
1035
+ {
1036
+ name: 'sidecar_do', // ui-copy-ignore
1037
+ description: 'Run several steps in one call and report one compact result: the whole of "click search, type a query, press Enter, wait for results" is one call instead of five. Steps run in order and stop at the first that cannot proceed. Each step: {ref, tap | text, submit}, {tap: "x,y"}, {tap: "Sign in"}, {field: "Email", text}, {text}, {key}, {scroll}, {wait_text}, {wait_gone}, {wait_url}, {wait_idle}, {eval}, {open}, {sleep_ms}. Waits know when the page has settled. A step that names a control clears a cookie banner in its way first.',
1038
+ parameters: {
1039
+ type: 'object', // ui-copy-ignore
1040
+ properties: {
1041
+ steps: { // ui-copy-ignore
1042
+ type: 'array', // ui-copy-ignore
1043
+ description: 'Steps, in order.',
1044
+ items: { // ui-copy-ignore
1045
+ type: 'object', // ui-copy-ignore
1046
+ properties: {
1047
+ ref: { type: 'integer', description: 'Control number for tap or text.' },
1048
+ tap: { type: 'string', description: 'Click: a ref number, "x,y", or the words on the control.' },
1049
+ text: { type: 'string', description: 'Type this text (into ref, into field, or wherever focus is).' },
1050
+ field: { type: 'string', description: 'Words on the field to fill before typing.' },
1051
+ submit: { type: 'boolean', description: 'Press Enter after typing.' },
1052
+ clear: { type: 'boolean', description: 'Empty the field before typing.' },
1053
+ key: { type: 'string', description: 'Key name.' },
1054
+ scroll: { type: 'integer', description: 'Wheel delta.' },
1055
+ wait_text: { type: 'string', description: 'Wait for this text to appear.' },
1056
+ wait_gone: { type: 'string', description: 'Wait for this text to disappear.' },
1057
+ wait_url: { type: 'string', description: 'Wait for the address to contain this.' },
1058
+ wait_idle: { type: 'boolean', description: 'Wait for the page to settle.' },
1059
+ eval: { type: 'string', description: 'Run JavaScript in the page.' },
1060
+ open: { type: 'string', description: 'Open an address.' },
1061
+ sleep_ms: { type: 'integer', description: 'Wait a fixed time.' },
1062
+ timeout: { type: 'integer', description: 'Ceiling for this step\'s wait, in milliseconds.' },
1063
+ },
1064
+ additionalProperties: false,
1065
+ },
1066
+ },
1067
+ read: { type: 'string', enum: ['none', 'text', 'links', 'forms', 'changed'], description: 'What to read at the end. Default text.' },
1068
+ max_chars: { type: 'integer', description: 'Character budget for that final read.' },
1069
+ },
1070
+ required: ['steps'], // ui-copy-ignore
1071
+ additionalProperties: false,
1072
+ },
1073
+ output: TEXT_OUTPUT, // ui-copy-ignore
1074
+ timeoutMs: 180_000,
1075
+ isConcurrencySafe: () => false,
1076
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} what happened and the page after. */
1077
+ execute: async (args) => withPage(async (page) => {
1078
+ const steps = Array.isArray(args?.steps) ? args.steps : []
1079
+ if (steps.length === 0) return 'Give at least one step.'
1080
+ const report = []
1081
+ for (const [index, step] of steps.entries()) {
1082
+ const said = await runStep(page, step ?? {}, refs?.refs)
1083
+ report.push(`${index + 1}. ${said}`)
1084
+ if (/^ref \d+: That control is gone/.test(said)) return `${report.join('\n')}\nStopped: the page changed. Call sidecar_view and use the new numbers.`
1085
+ /* A step can navigate straight onto Chrome's error page — a mistyped
1086
+ address, or a site that does not answer. The rest of the batch
1087
+ would then type into "This site can't be reached", so it stops
1088
+ here and says what really happened. */
1089
+ if (isErrorPage(page)) return `${report.join('\n')}\nStopped after step ${index + 1}: ${await unreachable(page)}`
1090
+ if (step?.sleep_ms === undefined && step?.wait_text === undefined && step?.wait_gone === undefined && step?.wait_url === undefined && step?.wait_idle === undefined) {
1091
+ await settle(page, Number(step?.timeout ?? 5000))
1092
+ }
1093
+ }
1094
+ const mode = String(args?.read ?? 'text')
1095
+ if (mode === 'none') return report.join('\n')
1096
+ const options = { mode: mode === 'changed' ? 'text' : mode, maxChars: Number(args?.max_chars ?? 3000) }
1097
+ const read = await remember(page, refs, options)
1098
+ if (mode === 'changed' && refs.last !== undefined) {
1099
+ const diff = diffRead(refs.last, read, options)
1100
+ refs.last = read
1101
+ return `${report.join('\n')}\n${diff}`
1102
+ }
1103
+ refs.last = read
1104
+ return `${report.join('\n')}\n\n${renderPage(read, options)}`
1105
+ }),
1106
+ },
1107
+ {
1108
+ name: 'sidecar_extract', // ui-copy-ignore
1109
+ description: 'Read the items a selector matches as one compact line each — lists, search results, tables, menus — instead of the whole page. Each row keeps its link when it has one. Use it when a page has more rows than the read budget allows; it costs a fraction of a full read.',
1110
+ parameters: {
1111
+ type: 'object', // ui-copy-ignore
1112
+ properties: {
1113
+ select: { type: 'string', description: 'CSS selector for the items, for example "ul.results > li".' },
1114
+ ref: { type: 'integer', description: 'Only look inside this control, as numbered by the last sidecar_view.' },
1115
+ limit: { type: 'integer', description: 'Most rows to return. Default 40, at most 200.' },
1116
+ },
1117
+ required: ['select'], // ui-copy-ignore
1118
+ additionalProperties: false,
1119
+ },
1120
+ output: TEXT_OUTPUT, // ui-copy-ignore
1121
+ timeoutMs: 30_000,
1122
+ isConcurrencySafe: () => true,
1123
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the rows. */
1124
+ execute: async (args) => withRead((page) => extractRows(page, args)),
1125
+ },
1126
+ {
1127
+ name: 'sidecar_eval', // ui-copy-ignore
1128
+ description: 'Run JavaScript in the sidecar page and return its value. For what the numbered controls cannot express: reading a table, setting storage, filling a widget. The page must be opened first.',
1129
+ parameters: {
1130
+ type: 'object', // ui-copy-ignore
1131
+ properties: {
1132
+ code: { type: 'string', description: 'Expression or statements. A returned value is handed back, trimmed.' },
1133
+ await: { type: 'boolean', description: 'Await the result if it is a promise.' },
1134
+ },
1135
+ required: ['code'], // ui-copy-ignore
1136
+ additionalProperties: false,
1137
+ },
1138
+ output: TEXT_OUTPUT, // ui-copy-ignore
1139
+ timeoutMs: 30_000,
1140
+ isConcurrencySafe: () => false,
1141
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the value. */
1142
+ execute: async (args) => withRead(async (page) => {
1143
+ const code = String(args?.code ?? '')
1144
+ if (code === '') return 'Give code to run.'
1145
+ const wrapped = args?.await === true ? `(async () => { ${code} })()` : code
1146
+ const { result, exceptionDetails } = await page.cdp.send('Runtime.evaluate', {
1147
+ expression: wrapped,
1148
+ returnByValue: true,
1149
+ awaitPromise: args?.await === true,
1150
+ userGesture: true,
1151
+ })
1152
+ if (exceptionDetails !== undefined) return String(exceptionDetails.exception?.description ?? exceptionDetails.text ?? 'evaluate failed').slice(0, 2000)
1153
+ const value = result?.value
1154
+ const text = typeof value === 'string' ? value : JSON.stringify(value)
1155
+ if (text === undefined) return 'undefined'
1156
+ return text.length > 4000 ? `${text.slice(0, 4000)}\n… ${text.length - 4000} more characters` : text
1157
+ }),
1158
+ },
1159
+ {
1160
+ name: 'sidecar_select', // ui-copy-ignore
1161
+ description: 'Choose an option in a dropdown, by visible label or value, then return the page.',
1162
+ parameters: {
1163
+ type: 'object', // ui-copy-ignore
1164
+ properties: {
1165
+ ref: { type: 'integer', description: 'Number of the select control from sidecar_view.' },
1166
+ value: { type: 'string', description: 'Option label or value to choose.' },
1167
+ },
1168
+ required: ['ref', 'value'], // ui-copy-ignore
1169
+ additionalProperties: false,
1170
+ },
1171
+ output: TEXT_OUTPUT, // ui-copy-ignore
1172
+ timeoutMs: 20_000,
1173
+ isConcurrencySafe: () => false,
1174
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
1175
+ execute: async (args) => withPage(async (page) => {
1176
+ const spot = await findAgain(page, Number(args?.ref))
1177
+ if (spot.gone === true) return REF_GONE
1178
+ const wanted = JSON.stringify(String(args?.value ?? ''))
1179
+ const route = await contextFor(page, Number(args?.ref))
1180
+ const local = route?.local ?? Number(args?.ref)
1181
+ const said = await evaluate(page, `(() => {
1182
+ const reg = window.__dshRead
1183
+ const hit = reg && reg.find(${local})
1184
+ if (!hit || hit.el.tagName !== 'SELECT') return 'that control is not a dropdown'
1185
+ const select = hit.el
1186
+ const wanted = ${wanted}
1187
+ const option = Array.from(select.options).find((item) => item.value === wanted || item.text.trim() === wanted || item.text.trim().toLowerCase() === wanted.toLowerCase())
1188
+ if (!option) return 'no option ' + wanted + '; options are ' + Array.from(select.options).slice(0, 12).map((item) => item.text.trim()).join(' | ')
1189
+ select.value = option.value
1190
+ select.dispatchEvent(new Event('input', { bubbles: true }))
1191
+ select.dispatchEvent(new Event('change', { bubbles: true }))
1192
+ return 'chose ' + option.text.trim()
1193
+ })()`, route)
1194
+ await settle(page, 3000)
1195
+ if (args?.read === true) return `${String(said)}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
1196
+ return `${String(said)} · ${await brief(page)}`
1197
+ }),
1198
+ },
1199
+ {
1200
+ name: 'sidecar_upload', // ui-copy-ignore
1201
+ description: 'Put files into a file input on the page, by absolute path, then return the page. The file chooser never opens.',
1202
+ parameters: {
1203
+ type: 'object', // ui-copy-ignore
1204
+ properties: {
1205
+ ref: { type: 'integer', description: 'Number of the file input from sidecar_view.' },
1206
+ files: { type: 'array', items: { type: 'string' }, description: 'Absolute paths on this machine.' },
1207
+ },
1208
+ required: ['ref', 'files'], // ui-copy-ignore
1209
+ additionalProperties: false,
1210
+ },
1211
+ output: TEXT_OUTPUT, // ui-copy-ignore
1212
+ timeoutMs: 30_000,
1213
+ isConcurrencySafe: () => false,
1214
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
1215
+ execute: async (args) => withPage(async (page) => {
1216
+ const ref = Number(args?.ref)
1217
+ const files = Array.isArray(args?.files) ? args.files.map(String) : []
1218
+ if (files.length === 0) return 'Give at least one absolute path.'
1219
+ const route = await contextFor(page, ref)
1220
+ const uploadParams = { expression: `(() => { const reg = window.__dshRead; const hit = reg && reg.find(${route?.local ?? ref}); return hit ? hit.el : null })()` }
1221
+ if (route?.contextId !== undefined) uploadParams.contextId = route.contextId
1222
+ const handle = await page.cdp.send('Runtime.evaluate', uploadParams, route?.sessionId)
1223
+ const objectId = handle.result?.objectId
1224
+ if (objectId === undefined) return REF_GONE
1225
+ const node = await page.cdp.send('DOM.describeNode', { objectId }, route?.sessionId)
1226
+ const backendNodeId = node?.node?.backendNodeId
1227
+ if (backendNodeId === undefined) return 'That control cannot take a file.'
1228
+ await page.cdp.send('DOM.setFileInputFiles', { files, backendNodeId }, route?.sessionId)
1229
+ await settle(page, 3000)
1230
+ return acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), `upload ${String(files.length)} file`) // ui-copy-ignore
1231
+ }),
1232
+ },
1233
+ {
1234
+ name: 'sidecar_shot', // ui-copy-ignore
1235
+ description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path. Default is the visible screen; full takes the whole page, ref takes one control.',
1236
+ parameters: {
1237
+ type: 'object', // ui-copy-ignore
1238
+ properties: {
1239
+ name: { type: 'string', description: 'File name without extension.' },
1240
+ full: { type: 'boolean', description: 'The whole page, not just the screen.' },
1241
+ ref: { type: 'integer', description: 'One control from sidecar_view.' },
1242
+ scale: { type: 'integer', description: 'Pixel ratio. Default 1; 2 is sharper and heavier.' },
1243
+ },
1244
+ additionalProperties: false,
1245
+ },
1246
+ output: TEXT_OUTPUT, // ui-copy-ignore
1247
+ timeoutMs: 40_000,
1248
+ isConcurrencySafe: () => false,
1249
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the saved path. */
1250
+ execute: async (args) => withRead(async (page) => {
1251
+ const scale = Math.max(1, Math.min(3, Number(args?.scale ?? 1)))
1252
+ let clip = undefined
1253
+ if (args?.ref !== undefined) {
1254
+ const spot = await findAgain(page, Number(args.ref))
1255
+ if (spot.gone === true) return REF_GONE
1256
+ clip = { x: Math.max(0, Number(spot.x) - 20), y: Math.max(0, Number(spot.y) - 20), width: 220, height: 140 }
1257
+ }
1258
+ const data = await page.shot({ full: args?.full === true, rect: clip, scale })
1259
+ const dir = join(homedir(), SHOT_DIR)
1260
+ await mkdir(dir, { recursive: true })
1261
+ const name = String(args?.name ?? '').replace(/[^\w.-]/g, '') || `sidecar-${Date.now()}`
1262
+ const file = join(dir, name.endsWith('.png') ? name : `${name}.png`)
1263
+ await writeFile(file, Buffer.from(data, 'base64'))
1264
+ return file
1265
+ }),
1266
+ },
1267
+ {
1268
+ name: 'sidecar_state', // ui-copy-ignore
1269
+ description: 'Report the sidecar browser: whether it runs, its profile, address, title, how many people are watching its mirror, and how many times a call had to find a control again after the page redrew it.',
1270
+ parameters: { type: 'object', properties: {}, additionalProperties: false },
1271
+ output: TEXT_OUTPUT, // ui-copy-ignore
1272
+ timeoutMs: 15_000,
1273
+ isConcurrencySafe: () => true,
1274
+ /** @returns {Promise<{text: string}>} the state. */
1275
+ execute: async () => {
1276
+ const state = await sidecar.state()
1277
+ const owner = String(host.owner?.() ?? '') // ui-copy-ignore
1278
+ const rows = [
1279
+ state.alive ? 'running' : 'stopped',
1280
+ state.headless === false ? 'showing a window' : 'no window',
1281
+ `profile ${String(state.profile)}`,
1282
+ `url ${String(state.url)}`,
1283
+ `title ${String(state.title)}`,
1284
+ `viewers ${String(state.viewers)}`,
1285
+ owner === '' ? 'no owner' : `owner ${owner}`,
1286
+ `refs followed ${String(follows.count)}`,
1287
+ holdText(sidecar) === '' ? 'person free' : `person using it (${String(Math.ceil(Number(sidecar.holding()) / 1000))} s left)`,
1288
+ ]
1289
+ if (state.alive === true && viewerNote(state.viewers) !== '') rows.push(NO_VIEWER)
1290
+ return { text: rows.join('\n') }
1291
+ },
1292
+ },
1293
+
1294
+ {
1295
+ name: 'sidecar_resume', // ui-copy-ignore
1296
+ description: 'Take the page back from the person at the mirror: clears the hold so the next action is sent at once, instead of waiting the few seconds out.',
1297
+ parameters: { type: 'object', properties: {}, additionalProperties: false },
1298
+ output: TEXT_OUTPUT, // ui-copy-ignore
1299
+ timeoutMs: 10_000,
1300
+ isConcurrencySafe: () => true,
1301
+ /** @returns {Promise<{text: string}>} confirmation. */
1302
+ execute: async () => {
1303
+ sidecar.release()
1304
+ return { text: 'ok — the page is the agent\'s again' }
1305
+ },
1306
+ },
1307
+ {
1308
+ name: 'sidecar_downloads', // ui-copy-ignore
1309
+ description: 'List what this browser has downloaded, newest first, with the file name, its size and when it arrived.',
1310
+ parameters: { type: 'object', properties: {}, additionalProperties: false },
1311
+ output: TEXT_OUTPUT, // ui-copy-ignore
1312
+ timeoutMs: 10_000,
1313
+ isConcurrencySafe: () => true,
1314
+ /** @returns {Promise<{text: string}>} the files. */
1315
+ execute: async () => {
1316
+ const files = sidecar.downloads()
1317
+ if (files.length === 0) return { text: 'no downloads yet' }
1318
+ return { text: files.map((file) => `${String(file.name)} · ${String(file.bytes)} B · ${String(file.at)}`).join('\n') }
1319
+ },
1320
+ },
1321
+
1322
+ {
1323
+ name: 'sidecar_drag', // ui-copy-ignore
1324
+ description: 'Drag from one place to another with real mouse steps: from a ref or a point, to a point or offset. Use it for sliders, dashboards and drag-and-drop lists.',
1325
+ parameters: {
1326
+ type: 'object', // ui-copy-ignore
1327
+ properties: {
1328
+ from: { type: 'integer', description: 'Ref to start on.' },
1329
+ x: { type: 'integer', description: 'Start x, when not using from.' },
1330
+ y: { type: 'integer', description: 'Start y, when not using from.' },
1331
+ to_x: { type: 'integer', description: 'Where to let go, x.' },
1332
+ to_y: { type: 'integer', description: 'Where to let go, y.' },
1333
+ steps: { type: 'integer', description: 'Mouse moves along the way. Default 12.' },
1334
+ read: { type: 'boolean', description: 'Read the page after the drag.' },
1335
+ },
1336
+ additionalProperties: false,
1337
+ },
1338
+ output: TEXT_OUTPUT, // ui-copy-ignore
1339
+ timeoutMs: 30_000,
1340
+ isConcurrencySafe: () => false,
1341
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} what happened. */
1342
+ execute: async (args) => withRawPage(async (page) => {
1343
+ let start = { x: Number(args?.x ?? NaN), y: Number(args?.y ?? NaN) }
1344
+ if (args?.from !== undefined) {
1345
+ const spot = await findAgain(page, Number(args.from))
1346
+ if (spot.gone === true) return REF_GONE
1347
+ start = { x: Number(spot.x), y: Number(spot.y) }
1348
+ }
1349
+ const end = { x: Number(args?.to_x ?? NaN), y: Number(args?.to_y ?? NaN) }
1350
+ if (![start.x, start.y, end.x, end.y].every(Number.isFinite)) return 'Give from (or x and y) and to_x and to_y.'
1351
+ const count = Math.max(2, Math.min(40, Number(args?.steps ?? 12)))
1352
+ await page.mouse('mouseMoved', start)
1353
+ await page.mouse('mousePressed', start)
1354
+ for (let index = 1; index <= count; index += 1) {
1355
+ await page.mouse('mouseMoved', {
1356
+ x: Math.round(start.x + ((end.x - start.x) * index) / count),
1357
+ y: Math.round(start.y + ((end.y - start.y) * index) / count),
1358
+ })
1359
+ }
1360
+ await page.mouse('mouseReleased', end)
1361
+ await settle(page, 3000)
1362
+ return acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), `drag ${start.x},${start.y} → ${end.x},${end.y}`)
1363
+ }),
1364
+ },
1365
+ {
1366
+ name: 'sidecar_check', // ui-copy-ignore
1367
+ description: 'Set a checkbox, radio or switch to a known state, and say whether it took. Clicking blindly flips a control that is already right; this one looks first.',
1368
+ parameters: {
1369
+ type: 'object', // ui-copy-ignore
1370
+ properties: {
1371
+ ref: { type: 'integer', description: 'Control number from sidecar_view.' },
1372
+ checked: { type: 'boolean', description: 'Wanted state. Omit to flip it.' },
1373
+ read: { type: 'boolean', description: 'Read the page after.' },
1374
+ },
1375
+ required: ['ref'], // ui-copy-ignore
1376
+ additionalProperties: false,
1377
+ },
1378
+ output: TEXT_OUTPUT, // ui-copy-ignore
1379
+ timeoutMs: 20_000,
1380
+ isConcurrencySafe: () => false,
1381
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the state now. */
1382
+ execute: async (args) => withRawPage(async (page) => {
1383
+ const checkRoute = await contextFor(page, Number(args?.ref))
1384
+ const checkParams = {
1385
+ expression: `(() => { const reg = window.__dshRead; const hit = reg && reg.find(${checkRoute?.local ?? Number(args?.ref)}); if (!hit) return null; const el = hit.el; return { checked: Boolean(el.checked), name: String(el.getAttribute('aria-label') || (el.labels && el.labels[0] && el.labels[0].innerText) || el.name || '').trim().slice(0, 60) } })()`,
1386
+ returnByValue: true,
1387
+ }
1388
+ if (checkRoute?.contextId !== undefined) checkParams.contextId = checkRoute.contextId
1389
+ const probe = await page.cdp.send('Runtime.evaluate', checkParams, checkRoute?.sessionId)
1390
+ const now = probe.result?.value
1391
+ if (now === null || now === undefined) return REF_GONE
1392
+ const want = args?.checked === undefined ? !Boolean(now.checked) : args.checked === true
1393
+ if (Boolean(now.checked) !== want) {
1394
+ const spot = await findAgain(page, Number(args.ref))
1395
+ if (spot.gone === true) return REF_GONE
1396
+ await page.mouse('mouseMoved', { x: Number(spot.x), y: Number(spot.y) })
1397
+ await page.mouse('mousePressed', { x: Number(spot.x), y: Number(spot.y) })
1398
+ await page.mouse('mouseReleased', { x: Number(spot.x), y: Number(spot.y) })
1399
+ await settle(page, 3000)
1400
+ }
1401
+ const line = `${now.name} is now ${want ? 'on' : 'off'}`
1402
+ if (args?.read === true) return `${line}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
1403
+ return line
1404
+ }),
1405
+ },
1406
+ {
1407
+ name: 'sidecar_wait', // ui-copy-ignore
1408
+ description: 'Wait until the page shows a piece of text, stops showing it, reaches an address, or goes quiet. Cheaper than reading in a loop.',
1409
+ parameters: {
1410
+ type: 'object', // ui-copy-ignore
1411
+ properties: {
1412
+ text: { type: 'string', description: 'Text to wait for.' },
1413
+ gone: { type: 'string', description: 'Text to wait to disappear.' },
1414
+ url: { type: 'string', description: 'Address piece to wait for.' },
1415
+ idle: { type: 'boolean', description: 'Wait for the page to settle.' },
1416
+ timeout: { type: 'integer', description: 'Ceiling in milliseconds. Default 8000.' },
1417
+ },
1418
+ additionalProperties: false,
1419
+ },
1420
+ output: TEXT_OUTPUT, // ui-copy-ignore
1421
+ timeoutMs: 30_000,
1422
+ isConcurrencySafe: () => false,
1423
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} whether it happened. */
1424
+ execute: async (args) => withRead(async (page) => {
1425
+ const timeout = Math.max(500, Math.min(30_000, Number(args?.timeout ?? 8000)))
1426
+ if (typeof args?.text === 'string' && args.text !== '') {
1427
+ const seen = await until(page, `document.body !== null && document.body.innerText.includes(${JSON.stringify(String(args.text))})`, timeout)
1428
+ return seen ? `saw ${String(args.text)}` : `did not see ${String(args.text)} within ${timeout} ms`
1429
+ }
1430
+ if (typeof args?.gone === 'string' && args.gone !== '') {
1431
+ const gone = await until(page, `document.body === null || !document.body.innerText.includes(${JSON.stringify(String(args.gone))})`, timeout)
1432
+ return gone ? `${String(args.gone)} is gone` : `${String(args.gone)} is still there after ${timeout} ms`
1433
+ }
1434
+ if (typeof args?.url === 'string' && args.url !== '') {
1435
+ const at = await until(page, `location.href.includes(${JSON.stringify(String(args.url))})`, timeout)
1436
+ return at ? `at ${String(await evaluate(page, 'location.href'))}` : `address never contained ${String(args.url)}`
1437
+ }
1438
+ await settle(page, timeout)
1439
+ return 'idle'
1440
+ }),
1441
+ },
1442
+ {
1443
+ name: 'sidecar_dialog', // ui-copy-ignore
1444
+ description: 'Answer the alert, confirm or prompt the page is showing, or read what it asks. A dialog blocks the page until it is answered, so nothing else works while one is open.',
1445
+ parameters: {
1446
+ type: 'object', // ui-copy-ignore
1447
+ properties: {
1448
+ accept: { type: 'boolean', description: 'Yes or OK. Default true; false dismisses it.' },
1449
+ text: { type: 'string', description: 'What to answer a prompt with.' },
1450
+ },
1451
+ additionalProperties: false,
1452
+ },
1453
+ output: TEXT_OUTPUT, // ui-copy-ignore
1454
+ timeoutMs: 15_000,
1455
+ isConcurrencySafe: () => false,
1456
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} what the dialog said. */
1457
+ execute: async (args) => withRawPage(async (page) => {
1458
+ const dialog = page.dialog
1459
+ if (dialog === null || dialog === undefined) return 'no dialog is open'
1460
+ const said = `${String(dialog.type ?? '')}: ${String(dialog.message ?? '')}`
1461
+ await page.cdp.send('Page.handleJavaScriptDialog', {
1462
+ accept: args?.accept !== false,
1463
+ promptText: String(args?.text ?? ''),
1464
+ })
1465
+ page.dialog = null
1466
+ await settle(page, 3000)
1467
+ return said
1468
+ }),
1469
+ },
1470
+ {
1471
+ name: 'sidecar_console', // ui-copy-ignore
1472
+ description: 'The console lines and errors this page has produced, newest last. Use it when a click seemed to do nothing.',
1473
+ parameters: {
1474
+ type: 'object', // ui-copy-ignore
1475
+ properties: { limit: { type: 'integer', description: 'How many lines. Default 30.' } },
1476
+ additionalProperties: false,
1477
+ },
1478
+ output: TEXT_OUTPUT, // ui-copy-ignore
1479
+ timeoutMs: 15_000,
1480
+ isConcurrencySafe: () => true,
1481
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the lines. */
1482
+ execute: async (args) => withRawPage(async (page) => {
1483
+ const limit = Math.max(1, Math.min(200, Number(args?.limit ?? 30)))
1484
+ const lines = page.console.slice(-limit).map((entry) => `${entry.type}: ${entry.text}`)
1485
+ return lines.length === 0 ? 'no console output' : lines.join('\n')
1486
+ }),
1487
+ },
1488
+ {
1489
+ name: 'sidecar_failed', // ui-copy-ignore
1490
+ description: 'Requests this page tried and that failed, newest last. A blank area is often one of these.',
1491
+ parameters: {
1492
+ type: 'object', // ui-copy-ignore
1493
+ properties: { limit: { type: 'integer', description: 'How many. Default 20.' } },
1494
+ additionalProperties: false,
1495
+ },
1496
+ output: TEXT_OUTPUT, // ui-copy-ignore
1497
+ timeoutMs: 15_000,
1498
+ isConcurrencySafe: () => true,
1499
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the failures. */
1500
+ execute: async (args) => withRawPage(async (page) => {
1501
+ const limit = Math.max(1, Math.min(100, Number(args?.limit ?? 20)))
1502
+ const rows = page.failed.slice(-limit).map((row) => `${row.type}: ${row.error}`)
1503
+ return rows.length === 0 ? 'no request failed' : rows.join('\n')
1504
+ }),
1505
+ },
1506
+ {
1507
+ name: 'sidecar_cookie',
1508
+ description: 'Read or change the cookies this browser keeps. Login state is a cookie, so this is how you check a sign-in and how you keep one.',
1509
+ parameters: {
1510
+ type: 'object', // ui-copy-ignore
1511
+ properties: {
1512
+ url: { type: 'string', description: 'Only cookies this address would send.' },
1513
+ set: { type: 'string', description: 'Write one cookie as name=value, for the current address or domain.' },
1514
+ domain: { type: 'string', description: 'Domain for a cookie being written.' },
1515
+ clear: { type: 'boolean', description: 'Drop every cookie in this browser.' },
1516
+ },
1517
+ additionalProperties: false,
1518
+ },
1519
+ output: TEXT_OUTPUT, // ui-copy-ignore
1520
+ timeoutMs: 20_000,
1521
+ /* Reading the jar is safe; writing one is not, and the tool can do both. */
1522
+ isConcurrencySafe: () => false,
1523
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} what the jar holds now. */
1524
+ execute: async (args) => withRawPage(async (page) => {
1525
+ if (args?.clear === true) {
1526
+ await page.clearCookies()
1527
+ return 'every cookie is gone'
1528
+ }
1529
+ if (typeof args?.set === 'string' && args.set !== '') {
1530
+ const at = args.set.indexOf('=')
1531
+ if (at < 1) return 'write one cookie as name=value'
1532
+ const name = args.set.slice(0, at)
1533
+ const value = args.set.slice(at + 1)
1534
+ const answer = await page.setCookie({
1535
+ name,
1536
+ value,
1537
+ domain: typeof args.domain === 'string' && args.domain !== '' ? args.domain : undefined,
1538
+ })
1539
+ return answer.ok ? `kept ${name}` : `the browser refused ${name}: ${answer.reason}`
1540
+ }
1541
+ const url = typeof args?.url === 'string' && args.url !== '' ? [args.url] : undefined
1542
+ const jar = await page.cookies(url)
1543
+ if (jar.length === 0) return 'no cookies'
1544
+ return jar
1545
+ .slice(0, 40)
1546
+ .map((c) => `${c.domain}${c.path ?? '/'} ${c.name}=${String(c.value).slice(0, 60)}`)
1547
+ .join('\n')
1548
+ }),
1549
+ },
1550
+ {
1551
+ name: 'sidecar_clipboard', // ui-copy-ignore
1552
+ description: 'Read the sidecar browser\'s clipboard, or put text on it and paste it with a key press. For a page that only copies through a button, click it then read here.',
1553
+ parameters: {
1554
+ type: 'object', // ui-copy-ignore
1555
+ properties: {
1556
+ set: { type: 'string', description: 'Text to put on the clipboard.' },
1557
+ },
1558
+ additionalProperties: false,
1559
+ },
1560
+ output: TEXT_OUTPUT, // ui-copy-ignore
1561
+ timeoutMs: 15_000,
1562
+ isConcurrencySafe: () => false,
1563
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} clipboard text. */
1564
+ execute: async (args) => withRawPage(async (page) => {
1565
+ if (typeof args?.set === 'string') {
1566
+ await page.clipboardWrite(args.set)
1567
+ return `clipboard set, ${String(args.set.length)} characters`
1568
+ }
1569
+ const text = await page.clipboardRead()
1570
+ return text === '' ? 'clipboard is empty, or the page did not allow reading it' : text
1571
+ }),
1572
+ },
1573
+ {
1574
+ name: 'sidecar_network', // ui-copy-ignore
1575
+ description: 'List the responses the page has received, newest first: method, status, type and address. Use it when a page fills itself from the network and the screen does not say why.',
1576
+ parameters: {
1577
+ type: 'object', // ui-copy-ignore
1578
+ properties: {
1579
+ filter: { type: 'string', description: 'Only rows whose address contains this text.' },
1580
+ limit: { type: 'integer', description: 'How many rows. Default 20, at most 60.' },
1581
+ },
1582
+ additionalProperties: false,
1583
+ },
1584
+ output: TEXT_OUTPUT, // ui-copy-ignore
1585
+ timeoutMs: 10_000,
1586
+ isConcurrencySafe: () => true,
1587
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the requests. */
1588
+ execute: async (args) => {
1589
+ const want = String(args?.filter ?? '').toLowerCase()
1590
+ const limit = Math.min(60, Math.max(1, Number(args?.limit ?? 20) || 20))
1591
+ const rows = sidecar.network()
1592
+ .filter((row) => want === '' || String(row.url).toLowerCase().includes(want))
1593
+ .slice(0, limit)
1594
+ if (rows.length === 0) return { text: 'no responses yet' }
1595
+ return { text: rows.map((row) => `${String(row.status)} ${String(row.method)} ${String(row.type)} ${String(row.url)}`).join('\n') }
1596
+ },
1597
+ },
1598
+
1599
+ {
1600
+ name: 'sidecar_tabs', // ui-copy-ignore
1601
+ description: 'List the pages open in the browser, switch the mirror to another one, or close one. Use it when a site opens a link in a tab of its own.',
1602
+ parameters: {
1603
+ type: 'object', // ui-copy-ignore
1604
+ properties: {
1605
+ action: { type: 'string', enum: ['list', 'switch', 'close'], description: 'Default list.' },
1606
+ index: { type: 'integer', description: 'A row from the list, counting from 0. Needed for switch and close.' },
1607
+ },
1608
+ additionalProperties: false,
1609
+ },
1610
+ output: TEXT_OUTPUT, // ui-copy-ignore
1611
+ timeoutMs: 20_000,
1612
+ isConcurrencySafe: () => false,
1613
+ /** @param {any} args - arguments. @returns {Promise<{text: string}>} the tabs or what happened. */
1614
+ execute: async (args) => withRawPage(async (page) => {
1615
+ const action = String(args?.action ?? 'list')
1616
+ if (action === 'list') {
1617
+ const rows = await sidecar.tabs()
1618
+ if (rows.length === 0) return 'no tabs'
1619
+ return rows.map((row, i) => `${i}${row.current ? ' *' : ''} ${row.title || row.url} · ${row.url}`).join('\n')
1620
+ }
1621
+ const index = Number(args?.index ?? 0)
1622
+ if (action === 'switch') {
1623
+ const next = await sidecar.useTab(index)
1624
+ return `tab ${index} · ${String(next?.url ?? '')}`
1625
+ }
1626
+ if (action === 'close') {
1627
+ await sidecar.closeTab(index)
1628
+ return `closed tab ${index} · ${String(sidecar.page?.url ?? '')}`
1629
+ }
1630
+ throw new Error(`sidecar: unknown action ${action}`)
1631
+ }),
1632
+ },
1633
+
1634
+ {
1635
+ name: 'sidecar_pdf', // ui-copy-ignore
1636
+ description: 'Print the page to a PDF and return its path. Useful when the person wants the document rather than a picture of it.',
1637
+ parameters: { type: 'object', properties: {}, additionalProperties: false },
1638
+ output: TEXT_OUTPUT, // ui-copy-ignore
1639
+ timeoutMs: 40_000,
1640
+ isConcurrencySafe: () => false,
1641
+ /** @returns {Promise<{text: string}>} where the PDF is. */
1642
+ execute: async () => withRawPage(async (page) => {
1643
+ const data = await page.pdf()
1644
+ const dir = join(homedir(), SHOT_DIR)
1645
+ await mkdir(dir, { recursive: true })
1646
+ const file = join(dir, `sidecar-${Date.now()}.pdf`)
1647
+ await writeFile(file, Buffer.from(data, 'base64'))
1648
+ return file
1649
+ }),
1650
+ },
1651
+ ]
1652
+ }