dsh-inner-browser-use 0.27.0 → 0.34.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -18
- package/README.zh.md +26 -14
- package/client.js +262 -28
- package/docs/ARCHITECTURE.md +15 -12
- package/docs/ARTIFACTS.md +19 -0
- package/docs/BENCH.md +568 -17
- package/docs/FLASH.md +84 -0
- package/docs/ROADMAP.md +107 -13
- package/docs/VERIFY.md +20 -10
- package/index.js +456 -45
- package/package.json +20 -3
- package/read.js +734 -29
- package/review.js +213 -0
- package/runtime/flash-app/main.js +232 -0
- package/runtime/flash-app/package.json +1 -0
- package/runtime/flash-setup.mjs +151 -0
- package/runtime/swf.mjs +233 -0
- package/scripts/check-flash-local.mjs +104 -0
- package/scripts/check-flash-mirror.mjs +73 -0
- package/scripts/check-flash-tools.mjs +74 -0
- package/scripts/check-flash-window.mjs +58 -0
- package/scripts/doctor.mjs +3 -1
- package/sidecar.js +1763 -106
- package/tools.js +1667 -191
package/tools.js
CHANGED
|
@@ -5,26 +5,179 @@
|
|
|
5
5
|
* Two costs decide whether this is usable: the tokens a read spends, and the
|
|
6
6
|
* round trips a task spends. The tool surface here is shaped around both — a
|
|
7
7
|
* read can be narrowed (`mode`, `max_chars`) and diffed (`changed_only`), and
|
|
8
|
-
* `
|
|
8
|
+
* `browser_do` runs a run of steps in one call with waits that know when a page
|
|
9
9
|
* has settled.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
13
|
+
import { createHash } from 'node:crypto'
|
|
12
14
|
import { homedir } from 'node:os'
|
|
13
|
-
import { join } from 'node:path'
|
|
14
|
-
import { mkdir, writeFile } from 'node:fs/promises'
|
|
15
|
+
import { basename, join } from 'node:path'
|
|
16
|
+
import { mkdir, readdir, stat, unlink, writeFile } from 'node:fs/promises'
|
|
15
17
|
import { VIEW } from './sidecar.js?v=1'
|
|
16
|
-
import { readPage, renderPage, locate, covering, contextFor, identityOf, sameThing, labelSpot, selectSpot, bounds, pictures, REF_GONE } from './read.js?v=1'
|
|
18
|
+
import { readPage, renderPage, readArticle, readRelevant, findOnPage, locate, covering, contextFor, identityOf, sameThing, differs, labelSpot, selectSpot, bounds, pictures, clickInDom, writeInDom, REF_GONE } from './read.js?v=1'
|
|
19
|
+
import { brief as askBrief, recordRound, lastRoundNote, lastRound, markRound, MARK_STATUS } from './review.js?v=1'
|
|
17
20
|
|
|
18
|
-
/**
|
|
19
|
-
export const
|
|
21
|
+
/** Which chat the running call belongs to, so each gets its own tab. */
|
|
22
|
+
export const sessionScope = new AsyncLocalStorage()
|
|
23
|
+
|
|
24
|
+
/** @returns {string} the chat id of the running call. */
|
|
25
|
+
function currentSession() {
|
|
26
|
+
return String(sessionScope.getStore() ?? '')
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** How a call asked to act: real mouse and keys, or the page's own DOM. */
|
|
30
|
+
const WAYS = new Set(['human', 'dom'])
|
|
31
|
+
|
|
32
|
+
/** The way each chat said to act, until it says otherwise. */
|
|
33
|
+
const chatWays = new Map()
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The way this call should act: what it asked for, else what its chat last set,
|
|
37
|
+
* else a real mouse and real keys, which no page can tell from a person.
|
|
38
|
+
* @param {any} args - the call's arguments.
|
|
39
|
+
* @returns {'human'|'dom'} which way.
|
|
40
|
+
*/
|
|
41
|
+
function wayOf(args) {
|
|
42
|
+
const said = String(args?.mode ?? '')
|
|
43
|
+
if (WAYS.has(said)) return /** @type {'human'|'dom'} */ (said)
|
|
44
|
+
const chat = currentSession()
|
|
45
|
+
return chatWays.get(chat) === 'dom' ? 'dom' : 'human'
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Which project the running call belongs to, so its files stay together. */
|
|
49
|
+
export const workScope = new AsyncLocalStorage()
|
|
50
|
+
|
|
51
|
+
/** Where files a call produces land, under the user's home. */
|
|
52
|
+
export const ARTIFACT_ROOT = String(process.env.DSH_INNER_BROWSER_ARTIFACTS ?? '') !== ''
|
|
53
|
+
? String(process.env.DSH_INNER_BROWSER_ARTIFACTS)
|
|
54
|
+
: join(homedir(), '.dsh-inner-browser-use', 'artifacts')
|
|
55
|
+
|
|
56
|
+
/** How many files one kind keeps per chat; older ones go. */
|
|
57
|
+
export const ARTIFACT_KEEP = 40
|
|
58
|
+
|
|
59
|
+
/** Project folder name: the folder's own name, plus a short hash of its path. */
|
|
60
|
+
function projectKey() {
|
|
61
|
+
const cwd = String(workScope.getStore() ?? '')
|
|
62
|
+
if (cwd === '') return 'no-project'
|
|
63
|
+
const named = basename(cwd).toLowerCase().replace(/[^a-z0-9._-]/g, '-').replace(/^-+|-+$/g, '')
|
|
64
|
+
return `${named === '' ? 'project' : named}-${createHash('sha1').update(cwd).digest('hex').slice(0, 6)}`
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Chat folder name: the short id a person can match to the chat. */
|
|
68
|
+
function sessionKey() {
|
|
69
|
+
const id = currentSession().replace(/^session-/, '').replace(/[^a-z0-9-]/gi, '')
|
|
70
|
+
return id === '' ? 'no-chat' : id.slice(0, 8)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Where one kind of file for this project and this chat goes.
|
|
75
|
+
* `~/.dsh-inner-browser-use/artifacts/<project>/<chat>/<kind>/`
|
|
76
|
+
* @param {string} kind - shots, pictures, pdf.
|
|
77
|
+
* @returns {string} absolute directory; not created yet.
|
|
78
|
+
*/
|
|
79
|
+
export function artifactDir(kind) {
|
|
80
|
+
return join(ARTIFACT_ROOT, projectKey(), sessionKey(), kind)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Write one file where the project and the chat expect it, and keep the folder
|
|
85
|
+
* from growing without a limit.
|
|
86
|
+
* @param {string} kind - shots, pictures, pdf.
|
|
87
|
+
* @param {string} body - base64 body.
|
|
88
|
+
* @param {{name?: string, ext?: string, prefix?: string}} [options] - file name.
|
|
89
|
+
* @returns {Promise<string>} absolute path.
|
|
90
|
+
*/
|
|
91
|
+
export async function saveArtifact(kind, body, options = {}) {
|
|
92
|
+
const dir = artifactDir(kind)
|
|
93
|
+
await mkdir(dir, { recursive: true })
|
|
94
|
+
const ext = String(options.ext ?? '.png')
|
|
95
|
+
/* A name is a name: no separators, no runs of dots that could read as a path
|
|
96
|
+
climb, and no leading dot to hide the file. */
|
|
97
|
+
const wanted = String(options.name ?? '').replace(/[^\w.-]/g, '').replace(/\.(png|jpe?g|webp|pdf)$/i, '').replace(/\.{2,}/g, '.').replace(/^\.+/, '')
|
|
98
|
+
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace('T', '-').slice(0, 15)
|
|
99
|
+
const prefix = String(options.prefix ?? kind)
|
|
100
|
+
const file = join(dir, `${wanted === '' ? `${prefix}-${stamp}` : wanted}${ext}`)
|
|
101
|
+
await writeFile(file, Buffer.from(body, 'base64'))
|
|
102
|
+
await pruneArtifacts(dir, ARTIFACT_KEEP)
|
|
103
|
+
return file
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Keep the newest `keep` files in one folder: a browser session that runs all
|
|
108
|
+
* day does not leave a folder nobody can look through.
|
|
109
|
+
* @param {string} dir - folder to trim.
|
|
110
|
+
* @param {number} keep - how many files survive.
|
|
111
|
+
* @returns {Promise<void>} done.
|
|
112
|
+
*/
|
|
113
|
+
export async function pruneArtifacts(dir, keep) {
|
|
114
|
+
let names
|
|
115
|
+
try {
|
|
116
|
+
names = await readdir(dir)
|
|
117
|
+
} catch {
|
|
118
|
+
return
|
|
119
|
+
}
|
|
120
|
+
const listed = []
|
|
121
|
+
for (const name of names) {
|
|
122
|
+
try {
|
|
123
|
+
listed.push({ name, at: (await stat(join(dir, name))).mtimeMs })
|
|
124
|
+
} catch {
|
|
125
|
+
/* a file that vanished between listing and reading is already gone */
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if (listed.length <= keep) return
|
|
129
|
+
listed.sort((left, right) => right.at - left.at)
|
|
130
|
+
for (const spare of listed.slice(keep)) {
|
|
131
|
+
try {
|
|
132
|
+
await unlink(join(dir, spare.name))
|
|
133
|
+
} catch {
|
|
134
|
+
/* best effort: a file in use stays */
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
20
138
|
|
|
21
139
|
/** How many times a call found a control again after the page redrew it. */
|
|
22
140
|
const follows = { count: 0 }
|
|
23
141
|
|
|
24
142
|
/** Shown when the sidecar has never been started in this run. */
|
|
25
|
-
export const SIDECAR_OFF = 'The sidecar browser is not running. Call
|
|
143
|
+
export const SIDECAR_OFF = 'The sidecar browser is not running. Call browser_open with the address first.'
|
|
26
144
|
|
|
27
145
|
/** Output schema every tool returns. */
|
|
146
|
+
/** The picture half of a screenshot result: shown only when the route can see it. */
|
|
147
|
+
const SHOT_IMAGE = {
|
|
148
|
+
type: 'object', // ui-copy-ignore
|
|
149
|
+
properties: {
|
|
150
|
+
attachmentId: { type: 'string' },
|
|
151
|
+
mediaType: { type: 'string' },
|
|
152
|
+
bytes: { type: 'integer' }, // ui-copy-ignore
|
|
153
|
+
width: { type: 'integer' }, // ui-copy-ignore
|
|
154
|
+
height: { type: 'integer' }, // ui-copy-ignore
|
|
155
|
+
name: { type: 'string' }, // ui-copy-ignore
|
|
156
|
+
},
|
|
157
|
+
required: ['attachmentId', 'mediaType', 'bytes', 'width', 'height'],
|
|
158
|
+
additionalProperties: false,
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** A screenshot's path, plus the picture when this model can look at it. */
|
|
162
|
+
export const SHOT_OUTPUT = {
|
|
163
|
+
schema: { // ui-copy-ignore
|
|
164
|
+
type: 'object', // ui-copy-ignore
|
|
165
|
+
properties: { text: { type: 'string' }, image: SHOT_IMAGE },
|
|
166
|
+
required: ['text'],
|
|
167
|
+
additionalProperties: false,
|
|
168
|
+
},
|
|
169
|
+
/**
|
|
170
|
+
* @param {unknown} _args - call arguments, unused.
|
|
171
|
+
* @param {any} value - tool value.
|
|
172
|
+
* @returns {any[]} the text, and the picture when there is one.
|
|
173
|
+
*/
|
|
174
|
+
render(_args, value) {
|
|
175
|
+
const blocks = [{ type: 'text', text: String(value?.text ?? '') }]
|
|
176
|
+
if (value?.image !== undefined) blocks.push({ type: 'image', attachment: value.image })
|
|
177
|
+
return blocks
|
|
178
|
+
},
|
|
179
|
+
}
|
|
180
|
+
|
|
28
181
|
const TEXT_OUTPUT = {
|
|
29
182
|
schema: { type: 'object', properties: { text: { type: 'string' } }, required: ['text'], additionalProperties: false },
|
|
30
183
|
/**
|
|
@@ -45,18 +198,38 @@ const TEXT_OUTPUT = {
|
|
|
45
198
|
* @param {any} page - sidecar page.
|
|
46
199
|
* @returns {Promise<string>} address, title, focus and any open dialog.
|
|
47
200
|
*/
|
|
201
|
+
export const BRIEF_JS = `(() => {
|
|
202
|
+
const a = document.activeElement
|
|
203
|
+
const secret = a !== null && typeof a === 'object' && (String(a.type ?? '') === 'password' || String(a.getAttribute === undefined ? '' : a.getAttribute('type') ?? '') === 'password')
|
|
204
|
+
const focus = a !== null && a !== document.body && a !== document.documentElement
|
|
205
|
+
? { name: String(a.getAttribute('aria-label') || a.getAttribute('placeholder') || (a.labels && a.labels[0] && a.labels[0].innerText) || a.innerText || a.name || a.tagName).trim().slice(0, 60), value: secret ? '' : String(a.value === undefined ? '' : a.value).slice(0, 60) }
|
|
206
|
+
: null
|
|
207
|
+
return { url: location.href, title: document.title.slice(0, 80), focus }
|
|
208
|
+
})()`
|
|
209
|
+
|
|
48
210
|
async function brief(page) {
|
|
49
211
|
const dialog = page.dialog
|
|
50
212
|
if (dialog !== null && dialog !== undefined) {
|
|
51
213
|
return `${String(page.url ?? '')} · dialog ${String(dialog.type ?? '')}: ${String(dialog.message ?? '')}`
|
|
52
214
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
215
|
+
/* On a Flash page the DOM around the plugin says nothing about what is on
|
|
216
|
+
screen, and reading it right after a click waits for the main thread the
|
|
217
|
+
plugin is holding. The bridge answers with the address and title instead. */
|
|
218
|
+
if (page.flashBridge !== null && page.flashBridge !== undefined) {
|
|
219
|
+
try {
|
|
220
|
+
const reply = await fetch(`http://127.0.0.1:${page.flashBridge}/info`, { signal: AbortSignal.timeout(600) })
|
|
221
|
+
if (reply.ok === true) {
|
|
222
|
+
const info = await reply.json()
|
|
223
|
+
const bits = [String(info?.url ?? page.url ?? '')]
|
|
224
|
+
if (String(info?.title ?? '') !== '') bits.push(String(info.title))
|
|
225
|
+
return bits.join(' · ')
|
|
226
|
+
}
|
|
227
|
+
} catch {
|
|
228
|
+
/* the cached address below is still a true answer */
|
|
229
|
+
}
|
|
230
|
+
return String(page.url ?? '')
|
|
231
|
+
}
|
|
232
|
+
const info = await evaluate(page, BRIEF_JS)
|
|
60
233
|
const bits = [String(info?.url ?? '')]
|
|
61
234
|
if (String(info?.title ?? '') !== '') bits.push(String(info.title))
|
|
62
235
|
if (info?.focus) bits.push(`focus ${info.focus.name}${info.focus.value === '' ? '' : ` = ${info.focus.value}`}`)
|
|
@@ -71,9 +244,117 @@ async function brief(page) {
|
|
|
71
244
|
* @param {string} label - what was done.
|
|
72
245
|
* @returns {Promise<string>} reply text.
|
|
73
246
|
*/
|
|
247
|
+
/** The address each page last reported to the model, so it is said once. */
|
|
248
|
+
const saidUrls = new WeakMap()
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* The address is worth one line the first time and every time it changes; the
|
|
252
|
+
* same address on every step is a cost with nothing to teach — the model read
|
|
253
|
+
* it a step ago and still has it. Titles and focus still come through.
|
|
254
|
+
* @param {any} page - sidecar page.
|
|
255
|
+
* @param {string} briefText - address, title and focus as `brief` built them.
|
|
256
|
+
* @returns {string} the line to send.
|
|
257
|
+
*/
|
|
258
|
+
/**
|
|
259
|
+
* A PDF is a document, not a blank page: say so, and put its bytes somewhere a
|
|
260
|
+
* reader can open them. The mirror cannot draw a PDF, and a model looking at an
|
|
261
|
+
* empty screen would otherwise think the address had nothing on it.
|
|
262
|
+
* @param {any} page - the tab.
|
|
263
|
+
* @returns {Promise<string>} a line to append to the read, or ''.
|
|
264
|
+
*/
|
|
265
|
+
async function pdfNote(page) {
|
|
266
|
+
const found = page?.pdf
|
|
267
|
+
if (found === null || found === undefined) return ''
|
|
268
|
+
let file = { path: '', bytes: 0 }
|
|
269
|
+
try {
|
|
270
|
+
file = await page.documentFile()
|
|
271
|
+
} catch (error) {
|
|
272
|
+
file = { path: '', bytes: 0 }
|
|
273
|
+
}
|
|
274
|
+
const size = file.bytes > 0 ? ` (${String(Math.max(1, Math.round(file.bytes / 1024)))} KB)` : ''
|
|
275
|
+
return file.path === ''
|
|
276
|
+
? `\nthis address is a PDF${size}: the mirror cannot draw it, and its bytes could not be saved. browser_downloads lists what the browser has downloaded.`
|
|
277
|
+
: `\nthis address is a PDF${size}: the mirror cannot draw it. Saved to ${file.path} — envoy_read can read that file, or another document reader.`
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* The numbered controls that were not on the page at the read before this one.
|
|
282
|
+
* A model that just acted cares about what its action brought in, and saying so
|
|
283
|
+
* costs a few numbers instead of a whole second read.
|
|
284
|
+
* @param {any} before - the read before this one, or undefined.
|
|
285
|
+
* @param {any} after - the read now.
|
|
286
|
+
* @returns {number[]} refs in `after` that `before` did not have.
|
|
287
|
+
*/
|
|
288
|
+
export function freshRefs(before, after) {
|
|
289
|
+
if (before === undefined || before === null) return []
|
|
290
|
+
if (after === undefined || after === null) return []
|
|
291
|
+
const had = new Set((Array.isArray(before.controls) ? before.controls : []).map((control) => Number(control?.ref ?? -1)))
|
|
292
|
+
const fresh = []
|
|
293
|
+
for (const control of Array.isArray(after.controls) ? after.controls : []) {
|
|
294
|
+
const ref = Number(control?.ref ?? -1)
|
|
295
|
+
if (ref < 0 || had.has(ref)) continue
|
|
296
|
+
if (!fresh.includes(ref)) fresh.push(ref)
|
|
297
|
+
}
|
|
298
|
+
return fresh
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/** What an action did, in one line: where it went, whether the page moved at all. */
|
|
302
|
+
export async function outcome(page, before) {
|
|
303
|
+
if (typeof before !== 'string' || before === '') return ''
|
|
304
|
+
const after = await pageMark(page)
|
|
305
|
+
if (after === '') return ''
|
|
306
|
+
/* A movie has no DOM to compare: its frame counter is the whole answer. */
|
|
307
|
+
if (before.startsWith('flash:') && after.startsWith('flash:')) {
|
|
308
|
+
return before === after ? ' · no frame change' : ' · screen updated'
|
|
309
|
+
}
|
|
310
|
+
const [wasTitle, wasUrl, wasText, wasNodes] = before.split('|')
|
|
311
|
+
const [nowTitle, nowUrl, nowText, nowNodes] = after.split('|')
|
|
312
|
+
if (after === before) return ' · no visible change'
|
|
313
|
+
if (nowUrl !== wasUrl) return ` · navigated to ${String(nowUrl).slice(0, 300)}`
|
|
314
|
+
const words = Math.abs(Number(nowText ?? 0) - Number(wasText ?? 0))
|
|
315
|
+
const nodes = Math.abs(Number(nowNodes ?? 0) - Number(wasNodes ?? 0))
|
|
316
|
+
if (nowTitle !== wasTitle) return ' · page moved on'
|
|
317
|
+
if (words >= 20 || nodes >= 3) return ' · content changed'
|
|
318
|
+
return ' · no visible change'
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** Read results this chat already paid for, keyed by what the page looks like. */
|
|
322
|
+
const readSeen = new WeakMap()
|
|
323
|
+
|
|
324
|
+
export function oncePerPage(page, briefText) {
|
|
325
|
+
const parts = String(briefText).split(' · ')
|
|
326
|
+
const url = String(parts[0] ?? '')
|
|
327
|
+
const known = page === null || page === undefined ? undefined : saidUrls.get(page)
|
|
328
|
+
if (url !== '' && known !== undefined) {
|
|
329
|
+
saidUrls.set(page, url)
|
|
330
|
+
if (url === known && parts.length > 1) return parts.slice(1).join(' · ')
|
|
331
|
+
return briefText
|
|
332
|
+
}
|
|
333
|
+
if (url !== '' && page !== null && page !== undefined) saidUrls.set(page, url)
|
|
334
|
+
return briefText
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* A read that starts with an address the model was already given opens with the
|
|
339
|
+
* page itself. The address is how a page says which page it is, not something
|
|
340
|
+
* to repeat every read; a page that moved says the new address in full.
|
|
341
|
+
* @param {any} page - sidecar page.
|
|
342
|
+
* @param {string} url - the address this read is from.
|
|
343
|
+
* @param {string} text - the rendered read.
|
|
344
|
+
* @returns {string} the read, without a leading address already said.
|
|
345
|
+
*/
|
|
346
|
+
export function withoutSeenAddress(page, url, text) {
|
|
347
|
+
const body = String(text ?? '')
|
|
348
|
+
const address = String(url ?? '')
|
|
349
|
+
if (address === '' || page === null || page === undefined) return body
|
|
350
|
+
const first = body.split('\n')[0]
|
|
351
|
+
if (first !== address || saidUrls.get(page) !== address) return body
|
|
352
|
+
return body.slice(first.length + 1)
|
|
353
|
+
}
|
|
354
|
+
|
|
74
355
|
async function acted(page, args, look, label) {
|
|
75
356
|
if (args?.read === true) return await look()
|
|
76
|
-
return `ok ${label} · ${await brief(page)}`
|
|
357
|
+
return `ok ${label} · ${oncePerPage(page, await brief(page))}`
|
|
77
358
|
}
|
|
78
359
|
|
|
79
360
|
/**
|
|
@@ -94,7 +375,7 @@ async function acted(page, args, look, label) {
|
|
|
94
375
|
/** Titles a site shows while it asks the browser to prove a person is there. */
|
|
95
376
|
const CHECK_TITLES = ['请稍候', 'just a moment', 'checking your browser', 'verifying you are human', 'attention required']
|
|
96
377
|
|
|
97
|
-
const CHECK_NOTE = 'That is the site asking a browser to prove a person is there, and a windowless one never gets past it.
|
|
378
|
+
const CHECK_NOTE = 'That is the site asking a browser to prove a person is there, and a windowless one never gets past it. browser_open with headless false opens a window on the same profile, with the same logins.'
|
|
98
379
|
|
|
99
380
|
/**
|
|
100
381
|
* A check page: the site has not refused, it is asking for proof. Reading it
|
|
@@ -156,27 +437,36 @@ export async function unreachable(page) {
|
|
|
156
437
|
if (!isErrorPage(page)) return ''
|
|
157
438
|
/* The code is the part a model can act on: ERR_NAME_NOT_RESOLVED is a typo,
|
|
158
439
|
ERR_PROXY_CONNECTION_FAILED is this machine's proxy, ERR_TIMED_OUT is slow. */
|
|
159
|
-
let detail =
|
|
440
|
+
let detail = ''
|
|
160
441
|
try {
|
|
161
442
|
const said = await evaluate(page, `(() => {
|
|
162
443
|
const text = String(document.body ? document.body.innerText : '')
|
|
163
444
|
const code = (String(text).match(/ERR_[A-Z0-9_]+/) || [''])[0]
|
|
164
|
-
return code
|
|
445
|
+
return code
|
|
165
446
|
})()`)
|
|
166
|
-
if (typeof said === 'string'
|
|
447
|
+
if (typeof said === 'string') detail = said.trim()
|
|
167
448
|
} catch {
|
|
168
449
|
/* The error page itself refused to be read; the address still says enough. */
|
|
169
450
|
}
|
|
170
|
-
|
|
451
|
+
/* Chrome often writes the address it could not reach where the code would
|
|
452
|
+
be: that is still the one thing the person can check. */
|
|
453
|
+
if (detail === '') {
|
|
454
|
+
const title = String(page.title ?? '').trim()
|
|
455
|
+
if (title !== '' && !/\s/.test(title)) detail = title
|
|
456
|
+
}
|
|
457
|
+
if (detail === '') detail = String(page.wantUrl ?? page.lastUrl ?? '').replace(/^https?:\/\//, '').replace(/\/.*$/, '')
|
|
458
|
+
return `the network could not reach that address — the browser stopped on its own error page (${detail === '' ? 'no details' : detail}), which is not the site. Nothing on it is page content. Use browser_nav with another address, or fix the connection this machine needs.`
|
|
171
459
|
}
|
|
172
460
|
|
|
173
|
-
/** Key names this driver can press
|
|
461
|
+
/** Key names this driver can press: DOM `key` names for the named keys, and
|
|
462
|
+
the character itself for a printable one (Space also answers to `Space`). */
|
|
174
463
|
export const KEYS = {
|
|
175
464
|
Enter: { key: 'Enter', code: 'Enter', keyCode: 13, text: '\r' },
|
|
176
465
|
Tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
|
|
177
466
|
Escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
|
|
178
467
|
Backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
|
|
179
468
|
Delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
|
|
469
|
+
Insert: { key: 'Insert', code: 'Insert', keyCode: 45 },
|
|
180
470
|
ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
|
|
181
471
|
ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
|
|
182
472
|
ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
|
|
@@ -186,6 +476,208 @@ export const KEYS = {
|
|
|
186
476
|
PageUp: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
|
|
187
477
|
PageDown: { key: 'PageDown', code: 'PageDown', keyCode: 34 },
|
|
188
478
|
Space: { key: ' ', code: 'Space', keyCode: 32, text: ' ' },
|
|
479
|
+
ContextMenu: { key: 'ContextMenu', code: 'ContextMenu', keyCode: 93 },
|
|
480
|
+
CapsLock: { key: 'CapsLock', code: 'CapsLock', keyCode: 20 },
|
|
481
|
+
NumLock: { key: 'NumLock', code: 'NumLock', keyCode: 144 },
|
|
482
|
+
ScrollLock: { key: 'ScrollLock', code: 'ScrollLock', keyCode: 145 },
|
|
483
|
+
PrintScreen: { key: 'PrintScreen', code: 'PrintScreen', keyCode: 44 },
|
|
484
|
+
Pause: { key: 'Pause', code: 'Pause', keyCode: 19 },
|
|
485
|
+
NumpadEnter: { key: 'Enter', code: 'NumpadEnter', keyCode: 13, text: '\r' },
|
|
486
|
+
AudioVolumeMute: { key: 'AudioVolumeMute', code: 'AudioVolumeMute', keyCode: 173 },
|
|
487
|
+
AudioVolumeDown: { key: 'AudioVolumeDown', code: 'AudioVolumeDown', keyCode: 174 },
|
|
488
|
+
AudioVolumeUp: { key: 'AudioVolumeUp', code: 'AudioVolumeUp', keyCode: 175 },
|
|
489
|
+
MediaPlayPause: { key: 'MediaPlayPause', code: 'MediaPlayPause', keyCode: 179 },
|
|
490
|
+
MediaTrackNext: { key: 'MediaTrackNext', code: 'MediaTrackNext', keyCode: 176 },
|
|
491
|
+
MediaTrackPrevious: { key: 'MediaTrackPrevious', code: 'MediaTrackPrevious', keyCode: 177 },
|
|
492
|
+
MediaStop: { key: 'MediaStop', code: 'MediaStop', keyCode: 178 },
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/* A game moves while a key is held, and the keys a game listens for are letters
|
|
496
|
+
and digits, not the ones a form needs. Each entry carries what a browser
|
|
497
|
+
reports: the `key` value, the physical `code`, and the legacy number old
|
|
498
|
+
engines still look at. */
|
|
499
|
+
for (const letter of 'abcdefghijklmnopqrstuvwxyz') {
|
|
500
|
+
KEYS[letter] = { key: letter, code: `Key${letter.toUpperCase()}`, keyCode: letter.toUpperCase().charCodeAt(0), text: letter }
|
|
501
|
+
/* A capital is its own key event: the page reads `A` with shift held, and a
|
|
502
|
+
form inserts `A`, not `a`. */
|
|
503
|
+
KEYS[letter.toUpperCase()] = {
|
|
504
|
+
key: letter.toUpperCase(),
|
|
505
|
+
code: `Key${letter.toUpperCase()}`,
|
|
506
|
+
keyCode: letter.toUpperCase().charCodeAt(0),
|
|
507
|
+
text: letter.toUpperCase(),
|
|
508
|
+
shift: true,
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
for (const digit of '0123456789') {
|
|
512
|
+
KEYS[digit] = { key: digit, code: `Digit${digit}`, keyCode: digit.charCodeAt(0), text: digit }
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
/* The symbols above the digits and around the alphabet, both halves of every
|
|
516
|
+
key: a page reads `?` as Slash with shift down, and a field inserts `?`.
|
|
517
|
+
`keyCode` is the legacy VK number Windows and old pages compare against. */
|
|
518
|
+
const PUNCTUATION = [
|
|
519
|
+
['Backquote', '`', 192], ['Minus', '-', 189], ['Equal', '=', 187],
|
|
520
|
+
['BracketLeft', '[', 219], ['BracketRight', ']', 221], ['Backslash', '\\', 220],
|
|
521
|
+
['Semicolon', ';', 186], ['Quote', "'", 222], ['Comma', ',', 188],
|
|
522
|
+
['Period', '.', 190], ['Slash', '/', 191],
|
|
523
|
+
]
|
|
524
|
+
const SHIFTED_SYMBOLS = [
|
|
525
|
+
['~', 'Backquote', 192], ['!', 'Digit1', 49], ['@', 'Digit2', 50], ['#', 'Digit3', 51],
|
|
526
|
+
['$', 'Digit4', 52], ['%', 'Digit5', 53], ['^', 'Digit6', 54], ['&', 'Digit7', 55],
|
|
527
|
+
['*', 'Digit8', 56], ['(', 'Digit9', 57], [')', 'Digit0', 48], ['_', 'Minus', 189],
|
|
528
|
+
['+', 'Equal', 187], ['{', 'BracketLeft', 219], ['}', 'BracketRight', 221], ['|', 'Backslash', 220],
|
|
529
|
+
[':', 'Semicolon', 186], ['"', 'Quote', 222], ['<', 'Comma', 188], ['>', 'Period', 190],
|
|
530
|
+
['?', 'Slash', 191],
|
|
531
|
+
]
|
|
532
|
+
for (const [code, char, keyCode] of PUNCTUATION) {
|
|
533
|
+
KEYS[char] = { key: char, code, keyCode, text: char }
|
|
534
|
+
}
|
|
535
|
+
for (const [char, code, keyCode] of SHIFTED_SYMBOLS) {
|
|
536
|
+
KEYS[char] = { key: char, code, keyCode, text: char, shift: true }
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/* Function keys, the numeric keypad a page can listen to, and the keys that do
|
|
540
|
+
nothing in a text field but everything in an app: a canvas game, a document
|
|
541
|
+
editor, a video player. */
|
|
542
|
+
for (let n = 1; n <= 12; n += 1) {
|
|
543
|
+
KEYS[`F${n}`] = { key: `F${n}`, code: `F${n}`, keyCode: 111 + n }
|
|
544
|
+
}
|
|
545
|
+
for (const digit of '0123456789') {
|
|
546
|
+
KEYS[`Numpad${digit}`] = { key: digit, code: `Numpad${digit}`, keyCode: 96 + Number(digit), text: digit }
|
|
547
|
+
}
|
|
548
|
+
KEYS.NumpadAdd = { key: '+', code: 'NumpadAdd', keyCode: 107, text: '+' }
|
|
549
|
+
KEYS.NumpadSubtract = { key: '-', code: 'NumpadSubtract', keyCode: 109, text: '-' }
|
|
550
|
+
KEYS.NumpadMultiply = { key: '*', code: 'NumpadMultiply', keyCode: 106, text: '*' }
|
|
551
|
+
KEYS.NumpadDivide = { key: '/', code: 'NumpadDivide', keyCode: 111, text: '/' }
|
|
552
|
+
KEYS.NumpadDecimal = { key: '.', code: 'NumpadDecimal', keyCode: 110, text: '.' }
|
|
553
|
+
|
|
554
|
+
/* The names people actually type, and the spelling a model guesses. */
|
|
555
|
+
const KEY_ALIASES = {
|
|
556
|
+
esc: 'Escape', return: 'Enter', del: 'Delete', ins: 'Insert',
|
|
557
|
+
up: 'ArrowUp', down: 'ArrowDown', left: 'ArrowLeft', right: 'ArrowRight',
|
|
558
|
+
pageup: 'PageUp', pagedown: 'PageDown', spacebar: 'Space', space: 'Space',
|
|
559
|
+
menu: 'ContextMenu', prtsc: 'PrintScreen', printscreen: 'PrintScreen',
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* The event a browser would report for a named key.
|
|
564
|
+
* @param {string} name - a key in {@link KEYS}, or something close to one.
|
|
565
|
+
* @returns {{key: string, code: string, keyCode: number, text?: string, shift?: boolean}} the event fields.
|
|
566
|
+
* @throws {Error} when the key does not exist.
|
|
567
|
+
*/
|
|
568
|
+
export function keySpec(name) {
|
|
569
|
+
const raw = String(name ?? '')
|
|
570
|
+
const exact = KEYS[raw]
|
|
571
|
+
if (exact !== undefined) return exact
|
|
572
|
+
if (raw === ' ') return KEYS.Space
|
|
573
|
+
const alias = KEY_ALIASES[raw.toLowerCase()]
|
|
574
|
+
if (alias !== undefined) return KEYS[alias]
|
|
575
|
+
const capital = raw.charAt(0).toUpperCase() + raw.slice(1)
|
|
576
|
+
if (KEYS[capital] !== undefined) return KEYS[capital]
|
|
577
|
+
const fn = /^f([1-9]|1[0-2])$/i.exec(raw)
|
|
578
|
+
if (fn !== null) return KEYS[`F${fn[1]}`]
|
|
579
|
+
throw new Error(`sidecar: no such key ${raw}`) // ui-copy-ignore
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* Turn `Ctrl+Shift` into the CDP modifier bits. `Mod` is the platform's own
|
|
584
|
+
* command key: Command on macOS, Control on Windows and Linux — one spelling
|
|
585
|
+
* for the shortcut a page treats as copy, undo, select-all.
|
|
586
|
+
* @param {string} held - modifiers joined by +, any of Alt/Option, Ctrl/Control,
|
|
587
|
+
* Meta/Cmd/Command/Super/Win, Shift, Mod.
|
|
588
|
+
* @param {string} [platform] - defaults to this process's platform.
|
|
589
|
+
* @returns {number} CDP modifier bits: Alt 1, Ctrl 2, Meta 4, Shift 8.
|
|
590
|
+
*/
|
|
591
|
+
export function modifierBits(held, platform = process.platform) {
|
|
592
|
+
let bits = 0
|
|
593
|
+
for (const raw of String(held ?? '').split('+')) {
|
|
594
|
+
const one = raw.trim().toLowerCase()
|
|
595
|
+
if (one === 'alt' || one === 'option' || one === 'opt' || one === '⌥') bits |= 1
|
|
596
|
+
else if (one === 'ctrl' || one === 'control' || one === '^') bits |= 2
|
|
597
|
+
else if (one === 'meta' || one === 'cmd' || one === 'command' || one === 'super' || one === 'win' || one === '⌘') bits |= 4
|
|
598
|
+
else if (one === 'shift' || one === '⇧') bits |= 8
|
|
599
|
+
else if (one === 'mod') bits |= platform === 'darwin' ? 4 : 2
|
|
600
|
+
}
|
|
601
|
+
return bits
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* Turn what the mirror's client posted into the event the page should see. A
|
|
606
|
+
* capital letter carries its shift, a shortcut becomes the browser command it
|
|
607
|
+
* names, and a release carries no text so the field does not type twice.
|
|
608
|
+
* @param {any} body - `{ key, code?, keyCode?, modifiers?, text?, down?, repeat? }`.
|
|
609
|
+
* @returns {{name: string, down: boolean, repeat: boolean, spec: any, command: string|null}} the event.
|
|
610
|
+
*/
|
|
611
|
+
export function keyEvent(body) {
|
|
612
|
+
const name = String(body?.key ?? '')
|
|
613
|
+
let known
|
|
614
|
+
try {
|
|
615
|
+
known = keySpec(name)
|
|
616
|
+
} catch {
|
|
617
|
+
known = null
|
|
618
|
+
}
|
|
619
|
+
const held = Number(body?.modifiers ?? 0)
|
|
620
|
+
const down = body?.down !== false
|
|
621
|
+
const repeat = body?.repeat === true
|
|
622
|
+
const capital = name.length === 1 && name !== name.toLowerCase()
|
|
623
|
+
const modifiers = capital ? held | 8 : held
|
|
624
|
+
const command = down === true && repeat === false ? commandOf(name, modifiers) : null
|
|
625
|
+
const text = command !== null ? undefined : body?.text === undefined ? known?.text : String(body.text)
|
|
626
|
+
return {
|
|
627
|
+
name,
|
|
628
|
+
down,
|
|
629
|
+
repeat,
|
|
630
|
+
spec: {
|
|
631
|
+
key: name,
|
|
632
|
+
code: String(body?.code ?? known?.code ?? name),
|
|
633
|
+
keyCode: Number(body?.keyCode ?? 0) || known?.keyCode || 0,
|
|
634
|
+
text,
|
|
635
|
+
modifiers,
|
|
636
|
+
},
|
|
637
|
+
command,
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* Read a chord the way a shortcut is written: `Ctrl+Shift+Tab` is the held
|
|
643
|
+
* modifiers Ctrl and Shift around one Tab, and `Mod+a` is Command+a on macOS.
|
|
644
|
+
* @param {string} chord - keys joined by +, modifiers may be in it.
|
|
645
|
+
* @param {number} [modifiers] - bits already held.
|
|
646
|
+
* @returns {{names: string[], modifiers: number}} the keys to press and the bits.
|
|
647
|
+
*/
|
|
648
|
+
export function splitChord(chord, modifiers = 0) {
|
|
649
|
+
const bits = { alt: 1, ctrl: 2, control: 2, meta: 4, cmd: 4, command: 4, super: 4, win: 4, shift: 8, option: 1, opt: 1, mod: process.platform === 'darwin' ? 4 : 2 }
|
|
650
|
+
const names = []
|
|
651
|
+
let held = modifiers
|
|
652
|
+
for (const raw of String(chord ?? '').split('+')) {
|
|
653
|
+
const one = raw.trim()
|
|
654
|
+
if (one === '') continue
|
|
655
|
+
const bit = bits[one.toLowerCase()]
|
|
656
|
+
if (bit === undefined) names.push(one)
|
|
657
|
+
else held |= bit
|
|
658
|
+
}
|
|
659
|
+
return { names, modifiers: held }
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/**
|
|
663
|
+
* The editing command a browser runs for a shortcut, if any: copy and paste are
|
|
664
|
+
* commands, not key events, and a key event alone leaves the clipboard alone.
|
|
665
|
+
* @param {string} name - the key name.
|
|
666
|
+
* @param {number} modifiers - CDP modifier bits.
|
|
667
|
+
* @returns {string|null} a CDP command name, or null.
|
|
668
|
+
*/
|
|
669
|
+
export function commandOf(name, modifiers) {
|
|
670
|
+
/* Either command key counts: a page listens for the one its platform uses,
|
|
671
|
+
and the mirror can be driven from a machine with the other one. */
|
|
672
|
+
if ((modifiers & 6) === 0) return null
|
|
673
|
+
const one = String(name ?? '').toLowerCase()
|
|
674
|
+
if (one === 'v') return 'paste'
|
|
675
|
+
if (one === 'c') return 'copy'
|
|
676
|
+
if (one === 'x') return 'cut'
|
|
677
|
+
if (one === 'a') return 'selectAll'
|
|
678
|
+
if (one === 'z') return (modifiers & 8) !== 0 ? 'redo' : 'undo'
|
|
679
|
+
if (one === 'y') return 'redo'
|
|
680
|
+
return null
|
|
189
681
|
}
|
|
190
682
|
|
|
191
683
|
/**
|
|
@@ -195,14 +687,41 @@ export const KEYS = {
|
|
|
195
687
|
* @param {number} [modifiers] - CDP modifier bits: Alt 1, Ctrl 2, Meta 4, Shift 8.
|
|
196
688
|
* @returns {Promise<void>} done.
|
|
197
689
|
*/
|
|
198
|
-
export async function pressKey(page, name, modifiers = 0) {
|
|
199
|
-
const key =
|
|
200
|
-
|
|
201
|
-
const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers }
|
|
202
|
-
|
|
690
|
+
export async function pressKey(page, name, modifiers = 0, holdMs = 0) {
|
|
691
|
+
const key = keySpec(name)
|
|
692
|
+
const withShift = (key.shift === true ? modifiers | 8 : modifiers)
|
|
693
|
+
const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers: withShift }
|
|
694
|
+
/* Copy, paste, undo and select-all are editor commands: a key event alone
|
|
695
|
+
leaves the clipboard and the field where they were. */
|
|
696
|
+
const command = commandOf(key.text ?? key.key, withShift)
|
|
697
|
+
await page.key('keyDown', command === null
|
|
698
|
+
? (key.text === undefined ? base : { ...base, text: key.text })
|
|
699
|
+
: { ...base, text: undefined, commands: [command] })
|
|
700
|
+
/* A key held down is how a game moves: down, wait, up — not a tap. */
|
|
701
|
+
if (holdMs > 0) await new Promise((settle) => setTimeout(settle, holdMs))
|
|
203
702
|
await page.key('keyUp', base)
|
|
204
703
|
}
|
|
205
704
|
|
|
705
|
+
/**
|
|
706
|
+
* Press several keys at once and hold them: the diagonal a game expects, and a
|
|
707
|
+
* chord such as Shift+ArrowLeft. Releasing in reverse order is what a pair of
|
|
708
|
+
* hands does, and some engines only notice the last key that went up.
|
|
709
|
+
* @param {any} page - sidecar page.
|
|
710
|
+
* @param {string[]} names - key names, for example ['ArrowLeft', 'ArrowUp'].
|
|
711
|
+
* @param {number} [modifiers] - CDP modifier bits.
|
|
712
|
+
* @param {number} [holdMs] - how long every key stays down.
|
|
713
|
+
* @returns {Promise<void>} done.
|
|
714
|
+
*/
|
|
715
|
+
export async function pressChord(page, names, modifiers = 0, holdMs = 0) {
|
|
716
|
+
const keys = names.map((name) => {
|
|
717
|
+
const key = keySpec(name)
|
|
718
|
+
return { key, base: { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode, modifiers: key.shift === true ? modifiers | 8 : modifiers } }
|
|
719
|
+
})
|
|
720
|
+
for (const one of keys) await page.key('keyDown', one.key.text === undefined ? one.base : { ...one.base, text: one.key.text })
|
|
721
|
+
if (holdMs > 0) await new Promise((settle) => setTimeout(settle, holdMs))
|
|
722
|
+
for (const one of [...keys].reverse()) await page.key('keyUp', one.base)
|
|
723
|
+
}
|
|
724
|
+
|
|
206
725
|
/**
|
|
207
726
|
* Empty the field that has focus, whatever this machine calls select-all:
|
|
208
727
|
* Command on macOS, Control elsewhere. A page can swallow the shortcut, so any
|
|
@@ -262,7 +781,7 @@ export function personNote(sidecar) {
|
|
|
262
781
|
export function holdText(sidecar) {
|
|
263
782
|
const left = Number(sidecar?.holding?.() ?? 0)
|
|
264
783
|
if (left <= 0) return ''
|
|
265
|
-
return `the person is using the mirror (${String(Math.ceil(left / 1000))} s more) — nothing was sent. Let them finish, then repeat this call;
|
|
784
|
+
return `the person is using the mirror (${String(Math.ceil(left / 1000))} s more) — nothing was sent. Let them finish, then repeat this call; browser_resume takes the page back now.`
|
|
266
785
|
}
|
|
267
786
|
|
|
268
787
|
/**
|
|
@@ -296,11 +815,15 @@ async function coverProbe(page, ref) {
|
|
|
296
815
|
* @param {ReturnType<typeof coverProbe>} probe - what the page said is on top.
|
|
297
816
|
* @returns {string} a sentence, or an empty string.
|
|
298
817
|
*/
|
|
299
|
-
function coverNote(probe) {
|
|
818
|
+
function coverNote(probe, moved) {
|
|
300
819
|
if (probe === null || probe === undefined) return ''
|
|
301
820
|
const what = probe.tag === 'div' || probe.tag === 'section' ? 'an overlay' : `a ${probe.tag}`
|
|
302
|
-
|
|
303
|
-
${what} covers it${probe.name === '' ? '' : `: ${probe.name}`}
|
|
821
|
+
const head = `
|
|
822
|
+
${what} covers it${probe.name === '' ? '' : `: ${probe.name}`}`
|
|
823
|
+
/* A banner that is about to be cleared is not a warning: the line below it
|
|
824
|
+
already says what was clicked on the way, so do not claim the click will
|
|
825
|
+
land under something that is leaving. */
|
|
826
|
+
return moved ? head : `${head} — the click would land beneath it`
|
|
304
827
|
}
|
|
305
828
|
|
|
306
829
|
/**
|
|
@@ -315,14 +838,19 @@ ${what} covers it${probe.name === '' ? '' : `: ${probe.name}`} — the click lan
|
|
|
315
838
|
async function clearCover(page, ref) {
|
|
316
839
|
const probe = await coverProbe(page, ref)
|
|
317
840
|
if (probe === null) return { note: '', cleared: true, name: '' }
|
|
318
|
-
const note = coverNote(probe)
|
|
841
|
+
const note = coverNote(probe, probe.dismiss !== null)
|
|
319
842
|
const away = probe.dismiss
|
|
320
843
|
if (away === null) return { note, cleared: false, name: '' }
|
|
321
844
|
await page.mouse('mouseMoved', { x: away.x, y: away.y })
|
|
322
845
|
await page.mouse('mousePressed', { x: away.x, y: away.y, clickCount: 1 })
|
|
323
846
|
await page.mouse('mouseReleased', { x: away.x, y: away.y, clickCount: 1 })
|
|
324
|
-
|
|
325
|
-
|
|
847
|
+
/* Prove the banner left instead of guessing a paint delay: poll the same
|
|
848
|
+
question the refusal would ask, and stop the moment it answers clear. */
|
|
849
|
+
let again = await coverProbe(page, ref)
|
|
850
|
+
for (let look = 0; look < 8 && again !== null; look += 1) {
|
|
851
|
+
await wait(20)
|
|
852
|
+
again = await coverProbe(page, ref)
|
|
853
|
+
}
|
|
326
854
|
if (again !== null) return { note: `${note}
|
|
327
855
|
- clicking ${away.name} on it did not clear it`, cleared: false, name: away.name } // ui-copy-ignore
|
|
328
856
|
return { note: `${note}
|
|
@@ -352,12 +880,19 @@ function wait(ms) {
|
|
|
352
880
|
* @param {{contextId?: number, sessionId?: string}|null} [route] - frame to run in.
|
|
353
881
|
* @returns {Promise<any>} the value.
|
|
354
882
|
*/
|
|
355
|
-
async function evaluate(page, expression, route) {
|
|
883
|
+
async function evaluate(page, expression, route, raceMs = 2000) {
|
|
356
884
|
const params = { expression, returnByValue: true, awaitPromise: true }
|
|
357
885
|
if (route?.contextId !== undefined) params.contextId = route.contextId
|
|
886
|
+
/* A page that navigates mid-call destroys the context this ran in; the
|
|
887
|
+
rejection arrives after the race is already lost. Handled, so it cannot
|
|
888
|
+
take the whole process down with it. */
|
|
358
889
|
const call = page.cdp.send('Runtime.evaluate', params, route?.sessionId)
|
|
359
|
-
|
|
890
|
+
.catch((error) => ({ threw: String(error?.message ?? error) }))
|
|
891
|
+
let timer = null
|
|
892
|
+
const raced = await Promise.race([call, new Promise((done) => { timer = setTimeout(() => done('timeout'), raceMs) })])
|
|
893
|
+
clearTimeout(timer)
|
|
360
894
|
if (raced === 'timeout') return undefined
|
|
895
|
+
if (raced?.threw !== undefined) return undefined
|
|
361
896
|
const { result, exceptionDetails } = raced
|
|
362
897
|
if (exceptionDetails !== undefined) {
|
|
363
898
|
return { error: String(exceptionDetails.exception?.description ?? exceptionDetails.text ?? 'evaluate failed') }
|
|
@@ -365,6 +900,36 @@ async function evaluate(page, expression, route) {
|
|
|
365
900
|
return result?.value
|
|
366
901
|
}
|
|
367
902
|
|
|
903
|
+
/**
|
|
904
|
+
* What the page looks like in one cheap string: address, title, how many
|
|
905
|
+
* elements and how much text. Two of these equal says the click did nothing
|
|
906
|
+
* visible — worth saying, so the model does not click a dead control twice.
|
|
907
|
+
* A Flash page has no DOM to ask, so its frame counter answers instead.
|
|
908
|
+
* @param {any} page - sidecar page.
|
|
909
|
+
* @returns {Promise<string>} a fingerprint, or '' when the page would not say.
|
|
910
|
+
*/
|
|
911
|
+
export async function pageMark(page) {
|
|
912
|
+
if (page?.flashBridge !== null && page?.flashBridge !== undefined) {
|
|
913
|
+
return `flash:${Number(page.seq ?? 0)}:${Number(page.lastFrameAt ?? 0)}`
|
|
914
|
+
}
|
|
915
|
+
const said = await evaluate(page, `(() => { const body = document.body; return String(document.title || '').slice(0, 80) + '|' + String(location.href).slice(0, 200) + '|' + (body ? body.innerText.length : 0) + '|' + document.querySelectorAll('*').length })()`).catch(() => undefined)
|
|
916
|
+
return typeof said === 'string' ? said : ''
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
/**
|
|
920
|
+
* The page's answer, as an object.
|
|
921
|
+
* @param {any} said - what evaluate returned.
|
|
922
|
+
* @returns {any} the answer, or null when the page did not answer.
|
|
923
|
+
*/
|
|
924
|
+
function asSpot(said) {
|
|
925
|
+
if (typeof said !== 'string') return null
|
|
926
|
+
try {
|
|
927
|
+
return JSON.parse(said)
|
|
928
|
+
} catch {
|
|
929
|
+
return null
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
|
|
368
933
|
/**
|
|
369
934
|
* Wait until the page has settled: loading finished and the address stopped
|
|
370
935
|
* changing. Acting on a page mid-navigation is how a driver clicks the old page.
|
|
@@ -372,21 +937,62 @@ async function evaluate(page, expression, route) {
|
|
|
372
937
|
* @param {number} [timeout] - ceiling in milliseconds.
|
|
373
938
|
* @returns {Promise<string>} the address it settled on.
|
|
374
939
|
*/
|
|
375
|
-
export async function settle(page, timeout = 5000) {
|
|
940
|
+
export async function settle(page, timeout = 5000, since = 0) {
|
|
941
|
+
/* A Flash page has no load events and no DOM of its own to poll: the next
|
|
942
|
+
picture is the only sign the click changed anything, and the bridge paints
|
|
943
|
+
every pump, so this ends at the frame instead of at the timeout. */
|
|
944
|
+
if (page.flashBridge !== null && page.flashBridge !== undefined) {
|
|
945
|
+
const until = Date.now() + Math.min(timeout, 700)
|
|
946
|
+
const before = Number(page.seq ?? 0)
|
|
947
|
+
while (Date.now() < until) {
|
|
948
|
+
if (Number(page.seq ?? 0) > before || Number(page.lastFrameAt ?? 0) > since) break
|
|
949
|
+
await new Promise((resolve) => setTimeout(resolve, 20))
|
|
950
|
+
}
|
|
951
|
+
return String(page.url ?? '')
|
|
952
|
+
}
|
|
376
953
|
/* A click may have opened a tab. The mirror follows it, so every tool that
|
|
377
954
|
waited on a page ends up on the page the person would be looking at. */
|
|
378
955
|
const opened = typeof page.adoptOpened === 'function' ? await page.adoptOpened().catch(() => '') : ''
|
|
379
956
|
const deadline = Date.now() + timeout
|
|
957
|
+
const started = Date.now()
|
|
380
958
|
let last = ''
|
|
381
959
|
let steady = 0
|
|
382
960
|
while (Date.now() < deadline) {
|
|
383
961
|
if (page.dialog !== null && page.dialog !== undefined) return String(page.url ?? '')
|
|
384
|
-
|
|
385
|
-
|
|
962
|
+
/* The browser already said the load it began has finished. That is the same
|
|
963
|
+
answer a readyState poll gives, and it stays true while a page's own
|
|
964
|
+
scripts hold the main thread, which is exactly when the poll stalls. */
|
|
965
|
+
const began = Number(page.navStarted ?? 0)
|
|
966
|
+
const fired = Number(page.loadFired ?? 0)
|
|
967
|
+
/* Two event-only answers: nothing began loading since the action, or the
|
|
968
|
+
navigation this action started has already finished. Neither needs the
|
|
969
|
+
renderer, whose answer can take seconds on a page that is busy. */
|
|
970
|
+
if (since > 0 && began <= since) return String(page.url ?? '')
|
|
971
|
+
if (fired > 0 && fired >= began && (since === 0 || fired >= since)) return String(page.url ?? '')
|
|
972
|
+
/* A click opened a page whose own requests hang: the browser began loading
|
|
973
|
+
and nothing has finished since. A person presses Esc and uses what
|
|
974
|
+
arrived, and waiting the whole timeout only to give up helps no one. */
|
|
975
|
+
if (since > 0 && began > since && Date.now() - began > 900) {
|
|
976
|
+
await page.cdp.send('Page.stopLoading', {}).catch(() => {})
|
|
977
|
+
return String(page.url ?? '')
|
|
978
|
+
}
|
|
979
|
+
const state = await evaluate(page, '({ ready: document.readyState, href: location.href, n: document.getElementsByTagName("*").length, c: (document.body?.textContent ?? "").length })', null, 400)
|
|
980
|
+
/* Nothing began loading since the action, and the page is complete: the
|
|
981
|
+
click changed state without navigating, so there is nothing to wait for. */
|
|
982
|
+
if (since > 0 && state?.ready === 'complete' && Number(page.navStarted ?? 0) <= since) return String(state?.href ?? '')
|
|
983
|
+
const key = `${String(state?.ready)}|${String(state?.href)}|${String(state?.n)}|${String(state?.c)}`
|
|
386
984
|
steady = key === last ? steady + 1 : 0
|
|
387
985
|
last = key
|
|
986
|
+
/* A page that already says complete needs one steady poll, not two: the
|
|
987
|
+
second look is what a redirect needs, and a click usually has none. */
|
|
988
|
+
if (state?.ready === 'complete' && steady >= 1) return String(state?.href ?? '')
|
|
388
989
|
if (state?.ready !== 'loading' && steady >= 2) return String(state?.href ?? '')
|
|
389
|
-
|
|
990
|
+
/* A loaded page can change on the next paint; a loading one is either
|
|
991
|
+
navigating or about to be. Right after an action that is usually the
|
|
992
|
+
page the action opened, which lands in tens of milliseconds, so the
|
|
993
|
+
first quarter second is watched closely and only then slows down. */
|
|
994
|
+
const fresh = Date.now() - started < 250
|
|
995
|
+
await wait(state?.ready === 'complete' ? (fresh ? 12 : 25) : (fresh ? 12 : 60))
|
|
390
996
|
}
|
|
391
997
|
return ''
|
|
392
998
|
}
|
|
@@ -416,7 +1022,29 @@ async function until(page, expression, timeout) {
|
|
|
416
1022
|
* @returns {Promise<string>} a compact report.
|
|
417
1023
|
*/
|
|
418
1024
|
/** The first control whose words match, scrolled into view. Text can beat a number: it costs no read. */
|
|
419
|
-
async function findWords(page, words) {
|
|
1025
|
+
export async function findWords(page, words, known) {
|
|
1026
|
+
/* The words a caller used usually came from a read that already happened, so
|
|
1027
|
+
look among those numbers first: one position check is a single evaluate,
|
|
1028
|
+
while a fresh read walks the whole page. A number the page no longer has
|
|
1029
|
+
falls through to the read, so a redraw is not a missed click. */
|
|
1030
|
+
const squeezeOf = (text) => String(text).toLowerCase().replace(/\s+/g, '')
|
|
1031
|
+
const wantedFirst = squeezeOf(words)
|
|
1032
|
+
const remembered = Array.isArray(known) ? known.filter((control) => control.view !== false && squeezeOf(control.name ?? '').includes(wantedFirst)) : []
|
|
1033
|
+
remembered.sort((a, b) => Number(squeezeOf(a.name ?? '') === wantedFirst) - Number(squeezeOf(b.name ?? '') === wantedFirst))
|
|
1034
|
+
/* A page that repeats one word down a list ("Add to cart" on every card) has
|
|
1035
|
+
one obvious answer: the first one a person would reach. The last-first
|
|
1036
|
+
order that keeps a scrolled page fast also picked whichever duplicate sat
|
|
1037
|
+
lowest, so visible matches are tried in reading order first and the near
|
|
1038
|
+
end is only the fallback. */
|
|
1039
|
+
const firstInOrder = remembered.filter((control) => control.view !== false).sort((a, b) => Number(a.ref) - Number(b.ref))
|
|
1040
|
+
const nearEnd = remembered.slice(-8).reverse()
|
|
1041
|
+
for (const control of [...firstInOrder.slice(0, 4), ...nearEnd]) {
|
|
1042
|
+
const spot = await locate(page, Number(control.ref))
|
|
1043
|
+
if (spot.gone === true) continue
|
|
1044
|
+
const name = String(spot.name ?? control.name ?? '')
|
|
1045
|
+
if (!squeezeOf(name).includes(wantedFirst)) continue
|
|
1046
|
+
return { ...control, ref: Number(control.ref), name, x: Number(spot.x), y: Number(spot.y) }
|
|
1047
|
+
}
|
|
420
1048
|
const read = await readPage(page, { mode: 'text', maxChars: 1200, max: 200 })
|
|
421
1049
|
const shown = read.controls.filter((control) => control.view !== false)
|
|
422
1050
|
/* The words may sit in separate inline nodes: <span>Sign</span><span>in</span>
|
|
@@ -454,21 +1082,90 @@ async function fieldSpot(page, words) {
|
|
|
454
1082
|
* Where the field called these words is, clearing a banner over it first.
|
|
455
1083
|
* @param {any} page - sidecar page.
|
|
456
1084
|
* @param {string} words - what the field is called on screen.
|
|
457
|
-
* @returns {Promise<{x: number, y: number, name: string, lead: string}|null>} the click.
|
|
1085
|
+
* @returns {Promise<{x: number, y: number, name: string, lead: string, ref?: number}|null>} the click.
|
|
458
1086
|
*/
|
|
459
|
-
async function fieldByName(page, words) {
|
|
460
|
-
const hit = await findWords(page, words)
|
|
1087
|
+
async function fieldByName(page, words, known) {
|
|
1088
|
+
const hit = await findWords(page, words, known)
|
|
461
1089
|
const lead = hit === null ? '' : await coverLead(page, hit.ref)
|
|
462
1090
|
if (hit !== null && EDITABLE_KINDS.includes(String(hit.kind))) {
|
|
463
1091
|
const spot = await locate(page, hit.ref)
|
|
464
|
-
if (spot.gone !== true) return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead }
|
|
1092
|
+
if (spot.gone !== true) return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead, ref: Number(hit.ref) }
|
|
465
1093
|
}
|
|
466
1094
|
const field = await fieldSpot(page, words)
|
|
467
1095
|
if (field !== null) return { ...field, lead }
|
|
468
1096
|
if (hit === null) return null
|
|
469
1097
|
const spot = await locate(page, hit.ref)
|
|
470
1098
|
if (spot.gone === true) return null
|
|
471
|
-
return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead }
|
|
1099
|
+
return { x: Number(spot.x), y: Number(spot.y), name: String(hit.name), lead, ref: Number(hit.ref) }
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
/**
|
|
1103
|
+
* A date-like field ignores human typing: Chrome's date input wants its own
|
|
1104
|
+
* segments, so keystrokes leave the value empty while the call reports success.
|
|
1105
|
+
* Set it the way a form would, and say the value the page now holds.
|
|
1106
|
+
* @param {any} page - sidecar page.
|
|
1107
|
+
* @param {string} text - what the caller wanted in it.
|
|
1108
|
+
* @returns {Promise<string>} a clause about the field, or '' when it is not one.
|
|
1109
|
+
*/
|
|
1110
|
+
/* The input kinds a person cannot type into: Chrome gives each of them its own
|
|
1111
|
+
segment editor, and keys typed at it leave the value empty. */
|
|
1112
|
+
const DATE_KINDS = ['date', 'time', 'month', 'week', 'datetime-local']
|
|
1113
|
+
|
|
1114
|
+
/**
|
|
1115
|
+
* The date-like kind the focus is on, if it is on one.
|
|
1116
|
+
* @param {any} page - sidecar page.
|
|
1117
|
+
* @returns {Promise<string>} date, time… or '' when it is an ordinary field.
|
|
1118
|
+
*/
|
|
1119
|
+
async function dateLikeFocus(page) {
|
|
1120
|
+
const said = await evaluate(page, `(() => {
|
|
1121
|
+
const el = document.activeElement
|
|
1122
|
+
if (!el || String(el.tagName).toUpperCase() !== 'INPUT') return ''
|
|
1123
|
+
const kind = String(el.type)
|
|
1124
|
+
if (!${JSON.stringify(DATE_KINDS)}.includes(kind)) return ''
|
|
1125
|
+
if (String(el.value) !== '') return ''
|
|
1126
|
+
return JSON.stringify({ kind })
|
|
1127
|
+
})()`).catch(() => undefined)
|
|
1128
|
+
return String(asSpot(said)?.kind ?? '')
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
async function fillDateLike(page, text) {
|
|
1132
|
+
const kind = await dateLikeFocus(page)
|
|
1133
|
+
if (kind === '') return ''
|
|
1134
|
+
const empty = { kind }
|
|
1135
|
+
const want = String(text).trim()
|
|
1136
|
+
const after = await evaluate(page, `(() => {
|
|
1137
|
+
const el = document.activeElement
|
|
1138
|
+
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set
|
|
1139
|
+
setter.call(el, ${JSON.stringify(want)})
|
|
1140
|
+
el.dispatchEvent(new Event('input', { bubbles: true }))
|
|
1141
|
+
el.dispatchEvent(new Event('change', { bubbles: true }))
|
|
1142
|
+
return String(el.value)
|
|
1143
|
+
})()`).catch(() => undefined)
|
|
1144
|
+
if (typeof after !== 'string' || after === '') {
|
|
1145
|
+
return `that ${empty.kind} field would not take "${want}" — it wants ${empty.kind === 'time' ? '09:30' : '2026-03-04'}`
|
|
1146
|
+
}
|
|
1147
|
+
return `set ${after} in the ${empty.kind} field`
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
/**
|
|
1151
|
+
* Put text in the focused box the cheapest way that still works there.
|
|
1152
|
+
* A box that reacts while you type (suggestion lists, live search) needs the
|
|
1153
|
+
* keys in order; anything else takes one paste event, and the box itself is
|
|
1154
|
+
* the judge of whether it took it.
|
|
1155
|
+
* @param {any} page - sidecar page.
|
|
1156
|
+
* @param {string} text - what to type.
|
|
1157
|
+
* @param {string} [hint] - the field's name, to spot a live-search box.
|
|
1158
|
+
* @returns {Promise<void>} resolves when the box holds the text.
|
|
1159
|
+
*/
|
|
1160
|
+
export async function typeInto(page, text, hint = '') {
|
|
1161
|
+
const words = String(text)
|
|
1162
|
+
if (words === '') return
|
|
1163
|
+
const live = /search|query|lookup|filter|搜索|筛选|查询/i.test(hint)
|
|
1164
|
+
if (!live && (await page.quickType(words)) === true) {
|
|
1165
|
+
await wait(30)
|
|
1166
|
+
return
|
|
1167
|
+
}
|
|
1168
|
+
await page.humanType(words)
|
|
472
1169
|
}
|
|
473
1170
|
|
|
474
1171
|
/**
|
|
@@ -479,13 +1176,31 @@ async function fieldByName(page, words) {
|
|
|
479
1176
|
* @param {{clear?: boolean, submit?: boolean}} options - how.
|
|
480
1177
|
*/
|
|
481
1178
|
async function writeAt(page, point, text, options = {}) {
|
|
1179
|
+
/* Approach the control the way a hand does: a couple of small moves that
|
|
1180
|
+
land in different places, not a teleport followed by a click. */
|
|
1181
|
+
const drift = (by) => ({ x: point.x + by, y: point.y + Math.round(by / 2) })
|
|
1182
|
+
/* The shape of the approach is what a page can see; the gaps between its
|
|
1183
|
+
points only buy latency. Three moves stay, the pauses are frames. */
|
|
1184
|
+
await page.mouse('mouseMoved', drift(-14 - Math.round(Math.random() * 10)))
|
|
1185
|
+
await wait(5 + Math.round(Math.random() * 8))
|
|
1186
|
+
await page.mouse('mouseMoved', drift(-4 - Math.round(Math.random() * 4)))
|
|
1187
|
+
await wait(4 + Math.round(Math.random() * 6))
|
|
482
1188
|
await page.mouse('mouseMoved', point)
|
|
1189
|
+
await wait(5 + Math.round(Math.random() * 8))
|
|
483
1190
|
await page.mouse('mousePressed', { ...point, clickCount: 1 })
|
|
1191
|
+
await wait(5 + Math.round(Math.random() * 9))
|
|
484
1192
|
await page.mouse('mouseReleased', { ...point, clickCount: 1 })
|
|
485
|
-
await wait(
|
|
1193
|
+
await wait(14 + Math.round(Math.random() * 18))
|
|
486
1194
|
if (options.clear === true) await clearFocus(page)
|
|
487
|
-
|
|
1195
|
+
/* A date or time input never takes keystrokes: noticing that before typing
|
|
1196
|
+
spends one round trip instead of fourteen keys and a failed readback. */
|
|
1197
|
+
if (await dateLikeFocus(page) !== '') return await fillDateLike(page, text)
|
|
1198
|
+
if (options.human === false) await page.insertText(text)
|
|
1199
|
+
else await typeInto(page, text, String(options.hint ?? ''))
|
|
1200
|
+
const fixed = await fillDateLike(page, text)
|
|
1201
|
+
if (fixed !== '') return fixed
|
|
488
1202
|
if (options.submit === true) await pressKey(page, 'Enter')
|
|
1203
|
+
return ''
|
|
489
1204
|
}
|
|
490
1205
|
|
|
491
1206
|
/**
|
|
@@ -498,8 +1213,11 @@ async function writeAt(page, point, text, options = {}) {
|
|
|
498
1213
|
*/
|
|
499
1214
|
async function followRef(page, ref, known) {
|
|
500
1215
|
const spot = await locate(page, ref)
|
|
501
|
-
if (spot.gone !== true) return spot
|
|
502
1216
|
const was = Array.isArray(known) ? known.find((control) => Number(control.ref) === Number(ref)) : undefined
|
|
1217
|
+
if (spot.gone !== true) {
|
|
1218
|
+
const drift = differs(was, spot)
|
|
1219
|
+
return drift === '' ? spot : { gone: true, why: drift }
|
|
1220
|
+
}
|
|
503
1221
|
if (was === undefined) return spot
|
|
504
1222
|
const want = identityOf(was)
|
|
505
1223
|
if (want.name === '') return spot
|
|
@@ -512,12 +1230,40 @@ async function followRef(page, ref, known) {
|
|
|
512
1230
|
return { ...next, followed: true, was: want.name }
|
|
513
1231
|
}
|
|
514
1232
|
|
|
1233
|
+
/**
|
|
1234
|
+
* One read state per chat, created the first time that chat reads.
|
|
1235
|
+
* @param {Map<string, any>} store - per-chat states.
|
|
1236
|
+
* @param {string} id - chat id.
|
|
1237
|
+
* @param {any} fallback - state for calls that name no chat.
|
|
1238
|
+
* @returns {any} the state of that chat.
|
|
1239
|
+
*/
|
|
1240
|
+
export function readStateFor(store, id, fallback) {
|
|
1241
|
+
const key = String(id ?? '')
|
|
1242
|
+
const known = store.get(key)
|
|
1243
|
+
if (known !== undefined) return known
|
|
1244
|
+
/* The one object the plugin was started with belongs to calls that name no
|
|
1245
|
+
chat, so a chat that does name itself gets an empty one of its own. */
|
|
1246
|
+
const fresh = key === '' ? fallback : { refs: [], at: 0 }
|
|
1247
|
+
store.set(key, fresh)
|
|
1248
|
+
return fresh
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
/**
|
|
1252
|
+
* A step that could not be carried out. The batch stops here: the next step was
|
|
1253
|
+
* written against a page that never arrived.
|
|
1254
|
+
* @param {string} text - what happened.
|
|
1255
|
+
* @returns {{text: string, blocked: boolean}} the step result.
|
|
1256
|
+
*/
|
|
1257
|
+
function stuck(text) {
|
|
1258
|
+
return { text, blocked: true }
|
|
1259
|
+
}
|
|
1260
|
+
|
|
515
1261
|
async function runStep(page, step, known) {
|
|
516
1262
|
if (step.ref !== undefined) {
|
|
517
1263
|
if (step.tap !== undefined || step.text !== undefined) {
|
|
518
1264
|
const lead = await coverLead(page, Number(step.ref))
|
|
519
1265
|
const spot = await followRef(page, Number(step.ref), known)
|
|
520
|
-
if (spot.gone === true) return `ref ${String(step.ref)}: ${REF_GONE}`
|
|
1266
|
+
if (spot.gone === true) return stuck(`ref ${String(step.ref)}: ${REF_GONE}`)
|
|
521
1267
|
const after = spot.followed === true ? ` (it was redrawn under a new number, used where it is now)` : ''
|
|
522
1268
|
const point = { x: Number(spot.x), y: Number(spot.y) }
|
|
523
1269
|
await page.mouse('mouseMoved', point)
|
|
@@ -526,17 +1272,17 @@ async function runStep(page, step, known) {
|
|
|
526
1272
|
if (step.tap !== undefined) return `${lead}clicked ${String(step.ref)} ${String(spot.name ?? '')}`.trim() + after
|
|
527
1273
|
await wait(80)
|
|
528
1274
|
if (step.clear === true) await clearFocus(page)
|
|
529
|
-
await page.
|
|
1275
|
+
await page.humanType(String(step.text))
|
|
530
1276
|
if (step.submit === true) await pressKey(page, 'Enter')
|
|
531
1277
|
return `${lead}typed into ${String(step.ref)} ${String(spot.name ?? '')}`.trim() + after
|
|
532
1278
|
}
|
|
533
1279
|
}
|
|
534
1280
|
if (step.tap !== undefined && !/^\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*$/.test(String(step.tap))) {
|
|
535
|
-
const hit = await findWords(page, String(step.tap))
|
|
536
|
-
if (hit === null) return `no control says ${JSON.stringify(String(step.tap))}`
|
|
1281
|
+
const hit = await findWords(page, String(step.tap), known)
|
|
1282
|
+
if (hit === null) return stuck(`no control says ${JSON.stringify(String(step.tap))}`)
|
|
537
1283
|
const lead = await coverLead(page, hit.ref)
|
|
538
1284
|
const spot = await locate(page, hit.ref)
|
|
539
|
-
if (spot.gone === true) return `ref ${String(hit.ref)}: ${REF_GONE}`
|
|
1285
|
+
if (spot.gone === true) return stuck(`ref ${String(hit.ref)}: ${REF_GONE}`)
|
|
540
1286
|
const point = { x: Number(spot.x), y: Number(spot.y) }
|
|
541
1287
|
await page.mouse('mouseMoved', point)
|
|
542
1288
|
await page.mouse('mousePressed', { ...point, clickCount: 1 })
|
|
@@ -546,9 +1292,10 @@ async function runStep(page, step, known) {
|
|
|
546
1292
|
if (step.field !== undefined) {
|
|
547
1293
|
/* The words may sit on the field or in a div above a bare input. Only a
|
|
548
1294
|
click that lands on the field itself can be trusted with the typing. */
|
|
549
|
-
const target = await fieldByName(page, String(step.field))
|
|
550
|
-
if (target === null) return `${String(step.field)}: ${REF_GONE}`
|
|
551
|
-
await writeAt(page, { x: target.x, y: target.y }, String(step.text ?? ''), { clear: step.clear, submit: step.submit })
|
|
1295
|
+
const target = await fieldByName(page, String(step.field), known)
|
|
1296
|
+
if (target === null) return stuck(`${String(step.field)}: ${REF_GONE}`)
|
|
1297
|
+
const dated = await writeAt(page, { x: target.x, y: target.y }, String(step.text ?? ''), { clear: step.clear, submit: step.submit })
|
|
1298
|
+
if (dated !== '') return dated
|
|
552
1299
|
return `${target.lead}typed into "${target.name.slice(0, 60)}"`
|
|
553
1300
|
}
|
|
554
1301
|
if (step.tap !== undefined) {
|
|
@@ -561,7 +1308,7 @@ async function runStep(page, step, known) {
|
|
|
561
1308
|
}
|
|
562
1309
|
if (step.text !== undefined) {
|
|
563
1310
|
if (step.clear === true) await clearFocus(page)
|
|
564
|
-
await page.
|
|
1311
|
+
await page.humanType(String(step.text))
|
|
565
1312
|
if (step.submit === true) await pressKey(page, 'Enter')
|
|
566
1313
|
return `typed ${String(step.text).slice(0, 60)}`
|
|
567
1314
|
}
|
|
@@ -577,17 +1324,17 @@ async function runStep(page, step, known) {
|
|
|
577
1324
|
if (step.wait_text !== undefined) {
|
|
578
1325
|
const needle = JSON.stringify(String(step.wait_text))
|
|
579
1326
|
const ok = await until(page, `document.body ? document.body.innerText.includes(${needle}) : false`, Number(step.timeout ?? 8000))
|
|
580
|
-
return ok ? `saw ${String(step.wait_text)}` : `did not see ${String(step.wait_text)}`
|
|
1327
|
+
return ok ? `saw ${String(step.wait_text)}` : stuck(`did not see ${String(step.wait_text)}`)
|
|
581
1328
|
}
|
|
582
1329
|
if (step.wait_gone !== undefined) {
|
|
583
1330
|
const needle = JSON.stringify(String(step.wait_gone))
|
|
584
1331
|
const ok = await until(page, `document.body ? !document.body.innerText.includes(${needle}) : true`, Number(step.timeout ?? 8000))
|
|
585
|
-
return ok ? `${String(step.wait_gone)} went away` : `${String(step.wait_gone)} stayed`
|
|
1332
|
+
return ok ? `${String(step.wait_gone)} went away` : stuck(`${String(step.wait_gone)} stayed`)
|
|
586
1333
|
}
|
|
587
1334
|
if (step.wait_url !== undefined) {
|
|
588
1335
|
const needle = JSON.stringify(String(step.wait_url))
|
|
589
1336
|
const ok = await until(page, `location.href.includes(${needle})`, Number(step.timeout ?? 8000))
|
|
590
|
-
return ok ? `at ${String(step.wait_url)}` : `still at a different address`
|
|
1337
|
+
return ok ? `at ${String(step.wait_url)}` : stuck(`still at a different address`)
|
|
591
1338
|
}
|
|
592
1339
|
if (step.open !== undefined) {
|
|
593
1340
|
const landed = await page.navigate(String(step.open))
|
|
@@ -617,13 +1364,85 @@ async function runStep(page, step, known) {
|
|
|
617
1364
|
* @param {{mode?: string, maxChars?: number}} options - narrowing.
|
|
618
1365
|
* @returns {Promise<any>} the read.
|
|
619
1366
|
*/
|
|
1367
|
+
/** A read with nothing in it often means the document is still being built:
|
|
1368
|
+
* samples of real pages showed zero controls when the read landed before
|
|
1369
|
+
* hydration, and the model then calls the page broken. */
|
|
1370
|
+
function looksEmpty(read) {
|
|
1371
|
+
return (read?.controls?.length ?? 0) === 0 && String(read?.text ?? '').trim().length < 40
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
/** @param {string} url - what the page claims to be. @returns {boolean} true when waiting can still help. */
|
|
1375
|
+
function canFill(url) {
|
|
1376
|
+
const said = String(url ?? '')
|
|
1377
|
+
return said.startsWith('http://') || said.startsWith('https://')
|
|
1378
|
+
}
|
|
1379
|
+
|
|
620
1380
|
async function remember(page, refs, options) {
|
|
621
|
-
|
|
1381
|
+
let read = await readPage(page, options)
|
|
1382
|
+
if (looksEmpty(read) && canFill(read.url)) {
|
|
1383
|
+
/* Hydration and client-side rendering finish a few hundred milliseconds
|
|
1384
|
+
after the address does. Read again until the page has something, but
|
|
1385
|
+
never longer than a person would wait for a page that stays empty. */
|
|
1386
|
+
const deadline = Date.now() + 1500
|
|
1387
|
+
while (looksEmpty(read) && canFill(read.url) && Date.now() < deadline) {
|
|
1388
|
+
await wait(120)
|
|
1389
|
+
const again = await readPage(page, options)
|
|
1390
|
+
if (!looksEmpty(again)) {
|
|
1391
|
+
read = again
|
|
1392
|
+
break
|
|
1393
|
+
}
|
|
1394
|
+
read = again
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
622
1397
|
refs.refs = read.controls
|
|
623
1398
|
refs.at = Date.now()
|
|
624
1399
|
return read
|
|
625
1400
|
}
|
|
626
1401
|
|
|
1402
|
+
/**
|
|
1403
|
+
* Two reads of the same page in the same shape say the same thing. Saying it
|
|
1404
|
+
* again costs tokens and teaches nothing, so the second read can be one line.
|
|
1405
|
+
* @param {any} before - the earlier read.
|
|
1406
|
+
* @param {any} after - the read now.
|
|
1407
|
+
* @returns {boolean} true when nothing a read would show has changed.
|
|
1408
|
+
*/
|
|
1409
|
+
export function sameRead(before, after) {
|
|
1410
|
+
if (before === undefined || before === null || after === undefined || after === null) return false
|
|
1411
|
+
if (before.url !== after.url || before.title !== after.title) return false
|
|
1412
|
+
if (before.text !== after.text) return false
|
|
1413
|
+
if (JSON.stringify(before.controls) !== JSON.stringify(after.controls)) return false
|
|
1414
|
+
if ((before.focused?.ref ?? null) !== (after.focused?.ref ?? null)) return false
|
|
1415
|
+
return true
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
/**
|
|
1419
|
+
* Where two readings of a page differ, said in one line. A counter moving, one
|
|
1420
|
+
* row appearing, a word edited: the page is mostly the same, and sending all of
|
|
1421
|
+
* it again is what makes a long session expensive.
|
|
1422
|
+
* @param {string} before - the text earlier.
|
|
1423
|
+
* @param {string} after - the text now.
|
|
1424
|
+
* @returns {string} a short line, or '' when the two have nothing in common.
|
|
1425
|
+
*/
|
|
1426
|
+
export function textChange(before, after) {
|
|
1427
|
+
const was = String(before ?? '')
|
|
1428
|
+
const now = String(after ?? '')
|
|
1429
|
+
let head = 0
|
|
1430
|
+
const shortest = Math.min(was.length, now.length)
|
|
1431
|
+
while (head < shortest && was[head] === now[head]) head += 1
|
|
1432
|
+
let tail = 0
|
|
1433
|
+
while (tail < shortest - head && was[was.length - 1 - tail] === now[now.length - 1 - tail]) tail += 1
|
|
1434
|
+
const gone = was.slice(head, was.length - tail)
|
|
1435
|
+
const came = now.slice(head, now.length - tail)
|
|
1436
|
+
/* A page that changed in one place is worth a line; one that shares almost no
|
|
1437
|
+
text is a different page, and a diff of it would be as long as the page. */
|
|
1438
|
+
if (head + tail < 12) return ''
|
|
1439
|
+
const clip = (text) => (text.length > 160 ? `${text.slice(0, 157)}…` : text)
|
|
1440
|
+
const near = clip(was.slice(Math.max(0, head - 24), head))
|
|
1441
|
+
const far = clip(now.slice(now.length - tail, now.length - tail + 24))
|
|
1442
|
+
const line = `${near === '' ? '' : `…${near} `}${gone === '' ? '' : `[${clip(gone)}] `}→ ${came === '' ? '' : `[${clip(came)}] `}${far === '' ? '' : `${far}…`}`
|
|
1443
|
+
return line.trim()
|
|
1444
|
+
}
|
|
1445
|
+
|
|
627
1446
|
/**
|
|
628
1447
|
* Say only what changed since the previous read.
|
|
629
1448
|
* @param {any} previous - the read before.
|
|
@@ -635,8 +1454,7 @@ export function diffRead(previous, next, options = {}) {
|
|
|
635
1454
|
const maxChars = Number(options.maxChars ?? 2000)
|
|
636
1455
|
const before = new Map(previous.controls.map((control) => [control.ref, control]))
|
|
637
1456
|
const after = new Map(next.controls.map((control) => [control.ref, control]))
|
|
638
|
-
const lines = [next.url]
|
|
639
|
-
if (previous.url !== next.url) lines.push(`address changed to ${next.url}`)
|
|
1457
|
+
const lines = [previous.url === next.url ? next.url : `address ${next.url}`]
|
|
640
1458
|
const added = next.controls.filter((control) => !before.has(control.ref))
|
|
641
1459
|
const removed = previous.controls.filter((control) => !after.has(control.ref))
|
|
642
1460
|
const changed = next.controls.filter((control) => {
|
|
@@ -652,9 +1470,14 @@ export function diffRead(previous, next, options = {}) {
|
|
|
652
1470
|
}
|
|
653
1471
|
if (added.length + removed.length + changed.length === 0) lines.push('no control changed')
|
|
654
1472
|
if (previous.text !== next.text) {
|
|
655
|
-
const
|
|
656
|
-
|
|
657
|
-
if (
|
|
1473
|
+
const said = textChange(previous.text, next.text)
|
|
1474
|
+
const room = Math.max(200, Math.min(1200, maxChars - 200))
|
|
1475
|
+
if (said !== '') lines.push(`text: ${said}`)
|
|
1476
|
+
else {
|
|
1477
|
+
const head = next.text.slice(0, room)
|
|
1478
|
+
lines.push('', 'text now:', head)
|
|
1479
|
+
if (next.text.length > head.length) lines.push(`… ${next.text.length - head.length} more characters`)
|
|
1480
|
+
}
|
|
658
1481
|
}
|
|
659
1482
|
if (next.focused !== null && previous.focused?.ref !== next.focused.ref) lines.push(`focus ${next.focused.ref} ${next.focused.name}`)
|
|
660
1483
|
return lines.join('\n')
|
|
@@ -709,7 +1532,7 @@ export async function extractRows(page, args) {
|
|
|
709
1532
|
const out = await page.cdp.send('Runtime.evaluate', { expression, returnByValue: true }).catch((error) => ({ error: String(error?.message ?? error) }))
|
|
710
1533
|
if (out?.error !== undefined && out.error !== null && typeof out.error === 'string') return `the page did not answer: ${out.error}`
|
|
711
1534
|
const value = out?.result?.value
|
|
712
|
-
if (value === undefined || value === null || value === '') return 'the page did not answer; try again, or read the page with
|
|
1535
|
+
if (value === undefined || value === null || value === '') return 'the page did not answer; try again, or read the page with browser_view.'
|
|
713
1536
|
if (value.gone === true) return `ref ${ref} is gone. Read the page again and use a new number.`
|
|
714
1537
|
if (value.bad === true) return `${select} is not a selector this browser understands.`
|
|
715
1538
|
const rows = Array.isArray(value.rows) ? value.rows : []
|
|
@@ -719,7 +1542,140 @@ export async function extractRows(page, args) {
|
|
|
719
1542
|
return `${rows.length} of ${Number(value.total)} rows matching ${select}:\n${said.join('\n')}${more}`
|
|
720
1543
|
}
|
|
721
1544
|
|
|
1545
|
+
/**
|
|
1546
|
+
* Say one thing to a box in the page and take the answer back. The wait ends
|
|
1547
|
+
* when the answer has stopped changing, so a streaming reply is not cut off and
|
|
1548
|
+
* a page that never answers does not hold the call open forever.
|
|
1549
|
+
* @param {any} page - sidecar page.
|
|
1550
|
+
* @param {{text?: string, ref?: number, submit?: boolean, timeout_ms?: number}} args - what to say.
|
|
1551
|
+
* @returns {Promise<string>} the answer, or why there is none.
|
|
1552
|
+
*/
|
|
1553
|
+
/**
|
|
1554
|
+
* Whether a box on the page still holds the words that were typed.
|
|
1555
|
+
* A box that keeps them never sent them, which is what a signed-out page does.
|
|
1556
|
+
* @param {any} page - sidecar page.
|
|
1557
|
+
* @param {string} words - what was typed.
|
|
1558
|
+
* @returns {Promise<boolean>} true when the words are still in a box.
|
|
1559
|
+
*/
|
|
1560
|
+
async function boxHolds(page, words) {
|
|
1561
|
+
const wanted = String(words).slice(0, 24)
|
|
1562
|
+
if (wanted === '') return false
|
|
1563
|
+
const held = await evaluate(page, `(() => {
|
|
1564
|
+
const wanted = ${JSON.stringify(wanted)}
|
|
1565
|
+
for (const el of document.querySelectorAll('textarea, [contenteditable=""], [contenteditable="true"], input[type="text"], input[type="search"], input:not([type])')) {
|
|
1566
|
+
if (String(el.value ?? el.innerText ?? '').trim().includes(wanted)) return true
|
|
1567
|
+
}
|
|
1568
|
+
return false
|
|
1569
|
+
})()`)
|
|
1570
|
+
return held === true
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
/**
|
|
1574
|
+
* Sends a line to whatever chat the page holds and waits for the answer to stop
|
|
1575
|
+
* growing. The ref is followed the same way a click follows it, so a reply box
|
|
1576
|
+
* that was redrawn since the read still gets the text.
|
|
1577
|
+
* @param {any} page - sidecar page.
|
|
1578
|
+
* @param {any} args - text, optional ref, optional timeout.
|
|
1579
|
+
* @returns {Promise<string>} the answer.
|
|
1580
|
+
*/
|
|
1581
|
+
export async function askInPage(page, args) {
|
|
1582
|
+
const said = String(args?.text ?? '').trim()
|
|
1583
|
+
if (said === '') return 'Give me something to say.'
|
|
1584
|
+
const budget = Math.max(3000, Math.min(180_000, Math.trunc(Number(args?.timeout_ms ?? 60_000)) || 60_000))
|
|
1585
|
+
const before = await readPage(page, { mode: 'text', maxChars: 4000 })
|
|
1586
|
+
let point = null
|
|
1587
|
+
if (args?.ref !== undefined) {
|
|
1588
|
+
const spot = await followRef(page, Number(args.ref))
|
|
1589
|
+
if (spot.gone === true) return spot.why ?? REF_GONE
|
|
1590
|
+
point = { x: Number(spot.x), y: Number(spot.y) }
|
|
1591
|
+
} else {
|
|
1592
|
+
const at = await evaluate(page, `(() => {
|
|
1593
|
+
let best = null
|
|
1594
|
+
for (const el of document.querySelectorAll('textarea, [contenteditable=""], [contenteditable="true"], input[type="text"], input[type="search"], input:not([type])')) {
|
|
1595
|
+
const b = el.getBoundingClientRect()
|
|
1596
|
+
const s = getComputedStyle(el)
|
|
1597
|
+
if (s.visibility === 'hidden' || s.display === 'none' || b.width < 80 || b.height < 18) continue
|
|
1598
|
+
if (b.bottom < 0 || b.top > innerHeight) continue
|
|
1599
|
+
if (best === null || b.width * b.height > best.area) best = { area: b.width * b.height, x: b.x + Math.min(24, b.width / 2), y: b.y + b.height / 2, name: String(el.getAttribute('placeholder') ?? el.getAttribute('aria-label') ?? '').slice(0, 40) }
|
|
1600
|
+
}
|
|
1601
|
+
return best === null ? null : { x: Math.round(best.x), y: Math.round(best.y), name: best.name }
|
|
1602
|
+
})()`)
|
|
1603
|
+
if (at === null || at === undefined) return 'No box to type in on this page. Call browser_view to see what is there.'
|
|
1604
|
+
point = { x: Number(at.x), y: Number(at.y) }
|
|
1605
|
+
}
|
|
1606
|
+
await page.mouse('mouseMoved', point)
|
|
1607
|
+
await page.mouse('mousePressed', { ...point, clickCount: 1 })
|
|
1608
|
+
await page.mouse('mouseReleased', { ...point, clickCount: 1 })
|
|
1609
|
+
await wait(60)
|
|
1610
|
+
await page.humanType(said)
|
|
1611
|
+
if (args?.submit !== false) await pressKey(page, 'Enter')
|
|
1612
|
+
await wait(500)
|
|
1613
|
+
if (args?.submit !== false && (await boxHolds(page, said))) {
|
|
1614
|
+
return `the words are still in the box 0.5s after pressing Enter: the page did not take the message. If it is waiting for a sign-in, sign in in the column, then ask again.`
|
|
1615
|
+
}
|
|
1616
|
+
const started = Date.now()
|
|
1617
|
+
const deadline = started + budget
|
|
1618
|
+
let last = before.text
|
|
1619
|
+
let fresh = ''
|
|
1620
|
+
let steady = 0
|
|
1621
|
+
let quiet = 0
|
|
1622
|
+
while (Date.now() < deadline) {
|
|
1623
|
+
await wait(500)
|
|
1624
|
+
const now = await readPage(page, { mode: 'text', maxChars: 4000 })
|
|
1625
|
+
if (now.text !== last) {
|
|
1626
|
+
last = now.text
|
|
1627
|
+
fresh = now.text
|
|
1628
|
+
steady = 0
|
|
1629
|
+
quiet = 0
|
|
1630
|
+
continue
|
|
1631
|
+
}
|
|
1632
|
+
if (fresh !== '') {
|
|
1633
|
+
steady += 1
|
|
1634
|
+
if (steady >= 3) break
|
|
1635
|
+
} else {
|
|
1636
|
+
quiet += 1
|
|
1637
|
+
if (quiet >= 4 && Date.now() - started > 4000) break
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
const took = Date.now() - started
|
|
1641
|
+
/* A page that shows neither the words in a box nor the message itself never
|
|
1642
|
+
took it: the chips moved, the menu redrew, and none of that is an answer. */
|
|
1643
|
+
const echo = String(said.slice(0, 24))
|
|
1644
|
+
if (fresh !== '' && echo.length >= 8 && !fresh.includes(echo)) {
|
|
1645
|
+
const asking = /(sign in|log in|登录|注册)/i.test(fresh)
|
|
1646
|
+
return `the page changed but does not show the message: it did not take what was sent.${asking ? ' It is asking for a sign-in — sign in in the column, then ask again.' : ' Try browser_view to see what is in the way.'}`
|
|
1647
|
+
}
|
|
1648
|
+
/* A box that still holds the words is the page saying it did not take them:
|
|
1649
|
+
a signed-out page leaves them there, and the old wording sent the reader
|
|
1650
|
+
looking for an answer that was never asked for. */
|
|
1651
|
+
const stuck = fresh === '' && (await boxHolds(page, said))
|
|
1652
|
+
if (fresh === '') {
|
|
1653
|
+
if (stuck) return `sent it — no answer after ${Math.round(took / 100) / 10}s, and the words are still in the box: the page did not take the message. If it is waiting for a sign-in, sign in in the column, then ask again.`
|
|
1654
|
+
return `sent it — no answer after ${Math.round(took / 100) / 10}s; the page still says: ${before.text.slice(0, 400)}`
|
|
1655
|
+
}
|
|
1656
|
+
/* The answer may replace the words that were there, so cut at the last
|
|
1657
|
+
character the two reads share, not at the old length. */
|
|
1658
|
+
let shared = 0
|
|
1659
|
+
while (shared < fresh.length && shared < before.text.length && fresh[shared] === before.text[shared]) shared += 1
|
|
1660
|
+
const added = fresh.slice(shared)
|
|
1661
|
+
/* Lines are kept: a reviewer's list is one point per line, and flattening it
|
|
1662
|
+
would leave the next round with a paragraph instead of items. */
|
|
1663
|
+
const words = added.replace(/[ \t]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim().slice(0, 2500)
|
|
1664
|
+
return `the answer came back in ${Math.round(took / 100) / 10}s:\n${words}`
|
|
1665
|
+
}
|
|
1666
|
+
|
|
722
1667
|
export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
1668
|
+
/* A ref belongs to the page it was read from, and two chats read two pages.
|
|
1669
|
+
One shared read state meant chat B's read replaced chat A's baseline, so A's
|
|
1670
|
+
next "what changed" was a diff against a page it never saw and a number from
|
|
1671
|
+
A could be matched against B's control. Each chat now keeps its own. */
|
|
1672
|
+
const byChat = new Map()
|
|
1673
|
+
/**
|
|
1674
|
+
* The read state of the chat this call speaks for.
|
|
1675
|
+
* @param {string} [id] - chat id; the current one by default.
|
|
1676
|
+
* @returns {{refs: any[], at: number, last?: any, lastShape?: string}} its state.
|
|
1677
|
+
*/
|
|
1678
|
+
const refsFor = (id = currentSession()) => readStateFor(byChat, id, refs)
|
|
723
1679
|
/* Reading, clicking and typing on Chrome's own error page is not driving the
|
|
724
1680
|
site: the text there is Chrome talking, the buttons reload. Those calls are
|
|
725
1681
|
refused with the error code instead. The tools that report the browser
|
|
@@ -727,18 +1683,46 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
727
1683
|
run, because they are how a stuck person finds out what happened. */
|
|
728
1684
|
const withRawPage = async (work) => {
|
|
729
1685
|
if (!sidecar.alive) return { text: SIDECAR_OFF }
|
|
730
|
-
const page = await sidecar.ensure()
|
|
1686
|
+
const page = await sidecar.ensure({ session: currentSession() })
|
|
731
1687
|
const text = await work(page)
|
|
732
1688
|
return { text: text + personNote(sidecar) }
|
|
733
1689
|
}
|
|
734
1690
|
/** Looking is still useful while the person has the page. */
|
|
735
1691
|
const withRead = async (work) => {
|
|
736
|
-
|
|
737
|
-
|
|
1692
|
+
/* A call starting means the last click is over: a tab that opens now is not
|
|
1693
|
+
going to be followed under this call's feet. */
|
|
1694
|
+
try {
|
|
1695
|
+
sidecar.page?.disarmFollow?.()
|
|
1696
|
+
} catch {
|
|
1697
|
+
/* the page is already gone */
|
|
1698
|
+
}
|
|
1699
|
+
/* A plugin reload makes a new driver while the browser it started keeps
|
|
1700
|
+
running. Refusing every read until someone opens a page again would throw
|
|
1701
|
+
away a browser that is still there: attach to it instead. */
|
|
1702
|
+
if (!sidecar.alive) {
|
|
1703
|
+
/* A renderer crash can take the whole browser with it, and the first
|
|
1704
|
+
relaunch sometimes loses the race with the old process exiting. One
|
|
1705
|
+
more try turns that into a fresh tab instead of a refusal. */
|
|
1706
|
+
try {
|
|
1707
|
+
await sidecar.ensure({ session: currentSession() })
|
|
1708
|
+
} catch {
|
|
1709
|
+
await wait(300)
|
|
1710
|
+
try {
|
|
1711
|
+
await sidecar.ensure({ session: currentSession() })
|
|
1712
|
+
} catch {
|
|
1713
|
+
return { text: SIDECAR_OFF }
|
|
1714
|
+
}
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
const page = await sidecar.ensure({ session: currentSession() })
|
|
738
1718
|
const dead = await unreachable(page)
|
|
739
1719
|
if (dead !== '') return { text: dead + personNote(sidecar) }
|
|
740
1720
|
const text = await work(page)
|
|
741
|
-
|
|
1721
|
+
/* Why a page stopped working is usually in its own log, and looking it up
|
|
1722
|
+
costs another call. This is the one place every read passes through, so
|
|
1723
|
+
say what went wrong since the previous one, once. */
|
|
1724
|
+
const trouble = typeof page.troubleOf === 'function' ? page.troubleOf() : ''
|
|
1725
|
+
return { text: text + (trouble === '' ? '' : `\n${trouble}`) + personNote(sidecar) }
|
|
742
1726
|
}
|
|
743
1727
|
const withPage = async (work) => {
|
|
744
1728
|
if (!sidecar.alive) return { text: SIDECAR_OFF }
|
|
@@ -747,7 +1731,9 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
747
1731
|
return withRead(work)
|
|
748
1732
|
}
|
|
749
1733
|
const look = async (page, options) => {
|
|
750
|
-
const read = await remember(page,
|
|
1734
|
+
const read = await remember(page, refsFor(), options)
|
|
1735
|
+
/* A read already showed the address: the next action does not repeat it. */
|
|
1736
|
+
if (String(read.url ?? '') !== '') saidUrls.set(page, String(read.url))
|
|
751
1737
|
const said = renderPage(read, options)
|
|
752
1738
|
return isCheckPage(read) ? `${said}\n${CHECK_NOTE}` : said
|
|
753
1739
|
}
|
|
@@ -768,8 +1754,11 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
768
1754
|
*/
|
|
769
1755
|
const findAgain = async (page, ref) => {
|
|
770
1756
|
const spot = await locate(page, ref)
|
|
771
|
-
|
|
772
|
-
|
|
1757
|
+
const known = (refsFor().refs ?? []).find((control) => Number(control.ref) === Number(ref))
|
|
1758
|
+
if (spot.gone !== true) {
|
|
1759
|
+
const drift = differs(known, spot)
|
|
1760
|
+
return drift === '' ? spot : { gone: true, why: drift }
|
|
1761
|
+
}
|
|
773
1762
|
if (known === undefined) return spot
|
|
774
1763
|
const want = identityOf(known)
|
|
775
1764
|
if (want.name === '') return spot
|
|
@@ -786,13 +1775,17 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
786
1775
|
|
|
787
1776
|
return [
|
|
788
1777
|
{
|
|
789
|
-
name: '
|
|
790
|
-
description: 'Start (or reuse) the
|
|
1778
|
+
name: 'browser_open', // ui-copy-ignore
|
|
1779
|
+
description: 'Start (or reuse) the background browser — a real Chrome with its own persistent profile — and open a URL in it. It is not the host\'s own browser tab: this Chrome runs on its own and only its picture is mirrored into this chat\'s right column, so it keeps working while the person looks elsewhere. Logins survive restarts.',
|
|
791
1780
|
parameters: {
|
|
792
1781
|
type: 'object', // ui-copy-ignore
|
|
793
1782
|
properties: {
|
|
794
1783
|
url: { type: 'string', description: 'Address to open. Omit to reuse the current page.' },
|
|
795
1784
|
headless: { type: 'boolean', description: 'Run without a window. Default true; turn off only if a site refuses a windowless browser.' },
|
|
1785
|
+
flash: {
|
|
1786
|
+
type: 'boolean',
|
|
1787
|
+
description: 'Play the page in the Flash-era browser instead of Chrome — for a site whose game is a Flash movie that never got an HTML5 version. Default off; switching either way restarts the browser.',
|
|
1788
|
+
},
|
|
796
1789
|
},
|
|
797
1790
|
additionalProperties: false,
|
|
798
1791
|
},
|
|
@@ -801,33 +1794,270 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
801
1794
|
isConcurrencySafe: () => false,
|
|
802
1795
|
/** @param {any} args - arguments. @param {any} exec - tool run. @returns {Promise<{text: string}>} what the page is. */
|
|
803
1796
|
execute: async (args, exec) => {
|
|
804
|
-
const
|
|
1797
|
+
const who = String(exec?.agent?.id ?? '')
|
|
1798
|
+
const page = await sidecar.ensure({ url: args?.url, headless: args?.headless, flash: args?.flash, session: who })
|
|
805
1799
|
if (args?.url === undefined) await page.readTitle()
|
|
806
1800
|
/* A window on the person's screen is their business, not a surprise: say
|
|
807
1801
|
it, so nobody wonders what popped up. A site that refuses a windowless
|
|
808
1802
|
browser is the only reason this happens. */
|
|
809
|
-
|
|
1803
|
+
/* The Flash runtime is windowless even when headless is false: its window
|
|
1804
|
+
is never shown, so nothing popped up to explain. */
|
|
1805
|
+
const windowed = sidecar.headless === false && sidecar.flash !== true ? 'The browser is showing a window this time — a site only opens for a windowed one. Same profile, same logins.\n' : ''
|
|
810
1806
|
/* The tab appears in the calling chat and nowhere else: the person may
|
|
811
1807
|
be reading a different chat, and a browser there would be a surprise. */
|
|
812
|
-
const who = String(exec?.agent?.id ?? '')
|
|
813
1808
|
if (who !== '') host.want?.(who) // ui-copy-ignore
|
|
814
1809
|
await settle(page, 8000)
|
|
815
1810
|
const dead = await unreachable(page)
|
|
816
1811
|
if (dead !== '') return { text: dead }
|
|
817
1812
|
/* The column may still be drawing nothing: say so here, where the
|
|
818
|
-
person is looking for a picture, rather than only in
|
|
1813
|
+
person is looking for a picture, rather than only in browser_state. */
|
|
819
1814
|
const unseen = who === '' ? '' : viewerNote(await watchForViewer(sidecar, page, 1200))
|
|
820
|
-
return { text: windowed + (unseen === '' ? '' : `${unseen}\n`) + await look(page, { mode: 'text' }) }
|
|
1815
|
+
return { text: windowed + (unseen === '' ? '' : `${unseen}\n`) + await look(page, { mode: 'text' }) + (await pdfNote(page)) }
|
|
1816
|
+
},
|
|
1817
|
+
},
|
|
1818
|
+
{
|
|
1819
|
+
name: 'browser_mode', // ui-copy-ignore
|
|
1820
|
+
description: 'Choose how this chat acts from now on. "human" moves the real mouse and presses real keys, which no page can tell from a person — games and pages that watch pointer movement need this. "dom" clicks and fills the controls the page registered, which reaches one that an overlay covers and leaves key handlers alone. A call can also pass its own mode, which wins for that call only.',
|
|
1821
|
+
parameters: {
|
|
1822
|
+
type: 'object', // ui-copy-ignore
|
|
1823
|
+
properties: {
|
|
1824
|
+
mode: { type: 'string', enum: ['human', 'dom'], description: 'Which way to act.' },
|
|
1825
|
+
},
|
|
1826
|
+
required: ['mode'], // ui-copy-ignore
|
|
1827
|
+
additionalProperties: false,
|
|
1828
|
+
},
|
|
1829
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
1830
|
+
timeoutMs: 10_000,
|
|
1831
|
+
isConcurrencySafe: () => true,
|
|
1832
|
+
/** @param {any} args - arguments. @returns {Promise<{text: string}>} which way is set. */
|
|
1833
|
+
execute: async (args) => {
|
|
1834
|
+
const said = String(args?.mode ?? '')
|
|
1835
|
+
if (!WAYS.has(said)) return 'Pass "human" or "dom".'
|
|
1836
|
+
const chat = currentSession()
|
|
1837
|
+
if (said === 'human') chatWays.delete(chat)
|
|
1838
|
+
else chatWays.set(chat, said)
|
|
1839
|
+
return said === 'human'
|
|
1840
|
+
? 'Real mouse and real keys from now on: pages cannot tell it from a person.'
|
|
1841
|
+
: "The page's own controls from now on: clicks and typing go through its DOM, so an overlay cannot swallow them."
|
|
1842
|
+
},
|
|
1843
|
+
},
|
|
1844
|
+
{
|
|
1845
|
+
name: 'browser_prepare', // ui-copy-ignore
|
|
1846
|
+
description: 'Get a game or a heavy app page ready to play, in one call: read what is on it, find the surface that takes input, say whether it needs a Flash player this browser does not have, measure its real frame rate, and test whether keys reach it. Ends with what to do next instead of a blank page and no explanation.',
|
|
1847
|
+
parameters: {
|
|
1848
|
+
type: 'object', // ui-copy-ignore
|
|
1849
|
+
properties: {
|
|
1850
|
+
seconds: { type: 'number', description: 'How long to count frames for. Default 1.2, at most 5.' },
|
|
1851
|
+
focus: { type: 'boolean', description: 'Click the surface to put keyboard focus in it. Default true.' },
|
|
1852
|
+
},
|
|
1853
|
+
additionalProperties: false,
|
|
1854
|
+
},
|
|
1855
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
1856
|
+
timeoutMs: 45_000,
|
|
1857
|
+
isConcurrencySafe: () => false,
|
|
1858
|
+
/** @param {any} args - arguments. @param {any} exec - tool run. @returns {Promise<{text: string}>} what the page is ready for. */
|
|
1859
|
+
execute: async (args, exec) => {
|
|
1860
|
+
const who = String(exec?.agent?.id ?? '')
|
|
1861
|
+
const page = await sidecar.ensure({ session: who })
|
|
1862
|
+
await settle(page, 8000)
|
|
1863
|
+
const seconds = Math.min(5, Math.max(0.5, Number(args?.seconds) || 1.2))
|
|
1864
|
+
const ms = Math.round(seconds * 1000)
|
|
1865
|
+
/* A page that never arrived is not a game that is not ready: say which. */
|
|
1866
|
+
const dead = await unreachable(page)
|
|
1867
|
+
if (dead !== '') return { text: dead }
|
|
1868
|
+
const read = await readPage(page, { mode: 'text', maxChars: 1500 })
|
|
1869
|
+
const said = asSpot(await evaluate(page, `(() => {
|
|
1870
|
+
const box = (el) => { const r = el.getBoundingClientRect(); return { w: Math.round(r.width), h: Math.round(r.height), x: Math.round(r.x + r.width / 2), y: Math.round(r.y + r.height / 2) } }
|
|
1871
|
+
const shown = (el) => {
|
|
1872
|
+
const r = el.getBoundingClientRect()
|
|
1873
|
+
const st = getComputedStyle(el)
|
|
1874
|
+
return r.width >= 120 && r.height >= 80 && st.visibility !== 'hidden' && st.display !== 'none' && Number(st.opacity) > 0.05
|
|
1875
|
+
}
|
|
1876
|
+
const spot = (el, tag) => {
|
|
1877
|
+
const first = box(el)
|
|
1878
|
+
const seen = first.y > 0 && first.y < innerHeight && first.x > 0 && first.x < innerWidth
|
|
1879
|
+
if (!seen) {
|
|
1880
|
+
/* A surface below the fold is not where the person is looking, and
|
|
1881
|
+
a click at its coordinate would land somewhere else. */
|
|
1882
|
+
el.scrollIntoView({ block: 'center', inline: 'center' })
|
|
1883
|
+
return { tag, src: String(el.getAttribute('src') ?? ''), ...box(el), scrolled: true }
|
|
1884
|
+
}
|
|
1885
|
+
return { tag, src: String(el.getAttribute('src') ?? ''), ...first }
|
|
1886
|
+
}
|
|
1887
|
+
const canvases = Array.from(document.querySelectorAll('canvas')).filter(shown).map((el) => spot(el, 'canvas'))
|
|
1888
|
+
const frames = Array.from(document.querySelectorAll('iframe')).filter(shown).map((el) => spot(el, 'frame'))
|
|
1889
|
+
const movies = document.querySelectorAll('embed, object').length
|
|
1890
|
+
return JSON.stringify({ canvases, frames, movies })
|
|
1891
|
+
})()`))
|
|
1892
|
+
const surface = said?.canvases?.[0] ?? said?.frames?.[0] ?? null
|
|
1893
|
+
/* Give a game the click a person gives it: some of them only start
|
|
1894
|
+
listening for keys once the surface has been touched. */
|
|
1895
|
+
let clicked = ''
|
|
1896
|
+
if (args?.focus !== false && surface !== null) {
|
|
1897
|
+
await page.mouse('mouseMoved', { x: surface.x - 6, y: surface.y - 4 }).catch(() => {})
|
|
1898
|
+
await wait(40)
|
|
1899
|
+
await page.mouse('mousePressed', { x: surface.x, y: surface.y, clickCount: 1 }).catch(() => {})
|
|
1900
|
+
await wait(30)
|
|
1901
|
+
await page.mouse('mouseReleased', { x: surface.x, y: surface.y, clickCount: 1 }).catch(() => {})
|
|
1902
|
+
clicked = `clicked the ${surface.tag} at ${surface.x},${surface.y}`
|
|
1903
|
+
}
|
|
1904
|
+
/* Count the page's own frames and listen for a key, in the page and in
|
|
1905
|
+
the biggest frame, because a game is usually not in the page itself. */
|
|
1906
|
+
const frames = await page.frameList().catch(() => [])
|
|
1907
|
+
const routeOfFrame = (frame) => {
|
|
1908
|
+
if (frame === undefined) return null
|
|
1909
|
+
const sessionId = frame.contextId === undefined && typeof page.childSession === 'function' ? page.childSession(frame.frameId) : undefined
|
|
1910
|
+
if (frame.contextId === undefined && sessionId === undefined) return null
|
|
1911
|
+
return { contextId: frame.contextId, sessionId }
|
|
1912
|
+
}
|
|
1913
|
+
const countFrames = `new Promise((done) => {
|
|
1914
|
+
window.__dshKeys = 0
|
|
1915
|
+
if (window.__dshKeyUp === undefined) {
|
|
1916
|
+
window.__dshKeyUp = true
|
|
1917
|
+
addEventListener('keydown', () => { window.__dshKeys += 1 }, true)
|
|
1918
|
+
}
|
|
1919
|
+
let n = 0
|
|
1920
|
+
const t0 = performance.now()
|
|
1921
|
+
const tick = () => { n += 1; if (performance.now() - t0 < ${ms}) requestAnimationFrame(tick) }
|
|
1922
|
+
requestAnimationFrame(tick)
|
|
1923
|
+
setTimeout(() => done(JSON.stringify({ fps: Math.round((n * 1000) / Math.max(1, performance.now() - t0)), keys: window.__dshKeys })), ${ms + 60})
|
|
1924
|
+
})`
|
|
1925
|
+
const mainRate = asSpot(await evaluate(page, countFrames))
|
|
1926
|
+
let frameRate = null
|
|
1927
|
+
const childFrames = frames.slice(1).filter((frame) => String(frame.url ?? '') !== 'chrome-error://chromewebdata/')
|
|
1928
|
+
const biggest = childFrames[0]
|
|
1929
|
+
const route = routeOfFrame(biggest)
|
|
1930
|
+
if (route !== null) frameRate = asSpot(await evaluate(page, countFrames, route))
|
|
1931
|
+
let keys = route === null ? 'none arrived — there is no frame to test' : 'none arrived in the frame'
|
|
1932
|
+
const keyDown = { type: 'keyDown', key: 'Shift', code: 'ShiftLeft', windowsVirtualKeyCode: 16, nativeVirtualKeyCode: 16 }
|
|
1933
|
+
await page.cdp.send('Input.dispatchKeyEvent', keyDown, route?.sessionId).catch(() => {})
|
|
1934
|
+
await page.cdp.send('Input.dispatchKeyEvent', { ...keyDown, key: 'Shift', type: 'keyUp' }, route?.sessionId).catch(() => {})
|
|
1935
|
+
await wait(80)
|
|
1936
|
+
const after = asSpot(await evaluate(page, 'JSON.stringify({ keys: window.__dshKeys ?? 0 })', route))
|
|
1937
|
+
const mainAfter = asSpot(await evaluate(page, 'JSON.stringify({ keys: window.__dshKeys ?? 0 })'))
|
|
1938
|
+
const arrived = Number(after?.keys ?? 0) + Number(mainAfter?.keys ?? 0) - Number(frameRate?.keys ?? 0) - Number(mainRate?.keys ?? 0)
|
|
1939
|
+
if (arrived > 0) keys = `arrived in the frame (${arrived} keydown event${arrived === 1 ? '' : 's'} from a real key)`
|
|
1940
|
+
else if (route !== null) keys = 'not in the frame — click inside it once, or the game only listens after it starts'
|
|
1941
|
+
const lines = []
|
|
1942
|
+
lines.push(`page: ${read.title === '' ? read.url : read.title} — ${read.controls.length} control(s), ${read.textChars} character(s)`)
|
|
1943
|
+
lines.push(surface === null ? 'surface: no canvas or frame takes the screen — the page itself takes input' : `surface: ${surface.tag} ${surface.w}x${surface.h} at ${surface.x},${surface.y}${surface.scrolled === true ? ' (scrolled it into view)' : ''}${clicked === '' ? '' : ` (${clicked})`}`)
|
|
1944
|
+
if (read.blind?.flash > 0 && sidecar.flash !== true) lines.push(`flash: ${read.blind.flash} movie(s) — open the same page with flash:true and it runs in this plugin's own player, in the column, with no window on the desktop`) // ui-copy-ignore
|
|
1945
|
+
else if (read.blind?.flash > 0) lines.push(`flash: ${read.blind.flash} movie(s) running on this plugin's own player`)
|
|
1946
|
+
else lines.push('flash: none found')
|
|
1947
|
+
lines.push(`frames: ${Math.max(0, frames.length - 1)} (${childFrames.filter((frame) => String(frame.url ?? '') === 'about:blank').length} of them about:blank)`)
|
|
1948
|
+
const rates = []
|
|
1949
|
+
if (mainRate !== null) rates.push(`page ${mainRate.fps} fps`)
|
|
1950
|
+
if (frameRate !== null) rates.push(`frame ${frameRate.fps} fps`)
|
|
1951
|
+
lines.push(`frame rate: ${rates.length === 0 ? 'the page could not count' : rates.join(', ')} over ${ms} ms`)
|
|
1952
|
+
lines.push(`keys: ${keys}`)
|
|
1953
|
+
lines.push(`mode: human${surface === null ? '' : ' — a game watches the mouse and the keys, and only real ones arrive trusted'}`)
|
|
1954
|
+
const trouble = read.blind?.flash > 0 && sidecar.flash !== true
|
|
1955
|
+
? 'call open with flash:true to run it here'
|
|
1956
|
+
: read.blind?.flash > 0
|
|
1957
|
+
? 'the movie needs a click inside it before it starts'
|
|
1958
|
+
: surface === null && Number(mainRate?.fps ?? 0) === 0
|
|
1959
|
+
? 'nothing on this page is animating — it may be waiting for a click, or the game is behind a login'
|
|
1960
|
+
: surface === null
|
|
1961
|
+
? 'ready to read and act on, though there is no canvas or frame to watch'
|
|
1962
|
+
: `ready: ${surface.tag} is live at ${mainRate?.fps ?? frameRate?.fps ?? 0} fps`
|
|
1963
|
+
lines.push(trouble)
|
|
1964
|
+
return { text: lines.join('\n') }
|
|
821
1965
|
},
|
|
822
1966
|
},
|
|
823
1967
|
{
|
|
824
|
-
name: '
|
|
825
|
-
description: '
|
|
1968
|
+
name: 'browser_find', // ui-copy-ignore
|
|
1969
|
+
description: 'Find words on the page without reading it all: returns matching numbered controls (click the ref) and matching text (click x/y), each with whether it is on screen. A phone-sized page has hundreds of lines; this answers "where does it say X" in one compact result.',
|
|
826
1970
|
parameters: {
|
|
827
1971
|
type: 'object', // ui-copy-ignore
|
|
828
1972
|
properties: {
|
|
829
|
-
|
|
830
|
-
|
|
1973
|
+
query: { type: 'string', description: 'Words to look for; case does not matter.' },
|
|
1974
|
+
regex: { type: 'boolean', description: 'Treat query as a regular expression.' },
|
|
1975
|
+
limit: { type: 'integer', description: 'Matches to return. Default 12, at most 40.' },
|
|
1976
|
+
},
|
|
1977
|
+
required: ['query'],
|
|
1978
|
+
additionalProperties: false,
|
|
1979
|
+
},
|
|
1980
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
1981
|
+
timeoutMs: 30_000,
|
|
1982
|
+
isConcurrencySafe: () => false,
|
|
1983
|
+
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the matches. */
|
|
1984
|
+
execute: async (args) => withRead(async (page) => {
|
|
1985
|
+
const query = String(args?.query ?? '').trim()
|
|
1986
|
+
if (query === '') return 'Give the words to look for.'
|
|
1987
|
+
const found = await findOnPage(page, query, { regex: args?.regex === true, limit: Number(args?.limit ?? 12) })
|
|
1988
|
+
if (found.fault !== undefined) return `the page did not answer when searched — ${String(found.fault).split('\n')[0].slice(0, 200)}`
|
|
1989
|
+
const matches = Array.isArray(found.matches) ? found.matches : []
|
|
1990
|
+
if (matches.length === 0) return `nothing on this page says ${JSON.stringify(query)} — try other words, or call browser_view for the whole page.`
|
|
1991
|
+
const lines = matches.map((one) => `${one.ref === 0 ? `at ${one.x},${one.y}` : one.ref} [${one.tag}] ${one.name}${one.view === true ? '' : ' (off screen)'}`)
|
|
1992
|
+
return `${matches.length} match(es) for ${JSON.stringify(query)}:\n${lines.join('\n')}`
|
|
1993
|
+
}),
|
|
1994
|
+
},
|
|
1995
|
+
{
|
|
1996
|
+
name: 'browser_read', // ui-copy-ignore
|
|
1997
|
+
description: 'Read the page as a person does: one Markdown block of headings, paragraphs, lists and links, no controls and no markup. Far fewer tokens than browser_view on long pages (articles, docs, threads, issue trackers), and it keeps links so a follow-up can be opened by address. Use browser_view when you need numbers to click or type; use browser_read when you only need the words.',
|
|
1998
|
+
parameters: {
|
|
1999
|
+
type: 'object', // ui-copy-ignore
|
|
2000
|
+
properties: {
|
|
2001
|
+
max_chars: { type: 'integer', description: 'Character ceiling. Default 6000; the result says when it was cut.' },
|
|
2002
|
+
query: { type: 'string', description: 'What you want to know. With a query, only the passages that answer it come back, ranked — the cheapest way to read a long page. Without one, the whole page comes back.' },
|
|
2003
|
+
force: { type: 'boolean', description: 'Read again even if the page has not changed since this chat last read it.' },
|
|
2004
|
+
},
|
|
2005
|
+
additionalProperties: false,
|
|
2006
|
+
},
|
|
2007
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
2008
|
+
timeoutMs: 30_000,
|
|
2009
|
+
isConcurrencySafe: () => false,
|
|
2010
|
+
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the words, as Markdown. */
|
|
2011
|
+
execute: async (args) => withRead(async (page) => {
|
|
2012
|
+
const asked = String(args?.query ?? '').trim()
|
|
2013
|
+
const ceiling = Number(args?.max_chars ?? (asked === '' ? 6000 : 4000))
|
|
2014
|
+
const mark = await pageMark(page)
|
|
2015
|
+
const key = mark === '' ? '' : `${mark}|${asked}|${ceiling}`
|
|
2016
|
+
const seen = page === null || page === undefined ? undefined : readSeen.get(page)
|
|
2017
|
+
/* The text this chat already has is not worth sending twice: a page that
|
|
2018
|
+
looks the same answers with where it is, not the same four thousand
|
|
2019
|
+
characters. force is for a caller that lost the earlier answer. */
|
|
2020
|
+
if (args?.force !== true && key !== '' && seen !== undefined && seen.key === key) {
|
|
2021
|
+
return `${mark.split('|')[1] ?? ''} · unchanged since this chat last read it (${String(seen.text.length)} chars) — the text is the one above. Call browser_view with changed_only to see what moved, or pass force for it again.`
|
|
2022
|
+
}
|
|
2023
|
+
const keep = (text) => {
|
|
2024
|
+
if (key !== '' && page !== null && page !== undefined) readSeen.set(page, { key, text })
|
|
2025
|
+
return text
|
|
2026
|
+
}
|
|
2027
|
+
if (asked !== '') {
|
|
2028
|
+
const found = await readRelevant(page, asked, { maxChars: ceiling })
|
|
2029
|
+
if (found.fault !== undefined) return `the page did not answer when read — ${String(found.fault).split('\n')[0].slice(0, 200)}`
|
|
2030
|
+
if (found.text === '') {
|
|
2031
|
+
/* The question missed every block: the page is still the answer, so
|
|
2032
|
+
hand it over whole rather than make the caller guess again. */
|
|
2033
|
+
const whole = await readArticle(page, { maxChars: ceiling })
|
|
2034
|
+
if (whole.fault === undefined && whole.text !== '') {
|
|
2035
|
+
return keep(`${whole.url}\n# ${whole.title === '' ? '(untitled)' : whole.title}\n\nnothing here answers ${JSON.stringify(asked)} (${String(found.total)} blocks read), so the whole page follows:\n\n${whole.text}${whole.truncated ? `\n\n(cut at ${String(whole.chars)} characters; raise max_chars for more)` : ''}`)
|
|
2036
|
+
}
|
|
2037
|
+
return `${found.url}\n# ${found.title === '' ? '(untitled)' : found.title}\n\nnothing here answers ${JSON.stringify(asked)} (${String(found.total)} blocks read). Call browser_view for the whole page, or try other words.`
|
|
2038
|
+
}
|
|
2039
|
+
return keep(`${found.url}\n# ${found.title === '' ? '(untitled)' : found.title}\n\n${found.text}\n\n(${String(found.matched)} of ${String(found.total)} blocks, ranked for ${JSON.stringify(asked)})`)
|
|
2040
|
+
}
|
|
2041
|
+
const read = await readArticle(page, { maxChars: ceiling })
|
|
2042
|
+
if (read.fault !== undefined) return `the page did not answer when read — ${read.fault.split('\n')[0].slice(0, 200)}`
|
|
2043
|
+
if (read.text === '') {
|
|
2044
|
+
const pdf = await pdfNote(page)
|
|
2045
|
+
if (pdf !== '') return `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}${pdf}`
|
|
2046
|
+
return `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}\n\nnothing to read here: the page draws its content, or it is still loading. Call browser_view to see what is on screen.`
|
|
2047
|
+
}
|
|
2048
|
+
const head = `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}`
|
|
2049
|
+
const tail = read.truncated ? `\n\n(cut at ${String(read.chars)} characters; raise max_chars for more)` : ''
|
|
2050
|
+
return keep(`${head}\n\n${read.text}${tail}`)
|
|
2051
|
+
}),
|
|
2052
|
+
},
|
|
2053
|
+
{
|
|
2054
|
+
name: 'browser_view', // ui-copy-ignore
|
|
2055
|
+
description: 'Read the sidecar browser: numbered controls and the words on screen. Numbers stay valid across reads and actions until the page navigates. Narrow it with mode (text, links, forms, full, map) and max_chars; ask for changed_only after an action to spend fewer tokens. Use map first on a page too big to read.',
|
|
2056
|
+
parameters: {
|
|
2057
|
+
type: 'object', // ui-copy-ignore
|
|
2058
|
+
properties: {
|
|
2059
|
+
mode: { type: 'string', enum: ['text', 'links', 'forms', 'full', 'map'], description: 'What to include. Default text: controls plus the visible words. map: a landmark summary of a big page — how many controls of each kind, the distinct ones with their numbers, and how many words there are, without the words.' },
|
|
2060
|
+
max_chars: { type: 'integer', description: 'Character budget for the page text. Default 4600.' },
|
|
831
2061
|
changed_only: { type: 'boolean', description: 'Only what changed since the last read.' },
|
|
832
2062
|
},
|
|
833
2063
|
additionalProperties: false,
|
|
@@ -838,25 +2068,56 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
838
2068
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the read. */
|
|
839
2069
|
execute: async (args) => withRead(async (page) => {
|
|
840
2070
|
const options = { mode: String(args?.mode ?? 'text'), maxChars: Number(args?.max_chars ?? 4600) }
|
|
841
|
-
|
|
2071
|
+
/* Each chat keeps its own read state: a shared one let another chat's
|
|
2072
|
+
read replace this chat's baseline, so its numbers and its "what
|
|
2073
|
+
changed" answered about a page it never saw. */
|
|
2074
|
+
const state = refsFor()
|
|
2075
|
+
const read = await remember(page, state, options)
|
|
842
2076
|
const note = isCheckPage(read) ? `\n${CHECK_NOTE}` : ''
|
|
843
|
-
if (args?.changed_only === true &&
|
|
844
|
-
|
|
845
|
-
|
|
2077
|
+
if (args?.changed_only === true && state.last !== undefined) {
|
|
2078
|
+
/* The diff becomes the new baseline: a page that moved zero -> one
|
|
2079
|
+
should not report that move again on the next read. */
|
|
2080
|
+
const text = diffRead(state.last, read, options) + note
|
|
2081
|
+
state.last = read
|
|
2082
|
+
state.lastShape = JSON.stringify(options)
|
|
2083
|
+
return text
|
|
2084
|
+
}
|
|
2085
|
+
/* The read before this one answers two questions: what is new on this
|
|
2086
|
+
one, and whether the page moved at all. Keep it before either answer
|
|
2087
|
+
is overwritten. */
|
|
2088
|
+
const before = state.last
|
|
2089
|
+
state.last = read
|
|
2090
|
+
const fresh = freshRefs(before, read)
|
|
2091
|
+
const freshLine = fresh.length === 0 || fresh.length > 12 ? '' : `\nnew since the read before: ${fresh.map(String).join(', ')}`
|
|
2092
|
+
/* Same page, same shape, same words: the numbers above still work, and
|
|
2093
|
+
repeating the page would only spend tokens. */
|
|
2094
|
+
if (sameRead(before, read) && state.lastShape === JSON.stringify(options)) {
|
|
2095
|
+
const moved = Number(before.scroll?.y ?? 0) !== Number(read.scroll?.y ?? 0) ? `, scrolled to ${Math.round(Number(read.scroll?.y ?? 0))}` : ''
|
|
2096
|
+
/* The address was said with the read above; "here" is the whole of it. */
|
|
2097
|
+
const where = saidUrls.get(page) === String(read.url ?? '') ? 'here' : read.url
|
|
2098
|
+
saidUrls.set(page, String(read.url ?? ''))
|
|
2099
|
+
return `still ${where}${moved} — nothing changed since the last read; the numbers above still work${note}`
|
|
2100
|
+
}
|
|
2101
|
+
state.lastShape = JSON.stringify(options)
|
|
2102
|
+
const said = withoutSeenAddress(page, read.url, renderPage(read, options))
|
|
2103
|
+
saidUrls.set(page, String(read.url ?? ''))
|
|
2104
|
+
return said + freshLine + note + (await pdfNote(page))
|
|
846
2105
|
}),
|
|
847
2106
|
},
|
|
848
2107
|
{
|
|
849
|
-
name: '
|
|
2108
|
+
name: 'browser_click', // ui-copy-ignore
|
|
850
2109
|
description: 'Click a numbered control, a point, or the first control whose words contain `text` — the last needs no read first. The click is real mouse input and the mirror shows where it landed.',
|
|
851
2110
|
parameters: {
|
|
852
2111
|
type: 'object', // ui-copy-ignore
|
|
853
2112
|
properties: {
|
|
854
|
-
ref: { type: 'integer', description: 'Number from
|
|
2113
|
+
ref: { type: 'integer', description: 'Number from browser_view.' },
|
|
855
2114
|
text: { type: 'string', description: 'Words on the control, when its number is not known.' },
|
|
856
2115
|
x: { type: 'integer', description: 'Viewport x, when clicking a point.' },
|
|
857
2116
|
y: { type: 'integer', description: 'Viewport y, when clicking a point.' },
|
|
858
2117
|
double: { type: 'boolean', description: 'Double-click.' },
|
|
859
2118
|
right: { type: 'boolean', description: 'Right-click, opening a context menu.' },
|
|
2119
|
+
hold_ms: { type: 'integer', description: 'Keep the button down this long before releasing: a press-and-hold a game or a drag handle needs.' },
|
|
2120
|
+
mode: { type: 'string', enum: ['human', 'dom'], description: '"human" moves the real mouse and is what a page cannot tell from a person; "dom" fires the page\'s own click on the numbered control, which reaches one an overlay covers. Omit to use what this chat set with browser_mode.' },
|
|
860
2121
|
read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
|
|
861
2122
|
},
|
|
862
2123
|
additionalProperties: false,
|
|
@@ -871,36 +2132,55 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
871
2132
|
let followed = ''
|
|
872
2133
|
if (args?.ref !== undefined) {
|
|
873
2134
|
const spot = await findAgain(page, Number(args.ref))
|
|
874
|
-
if (spot.gone === true) return REF_GONE
|
|
2135
|
+
if (spot.gone === true) return spot.why ?? REF_GONE
|
|
875
2136
|
if (spot.followed === true) followed = `the page redrew, so "${String(spot.was).slice(0, 50)}" has a new number — used where it is now`
|
|
876
2137
|
point = { x: Number(spot.x), y: Number(spot.y) }
|
|
877
2138
|
} else if (String(args?.text ?? '') !== '') {
|
|
878
|
-
const hit = await findWords(page, String(args.text))
|
|
879
|
-
if (hit === null) return `No control says ${JSON.stringify(String(args.text))}. Call
|
|
2139
|
+
const hit = await findWords(page, String(args.text), refsFor().refs)
|
|
2140
|
+
if (hit === null) return `No control says ${JSON.stringify(String(args.text))}. Call browser_view to see what is there.`
|
|
880
2141
|
point = { x: hit.x, y: hit.y }
|
|
881
2142
|
label = `"${hit.name.slice(0, 60)}"`
|
|
882
2143
|
}
|
|
883
|
-
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) return 'Give a ref from
|
|
2144
|
+
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) return 'Give a ref from browser_view, or both x and y.'
|
|
2145
|
+
if (wayOf(args) === 'dom') {
|
|
2146
|
+
if (args?.ref === undefined) return 'Acting on the page itself needs a number from browser_view: pass ref, or leave mode out to click like a person.'
|
|
2147
|
+
const pressedAt = Date.now()
|
|
2148
|
+
const said = await evaluate(page, clickInDom(Number(args.ref), { double: args?.double === true, right: args?.right === true })).catch(() => undefined)
|
|
2149
|
+
const hit = asSpot(said)
|
|
2150
|
+
if (hit?.ok !== true) return REF_GONE
|
|
2151
|
+
await settle(page, 5000, pressedAt)
|
|
2152
|
+
const called = await acted(page, args, () => look(page, { mode: 'text', maxChars: 3000 }), `click ${label} on the page itself`)
|
|
2153
|
+
return called + (followed === '' ? '' : ` (${followed})`)
|
|
2154
|
+
}
|
|
884
2155
|
const button = args?.right === true ? 'right' : 'left'
|
|
885
2156
|
let cover = ''
|
|
886
2157
|
if (args?.ref !== undefined && button === 'left') {
|
|
887
2158
|
const gate = await clearCover(page, Number(args.ref))
|
|
888
2159
|
cover = gate.note
|
|
2160
|
+
/* An overlay that would not go away leaves the control underneath:
|
|
2161
|
+
the click would land on the overlay, so it is not sent. */
|
|
2162
|
+
if (gate.cleared !== true) {
|
|
2163
|
+
return `${cover.trim()}\nNot clicked: the control is still covered. Call browser_view and handle the overlay first.`
|
|
2164
|
+
}
|
|
889
2165
|
if (gate.cleared && cover !== '') {
|
|
890
2166
|
const again = await locate(page, Number(args.ref))
|
|
891
2167
|
if (again.gone !== true) point = { x: Number(again.x), y: Number(again.y) }
|
|
892
2168
|
}
|
|
893
2169
|
}
|
|
2170
|
+
const before = args?.read === true ? '' : await pageMark(page)
|
|
2171
|
+
const pressedAt = Date.now()
|
|
894
2172
|
await page.mouse('mouseMoved', point)
|
|
895
2173
|
await page.mouse('mousePressed', { ...point, clickCount: args?.double === true ? 2 : 1, button })
|
|
2174
|
+
const hold = Math.max(0, Math.min(60_000, Number(args?.hold_ms ?? 0)))
|
|
2175
|
+
if (hold > 0) await new Promise((settle) => setTimeout(settle, hold))
|
|
896
2176
|
await page.mouse('mouseReleased', { ...point, clickCount: args?.double === true ? 2 : 1, button })
|
|
897
|
-
await settle(page)
|
|
2177
|
+
await settle(page, 5000, pressedAt)
|
|
898
2178
|
const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 3000 }), `click ${label}`)
|
|
899
|
-
return said + cover + (followed === '' ? '' : ` (${followed})`)
|
|
2179
|
+
return said + cover + (followed === '' ? '' : ` (${followed})`) + (await outcome(page, before))
|
|
900
2180
|
}),
|
|
901
2181
|
},
|
|
902
2182
|
{
|
|
903
|
-
name: '
|
|
2183
|
+
name: 'browser_type', // ui-copy-ignore
|
|
904
2184
|
description: 'Type text into the sidecar browser — Chinese included — optionally pressing Enter. Pass a ref to focus that control first. With clear, the field is emptied first.',
|
|
905
2185
|
parameters: {
|
|
906
2186
|
type: 'object', // ui-copy-ignore
|
|
@@ -910,6 +2190,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
910
2190
|
field: { type: 'string', description: 'The field is called this on screen, when you have no number for it.' },
|
|
911
2191
|
submit: { type: 'boolean', description: 'Press Enter after typing.' },
|
|
912
2192
|
clear: { type: 'boolean', description: 'Select all and delete before typing.' },
|
|
2193
|
+
mode: { type: 'string', enum: ['human', 'dom'], description: '"human" presses real keys and is what a page cannot tell from a person; "dom" sets the field and fires its input events, which leaves key handlers untouched. Omit to use what this chat set with browser_mode.' },
|
|
913
2194
|
read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
|
|
914
2195
|
},
|
|
915
2196
|
required: ['text'], // ui-copy-ignore
|
|
@@ -922,11 +2203,29 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
922
2203
|
execute: async (args) => withPage(async (page) => {
|
|
923
2204
|
let cover = ''
|
|
924
2205
|
let followed = ''
|
|
2206
|
+
if (wayOf(args) === 'dom') {
|
|
2207
|
+
let target = Number.isInteger(args?.ref) ? Number(args.ref) : undefined
|
|
2208
|
+
if (target === undefined && args?.field !== undefined) {
|
|
2209
|
+
const found = await fieldByName(page, String(args.field), refsFor().refs)
|
|
2210
|
+
if (found === null) return `${String(args.field)}: ${REF_GONE}`
|
|
2211
|
+
if (!Number.isInteger(found.ref)) return `${String(args.field)}: read the page and pass its number, since a field found by its label has none.`
|
|
2212
|
+
target = Number(found.ref)
|
|
2213
|
+
cover = found.lead
|
|
2214
|
+
}
|
|
2215
|
+
if (target === undefined) return 'Acting on the page itself needs a number from browser_view: pass ref, or leave mode out to type like a person.'
|
|
2216
|
+
const said = await evaluate(page, writeInDom(target, String(args?.text ?? ''), { clear: args?.clear === true, submit: args?.submit === true })).catch(() => undefined)
|
|
2217
|
+
const wrote = asSpot(said)
|
|
2218
|
+
if (wrote?.ok !== true) return REF_GONE
|
|
2219
|
+
await settle(page)
|
|
2220
|
+
const called = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type on the page itself')
|
|
2221
|
+
return `${called} (field now "${String(wrote.value).slice(0, 40)}")` + cover
|
|
2222
|
+
}
|
|
925
2223
|
if (args?.field !== undefined && args?.ref === undefined) {
|
|
926
|
-
const target = await fieldByName(page, String(args.field))
|
|
2224
|
+
const target = await fieldByName(page, String(args.field), refsFor().refs)
|
|
927
2225
|
if (target === null) return `${String(args.field)}: ${REF_GONE}`
|
|
928
2226
|
cover = target.lead
|
|
929
|
-
await writeAt(page, { x: target.x, y: target.y }, String(args?.text ?? ''), { clear: args?.clear, submit: args?.submit === true })
|
|
2227
|
+
const dated = await writeAt(page, { x: target.x, y: target.y }, String(args?.text ?? ''), { clear: args?.clear, submit: args?.submit === true, hint: target.name })
|
|
2228
|
+
if (dated !== '') return dated + cover
|
|
930
2229
|
await settle(page)
|
|
931
2230
|
const called = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type')
|
|
932
2231
|
return called + cover + ` (typed into "${target.name.slice(0, 50)}")` // ui-copy-ignore
|
|
@@ -934,8 +2233,11 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
934
2233
|
if (args?.ref !== undefined) {
|
|
935
2234
|
const gate = await clearCover(page, Number(args.ref))
|
|
936
2235
|
cover = gate.note
|
|
2236
|
+
if (gate.cleared !== true) {
|
|
2237
|
+
return `${cover.trim()}\nNot typed into: the field is still covered. Call browser_view and handle the overlay first.`
|
|
2238
|
+
}
|
|
937
2239
|
const spot = await findAgain(page, Number(args.ref))
|
|
938
|
-
if (spot.gone === true) return REF_GONE + cover
|
|
2240
|
+
if (spot.gone === true) return spot.why ?? REF_GONE + cover
|
|
939
2241
|
if (spot.followed === true) followed = `the page redrew, so "${String(spot.was).slice(0, 50)}" has a new number — typed where it is now`
|
|
940
2242
|
const point = { x: Number(spot.x), y: Number(spot.y) }
|
|
941
2243
|
await page.mouse('mouseMoved', point)
|
|
@@ -944,21 +2246,136 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
944
2246
|
await wait(80)
|
|
945
2247
|
}
|
|
946
2248
|
if (args?.clear === true) await clearFocus(page)
|
|
947
|
-
await page
|
|
2249
|
+
if (await dateLikeFocus(page) !== '') return (await fillDateLike(page, String(args?.text ?? ''))) + cover
|
|
2250
|
+
const words = String(args?.text ?? '')
|
|
2251
|
+
const before = args?.read === true ? '' : await pageMark(page)
|
|
2252
|
+
await typeInto(page, words, String(args?.field ?? ''))
|
|
2253
|
+
const dated = await fillDateLike(page, words)
|
|
2254
|
+
if (dated !== '') return dated + cover
|
|
948
2255
|
if (args?.submit === true) await pressKey(page, 'Enter')
|
|
949
2256
|
await settle(page)
|
|
950
2257
|
const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), 'type')
|
|
951
|
-
return said + cover + (followed === '' ? '' : ` (${followed})`) // ui-copy-ignore
|
|
2258
|
+
return said + cover + (followed === '' ? '' : ` (${followed})`) + (await outcome(page, before)) // ui-copy-ignore
|
|
952
2259
|
}),
|
|
953
2260
|
},
|
|
954
2261
|
{
|
|
955
|
-
name: '
|
|
956
|
-
description: '
|
|
2262
|
+
name: 'browser_ask', // ui-copy-ignore
|
|
2263
|
+
description: 'Say something to a chat that is open in the page — a model, a support window, a search box — and read its answer back in the same call. Types into the box, presses Enter, then waits until the answer stops growing. Use it to get a review or a picture prompt answered without a read between every step.',
|
|
957
2264
|
parameters: {
|
|
958
2265
|
type: 'object', // ui-copy-ignore
|
|
959
2266
|
properties: {
|
|
960
|
-
|
|
961
|
-
|
|
2267
|
+
text: { type: 'string', description: 'What to say.' },
|
|
2268
|
+
ref: { type: 'integer', description: 'The box to type in, from browser_view. Omit to use the page\u2019s own chat box.' },
|
|
2269
|
+
submit: { type: 'boolean', description: 'Press Enter to send. Default true.' },
|
|
2270
|
+
timeout_ms: { type: 'integer', description: 'How long to wait for the answer. Default 60000.' },
|
|
2271
|
+
},
|
|
2272
|
+
required: ['text'], // ui-copy-ignore
|
|
2273
|
+
additionalProperties: false,
|
|
2274
|
+
},
|
|
2275
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
2276
|
+
timeoutMs: 180_000,
|
|
2277
|
+
isConcurrencySafe: () => false,
|
|
2278
|
+
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the answer. */
|
|
2279
|
+
execute: async (args) => withPage((page) => askInPage(page, args)),
|
|
2280
|
+
},
|
|
2281
|
+
{
|
|
2282
|
+
name: 'browser_review', // ui-copy-ignore
|
|
2283
|
+
description: 'Ask the chat in the page to review a repository and take its review back as a list: builds the message, sends it, waits for the answer, and writes the round down so the next round knows what was already said. Use it for a review round, not for a one-off question (that is browser_ask).',
|
|
2284
|
+
parameters: {
|
|
2285
|
+
type: 'object', // ui-copy-ignore
|
|
2286
|
+
properties: {
|
|
2287
|
+
repo: { type: 'string', description: 'Address of the repository to review.' },
|
|
2288
|
+
commit: { type: 'string', description: 'The commit the review is about. Omit for the newest one.' },
|
|
2289
|
+
focus: { type: 'string', description: 'One thing to look at especially.' },
|
|
2290
|
+
diff: { type: 'string', description: 'The change to review, as a diff. Include it when the reviewer cannot open the repository itself.' },
|
|
2291
|
+
ref: { type: 'integer', description: 'The box to type in, from browser_view. Omit to use the page\u2019s own chat box.' },
|
|
2292
|
+
timeout_ms: { type: 'integer', description: 'How long to wait for the review. Default 120000.' },
|
|
2293
|
+
},
|
|
2294
|
+
required: ['repo'], // ui-copy-ignore
|
|
2295
|
+
additionalProperties: false,
|
|
2296
|
+
},
|
|
2297
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
2298
|
+
timeoutMs: 240_000,
|
|
2299
|
+
isConcurrencySafe: () => false,
|
|
2300
|
+
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the review. */
|
|
2301
|
+
execute: async (args) => withPage(async (page) => {
|
|
2302
|
+
const repo = String(args?.repo ?? '').trim()
|
|
2303
|
+
if (repo === '') return 'Which repository should the reviewer look at?'
|
|
2304
|
+
const commit = String(args?.commit ?? '').trim()
|
|
2305
|
+
/* What was settled last time travels with the question. */
|
|
2306
|
+
const before = lastRound()
|
|
2307
|
+
const already = Array.isArray(before?.marks) ? before.marks : []
|
|
2308
|
+
const asked = askBrief({ repo, commit, focus: String(args?.focus ?? ''), diff: String(args?.diff ?? ''), already })
|
|
2309
|
+
const started = Date.now()
|
|
2310
|
+
const told = await askInPage(page, { text: asked, ref: args?.ref, timeout_ms: args?.timeout_ms ?? 120_000 })
|
|
2311
|
+
const tookMs = Date.now() - started
|
|
2312
|
+
if (!String(told).startsWith('the answer came back in')) return told
|
|
2313
|
+
const answer = String(told).split('\n').slice(1).join('\n').trim()
|
|
2314
|
+
const { round } = recordRound({ commit, question: asked, answer, tookMs })
|
|
2315
|
+
/* A wall answers too, and its words are not advice: name it rather than
|
|
2316
|
+
call the login screen prose. */
|
|
2317
|
+
const wall = /(sign in|log in|登录|注册|verif|checking your browser)/i.test(answer)
|
|
2318
|
+
/* The question sitting there unanswered is not prose: say that instead. */
|
|
2319
|
+
const busy = asked.slice(0, 24)
|
|
2320
|
+
const waiting = round.points.length === 0 && busy.length >= 8 && answer.includes(busy)
|
|
2321
|
+
const list = round.points.length > 0
|
|
2322
|
+
? round.points.map((point, at) => `${at + 1}. ${point}`).join('\n')
|
|
2323
|
+
: waiting
|
|
2324
|
+
? `the page has the message but has not answered yet; it shows: ${answer.replace(/\s+/g, ' ').slice(0, 200)}`
|
|
2325
|
+
: wall
|
|
2326
|
+
? `the page is asking to sign in first, so there is no review yet: ${answer.replace(/\s+/g, ' ').slice(0, 200)}`
|
|
2327
|
+
: `It answered in prose; no list to work from. It said: ${answer.replace(/\s+/g, ' ').slice(0, 200)}`
|
|
2328
|
+
const settled = already.filter((mark) => mark.status !== 'open').length
|
|
2329
|
+
const carried = settled === 0 ? '' : `\n(${settled} earlier point${settled === 1 ? '' : 's'} left out as already handled)`
|
|
2330
|
+
return `review round: ${commit === '' ? 'newest commit' : commit.slice(0, 12)} in ${Math.round(tookMs / 100) / 10}s\n${list}${carried}`
|
|
2331
|
+
}),
|
|
2332
|
+
},
|
|
2333
|
+
{
|
|
2334
|
+
name: 'browser_review_verdict', // ui-copy-ignore
|
|
2335
|
+
description: 'Record what happened to each point of the last review round, so the next round leaves settled points out: point numbers are the ones browser_review printed, and each mark is fixed, verified (checked in the code and the point does not apply), invalid, or open.',
|
|
2336
|
+
parameters: {
|
|
2337
|
+
type: 'object', // ui-copy-ignore
|
|
2338
|
+
properties: {
|
|
2339
|
+
mark: { // ui-copy-ignore
|
|
2340
|
+
type: 'array',
|
|
2341
|
+
description: 'One entry per point: which point, its status, and the evidence that backs it.',
|
|
2342
|
+
items: { // ui-copy-ignore
|
|
2343
|
+
type: 'object', // ui-copy-ignore
|
|
2344
|
+
properties: { // ui-copy-ignore
|
|
2345
|
+
point: { type: 'integer', description: 'The point number browser_review printed, starting at 1.' },
|
|
2346
|
+
status: { type: 'string', enum: MARK_STATUS, description: 'fixed, verified, invalid, or open.' },
|
|
2347
|
+
evidence: { type: 'string', description: 'Where the code shows it: file, test, or commit.' },
|
|
2348
|
+
},
|
|
2349
|
+
required: ['point', 'status'], // ui-copy-ignore
|
|
2350
|
+
additionalProperties: false,
|
|
2351
|
+
},
|
|
2352
|
+
},
|
|
2353
|
+
},
|
|
2354
|
+
required: ['mark'], // ui-copy-ignore
|
|
2355
|
+
additionalProperties: false,
|
|
2356
|
+
},
|
|
2357
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
2358
|
+
timeoutMs: 10_000,
|
|
2359
|
+
isConcurrencySafe: () => false,
|
|
2360
|
+
/** @param {any} args - arguments. @returns {Promise<{text: string}>} what was recorded. */
|
|
2361
|
+
execute: async (args) => {
|
|
2362
|
+
const written = markRound({ marks: Array.isArray(args?.mark) ? args.mark : [] })
|
|
2363
|
+
if (written === null) return 'No review round to mark: play one with browser_review first.'
|
|
2364
|
+
const counts = new Map()
|
|
2365
|
+
for (const mark of written.marks) counts.set(mark.status, (counts.get(mark.status) ?? 0) + 1)
|
|
2366
|
+
const said = [...counts].map(([status, n]) => `${n} ${status}`).join(', ')
|
|
2367
|
+
return `review round ${String(written.round.commit || 'newest').slice(0, 12)}: recorded ${written.marks.length} of ${written.round.points.length} points (${said}).`
|
|
2368
|
+
},
|
|
2369
|
+
},
|
|
2370
|
+
{
|
|
2371
|
+
name: 'browser_key', // ui-copy-ignore
|
|
2372
|
+
description: 'Press one key in the background browser: Enter, Tab, Escape, Backspace, Delete, Insert, arrows, Home, End, PageUp, PageDown, Space, F1-F12, the numeric keypad (Numpad0-Numpad9, NumpadEnter, NumpadAdd, ...), printable characters and every symbol on a US keyboard (including shifted ones such as ? : @), plus the keys with a name (ContextMenu, CapsLock, PrintScreen, media keys). Join keys with + to hold them together (ArrowLeft+ArrowUp, Shift+Tab), and hold_ms to keep them down. Modifiers accept Ctrl, Alt, Shift, Meta/Cmd, or Mod for the platform command key; Ctrl+C/X/V/A/Z/Y run the real copy, cut, paste, select-all, undo and redo.',
|
|
2373
|
+
parameters: {
|
|
2374
|
+
type: 'object', // ui-copy-ignore
|
|
2375
|
+
properties: {
|
|
2376
|
+
key: { type: 'string', description: 'Key name, or several joined by + to hold them together (ArrowLeft+ArrowUp, Shift+ArrowLeft). Letters, digits, punctuation and shifted symbols work too.' },
|
|
2377
|
+
modifiers: { type: 'string', description: 'Held while pressing: Ctrl, Alt, Shift, Meta/Cmd, or Mod (Command on macOS, Control on Windows and Linux), joined by +.' },
|
|
2378
|
+
hold_ms: { type: 'integer', description: 'Keep the key down this long before releasing. Games move while a key is held.' },
|
|
962
2379
|
read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
|
|
963
2380
|
},
|
|
964
2381
|
required: ['key'], // ui-copy-ignore
|
|
@@ -970,15 +2387,22 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
970
2387
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
|
|
971
2388
|
execute: async (args) => withPage(async (page) => {
|
|
972
2389
|
const name = String(args?.key ?? 'Enter')
|
|
973
|
-
const
|
|
974
|
-
const mods =
|
|
975
|
-
|
|
976
|
-
|
|
2390
|
+
const chord = splitChord(name, modifierBits(args?.modifiers))
|
|
2391
|
+
const mods = chord.modifiers
|
|
2392
|
+
const hold = Math.max(0, Math.min(60_000, Number(args?.hold_ms ?? 0)))
|
|
2393
|
+
if (chord.names.length === 0) throw new Error(`sidecar: ${name} holds no key, only modifiers`) // ui-copy-ignore
|
|
2394
|
+
/* One key goes through the plain path so an editing shortcut is sent as
|
|
2395
|
+
the browser command it is; a chord holds every key down together. */
|
|
2396
|
+
if (chord.names.length > 1) await pressChord(page, chord.names, mods, hold)
|
|
2397
|
+
else await pressKey(page, chord.names[0], mods, hold)
|
|
2398
|
+
/* A held key is game input: waiting for the page to fall quiet after one
|
|
2399
|
+
costs more than the frame it was meant to drive. */
|
|
2400
|
+
if (hold === 0) await settle(page, 3000)
|
|
977
2401
|
return acted(page, args, () => look(page, { mode: 'text', maxChars: 2000 }), `key ${name}`)
|
|
978
2402
|
}),
|
|
979
2403
|
},
|
|
980
2404
|
{
|
|
981
|
-
name: '
|
|
2405
|
+
name: 'browser_scroll', // ui-copy-ignore
|
|
982
2406
|
description: 'Scroll the sidecar browser by wheel at its centre (negative goes up), then return what it shows.',
|
|
983
2407
|
parameters: {
|
|
984
2408
|
type: 'object', // ui-copy-ignore
|
|
@@ -996,18 +2420,21 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
996
2420
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
|
|
997
2421
|
execute: async (args) => withPage(async (page) => {
|
|
998
2422
|
await page.wheel({ x: Number(args?.x ?? VIEW.width / 2), y: Number(args?.y ?? VIEW.height / 2), dy: Number(args?.dy ?? 600) })
|
|
2423
|
+
/* A wheel is a request, not a result: a list that loads more rows on
|
|
2424
|
+
scroll has not answered yet when the wheel returns. Give the scroll
|
|
2425
|
+
handler its turn, then let the page fall quiet. */
|
|
999
2426
|
await wait(200)
|
|
1000
2427
|
const delta = Number(args?.dy ?? 600)
|
|
1001
2428
|
return acted(page, args, () => look(page, { mode: 'text', maxChars: 4000 }), `scroll ${delta > 0 ? '+' : ''}${delta}`)
|
|
1002
2429
|
}),
|
|
1003
2430
|
},
|
|
1004
2431
|
{
|
|
1005
|
-
name: '
|
|
2432
|
+
name: 'browser_hover', // ui-copy-ignore
|
|
1006
2433
|
description: 'Move the pointer onto a numbered control, so hover-only menus and tooltips open, then return what the page shows.',
|
|
1007
2434
|
parameters: {
|
|
1008
2435
|
type: 'object', // ui-copy-ignore
|
|
1009
2436
|
properties: {
|
|
1010
|
-
ref: { type: 'integer', description: 'Number from
|
|
2437
|
+
ref: { type: 'integer', description: 'Number from browser_view.' },
|
|
1011
2438
|
read: { type: 'boolean', description: 'Read the page after the action, instead of a one-line confirmation.' },
|
|
1012
2439
|
},
|
|
1013
2440
|
required: ['ref'], // ui-copy-ignore
|
|
@@ -1019,7 +2446,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1019
2446
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
|
|
1020
2447
|
execute: async (args) => withPage(async (page) => {
|
|
1021
2448
|
const spot = await findAgain(page, Number(args?.ref))
|
|
1022
|
-
if (spot.gone === true) return REF_GONE
|
|
2449
|
+
if (spot.gone === true) return spot.why ?? REF_GONE
|
|
1023
2450
|
const cursor = await page.hover({ x: Number(spot.x), y: Number(spot.y) })
|
|
1024
2451
|
await wait(400)
|
|
1025
2452
|
const said = await acted(page, args, () => look(page, { mode: 'text', maxChars: 2500 }), `hover ${String(args?.ref)} · ${cursor}`)
|
|
@@ -1027,7 +2454,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1027
2454
|
}),
|
|
1028
2455
|
},
|
|
1029
2456
|
{
|
|
1030
|
-
name: '
|
|
2457
|
+
name: 'browser_nav', // ui-copy-ignore
|
|
1031
2458
|
description: 'Change the sidecar browser\'s address: open a url, go back, go forward, or reload. Use this instead of clicking the browser chrome.',
|
|
1032
2459
|
parameters: {
|
|
1033
2460
|
type: 'object', // ui-copy-ignore
|
|
@@ -1057,7 +2484,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1057
2484
|
}),
|
|
1058
2485
|
},
|
|
1059
2486
|
{
|
|
1060
|
-
name: '
|
|
2487
|
+
name: 'browser_do', // ui-copy-ignore
|
|
1061
2488
|
description: 'Run several steps in one call and report one compact result: the whole of "click search, type a query, press Enter, wait for results" is one call instead of five. Steps run in order and stop at the first that cannot proceed. Each step: {ref, tap | text, submit}, {tap: "x,y"}, {tap: "Sign in"}, {field: "Email", text}, {text}, {key}, {scroll}, {wait_text}, {wait_gone}, {wait_url}, {wait_idle}, {eval}, {open}, {sleep_ms}. Waits know when the page has settled. A step that names a control clears a cookie banner in its way first.',
|
|
1062
2489
|
parameters: {
|
|
1063
2490
|
type: 'object', // ui-copy-ignore
|
|
@@ -1103,39 +2530,46 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1103
2530
|
if (steps.length === 0) return 'Give at least one step.'
|
|
1104
2531
|
const report = []
|
|
1105
2532
|
for (const [index, step] of steps.entries()) {
|
|
1106
|
-
const
|
|
1107
|
-
|
|
1108
|
-
|
|
2533
|
+
const stepAt = Date.now()
|
|
2534
|
+
const said = await runStep(page, step ?? {}, refsFor().refs)
|
|
2535
|
+
const text = typeof said === 'string' ? said : said.text
|
|
2536
|
+
report.push(`${index + 1}. ${text}`)
|
|
2537
|
+
/* A step that never landed makes every later step a guess, so the
|
|
2538
|
+
batch ends here instead of typing into a page that is not there. */
|
|
2539
|
+
if (typeof said !== 'string' && said.blocked === true) {
|
|
2540
|
+
return `${report.join('\n')}\nStopped after step ${index + 1}. Call browser_view and plan again from what the page shows now.`
|
|
2541
|
+
}
|
|
1109
2542
|
/* A step can navigate straight onto Chrome's error page — a mistyped
|
|
1110
2543
|
address, or a site that does not answer. The rest of the batch
|
|
1111
2544
|
would then type into "This site can't be reached", so it stops
|
|
1112
2545
|
here and says what really happened. */
|
|
1113
2546
|
if (isErrorPage(page)) return `${report.join('\n')}\nStopped after step ${index + 1}: ${await unreachable(page)}`
|
|
1114
2547
|
if (step?.sleep_ms === undefined && step?.wait_text === undefined && step?.wait_gone === undefined && step?.wait_url === undefined && step?.wait_idle === undefined) {
|
|
1115
|
-
await settle(page, Number(step?.timeout ?? 5000))
|
|
2548
|
+
await settle(page, Number(step?.timeout ?? 5000), stepAt)
|
|
1116
2549
|
}
|
|
1117
2550
|
}
|
|
1118
2551
|
const mode = String(args?.read ?? 'text')
|
|
1119
2552
|
if (mode === 'none') return report.join('\n')
|
|
1120
2553
|
const options = { mode: mode === 'changed' ? 'text' : mode, maxChars: Number(args?.max_chars ?? 3000) }
|
|
1121
|
-
const
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
2554
|
+
const state = refsFor()
|
|
2555
|
+
const read = await remember(page, state, options)
|
|
2556
|
+
if (mode === 'changed' && state.last !== undefined) {
|
|
2557
|
+
const diff = diffRead(state.last, read, options)
|
|
2558
|
+
state.last = read
|
|
1125
2559
|
return `${report.join('\n')}\n${diff}`
|
|
1126
2560
|
}
|
|
1127
|
-
|
|
2561
|
+
state.last = read
|
|
1128
2562
|
return `${report.join('\n')}\n\n${renderPage(read, options)}`
|
|
1129
2563
|
}),
|
|
1130
2564
|
},
|
|
1131
2565
|
{
|
|
1132
|
-
name: '
|
|
2566
|
+
name: 'browser_extract', // ui-copy-ignore
|
|
1133
2567
|
description: 'Read the items a selector matches as one compact line each — lists, search results, tables, menus — instead of the whole page. Each row keeps its link when it has one. Use it when a page has more rows than the read budget allows; it costs a fraction of a full read.',
|
|
1134
2568
|
parameters: {
|
|
1135
2569
|
type: 'object', // ui-copy-ignore
|
|
1136
2570
|
properties: {
|
|
1137
2571
|
select: { type: 'string', description: 'CSS selector for the items, for example "ul.results > li".' },
|
|
1138
|
-
ref: { type: 'integer', description: 'Only look inside this control, as numbered by the last
|
|
2572
|
+
ref: { type: 'integer', description: 'Only look inside this control, as numbered by the last browser_view.' },
|
|
1139
2573
|
limit: { type: 'integer', description: 'Most rows to return. Default 40, at most 200.' },
|
|
1140
2574
|
},
|
|
1141
2575
|
required: ['select'], // ui-copy-ignore
|
|
@@ -1148,7 +2582,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1148
2582
|
execute: async (args) => withRead((page) => extractRows(page, args)),
|
|
1149
2583
|
},
|
|
1150
2584
|
{
|
|
1151
|
-
name: '
|
|
2585
|
+
name: 'browser_eval', // ui-copy-ignore
|
|
1152
2586
|
description: 'Run JavaScript in the sidecar page and return its value. For what the numbered controls cannot express: reading a table, setting storage, filling a widget. The page must be opened first.',
|
|
1153
2587
|
parameters: {
|
|
1154
2588
|
type: 'object', // ui-copy-ignore
|
|
@@ -1181,12 +2615,12 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1181
2615
|
}),
|
|
1182
2616
|
},
|
|
1183
2617
|
{
|
|
1184
|
-
name: '
|
|
2618
|
+
name: 'browser_select', // ui-copy-ignore
|
|
1185
2619
|
description: 'Choose an option in a dropdown, by visible label or value, then return the page.',
|
|
1186
2620
|
parameters: {
|
|
1187
2621
|
type: 'object', // ui-copy-ignore
|
|
1188
2622
|
properties: {
|
|
1189
|
-
ref: { type: 'integer', description: 'Number of the select control from
|
|
2623
|
+
ref: { type: 'integer', description: 'Number of the select control from browser_view.' },
|
|
1190
2624
|
field: { type: 'string', description: 'The dropdown is called this on screen, when you have no number for it.' },
|
|
1191
2625
|
value: { type: 'string', description: 'Option label or value to choose.' },
|
|
1192
2626
|
},
|
|
@@ -1198,20 +2632,30 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1198
2632
|
isConcurrencySafe: () => false,
|
|
1199
2633
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the page after. */
|
|
1200
2634
|
execute: async (args) => withPage(async (page) => {
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
2635
|
+
let use = args
|
|
2636
|
+
/* One dropdown on the page needs no naming: the value alone says which
|
|
2637
|
+
one, and asking costs the caller a whole read. */
|
|
2638
|
+
if (use?.ref === undefined && use?.field === undefined) {
|
|
2639
|
+
const read = await readPage(page, { mode: 'text', maxChars: 1200, max: 200 })
|
|
2640
|
+
const drops = read.controls.filter((control) => control.view !== false && ['select', 'combobox', 'listbox'].includes(String(control.kind)))
|
|
2641
|
+
if (drops.length === 1) use = { ...use, ref: Number(drops[0].ref) }
|
|
2642
|
+
else if (drops.length > 1) return `which dropdown: ${drops.slice(0, 5).map((one) => `${String(one.name).slice(0, 30)} (${String(one.ref)})`).join(' · ')}`
|
|
2643
|
+
else return 'the page has no dropdown to choose in.' // ui-copy-ignore
|
|
2644
|
+
}
|
|
2645
|
+
if (use?.ref === undefined) {
|
|
2646
|
+
const field = String(use.field)
|
|
2647
|
+
const chosen = await evaluate(page, selectSpot(field, String(use?.value ?? '')))
|
|
1204
2648
|
const said = typeof chosen === 'string' ? chosen : chosen?.error === undefined ? '' : String(chosen.error)
|
|
1205
2649
|
if (said === '' || said.startsWith('no dropdown')) return said === '' ? 'the page did not answer; read it and try again.' : said // ui-copy-ignore
|
|
1206
2650
|
await settle(page, 3000)
|
|
1207
|
-
if (
|
|
2651
|
+
if (use?.read === true) return `${said}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
|
|
1208
2652
|
return `${said} · ${await brief(page)}`
|
|
1209
2653
|
}
|
|
1210
|
-
const spot = await findAgain(page, Number(
|
|
1211
|
-
if (spot.gone === true) return REF_GONE
|
|
1212
|
-
const wanted = JSON.stringify(String(
|
|
1213
|
-
const route = await contextFor(page, Number(
|
|
1214
|
-
const local = route?.local ?? Number(
|
|
2654
|
+
const spot = await findAgain(page, Number(use?.ref))
|
|
2655
|
+
if (spot.gone === true) return spot.why ?? REF_GONE
|
|
2656
|
+
const wanted = JSON.stringify(String(use?.value ?? ''))
|
|
2657
|
+
const route = await contextFor(page, Number(use?.ref))
|
|
2658
|
+
const local = route?.local ?? Number(use?.ref)
|
|
1215
2659
|
const said = await evaluate(page, `(() => {
|
|
1216
2660
|
const reg = window.__dshRead
|
|
1217
2661
|
const hit = reg && reg.find(${local})
|
|
@@ -1226,17 +2670,17 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1226
2670
|
return 'chose ' + option.text.trim()
|
|
1227
2671
|
})()`, route)
|
|
1228
2672
|
await settle(page, 3000)
|
|
1229
|
-
if (
|
|
2673
|
+
if (use?.read === true) return `${String(said)}\n${await look(page, { mode: 'text', maxChars: 2000 })}`
|
|
1230
2674
|
return `${String(said)} · ${await brief(page)}`
|
|
1231
2675
|
}),
|
|
1232
2676
|
},
|
|
1233
2677
|
{
|
|
1234
|
-
name: '
|
|
2678
|
+
name: 'browser_upload', // ui-copy-ignore
|
|
1235
2679
|
description: 'Put files into a file input on the page, by absolute path, then return the page. The file chooser never opens.',
|
|
1236
2680
|
parameters: {
|
|
1237
2681
|
type: 'object', // ui-copy-ignore
|
|
1238
2682
|
properties: {
|
|
1239
|
-
ref: { type: 'integer', description: 'Number of the file input from
|
|
2683
|
+
ref: { type: 'integer', description: 'Number of the file input from browser_view.' },
|
|
1240
2684
|
files: { type: 'array', items: { type: 'string' }, description: 'Absolute paths on this machine.' },
|
|
1241
2685
|
},
|
|
1242
2686
|
required: ['ref', 'files'], // ui-copy-ignore
|
|
@@ -1255,7 +2699,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1255
2699
|
if (route?.contextId !== undefined) uploadParams.contextId = route.contextId
|
|
1256
2700
|
const handle = await page.cdp.send('Runtime.evaluate', uploadParams, route?.sessionId)
|
|
1257
2701
|
const objectId = handle.result?.objectId
|
|
1258
|
-
if (objectId === undefined) return REF_GONE
|
|
2702
|
+
if (objectId === undefined) return spot.why ?? REF_GONE
|
|
1259
2703
|
const node = await page.cdp.send('DOM.describeNode', { objectId }, route?.sessionId)
|
|
1260
2704
|
const backendNodeId = node?.node?.backendNodeId
|
|
1261
2705
|
if (backendNodeId === undefined) return 'That control cannot take a file.'
|
|
@@ -1265,33 +2709,43 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1265
2709
|
}),
|
|
1266
2710
|
},
|
|
1267
2711
|
{
|
|
1268
|
-
name: '
|
|
1269
|
-
description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path. Default is the visible screen; full takes the whole page, ref takes one control.',
|
|
2712
|
+
name: 'browser_shot', // ui-copy-ignore
|
|
2713
|
+
description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path; when the current model accepts images the picture comes back in the same call, so a captcha, a chart or a broken layout can be looked at instead of guessed. Default is the visible screen; full takes the whole page, ref takes one control.',
|
|
1270
2714
|
parameters: {
|
|
1271
2715
|
type: 'object', // ui-copy-ignore
|
|
1272
2716
|
properties: {
|
|
1273
2717
|
name: { type: 'string', description: 'File name without extension.' },
|
|
1274
2718
|
full: { type: 'boolean', description: 'The whole page, not just the screen.' },
|
|
1275
|
-
ref: { type: 'integer', description: 'One control from
|
|
2719
|
+
ref: { type: 'integer', description: 'One control from browser_view.' },
|
|
1276
2720
|
scale: { type: 'integer', description: 'Pixel ratio. Default 1; 2 is sharper and heavier.' },
|
|
1277
2721
|
},
|
|
1278
2722
|
additionalProperties: false,
|
|
1279
2723
|
},
|
|
1280
|
-
output:
|
|
2724
|
+
output: SHOT_OUTPUT, // ui-copy-ignore
|
|
1281
2725
|
timeoutMs: 40_000,
|
|
1282
2726
|
isConcurrencySafe: () => false,
|
|
1283
|
-
/**
|
|
1284
|
-
|
|
2727
|
+
/**
|
|
2728
|
+
* A picture nobody can look at is only a file path: when the route declares
|
|
2729
|
+
* image input, the same call carries the PNG as an attachment so the next
|
|
2730
|
+
* step can be planned from what is actually on screen instead of guessing.
|
|
2731
|
+
* @param {any} args - arguments.
|
|
2732
|
+
* @param {any} [exec] - call context, for the route and its attachments.
|
|
2733
|
+
* @returns {Promise<any>} the saved path, and the picture when available.
|
|
2734
|
+
*/
|
|
2735
|
+
execute: async (args, exec) => {
|
|
2736
|
+
/** @type {{data: string, file: string}|undefined} */
|
|
2737
|
+
let taken
|
|
2738
|
+
const said = await withRead(async (page) => {
|
|
1285
2739
|
const scale = Math.max(1, Math.min(3, Number(args?.scale ?? 1)))
|
|
1286
2740
|
let clip = undefined
|
|
1287
2741
|
if (args?.ref !== undefined) {
|
|
1288
2742
|
const spot = await findAgain(page, Number(args.ref))
|
|
1289
|
-
if (spot.gone === true) return REF_GONE
|
|
2743
|
+
if (spot.gone === true) return spot.why ?? REF_GONE
|
|
1290
2744
|
/* The picture is of the control, not of a guess around its middle:
|
|
1291
2745
|
a canvas can be most of the page, a checkbox a few pixels. */
|
|
1292
2746
|
const box = await evaluate(page, bounds(Number(args.ref)))
|
|
1293
2747
|
const rect = typeof box === 'string' ? JSON.parse(box) : box
|
|
1294
|
-
if (rect?.gone === true) return REF_GONE
|
|
2748
|
+
if (rect?.gone === true) return spot.why ?? REF_GONE
|
|
1295
2749
|
const pad = 8
|
|
1296
2750
|
clip = {
|
|
1297
2751
|
x: Math.max(0, Math.round(Number(rect.x) - pad)),
|
|
@@ -1301,16 +2755,26 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1301
2755
|
}
|
|
1302
2756
|
}
|
|
1303
2757
|
const data = await page.shot({ full: args?.full === true, rect: clip, scale })
|
|
1304
|
-
const
|
|
1305
|
-
|
|
1306
|
-
const name = String(args?.name ?? '').replace(/[^\w.-]/g, '') || `sidecar-${Date.now()}`
|
|
1307
|
-
const file = join(dir, name.endsWith('.png') ? name : `${name}.png`)
|
|
1308
|
-
await writeFile(file, Buffer.from(data, 'base64'))
|
|
2758
|
+
const file = await saveArtifact('shots', data, { name: args?.name, prefix: 'shot' })
|
|
2759
|
+
taken = { data, file }
|
|
1309
2760
|
return file
|
|
1310
|
-
|
|
2761
|
+
})
|
|
2762
|
+
if (taken === undefined) return said
|
|
2763
|
+
let image
|
|
2764
|
+
try {
|
|
2765
|
+
image = await host.showShot?.(taken.data, taken.file.split('/').pop() ?? 'shot.png', exec) // ui-copy-ignore
|
|
2766
|
+
} catch {
|
|
2767
|
+
image = undefined
|
|
2768
|
+
}
|
|
2769
|
+
/* The route may not carry pictures at all: then the file path is all
|
|
2770
|
+
this call can give, and saying so is better than a silent path the
|
|
2771
|
+
next step assumes was looked at. */
|
|
2772
|
+
if (image === undefined) return { text: `${said.text}\nno picture reached this result — browser_view reads the words here, browser_grab returns a picture as a file` }
|
|
2773
|
+
return { text: said.text, image }
|
|
2774
|
+
},
|
|
1311
2775
|
},
|
|
1312
2776
|
{
|
|
1313
|
-
name: '
|
|
2777
|
+
name: 'browser_grab', // ui-copy-ignore
|
|
1314
2778
|
description: 'Save the picture on the page as a file: an image, a chart, a drawing surface. Picks the largest one, or the one inside a given control.',
|
|
1315
2779
|
parameters: {
|
|
1316
2780
|
type: 'object', // ui-copy-ignore
|
|
@@ -1334,9 +2798,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1334
2798
|
found = { pictures: [] }
|
|
1335
2799
|
}
|
|
1336
2800
|
const rows = []
|
|
1337
|
-
const
|
|
1338
|
-
await mkdir(dir, { recursive: true })
|
|
1339
|
-
const base = String(args?.name ?? '').replace(/[^\w.-]/g, '') || `sidecar-picture-${Date.now()}`
|
|
2801
|
+
const base = String(args?.name ?? '').replace(/[^\w.-]/g, '')
|
|
1340
2802
|
const list = Array.isArray(found?.pictures) ? found.pictures : []
|
|
1341
2803
|
for (let index = 0; index < list.length; index += 1) {
|
|
1342
2804
|
const item = list[index]
|
|
@@ -1345,8 +2807,11 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1345
2807
|
if (mark < 0) continue
|
|
1346
2808
|
const type = String(item?.type ?? 'image/png')
|
|
1347
2809
|
const ext = type.includes('jpeg') ? 'jpg' : type.includes('webp') ? 'webp' : type.includes('svg') ? 'svg' : 'png'
|
|
1348
|
-
const file =
|
|
1349
|
-
|
|
2810
|
+
const file = await saveArtifact('pictures', body.slice(mark + 1), {
|
|
2811
|
+
name: base === '' ? '' : `${base}${list.length > 1 ? `-${index + 1}` : ''}`,
|
|
2812
|
+
ext: `.${ext}`,
|
|
2813
|
+
prefix: 'picture',
|
|
2814
|
+
})
|
|
1350
2815
|
const size = `${Math.round(Number(item.width) || 0)}x${Math.round(Number(item.height) || 0)}`
|
|
1351
2816
|
/* Say what the picture looks like in words: a model that cannot open
|
|
1352
2817
|
a file still needs to know a chart came back blank. */
|
|
@@ -1362,7 +2827,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1362
2827
|
}),
|
|
1363
2828
|
},
|
|
1364
2829
|
{
|
|
1365
|
-
name: '
|
|
2830
|
+
name: 'browser_state', // ui-copy-ignore
|
|
1366
2831
|
description: 'Report the sidecar browser: whether it runs, its profile, address, title, how many people are watching its mirror, and how many times a call had to find a control again after the page redrew it.',
|
|
1367
2832
|
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
1368
2833
|
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
@@ -1372,13 +2837,24 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1372
2837
|
execute: async () => {
|
|
1373
2838
|
const state = await sidecar.state()
|
|
1374
2839
|
const owner = String(host.owner?.() ?? '') // ui-copy-ignore
|
|
2840
|
+
const session = currentSession()
|
|
2841
|
+
const seats = String(host.seats?.() ?? '?') // ui-copy-ignore
|
|
2842
|
+
const mine = sidecar.pageFor(session)
|
|
1375
2843
|
const rows = [
|
|
1376
2844
|
state.alive ? 'running' : 'stopped',
|
|
2845
|
+
`chat ${session === '' ? '(no chat id)' : session}`,
|
|
2846
|
+
`this chat's tab ${mine === null ? 'none' : String(mine.url)}`, // ui-copy-ignore
|
|
2847
|
+
`tabs in the browser ${String(sidecar.pages.size)}`,
|
|
2848
|
+
`tab list ${[...sidecar.pages.entries()].map(([key, each]) => `${key.slice(-8)}=${String(each.url)}=${each.cdp?.closed === true ? 'closed' : 'open'}`).join(' | ')}`, // ui-copy-ignore
|
|
2849
|
+
`default tab ${String(sidecar.page?.url)}`, // ui-copy-ignore
|
|
1377
2850
|
state.headless === false ? 'showing a window' : 'no window',
|
|
2851
|
+
sidecar.flash ? 'Flash-era runtime' : 'Chrome runtime',
|
|
1378
2852
|
`profile ${String(state.profile)}`,
|
|
1379
2853
|
`url ${String(state.url)}`,
|
|
1380
2854
|
`title ${String(state.title)}`,
|
|
1381
2855
|
`viewers ${String(state.viewers)}`,
|
|
2856
|
+
`listening chats ${seats}`, // ui-copy-ignore
|
|
2857
|
+
`mirror owner ${owner === '' ? 'none' : owner}`, // ui-copy-ignore
|
|
1382
2858
|
owner === '' ? 'no owner' : `owner ${owner}`,
|
|
1383
2859
|
`refs followed ${String(follows.count)}`,
|
|
1384
2860
|
holdText(sidecar) === '' ? 'person free' : `person using it (${String(Math.ceil(Number(sidecar.holding()) / 1000))} s left)`,
|
|
@@ -1389,7 +2865,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1389
2865
|
},
|
|
1390
2866
|
|
|
1391
2867
|
{
|
|
1392
|
-
name: '
|
|
2868
|
+
name: 'browser_resume', // ui-copy-ignore
|
|
1393
2869
|
description: 'Take the page back from the person at the mirror: clears the hold so the next action is sent at once, instead of waiting the few seconds out.',
|
|
1394
2870
|
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
1395
2871
|
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
@@ -1402,7 +2878,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1402
2878
|
},
|
|
1403
2879
|
},
|
|
1404
2880
|
{
|
|
1405
|
-
name: '
|
|
2881
|
+
name: 'browser_downloads', // ui-copy-ignore
|
|
1406
2882
|
description: 'List what this browser has downloaded, newest first, with the file name, its size and when it arrived.',
|
|
1407
2883
|
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
1408
2884
|
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
@@ -1417,7 +2893,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1417
2893
|
},
|
|
1418
2894
|
|
|
1419
2895
|
{
|
|
1420
|
-
name: '
|
|
2896
|
+
name: 'browser_drag', // ui-copy-ignore
|
|
1421
2897
|
description: 'Drag from one place to another with real mouse steps: from a ref or a point, to a point or offset. Use it for sliders, dashboards and drag-and-drop lists.',
|
|
1422
2898
|
parameters: {
|
|
1423
2899
|
type: 'object', // ui-copy-ignore
|
|
@@ -1440,7 +2916,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1440
2916
|
let start = { x: Number(args?.x ?? NaN), y: Number(args?.y ?? NaN) }
|
|
1441
2917
|
if (args?.from !== undefined) {
|
|
1442
2918
|
const spot = await findAgain(page, Number(args.from))
|
|
1443
|
-
if (spot.gone === true) return REF_GONE
|
|
2919
|
+
if (spot.gone === true) return spot.why ?? REF_GONE
|
|
1444
2920
|
start = { x: Number(spot.x), y: Number(spot.y) }
|
|
1445
2921
|
}
|
|
1446
2922
|
const end = { x: Number(args?.to_x ?? NaN), y: Number(args?.to_y ?? NaN) }
|
|
@@ -1460,12 +2936,12 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1460
2936
|
}),
|
|
1461
2937
|
},
|
|
1462
2938
|
{
|
|
1463
|
-
name: '
|
|
2939
|
+
name: 'browser_check', // ui-copy-ignore
|
|
1464
2940
|
description: 'Set a checkbox, radio or switch to a known state, and say whether it took. Clicking blindly flips a control that is already right; this one looks first.',
|
|
1465
2941
|
parameters: {
|
|
1466
2942
|
type: 'object', // ui-copy-ignore
|
|
1467
2943
|
properties: {
|
|
1468
|
-
ref: { type: 'integer', description: 'Control number from
|
|
2944
|
+
ref: { type: 'integer', description: 'Control number from browser_view.' },
|
|
1469
2945
|
checked: { type: 'boolean', description: 'Wanted state. Omit to flip it.' },
|
|
1470
2946
|
read: { type: 'boolean', description: 'Read the page after.' },
|
|
1471
2947
|
},
|
|
@@ -1485,11 +2961,11 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1485
2961
|
if (checkRoute?.contextId !== undefined) checkParams.contextId = checkRoute.contextId
|
|
1486
2962
|
const probe = await page.cdp.send('Runtime.evaluate', checkParams, checkRoute?.sessionId)
|
|
1487
2963
|
const now = probe.result?.value
|
|
1488
|
-
if (now === null || now === undefined) return REF_GONE
|
|
2964
|
+
if (now === null || now === undefined) return spot.why ?? REF_GONE
|
|
1489
2965
|
const want = args?.checked === undefined ? !Boolean(now.checked) : args.checked === true
|
|
1490
2966
|
if (Boolean(now.checked) !== want) {
|
|
1491
2967
|
const spot = await findAgain(page, Number(args.ref))
|
|
1492
|
-
if (spot.gone === true) return REF_GONE
|
|
2968
|
+
if (spot.gone === true) return spot.why ?? REF_GONE
|
|
1493
2969
|
await page.mouse('mouseMoved', { x: Number(spot.x), y: Number(spot.y) })
|
|
1494
2970
|
await page.mouse('mousePressed', { x: Number(spot.x), y: Number(spot.y) })
|
|
1495
2971
|
await page.mouse('mouseReleased', { x: Number(spot.x), y: Number(spot.y) })
|
|
@@ -1501,7 +2977,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1501
2977
|
}),
|
|
1502
2978
|
},
|
|
1503
2979
|
{
|
|
1504
|
-
name: '
|
|
2980
|
+
name: 'browser_wait', // ui-copy-ignore
|
|
1505
2981
|
description: 'Wait until the page shows a piece of text, stops showing it, reaches an address, or goes quiet. Cheaper than reading in a loop.',
|
|
1506
2982
|
parameters: {
|
|
1507
2983
|
type: 'object', // ui-copy-ignore
|
|
@@ -1537,7 +3013,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1537
3013
|
}),
|
|
1538
3014
|
},
|
|
1539
3015
|
{
|
|
1540
|
-
name: '
|
|
3016
|
+
name: 'browser_dialog', // ui-copy-ignore
|
|
1541
3017
|
description: 'Answer the alert, confirm or prompt the page is showing, or read what it asks. A dialog blocks the page until it is answered, so nothing else works while one is open.',
|
|
1542
3018
|
parameters: {
|
|
1543
3019
|
type: 'object', // ui-copy-ignore
|
|
@@ -1565,7 +3041,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1565
3041
|
}),
|
|
1566
3042
|
},
|
|
1567
3043
|
{
|
|
1568
|
-
name: '
|
|
3044
|
+
name: 'browser_console', // ui-copy-ignore
|
|
1569
3045
|
description: 'The console lines and errors this page has produced, newest last. Use it when a click seemed to do nothing.',
|
|
1570
3046
|
parameters: {
|
|
1571
3047
|
type: 'object', // ui-copy-ignore
|
|
@@ -1583,7 +3059,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1583
3059
|
}),
|
|
1584
3060
|
},
|
|
1585
3061
|
{
|
|
1586
|
-
name: '
|
|
3062
|
+
name: 'browser_failed', // ui-copy-ignore
|
|
1587
3063
|
description: 'Requests this page tried and that failed, newest last. A blank area is often one of these.',
|
|
1588
3064
|
parameters: {
|
|
1589
3065
|
type: 'object', // ui-copy-ignore
|
|
@@ -1601,7 +3077,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1601
3077
|
}),
|
|
1602
3078
|
},
|
|
1603
3079
|
{
|
|
1604
|
-
name: '
|
|
3080
|
+
name: 'browser_cookie',
|
|
1605
3081
|
description: 'Read or change the cookies this browser keeps. Login state is a cookie, so this is how you check a sign-in and how you keep one.',
|
|
1606
3082
|
parameters: {
|
|
1607
3083
|
type: 'object', // ui-copy-ignore
|
|
@@ -1645,7 +3121,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1645
3121
|
}),
|
|
1646
3122
|
},
|
|
1647
3123
|
{
|
|
1648
|
-
name: '
|
|
3124
|
+
name: 'browser_clipboard', // ui-copy-ignore
|
|
1649
3125
|
description: 'Read the sidecar browser\'s clipboard, or put text on it and paste it with a key press. For a page that only copies through a button, click it then read here.',
|
|
1650
3126
|
parameters: {
|
|
1651
3127
|
type: 'object', // ui-copy-ignore
|
|
@@ -1668,7 +3144,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1668
3144
|
}),
|
|
1669
3145
|
},
|
|
1670
3146
|
{
|
|
1671
|
-
name: '
|
|
3147
|
+
name: 'browser_network', // ui-copy-ignore
|
|
1672
3148
|
description: 'List the responses the page has received, newest first: method, status, type and address. Use it when a page fills itself from the network and the screen does not say why.',
|
|
1673
3149
|
parameters: {
|
|
1674
3150
|
type: 'object', // ui-copy-ignore
|
|
@@ -1694,7 +3170,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1694
3170
|
},
|
|
1695
3171
|
|
|
1696
3172
|
{
|
|
1697
|
-
name: '
|
|
3173
|
+
name: 'browser_tabs', // ui-copy-ignore
|
|
1698
3174
|
description: 'List the pages open in the browser, switch the mirror to another one, or close one. Use it when a site opens a link in a tab of its own.',
|
|
1699
3175
|
parameters: {
|
|
1700
3176
|
type: 'object', // ui-copy-ignore
|
|
@@ -1711,25 +3187,29 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1711
3187
|
execute: async (args) => withRawPage(async (page) => {
|
|
1712
3188
|
const action = String(args?.action ?? 'list')
|
|
1713
3189
|
if (action === 'list') {
|
|
1714
|
-
const rows = await sidecar.tabs()
|
|
3190
|
+
const rows = await sidecar.tabs(currentSession())
|
|
1715
3191
|
if (rows.length === 0) return 'no tabs'
|
|
1716
3192
|
return rows.map((row, i) => `${i}${row.current ? ' *' : ''} ${row.title || row.url} · ${row.url}`).join('\n')
|
|
1717
3193
|
}
|
|
1718
3194
|
const index = Number(args?.index ?? 0)
|
|
1719
3195
|
if (action === 'switch') {
|
|
1720
|
-
const next = await sidecar.useTab(index)
|
|
3196
|
+
const next = await sidecar.useTab(index, currentSession())
|
|
1721
3197
|
return `tab ${index} · ${String(next?.url ?? '')}`
|
|
1722
3198
|
}
|
|
1723
3199
|
if (action === 'close') {
|
|
1724
|
-
|
|
1725
|
-
|
|
3200
|
+
/* Say what was closed, not what is left: after the close the page
|
|
3201
|
+
that was at that row is gone and reads back as about:blank. */
|
|
3202
|
+
const before = await sidecar.tabs(currentSession())
|
|
3203
|
+
const gone = String(before[index]?.url ?? '')
|
|
3204
|
+
await sidecar.closeTab(index, currentSession())
|
|
3205
|
+
return `closed tab ${index} · ${gone}`
|
|
1726
3206
|
}
|
|
1727
3207
|
throw new Error(`sidecar: unknown action ${action}`)
|
|
1728
3208
|
}),
|
|
1729
3209
|
},
|
|
1730
3210
|
|
|
1731
3211
|
{
|
|
1732
|
-
name: '
|
|
3212
|
+
name: 'browser_pdf', // ui-copy-ignore
|
|
1733
3213
|
description: 'Print the page to a PDF and return its path. Useful when the person wants the document rather than a picture of it.',
|
|
1734
3214
|
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
1735
3215
|
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
@@ -1738,11 +3218,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
1738
3218
|
/** @returns {Promise<{text: string}>} where the PDF is. */
|
|
1739
3219
|
execute: async () => withRawPage(async (page) => {
|
|
1740
3220
|
const data = await page.pdf()
|
|
1741
|
-
|
|
1742
|
-
await mkdir(dir, { recursive: true })
|
|
1743
|
-
const file = join(dir, `sidecar-${Date.now()}.pdf`)
|
|
1744
|
-
await writeFile(file, Buffer.from(data, 'base64'))
|
|
1745
|
-
return file
|
|
3221
|
+
return saveArtifact('pdf', data, { ext: '.pdf', prefix: 'page' })
|
|
1746
3222
|
}),
|
|
1747
3223
|
},
|
|
1748
3224
|
]
|