dsh-inner-browser-use 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/README.zh.md +3 -0
- package/docs/ARCHITECTURE.md +36 -4
- package/docs/ARTIFACTS.md +19 -14
- package/docs/BENCH.md +30 -0
- package/docs/FLASH.md +10 -0
- package/docs/VERIFY.md +41 -0
- package/index.js +1 -1
- package/package.json +10 -3
- package/pixels.js +217 -0
- package/read.js +87 -17
- package/scripts/check-flash-tools.mjs +6 -2
- package/scripts/check-look.mjs +86 -0
- package/scripts/check-sites.mjs +94 -0
- package/scripts/soak.mjs +110 -0
- package/sidecar.js +45 -0
- package/tools.js +302 -23
package/tools.js
CHANGED
|
@@ -10,12 +10,12 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
13
|
-
import { createHash } from 'node:crypto'
|
|
14
13
|
import { homedir } from 'node:os'
|
|
15
|
-
import {
|
|
16
|
-
import { mkdir, readdir, stat, unlink, writeFile } from 'node:fs/promises'
|
|
14
|
+
import { join } from 'node:path'
|
|
15
|
+
import { mkdir, readdir, readFile, stat, unlink, writeFile } from 'node:fs/promises'
|
|
17
16
|
import { VIEW, DESKTOP_UA } from './sidecar.js?v=1'
|
|
18
17
|
import { planOf, gather, sizeText, coreRows } from './media.js?m=2'
|
|
18
|
+
import { around, decodePng, diffGrid, gridOf } from './pixels.js?v=1'
|
|
19
19
|
import { dropLogin, listLogins, readLogin, saveLogin, signInsFor, vaultState } from './vault.js'
|
|
20
20
|
import { readBrowserCookies, siteOf } from './chromecookies.js'
|
|
21
21
|
import { readPage, renderPage, readArticle, readRelevant, readTabs, findOnPage, locate, locateFirst, FRAME_BASE, covering, contextFor, identityOf, sameThing, differs, labelSpot, selectSpot, bounds, pictures, clickInDom, writeInDom, REF_GONE } from './read.js?v=1'
|
|
@@ -70,6 +70,9 @@ const WAYS = new Set(['human', 'dom'])
|
|
|
70
70
|
|
|
71
71
|
/** The way each chat said to act, until it says otherwise. */
|
|
72
72
|
const chatWays = new Map()
|
|
73
|
+
/* The last grid each chat saw, per screen and size: a look after a move is
|
|
74
|
+
worth more as a difference than as another grid. */
|
|
75
|
+
const lastLooks = new Map()
|
|
73
76
|
|
|
74
77
|
/**
|
|
75
78
|
* The way this call should act: what it asked for, else what its chat last set,
|
|
@@ -87,20 +90,34 @@ function wayOf(args) {
|
|
|
87
90
|
/** Which project the running call belongs to, so its files stay together. */
|
|
88
91
|
export const workScope = new AsyncLocalStorage()
|
|
89
92
|
|
|
90
|
-
/**
|
|
91
|
-
export const
|
|
92
|
-
|
|
93
|
-
|
|
93
|
+
/** Set when a caller wants artifacts somewhere other than the project. */
|
|
94
|
+
export const ARTIFACT_OVERRIDE = String(process.env.DSH_INNER_BROWSER_ARTIFACTS ?? '')
|
|
95
|
+
|
|
96
|
+
/** Where a call that is not inside a project still gets its files. */
|
|
97
|
+
export const ARTIFACT_HOME = join(homedir(), '.dsh-inner-browser-use', 'artifacts')
|
|
94
98
|
|
|
95
99
|
/** How many files one kind keeps per chat; older ones go. */
|
|
96
100
|
export const ARTIFACT_KEEP = 40
|
|
97
101
|
|
|
98
|
-
/**
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
102
|
+
/**
|
|
103
|
+
* The folder a project keeps its browser files in. A project is the place a
|
|
104
|
+
* person would look for what the agent did there, so screenshots and sniffed
|
|
105
|
+
* media land inside it instead of under the home directory.
|
|
106
|
+
* @param {string} cwd - the project folder, empty when there is none.
|
|
107
|
+
* @returns {string} absolute path.
|
|
108
|
+
*/
|
|
109
|
+
export function browserDirOf(cwd) {
|
|
110
|
+
return cwd === '' ? join(ARTIFACT_HOME, 'no-project') : join(cwd, '.dsh', 'browser')
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Where files a call produces land: inside the project the call belongs to.
|
|
115
|
+
* `DSH_INNER_BROWSER_ARTIFACTS` replaces the whole location when set.
|
|
116
|
+
* @returns {string} absolute path; not created yet.
|
|
117
|
+
*/
|
|
118
|
+
export function artifactRoot() {
|
|
119
|
+
if (ARTIFACT_OVERRIDE !== '') return ARTIFACT_OVERRIDE
|
|
120
|
+
return browserDirOf(String(workScope.getStore() ?? ''))
|
|
104
121
|
}
|
|
105
122
|
|
|
106
123
|
/** Chat folder name: the short id a person can match to the chat. */
|
|
@@ -111,18 +128,36 @@ function sessionKey() {
|
|
|
111
128
|
|
|
112
129
|
/**
|
|
113
130
|
* Where one kind of file for this project and this chat goes.
|
|
114
|
-
*
|
|
115
|
-
* @param {string} kind - shots, pictures, pdf.
|
|
131
|
+
* `<project>/.dsh/browser/<chat>/<kind>/`
|
|
132
|
+
* @param {string} kind - shots, pictures, pdf, media.
|
|
116
133
|
* @returns {string} absolute directory; not created yet.
|
|
117
134
|
*/
|
|
118
135
|
export function artifactDir(kind) {
|
|
119
|
-
return join(
|
|
136
|
+
return join(artifactRoot(), sessionKey(), kind)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* A project's browser folder is the agent's workspace, not part of the
|
|
141
|
+
* product: one ignore file keeps a run of screenshots out of `git status`.
|
|
142
|
+
* @param {string} root - the folder a call writes under.
|
|
143
|
+
* @returns {Promise<void>} done, or done without a file when it is not wanted.
|
|
144
|
+
*/
|
|
145
|
+
async function ignoreInProject(root) {
|
|
146
|
+
if (ARTIFACT_OVERRIDE !== '') return
|
|
147
|
+
/* With no project there is no repository to keep clean: the home folder this
|
|
148
|
+
falls back to sits outside every checkout. */
|
|
149
|
+
if (String(workScope.getStore() ?? '') === '') return
|
|
150
|
+
try {
|
|
151
|
+
await writeFile(join(root, '.gitignore'), '# browser files: screenshots, media, traces\n*\n')
|
|
152
|
+
} catch {
|
|
153
|
+
/* a folder that cannot take the file is still usable */
|
|
154
|
+
}
|
|
120
155
|
}
|
|
121
156
|
|
|
122
157
|
/**
|
|
123
158
|
* Write one file where the project and the chat expect it, and keep the folder
|
|
124
159
|
* from growing without a limit.
|
|
125
|
-
* @param {string} kind - shots, pictures, pdf.
|
|
160
|
+
* @param {string} kind - shots, pictures, pdf, media.
|
|
126
161
|
* @param {string} body - base64 body.
|
|
127
162
|
* @param {{name?: string, ext?: string, prefix?: string}} [options] - file name.
|
|
128
163
|
* @returns {Promise<string>} absolute path.
|
|
@@ -130,6 +165,7 @@ export function artifactDir(kind) {
|
|
|
130
165
|
export async function saveArtifact(kind, body, options = {}) {
|
|
131
166
|
const dir = artifactDir(kind)
|
|
132
167
|
await mkdir(dir, { recursive: true })
|
|
168
|
+
await ignoreInProject(artifactRoot())
|
|
133
169
|
const ext = String(options.ext ?? '.png')
|
|
134
170
|
/* A name is a name: no separators, no runs of dots that could read as a path
|
|
135
171
|
climb, and no leading dot to hide the file. */
|
|
@@ -175,6 +211,139 @@ export async function pruneArtifacts(dir, keep) {
|
|
|
175
211
|
}
|
|
176
212
|
}
|
|
177
213
|
|
|
214
|
+
/**
|
|
215
|
+
* What each site costs to read, remembered between calls.
|
|
216
|
+
*
|
|
217
|
+
* A wiki page and a web app shell are both "a page", but one is four thousand
|
|
218
|
+
* words of prose and the other is three hundred buttons: reading both the same
|
|
219
|
+
* way spends the same tokens on very different things. The first read of a
|
|
220
|
+
* site teaches the next one — that is the only reason this file exists.
|
|
221
|
+
*/
|
|
222
|
+
const SITE_FILE = 'sites.json'
|
|
223
|
+
/** Where the browser keeps its own state; the same folder the tabs list uses. */
|
|
224
|
+
export function tacticsDir() {
|
|
225
|
+
return process.env.DSH_INNER_BROWSER_STATE_DIR || join(homedir(), '.dsh', 'inner-browser-use')
|
|
226
|
+
}
|
|
227
|
+
/** The site a record belongs to: scheme plus host, nothing per-page. */
|
|
228
|
+
export function originOf(url) {
|
|
229
|
+
try {
|
|
230
|
+
const one = new URL(String(url ?? ''))
|
|
231
|
+
return one.protocol === 'http:' || one.protocol === 'https:' ? one.origin : ''
|
|
232
|
+
} catch {
|
|
233
|
+
return ''
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const siteTactics = new Map()
|
|
237
|
+
let siteLoaded = null
|
|
238
|
+
let siteWriting = Promise.resolve()
|
|
239
|
+
let siteWroteAt = 0
|
|
240
|
+
/**
|
|
241
|
+
* Read the record once per process. A file that is missing or half written by
|
|
242
|
+
* an older version is not a reason to fail a read: an empty memory still works.
|
|
243
|
+
* @returns {Promise<void>} done.
|
|
244
|
+
*/
|
|
245
|
+
export async function loadTactics() {
|
|
246
|
+
if (siteLoaded !== null) return siteLoaded
|
|
247
|
+
siteLoaded = (async () => {
|
|
248
|
+
try {
|
|
249
|
+
const said = JSON.parse(await readFile(join(tacticsDir(), SITE_FILE), 'utf8'))
|
|
250
|
+
for (const [key, value] of Object.entries(said ?? {})) {
|
|
251
|
+
if (originOf(key) === key && value !== null && typeof value === 'object') siteTactics.set(key, value)
|
|
252
|
+
}
|
|
253
|
+
} catch {
|
|
254
|
+
/* no memory yet */
|
|
255
|
+
}
|
|
256
|
+
})()
|
|
257
|
+
return siteLoaded
|
|
258
|
+
}
|
|
259
|
+
/** What is known about a site, or null. Call {@link loadTactics} first. */
|
|
260
|
+
export function tacticsOf(url) {
|
|
261
|
+
return siteTactics.get(originOf(url)) ?? null
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Whether a site has shown itself to be big: more text than a model wants
|
|
265
|
+
* twice, or more controls than a screen can hold. Two reads, so one odd page
|
|
266
|
+
* (a search result, a long error) does not decide for the site.
|
|
267
|
+
* @param {any} record - a record from {@link tacticsOf}.
|
|
268
|
+
* @returns {boolean} true when reads should start with a map.
|
|
269
|
+
*/
|
|
270
|
+
export function bigSite(record) {
|
|
271
|
+
if (record === null || record === undefined) return false
|
|
272
|
+
/* A read keeps at most ninety controls, so "many controls" has to be
|
|
273
|
+
measured on that scale: sixty of them already means a screen of numbers
|
|
274
|
+
is not what a model wants to read first. */
|
|
275
|
+
return Number(record.reads ?? 0) >= 2 && (Number(record.chars ?? 0) >= 3500 || Number(record.controls ?? 0) >= 60)
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Whether a site's own text is long enough that a full read is a spill: a
|
|
279
|
+
* docs page or a thread, where reading the whole thing again is the wrong
|
|
280
|
+
* starting move.
|
|
281
|
+
* @param {any} record - a record from {@link tacticsOf}.
|
|
282
|
+
* @returns {boolean} true when text should be asked for narrowly.
|
|
283
|
+
*/
|
|
284
|
+
export function longRead(record) {
|
|
285
|
+
if (record === null || record === undefined) return false
|
|
286
|
+
return Number(record.reads ?? 0) >= 2 && Number(record.text ?? 0) >= 6000
|
|
287
|
+
}
|
|
288
|
+
/** A running average that forgets: a site that changed is not judged forever. */
|
|
289
|
+
const drift = (was, next) => Math.round(Number(was ?? 0) * 0.6 + Number(next ?? 0) * 0.4)
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Record what one read cost this site. Writes are folded: a burst of reads in
|
|
293
|
+
* one task is one file write, and the last few hundred milliseconds of a
|
|
294
|
+
* process are not worth an extra syscall.
|
|
295
|
+
* @param {string} url - the page read.
|
|
296
|
+
* @param {{chars?: number, controls?: number, ms?: number}} cost - what it cost.
|
|
297
|
+
* @returns {Promise<void>} done.
|
|
298
|
+
*/
|
|
299
|
+
export async function noteRead(url, cost = {}) {
|
|
300
|
+
const key = originOf(url)
|
|
301
|
+
if (key === '') return
|
|
302
|
+
await loadTactics()
|
|
303
|
+
const was = siteTactics.get(key) ?? { reads: 0, chars: 0, controls: 0, text: 0, ms: 0 }
|
|
304
|
+
const reads = Number(was.reads ?? 0) + 1
|
|
305
|
+
siteTactics.set(key, {
|
|
306
|
+
reads,
|
|
307
|
+
chars: cost.chars === undefined ? Number(was.chars ?? 0) : reads === 1 ? Math.round(Number(cost.chars)) : drift(was.chars, cost.chars),
|
|
308
|
+
controls: cost.controls === undefined ? Number(was.controls ?? 0) : reads === 1 ? Math.round(Number(cost.controls)) : drift(was.controls, cost.controls),
|
|
309
|
+
text: cost.text === undefined ? Number(was.text ?? 0) : reads === 1 ? Math.round(Number(cost.text)) : drift(was.text, cost.text),
|
|
310
|
+
ms: cost.ms === undefined ? Number(was.ms ?? 0) : reads === 1 ? Math.round(Number(cost.ms)) : drift(was.ms, cost.ms),
|
|
311
|
+
lastAt: Date.now(),
|
|
312
|
+
})
|
|
313
|
+
if (Date.now() - siteWroteAt < 2000) return
|
|
314
|
+
return saveTactics()
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Write the whole record. Ordered after any write already in flight, so two
|
|
318
|
+
* chats reading at once cannot truncate each other's file.
|
|
319
|
+
* @returns {Promise<void>} done.
|
|
320
|
+
*/
|
|
321
|
+
export function saveTactics() {
|
|
322
|
+
siteWroteAt = Date.now()
|
|
323
|
+
const said = Object.fromEntries(siteTactics)
|
|
324
|
+
siteWriting = siteWriting.then(async () => {
|
|
325
|
+
try {
|
|
326
|
+
await mkdir(tacticsDir(), { recursive: true })
|
|
327
|
+
await writeFile(join(tacticsDir(), SITE_FILE), JSON.stringify(said))
|
|
328
|
+
} catch {
|
|
329
|
+
/* a state folder that cannot take the file still reads fine */
|
|
330
|
+
}
|
|
331
|
+
})
|
|
332
|
+
return siteWriting
|
|
333
|
+
}
|
|
334
|
+
/** One line about a site's shape, for a read that adapted to it. */
|
|
335
|
+
function siteLine(record) {
|
|
336
|
+
const reads = Number(record?.reads ?? 0)
|
|
337
|
+
const chars = Number(record?.chars ?? 0)
|
|
338
|
+
const controls = Number(record?.controls ?? 0)
|
|
339
|
+
const text = Number(record?.text ?? 0)
|
|
340
|
+
const parts = [`${String(reads)} read(s)`]
|
|
341
|
+
if (chars > 0) parts.push(`~${String(chars)} chars of page`)
|
|
342
|
+
else if (text > 0) parts.push(`~${String(text)} chars of text`)
|
|
343
|
+
if (controls > 0) parts.push(`~${String(controls)} controls`)
|
|
344
|
+
return parts.join(', ')
|
|
345
|
+
}
|
|
346
|
+
|
|
178
347
|
/** How many times a call found a control again after the page redrew it. */
|
|
179
348
|
const follows = { count: 0 }
|
|
180
349
|
|
|
@@ -2070,6 +2239,16 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2070
2239
|
const said = renderPage(read, options)
|
|
2071
2240
|
return isCheckPage(read) ? `${said}\n${CHECK_NOTE}` : said
|
|
2072
2241
|
}
|
|
2242
|
+
/* A runtime that opens a port is not a runtime that plays: the plugin was
|
|
2243
|
+
asked about at launch, and a missing one turns a movie into a black box
|
|
2244
|
+
nobody can explain. Say it where the person is looking. */
|
|
2245
|
+
const flashNote = () => {
|
|
2246
|
+
if (sidecar.flash !== true) return ''
|
|
2247
|
+
const check = sidecar.flashCheck
|
|
2248
|
+
if (check === null || check === undefined) return 'flash runtime: open, plugin not checked yet'
|
|
2249
|
+
if (check.ok === true) return `flash runtime: plugin loaded (${String(check.name)})`
|
|
2250
|
+
return 'flash runtime: open, but no Flash plugin is loaded — movies will not play; run scripts/flash-setup.mjs'
|
|
2251
|
+
}
|
|
2073
2252
|
|
|
2074
2253
|
/**
|
|
2075
2254
|
* Where the control behind a ref is now, following it across a re-render.
|
|
@@ -2145,7 +2324,8 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2145
2324
|
/* The column may still be drawing nothing: say so here, where the
|
|
2146
2325
|
person is looking for a picture, rather than only in browser_state. */
|
|
2147
2326
|
const unseen = who === '' ? '' : viewerNote(await watchForViewer(sidecar, page, 1200))
|
|
2148
|
-
|
|
2327
|
+
const runtime = flashNote()
|
|
2328
|
+
return { text: windowed + (unseen === '' ? '' : `${unseen}\n`) + (runtime === '' ? '' : `${runtime}\n`) + await look(page, { mode: 'text' }) + (await pdfNote(page)) }
|
|
2149
2329
|
},
|
|
2150
2330
|
},
|
|
2151
2331
|
{
|
|
@@ -2275,7 +2455,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2275
2455
|
lines.push(`page: ${read.title === '' ? read.url : read.title} — ${read.controls.length} control(s), ${read.textChars} character(s)`)
|
|
2276
2456
|
lines.push(surface === null ? 'surface: no canvas or frame takes the screen — the page itself takes input' : `surface: ${surface.tag} ${surface.w}x${surface.h} at ${surface.x},${surface.y}${surface.scrolled === true ? ' (scrolled it into view)' : ''}${clicked === '' ? '' : ` (${clicked})`}`)
|
|
2277
2457
|
if (read.blind?.flash > 0 && sidecar.flash !== true) lines.push(`flash: ${read.blind.flash} movie(s) — open the same page with flash:true and it runs in this plugin's own player, in the column, with no window on the desktop`) // ui-copy-ignore
|
|
2278
|
-
else if (read.blind?.flash > 0) lines.push(`flash: ${read.blind.flash} movie(s) running on this plugin's own player`)
|
|
2458
|
+
else if (read.blind?.flash > 0) lines.push(`flash: ${read.blind.flash} movie(s) running on this plugin's own player${sidecar.flash === true && sidecar.flashCheck?.ok === false ? ' — but no Flash plugin is loaded, so they cannot play' : ''}`)
|
|
2279
2459
|
else lines.push('flash: none found')
|
|
2280
2460
|
lines.push(`frames: ${Math.max(0, frames.length - 1)} (${childFrames.filter((frame) => String(frame.url ?? '') === 'about:blank').length} of them about:blank)`)
|
|
2281
2461
|
const rates = []
|
|
@@ -2294,6 +2474,85 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2294
2474
|
? 'ready to read and act on, though there is no canvas or frame to watch'
|
|
2295
2475
|
: `ready: ${surface.tag} is live at ${mainRate?.fps ?? frameRate?.fps ?? 0} fps`
|
|
2296
2476
|
lines.push(trouble)
|
|
2477
|
+
if (surface !== null) lines.push('watch: browser_look turns the screen into a grid of letters, so a move can be checked without a picture')
|
|
2478
|
+
return { text: lines.join('\n') }
|
|
2479
|
+
},
|
|
2480
|
+
},
|
|
2481
|
+
{
|
|
2482
|
+
name: 'browser_look', // ui-copy-ignore
|
|
2483
|
+
description: 'Look at a page that draws itself — a game, a canvas app, a Flash movie — as a grid of letters instead of a picture: one character per block of screen, the same character for the same colour. Read what is on screen, then look again after a move to see only what changed, with no image tokens. For pages with text, browser_view and browser_read are cheaper and exact.',
|
|
2484
|
+
parameters: {
|
|
2485
|
+
type: 'object', // ui-copy-ignore
|
|
2486
|
+
properties: {
|
|
2487
|
+
cols: { type: 'integer', description: 'Cells across. Default 24; more cells show more detail and cost more text.' },
|
|
2488
|
+
region: { type: 'array', items: { type: 'number' }, description: 'Look at one box instead of the whole screen: [x, y, width, height] in page pixels, as browser_prepare reports.' },
|
|
2489
|
+
diff: { type: 'boolean', description: 'Show only what changed since the last look at this screen. Default true; pass false to always see the whole grid.' },
|
|
2490
|
+
},
|
|
2491
|
+
additionalProperties: false,
|
|
2492
|
+
},
|
|
2493
|
+
output: TEXT_OUTPUT, // ui-copy-ignore
|
|
2494
|
+
timeoutMs: 30_000,
|
|
2495
|
+
isConcurrencySafe: () => false,
|
|
2496
|
+
/** @param {any} args - arguments. @param {any} exec - tool run. @returns {Promise<{text: string}>} the grid and what moved. */
|
|
2497
|
+
execute: async (args, exec) => {
|
|
2498
|
+
const who = String(exec?.agent?.id ?? '')
|
|
2499
|
+
const page = await sidecar.ensure({ session: who })
|
|
2500
|
+
const dead = await unreachable(page)
|
|
2501
|
+
if (dead !== '') return { text: dead }
|
|
2502
|
+
const screen = asSpot(await evaluate(page, `JSON.stringify({ w: innerWidth, h: innerHeight, canvases: document.querySelectorAll('canvas').length, frames: document.querySelectorAll('iframe').length, movies: document.querySelectorAll('embed, object').length })`))
|
|
2503
|
+
if (screen === null) return { text: 'The page did not answer — it may be loading or navigating. Wait a moment with browser_wait, then look again.' }
|
|
2504
|
+
const width = Math.max(1, Math.round(Number(screen.w) || 1))
|
|
2505
|
+
const height = Math.max(1, Math.round(Number(screen.h) || 1))
|
|
2506
|
+
let box = { x: 0, y: 0, width, height }
|
|
2507
|
+
if (Array.isArray(args?.region) && args.region.length === 4) {
|
|
2508
|
+
const parts = args.region.map((part) => Math.round(Number(part)))
|
|
2509
|
+
box = {
|
|
2510
|
+
x: Math.max(0, Math.min(width - 1, Number.isFinite(parts[0]) ? parts[0] : 0)),
|
|
2511
|
+
y: Math.max(0, Math.min(height - 1, Number.isFinite(parts[1]) ? parts[1] : 0)),
|
|
2512
|
+
width: Math.max(8, Math.min(width, Number.isFinite(parts[2]) ? parts[2] : width)),
|
|
2513
|
+
height: Math.max(8, Math.min(height, Number.isFinite(parts[3]) ? parts[3] : height)),
|
|
2514
|
+
}
|
|
2515
|
+
box.width = Math.min(box.width, width - box.x)
|
|
2516
|
+
box.height = Math.min(box.height, height - box.y)
|
|
2517
|
+
}
|
|
2518
|
+
const cols = Math.min(48, Math.max(8, Math.round(Number(args?.cols) || 24)))
|
|
2519
|
+
/* Only as much picture as the grid needs: six pixels per cell is enough
|
|
2520
|
+
for the average of a block to be its colour, and the PNG stays small
|
|
2521
|
+
enough to read in the same call instead of writing a file. */
|
|
2522
|
+
const scale = Math.min(1, (cols * 6) / Math.max(1, box.width))
|
|
2523
|
+
let grid
|
|
2524
|
+
try {
|
|
2525
|
+
const data = await page.shot({ rect: box, scale })
|
|
2526
|
+
grid = gridOf(decodePng(Buffer.from(data, 'base64')), cols)
|
|
2527
|
+
} catch (error) {
|
|
2528
|
+
return { text: `The screen could not be read as pixels (${String(error?.message ?? error)}). browser_shot still saves a picture of it.` }
|
|
2529
|
+
}
|
|
2530
|
+
const key = `${currentSession()}|${String(page.url ?? '')}|${box.x},${box.y},${box.width},${box.height}|${cols}`
|
|
2531
|
+
const before = lastLooks.get(key)
|
|
2532
|
+
lastLooks.set(key, grid.rows)
|
|
2533
|
+
/* Keep this small: a chat that looks at a hundred screens should not
|
|
2534
|
+
keep a hundred grids alive. */
|
|
2535
|
+
if (lastLooks.size > 24) lastLooks.delete([...lastLooks.keys()][0])
|
|
2536
|
+
const lines = []
|
|
2537
|
+
const whole = box.x === 0 && box.y === 0 && box.width === width && box.height === height
|
|
2538
|
+
const shape = []
|
|
2539
|
+
if (Number(screen.canvases) > 0) shape.push(`${screen.canvases} canvas`)
|
|
2540
|
+
if (Number(screen.frames) > 0) shape.push(`${screen.frames} frame${Number(screen.frames) === 1 ? '' : 's'}`)
|
|
2541
|
+
if (Number(screen.movies) > 0) shape.push(`${screen.movies} movie${Number(screen.movies) === 1 ? '' : 's'}`)
|
|
2542
|
+
lines.push(`look ${grid.cols}x${grid.rowCount} of ${whole ? 'the screen' : `box ${box.x},${box.y} ${box.width}x${box.height}`} (${width}x${height}) — ${shape.length === 0 ? 'no canvas or frame, this is the page itself' : shape.join(', ')}`)
|
|
2543
|
+
lines.push(...grid.rows)
|
|
2544
|
+
if (args?.diff !== false && before !== undefined) {
|
|
2545
|
+
const change = diffGrid(before, grid.rows)
|
|
2546
|
+
if (change.cells === 0) lines.push('nothing changed since the last look')
|
|
2547
|
+
else if (change.box === null || change.cells > grid.cols * grid.rowCount * 0.5) lines.push('most of the screen changed since the last look')
|
|
2548
|
+
else {
|
|
2549
|
+
const cut = around(grid.rows, change.box)
|
|
2550
|
+
lines.push(`changed: ${change.cells} cell${change.cells === 1 ? '' : 's'} in box ${change.box.x},${change.box.y} ${change.box.w}x${change.box.h} — that part again:`)
|
|
2551
|
+
lines.push(...cut.rows)
|
|
2552
|
+
}
|
|
2553
|
+
} else if (args?.diff !== false) {
|
|
2554
|
+
lines.push('first look at this screen: the next look after a move says what changed')
|
|
2555
|
+
}
|
|
2297
2556
|
return { text: lines.join('\n') }
|
|
2298
2557
|
},
|
|
2299
2558
|
},
|
|
@@ -2345,7 +2604,13 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2345
2604
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the words, as Markdown. */
|
|
2346
2605
|
execute: async (args) => withRead(async (page) => {
|
|
2347
2606
|
const asked = String(args?.query ?? '').trim()
|
|
2348
|
-
|
|
2607
|
+
await loadTactics()
|
|
2608
|
+
const learned = tacticsOf(page.url)
|
|
2609
|
+
/* A site that already gave a long answer is asked for less unless the
|
|
2610
|
+
caller said how much: the line at the end says so, and max_chars
|
|
2611
|
+
overrides it. */
|
|
2612
|
+
const longSite = asked === '' && args?.max_chars === undefined && longRead(learned)
|
|
2613
|
+
const ceiling = Number(args?.max_chars ?? (asked === '' ? (longSite ? 3500 : 6000) : 4000))
|
|
2349
2614
|
const mark = await pageMark(page)
|
|
2350
2615
|
const key = mark === '' ? '' : `${mark}|${asked}|${ceiling}`
|
|
2351
2616
|
const seen = page === null || page === undefined ? undefined : readSeen.get(page)
|
|
@@ -2357,6 +2622,9 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2357
2622
|
}
|
|
2358
2623
|
const keep = (text) => {
|
|
2359
2624
|
if (key !== '' && page !== null && page !== undefined) readSeen.set(page, { key, text })
|
|
2625
|
+
/* Cutting a long site short is only honest if it is said: the count
|
|
2626
|
+
and the way past it go with the text, not in a log. */
|
|
2627
|
+
if (longSite && text.length >= ceiling * 0.9) return `${text}\n\nthis site reads long (${siteLine(learned)}): cut at ${String(ceiling)} characters — pass query for one answer, or max_chars for more`
|
|
2360
2628
|
return text
|
|
2361
2629
|
}
|
|
2362
2630
|
if (asked !== '') {
|
|
@@ -2427,6 +2695,9 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2427
2695
|
}
|
|
2428
2696
|
const read = await readArticle(page, { maxChars: ceiling })
|
|
2429
2697
|
if (read.fault !== undefined) return `the page did not answer when read — ${read.fault.split('\n')[0].slice(0, 200)}`
|
|
2698
|
+
/* Measured before the budget cuts it: a cut says how much was sent, not
|
|
2699
|
+
how much text the site has. */
|
|
2700
|
+
await noteRead(String(read.url ?? page.url ?? ''), { text: read.chars })
|
|
2430
2701
|
if (read.text === '') {
|
|
2431
2702
|
const pdf = await pdfNote(page)
|
|
2432
2703
|
if (pdf !== '') return `${read.url}\n# ${read.title === '' ? '(untitled)' : read.title}${pdf}`
|
|
@@ -2464,13 +2735,20 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
2464
2735
|
isConcurrencySafe: () => false,
|
|
2465
2736
|
/** @param {any} args - arguments. @returns {Promise<{text: string}>} the read. */
|
|
2466
2737
|
execute: async (args) => withRead(async (page) => {
|
|
2467
|
-
|
|
2738
|
+
await loadTactics()
|
|
2739
|
+
const learned = tacticsOf(page.url)
|
|
2740
|
+
/* A site already known to be big opens as a map: a screen of landmarks
|
|
2741
|
+
first, words on request. The caller's own mode always wins. */
|
|
2742
|
+
const autoMap = args?.mode === undefined && args?.changed_only !== true && bigSite(learned)
|
|
2743
|
+
const options = { mode: String(args?.mode ?? (autoMap ? 'map' : 'text')), maxChars: Number(args?.max_chars ?? 4600) }
|
|
2468
2744
|
/* Each chat keeps its own read state: a shared one let another chat's
|
|
2469
2745
|
read replace this chat's baseline, so its numbers and its "what
|
|
2470
2746
|
changed" answered about a page it never saw. */
|
|
2471
2747
|
const state = refsFor()
|
|
2748
|
+
const startedAt = Date.now()
|
|
2472
2749
|
const read = await remember(page, state, options)
|
|
2473
|
-
|
|
2750
|
+
await noteRead(String(read.url ?? page.url ?? ''), { chars: read.textChars, controls: read.controls?.length ?? 0, ms: Date.now() - startedAt })
|
|
2751
|
+
const note = `${isCheckPage(read) ? `\n${CHECK_NOTE}` : ''}${autoMap ? `\nthis site reads big (${siteLine(learned)}): opened as a map — pass mode text for the words` : ''}`
|
|
2474
2752
|
if (args?.changed_only === true && state.last !== undefined) {
|
|
2475
2753
|
/* The diff becomes the new baseline: a page that moved zero -> one
|
|
2476
2754
|
should not report that move again on the next read. */
|
|
@@ -3257,7 +3535,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
3257
3535
|
`tab list ${[...sidecar.pages.entries()].map(([key, each]) => `${key.slice(-8)}=${String(each.url)}=${each.cdp?.closed === true ? 'closed' : 'open'}`).join(' | ')}`, // ui-copy-ignore
|
|
3258
3536
|
`default tab ${String(sidecar.page?.url)}`, // ui-copy-ignore
|
|
3259
3537
|
state.headless === false ? 'showing a window' : 'no window',
|
|
3260
|
-
sidecar.flash ?
|
|
3538
|
+
sidecar.flash ? `Flash-era runtime — ${flashNote().replace('flash runtime: ', '')}` : 'Chrome runtime',
|
|
3261
3539
|
`profile ${String(state.profile)}`,
|
|
3262
3540
|
`url ${String(state.url)}`,
|
|
3263
3541
|
`title ${String(state.title)}`,
|
|
@@ -3266,6 +3544,7 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
|
|
|
3266
3544
|
`mirror owner ${owner === '' ? 'none' : owner}`, // ui-copy-ignore
|
|
3267
3545
|
owner === '' ? 'no owner' : `owner ${owner}`,
|
|
3268
3546
|
`refs followed ${String(follows.count)}`,
|
|
3547
|
+
`files ${artifactRoot()}`,
|
|
3269
3548
|
/* What it costs and when it would let go: the two facts that decide
|
|
3270
3549
|
whether to keep using it or let it rest. */
|
|
3271
3550
|
state.alive === true ? `memory ${Math.max(1, Math.round(Number(state.memory ?? 0) / 1048576))} MB` : 'memory 0 MB',
|