dsh-inner-browser-use 0.27.0 → 0.34.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -18
- package/README.zh.md +26 -14
- package/client.js +262 -28
- package/docs/ARCHITECTURE.md +15 -12
- package/docs/ARTIFACTS.md +19 -0
- package/docs/BENCH.md +568 -17
- package/docs/FLASH.md +84 -0
- package/docs/ROADMAP.md +107 -13
- package/docs/VERIFY.md +20 -10
- package/index.js +456 -45
- package/package.json +20 -3
- package/read.js +734 -29
- package/review.js +213 -0
- package/runtime/flash-app/main.js +232 -0
- package/runtime/flash-app/package.json +1 -0
- package/runtime/flash-setup.mjs +151 -0
- package/runtime/swf.mjs +233 -0
- package/scripts/check-flash-local.mjs +104 -0
- package/scripts/check-flash-mirror.mjs +73 -0
- package/scripts/check-flash-tools.mjs +74 -0
- package/scripts/check-flash-window.mjs +58 -0
- package/scripts/doctor.mjs +3 -1
- package/sidecar.js +1763 -106
- package/tools.js +1667 -191
package/index.js
CHANGED
|
@@ -20,8 +20,36 @@
|
|
|
20
20
|
* learns no port and no secret. // ui-copy-ignore
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
|
+
import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
|
24
|
+
import { homedir } from 'node:os'
|
|
25
|
+
import { join } from 'node:path'
|
|
23
26
|
import { Sidecar } from './sidecar.js?v=1'
|
|
24
27
|
|
|
28
|
+
/** Where the mirror's owner is remembered across a reload: the column should not go dark because the code reloaded. */
|
|
29
|
+
/** Where the mirror's note lives; a test points this elsewhere so it never reads a real chat. */
|
|
30
|
+
const STATE_DIR = process.env.DSH_INNER_BROWSER_STATE_DIR || join(homedir(), '.dsh', 'inner-browser-use')
|
|
31
|
+
const MIRROR_FILE = join(STATE_DIR, 'mirror.json')
|
|
32
|
+
|
|
33
|
+
/** @returns {{owner: string, session: string, at: number}} the last owner a chat asked to keep. */
|
|
34
|
+
function readMirror() {
|
|
35
|
+
try {
|
|
36
|
+
const saved = JSON.parse(readFileSync(MIRROR_FILE, 'utf8'))
|
|
37
|
+
return { owner: String(saved?.owner ?? ''), session: String(saved?.session ?? ''), at: Number(saved?.at ?? 0) }
|
|
38
|
+
} catch {
|
|
39
|
+
return { owner: '', session: '', at: 0 }
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** @param {{owner: string, session: string, at: number}} value - what to keep. */
|
|
44
|
+
function writeMirror(value) {
|
|
45
|
+
try {
|
|
46
|
+
mkdirSync(STATE_DIR, { recursive: true })
|
|
47
|
+
writeFileSync(MIRROR_FILE, JSON.stringify(value))
|
|
48
|
+
} catch {
|
|
49
|
+
/* an unwritable note must not take the mirror down */
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
25
53
|
/** Route prefix owned by this plugin. Never under `/api`, which is the Host's own channel. */ // ui-copy-ignore
|
|
26
54
|
export const ROUTE = '/dsh-inner-browser-use'
|
|
27
55
|
|
|
@@ -57,12 +85,36 @@ function readBody(req) {
|
|
|
57
85
|
* Nothing else lives here; the browser itself is the Host's. // ui-copy-ignore
|
|
58
86
|
*/
|
|
59
87
|
export class Bridge {
|
|
60
|
-
|
|
88
|
+
/** @param {{persist?: boolean}} [options] - persist the owner across reloads. */
|
|
89
|
+
constructor(options = {}) {
|
|
61
90
|
/** @type {Map<string, import('node:http').ServerResponse>} */
|
|
62
91
|
this.seats = new Map()
|
|
63
92
|
/* The one chat that owns the mirror. The column is global, so the picture
|
|
64
93
|
may only ever be drawn for this chat; every other chat shows a note. */
|
|
65
94
|
this.owner = ''
|
|
95
|
+
/* The tab asked for, kept until the chat comes back into view. A chat that
|
|
96
|
+
is not on screen has no stream, so a push at ask time is lost; remembering
|
|
97
|
+
the ask is what makes the mirror appear when the person returns. */
|
|
98
|
+
this.wanted = { session: '', at: 0 }
|
|
99
|
+
/* A running plugin keeps the note; a bare Bridge in a test does not. */
|
|
100
|
+
this.persist = options.persist === true
|
|
101
|
+
const kept = this.persist ? readMirror() : { owner: '', session: '', at: 0 }
|
|
102
|
+
/* Only a real chat id is restored: a stray note must not hand the mirror to
|
|
103
|
+
a chat that no longer exists. */
|
|
104
|
+
if (/^session-/.test(kept.owner) !== true) kept.owner = ''
|
|
105
|
+
this.owner = kept.owner
|
|
106
|
+
this.wanted = { session: kept.session, at: kept.at }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* A chat is the chat that asked, and nothing else. Guessing at an alias once
|
|
111
|
+
* put the mirror in a stranger's column: two conversations are never the same
|
|
112
|
+
* conversation, however few pages happen to be on screen.
|
|
113
|
+
* @param {string} session - whatever id the caller has.
|
|
114
|
+
* @returns {string} the same id.
|
|
115
|
+
*/
|
|
116
|
+
canon(session) {
|
|
117
|
+
return String(session ?? '')
|
|
66
118
|
}
|
|
67
119
|
|
|
68
120
|
/**
|
|
@@ -70,7 +122,24 @@ export class Bridge {
|
|
|
70
122
|
* @param {string} sessionId - chat that asked for it.
|
|
71
123
|
*/
|
|
72
124
|
claim(sessionId) {
|
|
73
|
-
if (sessionId
|
|
125
|
+
if (sessionId === '') return
|
|
126
|
+
this.owner = sessionId
|
|
127
|
+
if (this.persist === true) writeMirror({ owner: this.owner, session: this.wanted.session, at: this.wanted.at })
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Remember that a chat wants the mirror, and tell it now if it is listening.
|
|
132
|
+
* @param {string} sessionId - chat that asked.
|
|
133
|
+
* @param {number} [at] - when it asked; older asks are not replayed.
|
|
134
|
+
* @returns {boolean} whether a live tab took the frame.
|
|
135
|
+
*/
|
|
136
|
+
want(sessionId, at = Date.now()) {
|
|
137
|
+
const id = this.canon(sessionId)
|
|
138
|
+
if (id === '') return false
|
|
139
|
+
this.claim(id)
|
|
140
|
+
this.wanted = { session: id, at }
|
|
141
|
+
if (this.persist === true) writeMirror({ owner: this.owner, session: id, at })
|
|
142
|
+
return this.push(id, { type: 'sidecar-wanted', session: id, at })
|
|
74
143
|
}
|
|
75
144
|
|
|
76
145
|
/**
|
|
@@ -81,6 +150,13 @@ export class Bridge {
|
|
|
81
150
|
attach(sessionId, res) {
|
|
82
151
|
this.seats.get(sessionId)?.end()
|
|
83
152
|
this.seats.set(sessionId, res)
|
|
153
|
+
/* A tab that connects after the ask still gets the mirror: this is the case
|
|
154
|
+
where the person was looking at another chat, or reloaded the page. An
|
|
155
|
+
ask older than two minutes is spent and is not replayed. */
|
|
156
|
+
const ask = this.wanted
|
|
157
|
+
if (ask.session === sessionId && Date.now() - ask.at <= 120_000) {
|
|
158
|
+
this.push(sessionId, { type: 'sidecar-wanted', session: sessionId, at: ask.at })
|
|
159
|
+
}
|
|
84
160
|
}
|
|
85
161
|
|
|
86
162
|
/**
|
|
@@ -110,6 +186,17 @@ export class Bridge {
|
|
|
110
186
|
return [...this.seats.keys()]
|
|
111
187
|
}
|
|
112
188
|
|
|
189
|
+
/**
|
|
190
|
+
* Forget a mirror nobody is waiting for any more.
|
|
191
|
+
* @param {string} sessionId - chat that closed its tab.
|
|
192
|
+
* @returns {boolean} whether the ask was the remembered one.
|
|
193
|
+
*/
|
|
194
|
+
forget(sessionId) {
|
|
195
|
+
if (sessionId === '' || this.wanted.session !== sessionId) return false
|
|
196
|
+
this.wanted = { session: '', at: 0 }
|
|
197
|
+
return true
|
|
198
|
+
}
|
|
199
|
+
|
|
113
200
|
/** Close every stream. */
|
|
114
201
|
dispose() {
|
|
115
202
|
for (const res of this.seats.values()) res.end()
|
|
@@ -120,7 +207,92 @@ export class Bridge {
|
|
|
120
207
|
/** Page read: address, title, controls, then text, as JSON the Host parses. */ // ui-copy-ignore
|
|
121
208
|
/* The read model and the tool surface live in their own files: this module is
|
|
122
209
|
the Host half (routes, bridge, lifetimes), not the model's vocabulary. */ // ui-copy-ignore
|
|
123
|
-
import { sidecarTools } from './tools.js?v=1'
|
|
210
|
+
import { keyEvent, sidecarTools, sessionScope, workScope } from './tools.js?v=1'
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* A state poll may pick up a browser that outlived a reload, and only for the
|
|
214
|
+
* chat that owns it: any chat polling would otherwise start a browser nobody
|
|
215
|
+
* asked for, and a chat with no column would look like it had one.
|
|
216
|
+
* @param {{alive?: boolean}} said - what the browser half answered.
|
|
217
|
+
* @param {string} chat - the chat asking.
|
|
218
|
+
* @param {string} owner - the chat that owns the mirror.
|
|
219
|
+
* @returns {boolean} whether this poll should attach to a running browser.
|
|
220
|
+
*/
|
|
221
|
+
/**
|
|
222
|
+
* What the state poll hands the browser half when it picks a surviving browser
|
|
223
|
+
* up: the chat's own tab, at the address that chat last had. Without the chat
|
|
224
|
+
* the browser comes back with no tab for anybody, and the column stays empty.
|
|
225
|
+
* @param {string} chat - the chat that is polling.
|
|
226
|
+
* @param {string} remembered - where that chat left off, or ''.
|
|
227
|
+
* @returns {{session: string, url?: string}} options for `ensure`.
|
|
228
|
+
*/
|
|
229
|
+
/**
|
|
230
|
+
* Whether this chat's poll should put its tab back: the browser is running, the
|
|
231
|
+
* chat holds the mirror, and its tab is not there.
|
|
232
|
+
* @param {any} chatState - what `chatState(chat)` said.
|
|
233
|
+
* @param {string} chat - the chat that is polling.
|
|
234
|
+
* @param {string} owner - the chat that holds the mirror.
|
|
235
|
+
* @returns {boolean} true when a tab has to be opened for this chat.
|
|
236
|
+
*/
|
|
237
|
+
/**
|
|
238
|
+
* The folder the calling session works in, wherever it is kept.
|
|
239
|
+
* @param {any} execution - tool execution context.
|
|
240
|
+
* @returns {string} absolute path, or empty when nothing was said.
|
|
241
|
+
*/
|
|
242
|
+
export function cwdOf(execution) {
|
|
243
|
+
try {
|
|
244
|
+
const agent = execution?.agent
|
|
245
|
+
const value = agent?.cwd ?? agent?.session?.header?.cwd ?? agent?.header?.cwd
|
|
246
|
+
if (typeof value === 'string' && value !== '') return value
|
|
247
|
+
} catch {
|
|
248
|
+
/* a context that cannot be read is a context without a folder */
|
|
249
|
+
}
|
|
250
|
+
return ''
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
export function needsTab(chatState, chat, owner) {
|
|
254
|
+
if (String(chat ?? '') === '' || owner !== chat) return false
|
|
255
|
+
return chatState?.alive !== true
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export function adoptOptions(chat, remembered) {
|
|
259
|
+
const back = String(remembered ?? '')
|
|
260
|
+
return back === '' ? { session: chat } : { session: chat, url: back }
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
export function shouldAdoptOnState(said, chat, owner) {
|
|
264
|
+
return said?.alive === false && chat !== '' && owner === chat
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Whether a running browser still needs its column told to draw it.
|
|
269
|
+
* @param {{alive: boolean, owner: string, wanted: string, listeners: number}} what - current facts.
|
|
270
|
+
* @returns {boolean} true when the owner is known, the browser runs, and nothing listens.
|
|
271
|
+
*/
|
|
272
|
+
export function wantsMirrorAgain(what) {
|
|
273
|
+
if (what?.alive !== true) return false
|
|
274
|
+
const owner = String(what?.owner ?? '')
|
|
275
|
+
if (owner === '') return false
|
|
276
|
+
if (String(what?.wanted ?? '') !== owner) return false
|
|
277
|
+
return Number(what?.listeners ?? 0) <= 0
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Whether asking again can achieve anything. A pane that is polling this very
|
|
282
|
+
* endpoint is awake and already knows the frames are missing, so a second ask
|
|
283
|
+
* every few seconds only writes log lines; a pane that went quiet may have lost
|
|
284
|
+
* the stream and does need one. One ask per gap at most.
|
|
285
|
+
* @param {any} what - { askedAgo, sinceLast, quiet, gap } in milliseconds.
|
|
286
|
+
* @returns {boolean} true when the ask should be sent now.
|
|
287
|
+
*/
|
|
288
|
+
export function nudgeDue(what) {
|
|
289
|
+
const askedAgo = Number(what?.askedAgo ?? -1)
|
|
290
|
+
const sinceLast = Number(what?.sinceLast ?? Infinity)
|
|
291
|
+
const quiet = Number(what?.quiet ?? 15_000)
|
|
292
|
+
const gap = Number(what?.gap ?? 10_000)
|
|
293
|
+
if (!Number.isFinite(askedAgo) || askedAgo < quiet) return false
|
|
294
|
+
return sinceLast >= gap
|
|
295
|
+
}
|
|
124
296
|
|
|
125
297
|
export const name = 'dsh-inner-browser-use'
|
|
126
298
|
export const inject = ['webServer']
|
|
@@ -130,7 +302,7 @@ export const inject = ['webServer']
|
|
|
130
302
|
* @param {any} ctx - plugin context.
|
|
131
303
|
*/
|
|
132
304
|
export function apply(ctx) {
|
|
133
|
-
const bridge = new Bridge()
|
|
305
|
+
const bridge = new Bridge({ persist: true })
|
|
134
306
|
/* The sidecar is a browser the Host owns outright: a real Chrome with its own // ui-copy-ignore
|
|
135
307
|
profile, shown in the column as a screencast. It outlives a plugin reload,
|
|
136
308
|
and it does not need the chat to be on screen to do work. */
|
|
@@ -138,6 +310,31 @@ export function apply(ctx) {
|
|
|
138
310
|
/** @type {{refs: any[], at: number, last?: any}} */
|
|
139
311
|
const sidecarRefs = { refs: [], at: 0 }
|
|
140
312
|
ctx.effect(() => () => bridge.dispose(), 'dsh-inner-browser-use: bridge lifetime')
|
|
313
|
+
/* A live picture that nobody is drawing is the one failure people notice
|
|
314
|
+
first: the browser is fine, the column just lost its pane. The chat that
|
|
315
|
+
owns the browser is asked again, on a slow beat, until the frames stream
|
|
316
|
+
has a listener — the client keeps the rule that only the mounted chat's
|
|
317
|
+
column may open. */
|
|
318
|
+
/** @type {Map<string, number>} when each chat last polled for state. */
|
|
319
|
+
const lastAsk = new Map()
|
|
320
|
+
ctx.effect(() => {
|
|
321
|
+
let last = 0
|
|
322
|
+
const timer = setInterval(() => {
|
|
323
|
+
try {
|
|
324
|
+
const owner = bridge.owner
|
|
325
|
+
const listeners = sidecar.page?.frames?.size ?? 0
|
|
326
|
+
const again = wantsMirrorAgain({ alive: sidecar.page !== null && sidecar.page !== undefined, owner, wanted: bridge.wanted.session, listeners }) // ui-copy-ignore
|
|
327
|
+
if (again !== true) return
|
|
328
|
+
const due = nudgeDue({ askedAgo: Date.now() - (lastAsk.get(owner) ?? 0), sinceLast: Date.now() - last }) // ui-copy-ignore
|
|
329
|
+
if (due !== true) return
|
|
330
|
+
last = Date.now()
|
|
331
|
+
bridge.want(owner)
|
|
332
|
+
appendFileSync(join(STATE_DIR, 'asks.log'), `${new Date().toISOString()} asked ${owner.slice(-8)} to draw the mirror again\n`)
|
|
333
|
+
} catch {}
|
|
334
|
+
}, 4000)
|
|
335
|
+
timer.unref?.()
|
|
336
|
+
return () => clearInterval(timer)
|
|
337
|
+
}, 'dsh-inner-browser-use: mirror keepalive')
|
|
141
338
|
/* Closing the browser cleanly on quit is what keeps a sign-in alive: a
|
|
142
339
|
killed Chrome never writes the cookies a page set this minute. */
|
|
143
340
|
ctx.effect(() => {
|
|
@@ -148,15 +345,25 @@ export function apply(ctx) {
|
|
|
148
345
|
timer.unref?.()
|
|
149
346
|
return () => {
|
|
150
347
|
clearInterval(timer)
|
|
151
|
-
void sidecar.
|
|
348
|
+
void sidecar.handoff().catch(() => {})
|
|
152
349
|
}
|
|
153
350
|
}, 'dsh-inner-browser-use: sidecar lifetime')
|
|
154
351
|
|
|
155
352
|
/** @param {import('node:http').IncomingMessage} req - request. @param {import('node:http').ServerResponse} res - response. */
|
|
353
|
+
/** @type {{at: number, path: string, session: string}[]} the last few calls the column made. */
|
|
354
|
+
const recent = []
|
|
355
|
+
/* Whether the pane is asking at all is the difference between "the browser
|
|
356
|
+
stopped" and "nothing is drawing it": the poll is every 2.5 s, so a chat
|
|
357
|
+
that asks is a chat with a mounted pane. Kept on disk because the answer
|
|
358
|
+
has to survive a plugin reload, which is when the question is asked. */
|
|
156
359
|
const handler = async (req, res) => {
|
|
157
360
|
const url = new URL(req.url ?? ROUTE, 'http://dsh.invalid')
|
|
158
361
|
const path = url.pathname.slice(ROUTE.length) || '/'
|
|
159
362
|
const sessionId = url.searchParams.get('session') ?? ''
|
|
363
|
+
/* Who came in and when: an empty picture has two very different causes, and
|
|
364
|
+
this is what tells a column that never connected from a page that is gone. */
|
|
365
|
+
recent.push({ at: Date.now(), path, session: sessionId.slice(-8) })
|
|
366
|
+
if (recent.length > 40) recent.splice(0, recent.length - 40)
|
|
160
367
|
|
|
161
368
|
/** @param {number} code - status. @param {unknown} value - body. */
|
|
162
369
|
const json = (code, value) => {
|
|
@@ -164,14 +371,42 @@ export function apply(ctx) {
|
|
|
164
371
|
res.end(JSON.stringify(value))
|
|
165
372
|
}
|
|
166
373
|
|
|
374
|
+
/* One answer to one question: may this chat act on the mirror? A browser
|
|
375
|
+
nobody holds is anybody's to take; once a chat holds it, another chat is
|
|
376
|
+
refused here instead of being trusted to check itself. */
|
|
377
|
+
/** @param {string} session - the chat a request speaks for. */
|
|
378
|
+
const held = (session) => session !== '' && bridge.owner !== '' && session !== bridge.owner
|
|
379
|
+
/** @returns {boolean} true when the request was refused. */
|
|
380
|
+
const refuse = () => {
|
|
381
|
+
json(403, { ok: false, text: 'this chat does not own the browser', owner: bridge.owner }) // ui-copy-ignore
|
|
382
|
+
return true
|
|
383
|
+
}
|
|
384
|
+
/* Every route that changes the page goes through one check: it must say
|
|
385
|
+
which chat it is for, and that chat must be the one holding the mirror.
|
|
386
|
+
Two rules in one place, so a new route cannot quietly skip one. */
|
|
387
|
+
/**
|
|
388
|
+
* @param {any} body - parsed request body, if any.
|
|
389
|
+
* @returns {string} the chat that may act, or '' when it was refused.
|
|
390
|
+
*/
|
|
391
|
+
const acting = (body) => {
|
|
392
|
+
const who = String(body?.session ?? '') || sessionId
|
|
393
|
+
if (who === '') {
|
|
394
|
+
json(400, { ok: false, text: 'which chat asked for it' }) // ui-copy-ignore
|
|
395
|
+
return ''
|
|
396
|
+
}
|
|
397
|
+
if (held(who) && refuse()) return ''
|
|
398
|
+
return who
|
|
399
|
+
}
|
|
400
|
+
|
|
167
401
|
if (path === '/sidecar/frames' && req.method === 'GET') {
|
|
168
402
|
/* The column is global and only one chat may hold the mirror: a chat that
|
|
169
403
|
does not own it is refused here, so a stale tab cannot stream a picture
|
|
170
404
|
of somebody else's browser. */
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
405
|
+
const asked = String(url.searchParams.get('session') ?? '')
|
|
406
|
+
const watching = asked !== '' ? asked : sessionId
|
|
407
|
+
if (held(watching) && refuse()) return
|
|
408
|
+
/* Only the owner's own page may stream the picture. */
|
|
409
|
+
const streaming = watching
|
|
175
410
|
res.writeHead(200, {
|
|
176
411
|
'content-type': 'text/event-stream; charset=utf-8',
|
|
177
412
|
'cache-control': 'no-cache, no-transform',
|
|
@@ -191,9 +426,13 @@ export function apply(ctx) {
|
|
|
191
426
|
page wait: the newest picture is the only one worth sending. */
|
|
192
427
|
if (res.writableLength > 3 * 1024 * 1024) return
|
|
193
428
|
res.write(`data: ${JSON.stringify(frame)}\n\n`)
|
|
194
|
-
})
|
|
429
|
+
}, streaming)
|
|
195
430
|
const info = await sidecar.state()
|
|
196
|
-
|
|
431
|
+
/* The pane needs one address to offer back when this chat's own page
|
|
432
|
+
went away; the state route answers the same question for the tab
|
|
433
|
+
header, so both carry it. */
|
|
434
|
+
const chat = watching === '' ? {} : { chat: await sidecar.chatState(watching) }
|
|
435
|
+
res.write(`data: ${JSON.stringify({ type: 'hello', ...info, ...chat })}\n\n`)
|
|
197
436
|
} catch (error) {
|
|
198
437
|
res.write(`data: ${JSON.stringify({ type: 'failed', text: String(error?.message ?? error) })}\n\n`)
|
|
199
438
|
}
|
|
@@ -203,8 +442,73 @@ export function apply(ctx) {
|
|
|
203
442
|
return
|
|
204
443
|
}
|
|
205
444
|
|
|
445
|
+
if (path === '/sidecar/claim' && req.method === 'POST') {
|
|
446
|
+
/* The pane itself asks back what it was showing: after a reload the
|
|
447
|
+
forgot who owned the mirror, and the chat that still has the tab open is
|
|
448
|
+
the only one allowed to say so. A different owner is refused. */
|
|
449
|
+
let body
|
|
450
|
+
try {
|
|
451
|
+
body = JSON.parse(await readBody(req))
|
|
452
|
+
} catch {
|
|
453
|
+
json(400, { ok: false, text: 'bad frame' })
|
|
454
|
+
return
|
|
455
|
+
}
|
|
456
|
+
const id = String(body.session ?? '')
|
|
457
|
+
if (id === '') {
|
|
458
|
+
json(400, { ok: false, text: 'no chat' })
|
|
459
|
+
return
|
|
460
|
+
}
|
|
461
|
+
if (bridge.owner !== '' && bridge.owner !== id) {
|
|
462
|
+
json(409, { ok: false, owner: bridge.owner })
|
|
463
|
+
return
|
|
464
|
+
}
|
|
465
|
+
bridge.claim(id)
|
|
466
|
+
json(200, { ok: true, owner: bridge.owner })
|
|
467
|
+
return
|
|
468
|
+
}
|
|
469
|
+
|
|
206
470
|
if (path === '/sidecar/state' && req.method === 'GET') {
|
|
207
|
-
|
|
471
|
+
/* Which chats have a column open right now: 0 means nobody is watching,
|
|
472
|
+
so an empty picture is the column's own problem, not the browser's. */
|
|
473
|
+
const chat = String(url.searchParams.get('session') ?? '')
|
|
474
|
+
/* Diagnostics belong to the chat that made them: a caller sees its own
|
|
475
|
+
calls, and one chat's browsing never explains another's. */
|
|
476
|
+
const mine = chat === '' ? recent : recent.filter((entry) => entry.session === chat.slice(-8))
|
|
477
|
+
let said = await sidecar.state()
|
|
478
|
+
/* The browser can be running and still have no tab for this chat: a reload
|
|
479
|
+
replaces the half that remembered which tab belonged to whom. The poll
|
|
480
|
+
is this chat asking, so it answers by putting that chat's tab back
|
|
481
|
+
before the column draws an empty picture — waiting for a claim first
|
|
482
|
+
left a blank frame in front of the person. */
|
|
483
|
+
let mineChat = chat === '' ? undefined : await sidecar.chatState(chat)
|
|
484
|
+
if (needsTab(mineChat, chat, bridge.owner)) {
|
|
485
|
+
try {
|
|
486
|
+
await sidecar.ensure(adoptOptions(chat, sidecar.rememberedTab?.(chat)))
|
|
487
|
+
said = await sidecar.state()
|
|
488
|
+
mineChat = await sidecar.chatState(chat)
|
|
489
|
+
} catch {}
|
|
490
|
+
}
|
|
491
|
+
/* A plugin reload leaves the browser running and this half brand new, so
|
|
492
|
+
the column's first poll would answer "nothing open" until somebody
|
|
493
|
+
called open again: the mirror would come back empty in a chat that
|
|
494
|
+
never closed it. The chat that owned the browser asks for its state, so
|
|
495
|
+
answering honestly means picking the browser up first. */
|
|
496
|
+
if (shouldAdoptOnState(said, chat, bridge.owner)) {
|
|
497
|
+
try {
|
|
498
|
+
await sidecar.ensure(adoptOptions(chat, sidecar.rememberedTab?.(chat)))
|
|
499
|
+
said = await sidecar.state()
|
|
500
|
+
} catch {}
|
|
501
|
+
}
|
|
502
|
+
if (chat !== '') {
|
|
503
|
+
const before = lastAsk.get(chat) ?? 0
|
|
504
|
+
if (Date.now() - before > 2000) {
|
|
505
|
+
lastAsk.set(chat, Date.now())
|
|
506
|
+
try {
|
|
507
|
+
appendFileSync(join(STATE_DIR, 'asks.log'), `${new Date().toISOString()} ${chat.slice(-8)} alive=${String(said?.alive === true)} viewers=${String(said?.viewers ?? 0)}\n`)
|
|
508
|
+
} catch {}
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
json(200, { ...said, ...(chat === '' ? {} : { chat: mineChat ?? (await sidecar.chatState(chat)) }), owner: bridge.owner, chats: [...bridge.seats.keys()], recent: mine })
|
|
208
512
|
return
|
|
209
513
|
}
|
|
210
514
|
|
|
@@ -219,19 +523,21 @@ export function apply(ctx) {
|
|
|
219
523
|
const address = String(body.url ?? '')
|
|
220
524
|
/* The ask must name the chat it belongs to: a request that cannot say
|
|
221
525
|
who it is for never gets the mirror. */
|
|
222
|
-
|
|
526
|
+
/* An open names its chat and then takes the mirror over: that is what
|
|
527
|
+
asking for it means, so another owner is not a refusal here. */
|
|
528
|
+
const who = String(body.session ?? '') || sessionId
|
|
223
529
|
if (who === '') {
|
|
224
|
-
json(400, { ok: false, text: 'which chat asked for it' })
|
|
530
|
+
json(400, { ok: false, text: 'which chat asked for it' }) // ui-copy-ignore
|
|
225
531
|
return
|
|
226
532
|
}
|
|
227
533
|
await sidecar.ensure({
|
|
228
534
|
url: address === '' ? undefined : address,
|
|
229
535
|
headless: body.headless === undefined ? undefined : body.headless === true,
|
|
536
|
+
session: who,
|
|
230
537
|
})
|
|
231
538
|
/* The mirror belongs to the chat that asked for it; any earlier owner
|
|
232
539
|
loses it and its tab. */
|
|
233
|
-
bridge.
|
|
234
|
-
bridge.push(who, { type: 'sidecar-wanted', session: who })
|
|
540
|
+
bridge.want(who)
|
|
235
541
|
json(200, { ok: true, ...(await sidecar.state()), owner: bridge.owner })
|
|
236
542
|
} catch (error) {
|
|
237
543
|
json(500, { ok: false, text: String(error?.message ?? error) })
|
|
@@ -239,6 +545,35 @@ export function apply(ctx) {
|
|
|
239
545
|
return
|
|
240
546
|
}
|
|
241
547
|
|
|
548
|
+
if (path === '/sidecar/viewport' && req.method === 'POST') {
|
|
549
|
+
/* The pane tells the page how big it is, so the picture fills the column
|
|
550
|
+
instead of being letterboxed inside it. Only the owner may resize what
|
|
551
|
+
its own mirror shows. */
|
|
552
|
+
let body
|
|
553
|
+
try {
|
|
554
|
+
body = JSON.parse(await readBody(req))
|
|
555
|
+
} catch {
|
|
556
|
+
json(400, { ok: false, text: 'bad frame' }) // ui-copy-ignore
|
|
557
|
+
return
|
|
558
|
+
}
|
|
559
|
+
const actor = acting(body)
|
|
560
|
+
if (actor === '') {
|
|
561
|
+
json(403, { ok: false, text: 'this chat does not own the browser' }) // ui-copy-ignore
|
|
562
|
+
return
|
|
563
|
+
}
|
|
564
|
+
try {
|
|
565
|
+
const page = sidecar.page
|
|
566
|
+
if (page === null || page === undefined) {
|
|
567
|
+
json(409, { ok: false, text: 'the browser is not open' }) // ui-copy-ignore
|
|
568
|
+
return
|
|
569
|
+
}
|
|
570
|
+
json(200, { ok: true, view: await page.setViewport(body.w, body.h) })
|
|
571
|
+
} catch (error) {
|
|
572
|
+
json(500, { ok: false, text: String(error?.message ?? error) })
|
|
573
|
+
}
|
|
574
|
+
return
|
|
575
|
+
}
|
|
576
|
+
|
|
242
577
|
if (path === '/sidecar/input' && req.method === 'POST') {
|
|
243
578
|
let body
|
|
244
579
|
try {
|
|
@@ -249,12 +584,10 @@ export function apply(ctx) {
|
|
|
249
584
|
}
|
|
250
585
|
/* Input follows the owner: a chat that lost the mirror cannot act on the
|
|
251
586
|
page from a stale tab either. */
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
return
|
|
255
|
-
}
|
|
587
|
+
const actor = acting(body)
|
|
588
|
+
if (actor === '') return
|
|
256
589
|
try {
|
|
257
|
-
const page = await sidecar.ensure()
|
|
590
|
+
const page = await sidecar.ensure({ session: actor })
|
|
258
591
|
const kind = String(body.kind ?? '')
|
|
259
592
|
const x = Number(body.x ?? 0)
|
|
260
593
|
const y = Number(body.y ?? 0)
|
|
@@ -273,21 +606,26 @@ export function apply(ctx) {
|
|
|
273
606
|
} else if (kind === 'wheel') await page.wheel({ x, y, dx: Number(body.dx ?? 0), dy: Number(body.dy ?? 0) })
|
|
274
607
|
else if (kind === 'text') await page.insertText(String(body.text ?? ''))
|
|
275
608
|
else if (kind === 'key') {
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
modifiers: Number(body.modifiers ?? 0),
|
|
284
|
-
}
|
|
285
|
-
await page.key('keyDown', spec)
|
|
286
|
-
await page.key('keyUp', { ...spec, text: undefined })
|
|
609
|
+
const key = keyEvent(body)
|
|
610
|
+
if (key.down !== true) await page.key('keyUp', { ...key.spec, text: undefined })
|
|
611
|
+
else if (key.repeat === true) {
|
|
612
|
+
/* Typing repeat: the page draws a held key as a fresh keydown. */
|
|
613
|
+
await page.key('keyDown', { ...key.spec, autoRepeat: true })
|
|
614
|
+
} else if (key.command === null) await page.key('keyDown', key.spec)
|
|
615
|
+
else await page.key('keyDown', { ...key.spec, text: undefined, commands: [key.command] })
|
|
287
616
|
}
|
|
288
617
|
/* Hovering and moving are looking, not acting: only real input counts as
|
|
289
618
|
the person taking the page. */
|
|
290
|
-
if (kind !== 'hover' && kind !== 'move')
|
|
619
|
+
if (kind !== 'hover' && kind !== 'move') {
|
|
620
|
+
sidecar.humanUse({ kind, x, y })
|
|
621
|
+
/* The arrow goes now, not on the next repaint: a still page may not
|
|
622
|
+
paint again for a second and a half. */
|
|
623
|
+
try {
|
|
624
|
+
page.announceHold()
|
|
625
|
+
} catch {
|
|
626
|
+
/* the stream is gone; the next frame carries the same answer */
|
|
627
|
+
}
|
|
628
|
+
}
|
|
291
629
|
json(200, cursor === undefined ? { ok: true } : { ok: true, cursor })
|
|
292
630
|
} catch (error) {
|
|
293
631
|
json(500, { ok: false, text: String(error?.message ?? error) })
|
|
@@ -304,12 +642,10 @@ export function apply(ctx) {
|
|
|
304
642
|
return
|
|
305
643
|
}
|
|
306
644
|
/* Navigation follows the owner too: a lost tab cannot steer the page. */
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
return
|
|
310
|
-
}
|
|
645
|
+
const actor = acting(body)
|
|
646
|
+
if (actor === '') return
|
|
311
647
|
try {
|
|
312
|
-
const page = await sidecar.ensure()
|
|
648
|
+
const page = await sidecar.ensure({ session: actor })
|
|
313
649
|
const op = String(body.op ?? 'goto')
|
|
314
650
|
if (op === 'reload') json(200, { ok: true, ...(await page.reload()) })
|
|
315
651
|
else if (op === 'back') json(200, { ok: true, ...(await page.history(-1)) })
|
|
@@ -325,7 +661,22 @@ export function apply(ctx) {
|
|
|
325
661
|
return
|
|
326
662
|
}
|
|
327
663
|
|
|
664
|
+
if (path === '/sidecar/forget' && req.method === 'POST') {
|
|
665
|
+
/* Closing the tab is an answer: coming back should not reopen it. Only the
|
|
666
|
+
chat that holds the mirror may withdraw its own ask. */
|
|
667
|
+
const body = await readBody(req).then((raw) => JSON.parse(raw || '{}')).catch(() => ({}))
|
|
668
|
+
const who = acting(body)
|
|
669
|
+
if (who === '') return
|
|
670
|
+
json(200, { ok: true, forgot: bridge.forget(who) })
|
|
671
|
+
return
|
|
672
|
+
}
|
|
673
|
+
|
|
328
674
|
if (path === '/sidecar/stop' && req.method === 'POST') {
|
|
675
|
+
/* Closing the browser is a write as well: a chat that does not hold the
|
|
676
|
+
mirror cannot close another chat's page. */
|
|
677
|
+
const body = await readBody(req).then((raw) => JSON.parse(raw || '{}')).catch(() => ({}))
|
|
678
|
+
const who = String(body.session ?? '') || sessionId
|
|
679
|
+
if (who !== '' && held(who) && refuse()) return
|
|
329
680
|
await sidecar.shutdown()
|
|
330
681
|
json(200, { ok: true })
|
|
331
682
|
return
|
|
@@ -359,13 +710,69 @@ export function apply(ctx) {
|
|
|
359
710
|
|
|
360
711
|
ctx.inject(['tools'], (scope) => {
|
|
361
712
|
const disposers = sidecarTools(sidecar, sidecarRefs, {
|
|
713
|
+
/* A screenshot the model can look at is the difference between planning
|
|
714
|
+
from the picture and guessing from a path. The store and the route that
|
|
715
|
+
can carry it are Harness services, so the tool asks here for it; when
|
|
716
|
+
either is missing the picture stays a file and the call still works. */
|
|
717
|
+
/**
|
|
718
|
+
* @param {string} data - base64 PNG.
|
|
719
|
+
* @param {string} name - file name for the attachment.
|
|
720
|
+
* @param {any} exec - the tool call's context.
|
|
721
|
+
* @returns {Promise<any>} the attachment value, or undefined to stay text-only.
|
|
722
|
+
*/
|
|
723
|
+
async showShot(data, name, exec) {
|
|
724
|
+
try {
|
|
725
|
+
const attachments = ctx.get('attachments')
|
|
726
|
+
const llm = ctx.get('llm')
|
|
727
|
+
if (attachments === undefined || llm === undefined) return undefined
|
|
728
|
+
const route = exec?.agent?.session?.requestHeader?.()?.config
|
|
729
|
+
const provider = route?.provider ?? exec?.agent?.options?.provider
|
|
730
|
+
const model = route?.model ?? exec?.agent?.options?.model
|
|
731
|
+
if (provider === undefined || model === undefined) return undefined
|
|
732
|
+
const info = await llm.resolveModelInfo(provider, model, exec?.signal)
|
|
733
|
+
if (!Array.isArray(info?.inputModalities) || !info.inputModalities.includes('image')) return undefined
|
|
734
|
+
const bytes = Buffer.from(data, 'base64')
|
|
735
|
+
const cap = Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
|
|
736
|
+
if (!(bytes.length > 0) || bytes.length > cap) return undefined
|
|
737
|
+
const ref = await attachments.saveImage({ data: bytes, mediaType: 'image/png', name })
|
|
738
|
+
return {
|
|
739
|
+
attachmentId: ref.attachmentId, // ui-copy-ignore
|
|
740
|
+
mediaType: ref.mediaType, // ui-copy-ignore
|
|
741
|
+
bytes: ref.bytes, // ui-copy-ignore
|
|
742
|
+
width: ref.width, // ui-copy-ignore
|
|
743
|
+
height: ref.height, // ui-copy-ignore
|
|
744
|
+
...(ref.name === undefined ? {} : { name: ref.name }),
|
|
745
|
+
}
|
|
746
|
+
} catch {
|
|
747
|
+
return undefined
|
|
748
|
+
}
|
|
749
|
+
},
|
|
362
750
|
/* The mirror changes hands here: the calling chat owns it from now on,
|
|
363
751
|
and every earlier owner loses it. */
|
|
364
752
|
want: (sessionId) => {
|
|
365
|
-
bridge.
|
|
366
|
-
bridge.push(sessionId, { type: 'sidecar-wanted', session: sessionId })
|
|
753
|
+
bridge.want(sessionId)
|
|
367
754
|
},
|
|
368
|
-
|
|
755
|
+
/* Who holds the mirror, and whether a chat is listening right now. */
|
|
756
|
+
owner: () => bridge.owner, // ui-copy-ignore
|
|
757
|
+
seats: () => [...bridge.seats.keys()].map((k) => String(k).slice(-8)).join(',') || 'none', // ui-copy-ignore
|
|
758
|
+
}).map((definition) => {
|
|
759
|
+
/* A reload can mount this surface twice before the old one is torn down.
|
|
760
|
+
A name already in the registry is kept, not fought over: a duplicate
|
|
761
|
+
registration is a warning, never a crash. */
|
|
762
|
+
try {
|
|
763
|
+
const run = definition.execute
|
|
764
|
+
const scoped = {
|
|
765
|
+
...definition,
|
|
766
|
+
/* Every tool call runs inside the chat that made it, so the browser
|
|
767
|
+
hands that chat its own tab and no other. */
|
|
768
|
+
execute: async (args, exec) => sessionScope.run(String(exec?.agent?.id ?? ''), () => workScope.run(cwdOf(exec), () => run(args, exec))),
|
|
769
|
+
}
|
|
770
|
+
return scope.tools.register(scoped)
|
|
771
|
+
} catch (error) {
|
|
772
|
+
ctx.logger?.warn?.(`browser tool ${definition.name} already registered: ${String(error?.message ?? error)}`)
|
|
773
|
+
return () => {}
|
|
774
|
+
}
|
|
775
|
+
})
|
|
369
776
|
scope.effect(() => () => {
|
|
370
777
|
for (const dispose of disposers) dispose()
|
|
371
778
|
}, 'dsh-inner-browser-use: tool surface')
|
|
@@ -377,11 +784,15 @@ export function apply(ctx) {
|
|
|
377
784
|
name: 'browser-in-this-chat',
|
|
378
785
|
order: 858,
|
|
379
786
|
text: [
|
|
380
|
-
'##
|
|
787
|
+
'## 网页任务:人明确点名才开右栏浏览器',
|
|
788
|
+
'右栏「后台浏览器」是一台独立的后台 Chrome,模型能点、能读、能执行脚本;画面只出现在本会话右栏,不占别的会话。它只做「必须动手」的活,默认不开。',
|
|
381
789
|
'',
|
|
382
|
-
'
|
|
383
|
-
'
|
|
384
|
-
'
|
|
790
|
+
'只有人明确说要看网页/用浏览器时才开始:「用网页端 / 在浏览器里 / 打开某个网站 / 上 ChatGPT 生成 / 帮我在这个页面点…」。人只给了链接、没说要用浏览器时,先用 `web_fetch` 或 envoy 读;读不到(登录墙、空壳、要交互)就说明原因再问一句,不要自己开。',
|
|
791
|
+
'不要用它:读公开文章、单个链接正文、文档、搜索结果摘要、GitHub 仓库或 PR 页面——这些能用 `web_fetch` / envoy 读就不用浏览器;一打开就是一个后台 Chrome 进程加一个页签,会话多了很占内存。',
|
|
792
|
+
'人点名之后不问、不另起外部 Chrome;画面留在本会话右栏,做完保留,不主动关。',
|
|
793
|
+
'先接着用已有的页:`browser_view`(拿编号控件与可见文字)→ `browser_click` / `browser_type` → 再 `browser_view` 直到出现要的结果;页面要等就重复读,不 sleep。`browser_open` 只用于没有页、要换新地址、或要明确刷新;同一个地址不要反复重开。',
|
|
794
|
+
'页面很长、只想确认某句话/按钮在哪时用 `browser_find`,别整页读。要读长文(文章、文档、issue、论坛帖)用 `browser_read`:它回 Markdown,只花几十分之一的 token,链接可点;要动手点、填表才用 `browser_view` 拿编号。',
|
|
795
|
+
'要保存页面样子用 `browser_shot`,它会给出 PNG 绝对路径;要交给人看的文件,把绝对路径写进回复。',
|
|
385
796
|
].join('\n'),
|
|
386
797
|
}), 'dsh-inner-browser-use: system prompt section')
|
|
387
798
|
}
|