dsh-inner-browser-use 0.27.0 → 0.34.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -20,8 +20,36 @@
20
20
  * learns no port and no secret. // ui-copy-ignore
21
21
  */
22
22
 
23
+ import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
24
+ import { homedir } from 'node:os'
25
+ import { join } from 'node:path'
23
26
  import { Sidecar } from './sidecar.js?v=1'
24
27
 
28
+ /** Where the mirror's owner is remembered across a reload: the column should not go dark because the code reloaded. */
29
+ /** Where the mirror's note lives; a test points this elsewhere so it never reads a real chat. */
30
+ const STATE_DIR = process.env.DSH_INNER_BROWSER_STATE_DIR || join(homedir(), '.dsh', 'inner-browser-use')
31
+ const MIRROR_FILE = join(STATE_DIR, 'mirror.json')
32
+
33
+ /** @returns {{owner: string, session: string, at: number}} the last owner a chat asked to keep. */
34
+ function readMirror() {
35
+ try {
36
+ const saved = JSON.parse(readFileSync(MIRROR_FILE, 'utf8'))
37
+ return { owner: String(saved?.owner ?? ''), session: String(saved?.session ?? ''), at: Number(saved?.at ?? 0) }
38
+ } catch {
39
+ return { owner: '', session: '', at: 0 }
40
+ }
41
+ }
42
+
43
+ /** @param {{owner: string, session: string, at: number}} value - what to keep. */
44
+ function writeMirror(value) {
45
+ try {
46
+ mkdirSync(STATE_DIR, { recursive: true })
47
+ writeFileSync(MIRROR_FILE, JSON.stringify(value))
48
+ } catch {
49
+ /* an unwritable note must not take the mirror down */
50
+ }
51
+ }
52
+
25
53
  /** Route prefix owned by this plugin. Never under `/api`, which is the Host's own channel. */ // ui-copy-ignore
26
54
  export const ROUTE = '/dsh-inner-browser-use'
27
55
 
@@ -57,12 +85,36 @@ function readBody(req) {
57
85
  * Nothing else lives here; the browser itself is the Host's. // ui-copy-ignore
58
86
  */
59
87
  export class Bridge {
60
- constructor() {
88
+ /** @param {{persist?: boolean}} [options] - persist the owner across reloads. */
89
+ constructor(options = {}) {
61
90
  /** @type {Map<string, import('node:http').ServerResponse>} */
62
91
  this.seats = new Map()
63
92
  /* The one chat that owns the mirror. The column is global, so the picture
64
93
  may only ever be drawn for this chat; every other chat shows a note. */
65
94
  this.owner = ''
95
+ /* The tab asked for, kept until the chat comes back into view. A chat that
96
+ is not on screen has no stream, so a push at ask time is lost; remembering
97
+ the ask is what makes the mirror appear when the person returns. */
98
+ this.wanted = { session: '', at: 0 }
99
+ /* A running plugin keeps the note; a bare Bridge in a test does not. */
100
+ this.persist = options.persist === true
101
+ const kept = this.persist ? readMirror() : { owner: '', session: '', at: 0 }
102
+ /* Only a real chat id is restored: a stray note must not hand the mirror to
103
+ a chat that no longer exists. */
104
+ if (/^session-/.test(kept.owner) !== true) kept.owner = ''
105
+ this.owner = kept.owner
106
+ this.wanted = { session: kept.session, at: kept.at }
107
+ }
108
+
109
+ /**
110
+ * A chat is the chat that asked, and nothing else. Guessing at an alias once
111
+ * put the mirror in a stranger's column: two conversations are never the same
112
+ * conversation, however few pages happen to be on screen.
113
+ * @param {string} session - whatever id the caller has.
114
+ * @returns {string} the same id.
115
+ */
116
+ canon(session) {
117
+ return String(session ?? '')
66
118
  }
67
119
 
68
120
  /**
@@ -70,7 +122,24 @@ export class Bridge {
70
122
  * @param {string} sessionId - chat that asked for it.
71
123
  */
72
124
  claim(sessionId) {
73
- if (sessionId !== '') this.owner = sessionId
125
+ if (sessionId === '') return
126
+ this.owner = sessionId
127
+ if (this.persist === true) writeMirror({ owner: this.owner, session: this.wanted.session, at: this.wanted.at })
128
+ }
129
+
130
+ /**
131
+ * Remember that a chat wants the mirror, and tell it now if it is listening.
132
+ * @param {string} sessionId - chat that asked.
133
+ * @param {number} [at] - when it asked; older asks are not replayed.
134
+ * @returns {boolean} whether a live tab took the frame.
135
+ */
136
+ want(sessionId, at = Date.now()) {
137
+ const id = this.canon(sessionId)
138
+ if (id === '') return false
139
+ this.claim(id)
140
+ this.wanted = { session: id, at }
141
+ if (this.persist === true) writeMirror({ owner: this.owner, session: id, at })
142
+ return this.push(id, { type: 'sidecar-wanted', session: id, at })
74
143
  }
75
144
 
76
145
  /**
@@ -81,6 +150,13 @@ export class Bridge {
81
150
  attach(sessionId, res) {
82
151
  this.seats.get(sessionId)?.end()
83
152
  this.seats.set(sessionId, res)
153
+ /* A tab that connects after the ask still gets the mirror: this is the case
154
+ where the person was looking at another chat, or reloaded the page. An
155
+ ask older than two minutes is spent and is not replayed. */
156
+ const ask = this.wanted
157
+ if (ask.session === sessionId && Date.now() - ask.at <= 120_000) {
158
+ this.push(sessionId, { type: 'sidecar-wanted', session: sessionId, at: ask.at })
159
+ }
84
160
  }
85
161
 
86
162
  /**
@@ -110,6 +186,17 @@ export class Bridge {
110
186
  return [...this.seats.keys()]
111
187
  }
112
188
 
189
+ /**
190
+ * Forget a mirror nobody is waiting for any more.
191
+ * @param {string} sessionId - chat that closed its tab.
192
+ * @returns {boolean} whether the ask was the remembered one.
193
+ */
194
+ forget(sessionId) {
195
+ if (sessionId === '' || this.wanted.session !== sessionId) return false
196
+ this.wanted = { session: '', at: 0 }
197
+ return true
198
+ }
199
+
113
200
  /** Close every stream. */
114
201
  dispose() {
115
202
  for (const res of this.seats.values()) res.end()
@@ -120,7 +207,92 @@ export class Bridge {
120
207
  /** Page read: address, title, controls, then text, as JSON the Host parses. */ // ui-copy-ignore
121
208
  /* The read model and the tool surface live in their own files: this module is
122
209
  the Host half (routes, bridge, lifetimes), not the model's vocabulary. */ // ui-copy-ignore
123
- import { sidecarTools } from './tools.js?v=1'
210
+ import { keyEvent, sidecarTools, sessionScope, workScope } from './tools.js?v=1'
211
+
212
+ /**
213
+ * A state poll may pick up a browser that outlived a reload, and only for the
214
+ * chat that owns it: any chat polling would otherwise start a browser nobody
215
+ * asked for, and a chat with no column would look like it had one.
216
+ * @param {{alive?: boolean}} said - what the browser half answered.
217
+ * @param {string} chat - the chat asking.
218
+ * @param {string} owner - the chat that owns the mirror.
219
+ * @returns {boolean} whether this poll should attach to a running browser.
220
+ */
221
+ /**
222
+ * What the state poll hands the browser half when it picks a surviving browser
223
+ * up: the chat's own tab, at the address that chat last had. Without the chat
224
+ * the browser comes back with no tab for anybody, and the column stays empty.
225
+ * @param {string} chat - the chat that is polling.
226
+ * @param {string} remembered - where that chat left off, or ''.
227
+ * @returns {{session: string, url?: string}} options for `ensure`.
228
+ */
229
+ /**
230
+ * Whether this chat's poll should put its tab back: the browser is running, the
231
+ * chat holds the mirror, and its tab is not there.
232
+ * @param {any} chatState - what `chatState(chat)` said.
233
+ * @param {string} chat - the chat that is polling.
234
+ * @param {string} owner - the chat that holds the mirror.
235
+ * @returns {boolean} true when a tab has to be opened for this chat.
236
+ */
237
+ /**
238
+ * The folder the calling session works in, wherever it is kept.
239
+ * @param {any} execution - tool execution context.
240
+ * @returns {string} absolute path, or empty when nothing was said.
241
+ */
242
+ export function cwdOf(execution) {
243
+ try {
244
+ const agent = execution?.agent
245
+ const value = agent?.cwd ?? agent?.session?.header?.cwd ?? agent?.header?.cwd
246
+ if (typeof value === 'string' && value !== '') return value
247
+ } catch {
248
+ /* a context that cannot be read is a context without a folder */
249
+ }
250
+ return ''
251
+ }
252
+
253
+ export function needsTab(chatState, chat, owner) {
254
+ if (String(chat ?? '') === '' || owner !== chat) return false
255
+ return chatState?.alive !== true
256
+ }
257
+
258
+ export function adoptOptions(chat, remembered) {
259
+ const back = String(remembered ?? '')
260
+ return back === '' ? { session: chat } : { session: chat, url: back }
261
+ }
262
+
263
+ export function shouldAdoptOnState(said, chat, owner) {
264
+ return said?.alive === false && chat !== '' && owner === chat
265
+ }
266
+
267
+ /**
268
+ * Whether a running browser still needs its column told to draw it.
269
+ * @param {{alive: boolean, owner: string, wanted: string, listeners: number}} what - current facts.
270
+ * @returns {boolean} true when the owner is known, the browser runs, and nothing listens.
271
+ */
272
+ export function wantsMirrorAgain(what) {
273
+ if (what?.alive !== true) return false
274
+ const owner = String(what?.owner ?? '')
275
+ if (owner === '') return false
276
+ if (String(what?.wanted ?? '') !== owner) return false
277
+ return Number(what?.listeners ?? 0) <= 0
278
+ }
279
+
280
+ /**
281
+ * Whether asking again can achieve anything. A pane that is polling this very
282
+ * endpoint is awake and already knows the frames are missing, so a second ask
283
+ * every few seconds only writes log lines; a pane that went quiet may have lost
284
+ * the stream and does need one. One ask per gap at most.
285
+ * @param {any} what - { askedAgo, sinceLast, quiet, gap } in milliseconds.
286
+ * @returns {boolean} true when the ask should be sent now.
287
+ */
288
+ export function nudgeDue(what) {
289
+ const askedAgo = Number(what?.askedAgo ?? -1)
290
+ const sinceLast = Number(what?.sinceLast ?? Infinity)
291
+ const quiet = Number(what?.quiet ?? 15_000)
292
+ const gap = Number(what?.gap ?? 10_000)
293
+ if (!Number.isFinite(askedAgo) || askedAgo < quiet) return false
294
+ return sinceLast >= gap
295
+ }
124
296
 
125
297
  export const name = 'dsh-inner-browser-use'
126
298
  export const inject = ['webServer']
@@ -130,7 +302,7 @@ export const inject = ['webServer']
130
302
  * @param {any} ctx - plugin context.
131
303
  */
132
304
  export function apply(ctx) {
133
- const bridge = new Bridge()
305
+ const bridge = new Bridge({ persist: true })
134
306
  /* The sidecar is a browser the Host owns outright: a real Chrome with its own // ui-copy-ignore
135
307
  profile, shown in the column as a screencast. It outlives a plugin reload,
136
308
  and it does not need the chat to be on screen to do work. */
@@ -138,6 +310,31 @@ export function apply(ctx) {
138
310
  /** @type {{refs: any[], at: number, last?: any}} */
139
311
  const sidecarRefs = { refs: [], at: 0 }
140
312
  ctx.effect(() => () => bridge.dispose(), 'dsh-inner-browser-use: bridge lifetime')
313
+ /* A live picture that nobody is drawing is the one failure people notice
314
+ first: the browser is fine, the column just lost its pane. The chat that
315
+ owns the browser is asked again, on a slow beat, until the frames stream
316
+ has a listener — the client keeps the rule that only the mounted chat's
317
+ column may open. */
318
+ /** @type {Map<string, number>} when each chat last polled for state. */
319
+ const lastAsk = new Map()
320
+ ctx.effect(() => {
321
+ let last = 0
322
+ const timer = setInterval(() => {
323
+ try {
324
+ const owner = bridge.owner
325
+ const listeners = sidecar.page?.frames?.size ?? 0
326
+ const again = wantsMirrorAgain({ alive: sidecar.page !== null && sidecar.page !== undefined, owner, wanted: bridge.wanted.session, listeners }) // ui-copy-ignore
327
+ if (again !== true) return
328
+ const due = nudgeDue({ askedAgo: Date.now() - (lastAsk.get(owner) ?? 0), sinceLast: Date.now() - last }) // ui-copy-ignore
329
+ if (due !== true) return
330
+ last = Date.now()
331
+ bridge.want(owner)
332
+ appendFileSync(join(STATE_DIR, 'asks.log'), `${new Date().toISOString()} asked ${owner.slice(-8)} to draw the mirror again\n`)
333
+ } catch {}
334
+ }, 4000)
335
+ timer.unref?.()
336
+ return () => clearInterval(timer)
337
+ }, 'dsh-inner-browser-use: mirror keepalive')
141
338
  /* Closing the browser cleanly on quit is what keeps a sign-in alive: a
142
339
  killed Chrome never writes the cookies a page set this minute. */
143
340
  ctx.effect(() => {
@@ -148,15 +345,25 @@ export function apply(ctx) {
148
345
  timer.unref?.()
149
346
  return () => {
150
347
  clearInterval(timer)
151
- void sidecar.shutdown().catch(() => {})
348
+ void sidecar.handoff().catch(() => {})
152
349
  }
153
350
  }, 'dsh-inner-browser-use: sidecar lifetime')
154
351
 
155
352
  /** @param {import('node:http').IncomingMessage} req - request. @param {import('node:http').ServerResponse} res - response. */
353
+ /** @type {{at: number, path: string, session: string}[]} the last few calls the column made. */
354
+ const recent = []
355
+ /* Whether the pane is asking at all is the difference between "the browser
356
+ stopped" and "nothing is drawing it": the poll is every 2.5 s, so a chat
357
+ that asks is a chat with a mounted pane. Kept on disk because the answer
358
+ has to survive a plugin reload, which is when the question is asked. */
156
359
  const handler = async (req, res) => {
157
360
  const url = new URL(req.url ?? ROUTE, 'http://dsh.invalid')
158
361
  const path = url.pathname.slice(ROUTE.length) || '/'
159
362
  const sessionId = url.searchParams.get('session') ?? ''
363
+ /* Who came in and when: an empty picture has two very different causes, and
364
+ this is what tells a column that never connected from a page that is gone. */
365
+ recent.push({ at: Date.now(), path, session: sessionId.slice(-8) })
366
+ if (recent.length > 40) recent.splice(0, recent.length - 40)
160
367
 
161
368
  /** @param {number} code - status. @param {unknown} value - body. */
162
369
  const json = (code, value) => {
@@ -164,14 +371,42 @@ export function apply(ctx) {
164
371
  res.end(JSON.stringify(value))
165
372
  }
166
373
 
374
+ /* One answer to one question: may this chat act on the mirror? A browser
375
+ nobody holds is anybody's to take; once a chat holds it, another chat is
376
+ refused here instead of being trusted to check itself. */
377
+ /** @param {string} session - the chat a request speaks for. */
378
+ const held = (session) => session !== '' && bridge.owner !== '' && session !== bridge.owner
379
+ /** @returns {boolean} true when the request was refused. */
380
+ const refuse = () => {
381
+ json(403, { ok: false, text: 'this chat does not own the browser', owner: bridge.owner }) // ui-copy-ignore
382
+ return true
383
+ }
384
+ /* Every route that changes the page goes through one check: it must say
385
+ which chat it is for, and that chat must be the one holding the mirror.
386
+ Two rules in one place, so a new route cannot quietly skip one. */
387
+ /**
388
+ * @param {any} body - parsed request body, if any.
389
+ * @returns {string} the chat that may act, or '' when it was refused.
390
+ */
391
+ const acting = (body) => {
392
+ const who = String(body?.session ?? '') || sessionId
393
+ if (who === '') {
394
+ json(400, { ok: false, text: 'which chat asked for it' }) // ui-copy-ignore
395
+ return ''
396
+ }
397
+ if (held(who) && refuse()) return ''
398
+ return who
399
+ }
400
+
167
401
  if (path === '/sidecar/frames' && req.method === 'GET') {
168
402
  /* The column is global and only one chat may hold the mirror: a chat that
169
403
  does not own it is refused here, so a stale tab cannot stream a picture
170
404
  of somebody else's browser. */
171
- if (bridge.owner !== '' && sessionId !== bridge.owner) {
172
- json(403, { ok: false, text: 'this chat does not own the browser', owner: bridge.owner })
173
- return
174
- }
405
+ const asked = String(url.searchParams.get('session') ?? '')
406
+ const watching = asked !== '' ? asked : sessionId
407
+ if (held(watching) && refuse()) return
408
+ /* Only the owner's own page may stream the picture. */
409
+ const streaming = watching
175
410
  res.writeHead(200, {
176
411
  'content-type': 'text/event-stream; charset=utf-8',
177
412
  'cache-control': 'no-cache, no-transform',
@@ -191,9 +426,13 @@ export function apply(ctx) {
191
426
  page wait: the newest picture is the only one worth sending. */
192
427
  if (res.writableLength > 3 * 1024 * 1024) return
193
428
  res.write(`data: ${JSON.stringify(frame)}\n\n`)
194
- })
429
+ }, streaming)
195
430
  const info = await sidecar.state()
196
- res.write(`data: ${JSON.stringify({ type: 'hello', ...info })}\n\n`)
431
+ /* The pane needs one address to offer back when this chat's own page
432
+ went away; the state route answers the same question for the tab
433
+ header, so both carry it. */
434
+ const chat = watching === '' ? {} : { chat: await sidecar.chatState(watching) }
435
+ res.write(`data: ${JSON.stringify({ type: 'hello', ...info, ...chat })}\n\n`)
197
436
  } catch (error) {
198
437
  res.write(`data: ${JSON.stringify({ type: 'failed', text: String(error?.message ?? error) })}\n\n`)
199
438
  }
@@ -203,8 +442,73 @@ export function apply(ctx) {
203
442
  return
204
443
  }
205
444
 
445
+ if (path === '/sidecar/claim' && req.method === 'POST') {
446
+ /* The pane itself asks back what it was showing: after a reload the
447
+ forgot who owned the mirror, and the chat that still has the tab open is
448
+ the only one allowed to say so. A different owner is refused. */
449
+ let body
450
+ try {
451
+ body = JSON.parse(await readBody(req))
452
+ } catch {
453
+ json(400, { ok: false, text: 'bad frame' })
454
+ return
455
+ }
456
+ const id = String(body.session ?? '')
457
+ if (id === '') {
458
+ json(400, { ok: false, text: 'no chat' })
459
+ return
460
+ }
461
+ if (bridge.owner !== '' && bridge.owner !== id) {
462
+ json(409, { ok: false, owner: bridge.owner })
463
+ return
464
+ }
465
+ bridge.claim(id)
466
+ json(200, { ok: true, owner: bridge.owner })
467
+ return
468
+ }
469
+
206
470
  if (path === '/sidecar/state' && req.method === 'GET') {
207
- json(200, { ...(await sidecar.state()), owner: bridge.owner })
471
+ /* Which chats have a column open right now: 0 means nobody is watching,
472
+ so an empty picture is the column's own problem, not the browser's. */
473
+ const chat = String(url.searchParams.get('session') ?? '')
474
+ /* Diagnostics belong to the chat that made them: a caller sees its own
475
+ calls, and one chat's browsing never explains another's. */
476
+ const mine = chat === '' ? recent : recent.filter((entry) => entry.session === chat.slice(-8))
477
+ let said = await sidecar.state()
478
+ /* The browser can be running and still have no tab for this chat: a reload
479
+ replaces the half that remembered which tab belonged to whom. The poll
480
+ is this chat asking, so it answers by putting that chat's tab back
481
+ before the column draws an empty picture — waiting for a claim first
482
+ left a blank frame in front of the person. */
483
+ let mineChat = chat === '' ? undefined : await sidecar.chatState(chat)
484
+ if (needsTab(mineChat, chat, bridge.owner)) {
485
+ try {
486
+ await sidecar.ensure(adoptOptions(chat, sidecar.rememberedTab?.(chat)))
487
+ said = await sidecar.state()
488
+ mineChat = await sidecar.chatState(chat)
489
+ } catch {}
490
+ }
491
+ /* A plugin reload leaves the browser running and this half brand new, so
492
+ the column's first poll would answer "nothing open" until somebody
493
+ called open again: the mirror would come back empty in a chat that
494
+ never closed it. The chat that owned the browser asks for its state, so
495
+ answering honestly means picking the browser up first. */
496
+ if (shouldAdoptOnState(said, chat, bridge.owner)) {
497
+ try {
498
+ await sidecar.ensure(adoptOptions(chat, sidecar.rememberedTab?.(chat)))
499
+ said = await sidecar.state()
500
+ } catch {}
501
+ }
502
+ if (chat !== '') {
503
+ const before = lastAsk.get(chat) ?? 0
504
+ if (Date.now() - before > 2000) {
505
+ lastAsk.set(chat, Date.now())
506
+ try {
507
+ appendFileSync(join(STATE_DIR, 'asks.log'), `${new Date().toISOString()} ${chat.slice(-8)} alive=${String(said?.alive === true)} viewers=${String(said?.viewers ?? 0)}\n`)
508
+ } catch {}
509
+ }
510
+ }
511
+ json(200, { ...said, ...(chat === '' ? {} : { chat: mineChat ?? (await sidecar.chatState(chat)) }), owner: bridge.owner, chats: [...bridge.seats.keys()], recent: mine })
208
512
  return
209
513
  }
210
514
 
@@ -219,19 +523,21 @@ export function apply(ctx) {
219
523
  const address = String(body.url ?? '')
220
524
  /* The ask must name the chat it belongs to: a request that cannot say
221
525
  who it is for never gets the mirror. */
222
- const who = String(body.session ?? '')
526
+ /* An open names its chat and then takes the mirror over: that is what
527
+ asking for it means, so another owner is not a refusal here. */
528
+ const who = String(body.session ?? '') || sessionId
223
529
  if (who === '') {
224
- json(400, { ok: false, text: 'which chat asked for it' })
530
+ json(400, { ok: false, text: 'which chat asked for it' }) // ui-copy-ignore
225
531
  return
226
532
  }
227
533
  await sidecar.ensure({
228
534
  url: address === '' ? undefined : address,
229
535
  headless: body.headless === undefined ? undefined : body.headless === true,
536
+ session: who,
230
537
  })
231
538
  /* The mirror belongs to the chat that asked for it; any earlier owner
232
539
  loses it and its tab. */
233
- bridge.claim(who)
234
- bridge.push(who, { type: 'sidecar-wanted', session: who })
540
+ bridge.want(who)
235
541
  json(200, { ok: true, ...(await sidecar.state()), owner: bridge.owner })
236
542
  } catch (error) {
237
543
  json(500, { ok: false, text: String(error?.message ?? error) })
@@ -239,6 +545,35 @@ export function apply(ctx) {
239
545
  return
240
546
  }
241
547
 
548
+ if (path === '/sidecar/viewport' && req.method === 'POST') {
549
+ /* The pane tells the page how big it is, so the picture fills the column
550
+ instead of being letterboxed inside it. Only the owner may resize what
551
+ its own mirror shows. */
552
+ let body
553
+ try {
554
+ body = JSON.parse(await readBody(req))
555
+ } catch {
556
+ json(400, { ok: false, text: 'bad frame' }) // ui-copy-ignore
557
+ return
558
+ }
559
+ const actor = acting(body)
560
+ if (actor === '') {
561
+ json(403, { ok: false, text: 'this chat does not own the browser' }) // ui-copy-ignore
562
+ return
563
+ }
564
+ try {
565
+ const page = sidecar.page
566
+ if (page === null || page === undefined) {
567
+ json(409, { ok: false, text: 'the browser is not open' }) // ui-copy-ignore
568
+ return
569
+ }
570
+ json(200, { ok: true, view: await page.setViewport(body.w, body.h) })
571
+ } catch (error) {
572
+ json(500, { ok: false, text: String(error?.message ?? error) })
573
+ }
574
+ return
575
+ }
576
+
242
577
  if (path === '/sidecar/input' && req.method === 'POST') {
243
578
  let body
244
579
  try {
@@ -249,12 +584,10 @@ export function apply(ctx) {
249
584
  }
250
585
  /* Input follows the owner: a chat that lost the mirror cannot act on the
251
586
  page from a stale tab either. */
252
- if (body.session !== undefined && String(body.session) !== '' && bridge.owner !== '' && String(body.session) !== bridge.owner) {
253
- json(403, { ok: false, text: 'this chat does not own the browser', owner: bridge.owner })
254
- return
255
- }
587
+ const actor = acting(body)
588
+ if (actor === '') return
256
589
  try {
257
- const page = await sidecar.ensure()
590
+ const page = await sidecar.ensure({ session: actor })
258
591
  const kind = String(body.kind ?? '')
259
592
  const x = Number(body.x ?? 0)
260
593
  const y = Number(body.y ?? 0)
@@ -273,21 +606,26 @@ export function apply(ctx) {
273
606
  } else if (kind === 'wheel') await page.wheel({ x, y, dx: Number(body.dx ?? 0), dy: Number(body.dy ?? 0) })
274
607
  else if (kind === 'text') await page.insertText(String(body.text ?? ''))
275
608
  else if (kind === 'key') {
276
- const name = String(body.key ?? '')
277
- const known = KEYS[name]
278
- const spec = {
279
- key: name,
280
- code: String(body.code ?? known?.code ?? name),
281
- keyCode: Number(body.keyCode ?? 0) || known?.keyCode || 0,
282
- text: body.text === undefined ? known?.text : String(body.text),
283
- modifiers: Number(body.modifiers ?? 0),
284
- }
285
- await page.key('keyDown', spec)
286
- await page.key('keyUp', { ...spec, text: undefined })
609
+ const key = keyEvent(body)
610
+ if (key.down !== true) await page.key('keyUp', { ...key.spec, text: undefined })
611
+ else if (key.repeat === true) {
612
+ /* Typing repeat: the page draws a held key as a fresh keydown. */
613
+ await page.key('keyDown', { ...key.spec, autoRepeat: true })
614
+ } else if (key.command === null) await page.key('keyDown', key.spec)
615
+ else await page.key('keyDown', { ...key.spec, text: undefined, commands: [key.command] })
287
616
  }
288
617
  /* Hovering and moving are looking, not acting: only real input counts as
289
618
  the person taking the page. */
290
- if (kind !== 'hover' && kind !== 'move') sidecar.humanUse({ kind, x, y })
619
+ if (kind !== 'hover' && kind !== 'move') {
620
+ sidecar.humanUse({ kind, x, y })
621
+ /* The arrow goes now, not on the next repaint: a still page may not
622
+ paint again for a second and a half. */
623
+ try {
624
+ page.announceHold()
625
+ } catch {
626
+ /* the stream is gone; the next frame carries the same answer */
627
+ }
628
+ }
291
629
  json(200, cursor === undefined ? { ok: true } : { ok: true, cursor })
292
630
  } catch (error) {
293
631
  json(500, { ok: false, text: String(error?.message ?? error) })
@@ -304,12 +642,10 @@ export function apply(ctx) {
304
642
  return
305
643
  }
306
644
  /* Navigation follows the owner too: a lost tab cannot steer the page. */
307
- if (body.session !== undefined && String(body.session) !== '' && bridge.owner !== '' && String(body.session) !== bridge.owner) {
308
- json(403, { ok: false, text: 'this chat does not own the browser', owner: bridge.owner })
309
- return
310
- }
645
+ const actor = acting(body)
646
+ if (actor === '') return
311
647
  try {
312
- const page = await sidecar.ensure()
648
+ const page = await sidecar.ensure({ session: actor })
313
649
  const op = String(body.op ?? 'goto')
314
650
  if (op === 'reload') json(200, { ok: true, ...(await page.reload()) })
315
651
  else if (op === 'back') json(200, { ok: true, ...(await page.history(-1)) })
@@ -325,7 +661,22 @@ export function apply(ctx) {
325
661
  return
326
662
  }
327
663
 
664
+ if (path === '/sidecar/forget' && req.method === 'POST') {
665
+ /* Closing the tab is an answer: coming back should not reopen it. Only the
666
+ chat that holds the mirror may withdraw its own ask. */
667
+ const body = await readBody(req).then((raw) => JSON.parse(raw || '{}')).catch(() => ({}))
668
+ const who = acting(body)
669
+ if (who === '') return
670
+ json(200, { ok: true, forgot: bridge.forget(who) })
671
+ return
672
+ }
673
+
328
674
  if (path === '/sidecar/stop' && req.method === 'POST') {
675
+ /* Closing the browser is a write as well: a chat that does not hold the
676
+ mirror cannot close another chat's page. */
677
+ const body = await readBody(req).then((raw) => JSON.parse(raw || '{}')).catch(() => ({}))
678
+ const who = String(body.session ?? '') || sessionId
679
+ if (who !== '' && held(who) && refuse()) return
329
680
  await sidecar.shutdown()
330
681
  json(200, { ok: true })
331
682
  return
@@ -359,13 +710,69 @@ export function apply(ctx) {
359
710
 
360
711
  ctx.inject(['tools'], (scope) => {
361
712
  const disposers = sidecarTools(sidecar, sidecarRefs, {
713
+ /* A screenshot the model can look at is the difference between planning
714
+ from the picture and guessing from a path. The store and the route that
715
+ can carry it are Harness services, so the tool asks here for it; when
716
+ either is missing the picture stays a file and the call still works. */
717
+ /**
718
+ * @param {string} data - base64 PNG.
719
+ * @param {string} name - file name for the attachment.
720
+ * @param {any} exec - the tool call's context.
721
+ * @returns {Promise<any>} the attachment value, or undefined to stay text-only.
722
+ */
723
+ async showShot(data, name, exec) {
724
+ try {
725
+ const attachments = ctx.get('attachments')
726
+ const llm = ctx.get('llm')
727
+ if (attachments === undefined || llm === undefined) return undefined
728
+ const route = exec?.agent?.session?.requestHeader?.()?.config
729
+ const provider = route?.provider ?? exec?.agent?.options?.provider
730
+ const model = route?.model ?? exec?.agent?.options?.model
731
+ if (provider === undefined || model === undefined) return undefined
732
+ const info = await llm.resolveModelInfo(provider, model, exec?.signal)
733
+ if (!Array.isArray(info?.inputModalities) || !info.inputModalities.includes('image')) return undefined
734
+ const bytes = Buffer.from(data, 'base64')
735
+ const cap = Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
736
+ if (!(bytes.length > 0) || bytes.length > cap) return undefined
737
+ const ref = await attachments.saveImage({ data: bytes, mediaType: 'image/png', name })
738
+ return {
739
+ attachmentId: ref.attachmentId, // ui-copy-ignore
740
+ mediaType: ref.mediaType, // ui-copy-ignore
741
+ bytes: ref.bytes, // ui-copy-ignore
742
+ width: ref.width, // ui-copy-ignore
743
+ height: ref.height, // ui-copy-ignore
744
+ ...(ref.name === undefined ? {} : { name: ref.name }),
745
+ }
746
+ } catch {
747
+ return undefined
748
+ }
749
+ },
362
750
  /* The mirror changes hands here: the calling chat owns it from now on,
363
751
  and every earlier owner loses it. */
364
752
  want: (sessionId) => {
365
- bridge.claim(sessionId)
366
- bridge.push(sessionId, { type: 'sidecar-wanted', session: sessionId })
753
+ bridge.want(sessionId)
367
754
  },
368
- }).map((definition) => scope.tools.register(definition))
755
+ /* Who holds the mirror, and whether a chat is listening right now. */
756
+ owner: () => bridge.owner, // ui-copy-ignore
757
+ seats: () => [...bridge.seats.keys()].map((k) => String(k).slice(-8)).join(',') || 'none', // ui-copy-ignore
758
+ }).map((definition) => {
759
+ /* A reload can mount this surface twice before the old one is torn down.
760
+ A name already in the registry is kept, not fought over: a duplicate
761
+ registration is a warning, never a crash. */
762
+ try {
763
+ const run = definition.execute
764
+ const scoped = {
765
+ ...definition,
766
+ /* Every tool call runs inside the chat that made it, so the browser
767
+ hands that chat its own tab and no other. */
768
+ execute: async (args, exec) => sessionScope.run(String(exec?.agent?.id ?? ''), () => workScope.run(cwdOf(exec), () => run(args, exec))),
769
+ }
770
+ return scope.tools.register(scoped)
771
+ } catch (error) {
772
+ ctx.logger?.warn?.(`browser tool ${definition.name} already registered: ${String(error?.message ?? error)}`)
773
+ return () => {}
774
+ }
775
+ })
369
776
  scope.effect(() => () => {
370
777
  for (const dispose of disposers) dispose()
371
778
  }, 'dsh-inner-browser-use: tool surface')
@@ -377,11 +784,15 @@ export function apply(ctx) {
377
784
  name: 'browser-in-this-chat',
378
785
  order: 858,
379
786
  text: [
380
- '## 网页任务:用 sidecar 浏览器做,不问直接开始',
787
+ '## 网页任务:人明确点名才开右栏浏览器',
788
+ '右栏「后台浏览器」是一台独立的后台 Chrome,模型能点、能读、能执行脚本;画面只出现在本会话右栏,不占别的会话。它只做「必须动手」的活,默认不开。',
381
789
  '',
382
- '人说「用网页端 / 在浏览器里 / 打开某个网站 / 帮我上 ChatGPT 生成 …」时,直接用 `sidecar_*` 工具;不要问要不要打开,也不要另起一个外部 Chrome。它的画面会出现在本会话右栏,做完保留,不主动关。',
383
- '先接着用已有的页:`sidecar_view`(拿编号控件与可见文字)→ `sidecar_click` / `sidecar_type` → 再 `sidecar_view` 直到出现要的结果;页面要等就重复读,不 sleep。`sidecar_open` 只用于没有页、要换新地址、或要明确刷新;同一个地址不要反复重开。',
384
- '要保存页面样子用 `sidecar_shot`,它会给出 PNG 绝对路径;要交给人看的文件,把绝对路径写进回复。',
790
+ '只有人明确说要看网页/用浏览器时才开始:「用网页端 / 在浏览器里 / 打开某个网站 / 上 ChatGPT 生成 / 帮我在这个页面点…」。人只给了链接、没说要用浏览器时,先用 `web_fetch` 或 envoy 读;读不到(登录墙、空壳、要交互)就说明原因再问一句,不要自己开。',
791
+ '不要用它:读公开文章、单个链接正文、文档、搜索结果摘要、GitHub 仓库或 PR 页面——这些能用 `web_fetch` / envoy 读就不用浏览器;一打开就是一个后台 Chrome 进程加一个页签,会话多了很占内存。',
792
+ '人点名之后不问、不另起外部 Chrome;画面留在本会话右栏,做完保留,不主动关。',
793
+ '先接着用已有的页:`browser_view`(拿编号控件与可见文字)→ `browser_click` / `browser_type` → 再 `browser_view` 直到出现要的结果;页面要等就重复读,不 sleep。`browser_open` 只用于没有页、要换新地址、或要明确刷新;同一个地址不要反复重开。',
794
+ '页面很长、只想确认某句话/按钮在哪时用 `browser_find`,别整页读。要读长文(文章、文档、issue、论坛帖)用 `browser_read`:它回 Markdown,只花几十分之一的 token,链接可点;要动手点、填表才用 `browser_view` 拿编号。',
795
+ '要保存页面样子用 `browser_shot`,它会给出 PNG 绝对路径;要交给人看的文件,把绝对路径写进回复。',
385
796
  ].join('\n'),
386
797
  }), 'dsh-inner-browser-use: system prompt section')
387
798
  }