dsh-inner-browser-use 0.27.0 → 0.34.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -18
- package/README.zh.md +26 -14
- package/client.js +262 -28
- package/docs/ARCHITECTURE.md +15 -12
- package/docs/ARTIFACTS.md +19 -0
- package/docs/BENCH.md +568 -17
- package/docs/FLASH.md +84 -0
- package/docs/ROADMAP.md +107 -13
- package/docs/VERIFY.md +20 -10
- package/index.js +456 -45
- package/package.json +20 -3
- package/read.js +734 -29
- package/review.js +213 -0
- package/runtime/flash-app/main.js +232 -0
- package/runtime/flash-app/package.json +1 -0
- package/runtime/flash-setup.mjs +151 -0
- package/runtime/swf.mjs +233 -0
- package/scripts/check-flash-local.mjs +104 -0
- package/scripts/check-flash-mirror.mjs +73 -0
- package/scripts/check-flash-tools.mjs +74 -0
- package/scripts/check-flash-window.mjs +58 -0
- package/scripts/doctor.mjs +3 -1
- package/sidecar.js +1763 -106
- package/tools.js +1667 -191
package/client.js
CHANGED
|
@@ -32,7 +32,8 @@ window.__ModuleLoader__.load({
|
|
|
32
32
|
const DICT = {
|
|
33
33
|
zh: {
|
|
34
34
|
'side.label': '后台浏览器',
|
|
35
|
-
'side.empty': '
|
|
35
|
+
'side.empty': '后台的独立 Chrome 还没有页面,在上方地址栏输入网址',
|
|
36
|
+
'side.restore': '恢复上次的页面',
|
|
36
37
|
'side.connecting': '正在连接…',
|
|
37
38
|
'side.failed': '画面没有连上',
|
|
38
39
|
'side.foreign': '这个浏览器属于发起它的会话,回到那个会话继续。',
|
|
@@ -44,7 +45,8 @@ window.__ModuleLoader__.load({
|
|
|
44
45
|
},
|
|
45
46
|
en: {
|
|
46
47
|
'side.label': 'Background browser',
|
|
47
|
-
'side.empty': '
|
|
48
|
+
'side.empty': 'The background Chrome has no page yet; type an address above',
|
|
49
|
+
'side.restore': 'Reopen the last page',
|
|
48
50
|
'side.connecting': 'Connecting…',
|
|
49
51
|
'side.failed': 'The picture did not connect',
|
|
50
52
|
'side.foreign': 'This browser belongs to the chat that opened it.',
|
|
@@ -73,6 +75,8 @@ window.__ModuleLoader__.load({
|
|
|
73
75
|
on screen at once, and moving the mouse never takes the arrow over. */
|
|
74
76
|
'.dshib-side-ptr{position:absolute;width:20px;height:20px;pointer-events:none;z-index:2;transform:translate(-2px,-2px);transition:left 60ms linear,top 60ms linear;filter:drop-shadow(0 1px 2px rgba(0,0,0,.45))}',
|
|
75
77
|
'.dshib-side-note{font-size:12px;color:var(--dsw-alias-label-tertiary);padding:0 16px;text-align:center}',
|
|
78
|
+
'.dshib-side-back{border:0;border-radius:8px;background:transparent;color:var(--dsw-alias-label-primary);font:inherit;font-size:12px;padding:6px 12px;cursor:pointer}',
|
|
79
|
+
'.dshib-side-back:hover{background:var(--dsw-alias-bg-subtle)}',
|
|
76
80
|
].join('')
|
|
77
81
|
|
|
78
82
|
return {
|
|
@@ -84,6 +88,9 @@ window.__ModuleLoader__.load({
|
|
|
84
88
|
/* one locale service or none */
|
|
85
89
|
}
|
|
86
90
|
ctx.effect(() => {
|
|
91
|
+
/* A reload keeps one sheet, not one per lifetime: the same id is
|
|
92
|
+
replaced instead of stacked. */
|
|
93
|
+
document.getElementById?.(STYLE_ID)?.remove()
|
|
87
94
|
const style = document.createElement('style')
|
|
88
95
|
style.id = STYLE_ID
|
|
89
96
|
/* A sheet is keyed by the plugin that brought it. */
|
|
@@ -109,6 +116,12 @@ window.__ModuleLoader__.load({
|
|
|
109
116
|
}
|
|
110
117
|
}
|
|
111
118
|
const closeHere = (sessionId) => {
|
|
119
|
+
/* A tab the person closed is not a tab they want back on return. */
|
|
120
|
+
try {
|
|
121
|
+
void fetch(`${ROUTE}/sidecar/forget?session=${encodeURIComponent(sessionId)}`, { method: 'POST' }) // ui-copy-ignore
|
|
122
|
+
} catch {
|
|
123
|
+
/* a page that cannot hear it keeps its note */
|
|
124
|
+
}
|
|
112
125
|
let records = []
|
|
113
126
|
try {
|
|
114
127
|
records = surface?.tabsIn?.(sessionId) ?? []
|
|
@@ -138,14 +151,29 @@ window.__ModuleLoader__.load({
|
|
|
138
151
|
/* a column that refuses keeps its own tabs */
|
|
139
152
|
}
|
|
140
153
|
}
|
|
141
|
-
|
|
154
|
+
/* The same rule on this side: a second mount of the same tab is a
|
|
155
|
+
no-op, not a thrown error that takes the pane down. */
|
|
156
|
+
const safely = (register) => {
|
|
157
|
+
try {
|
|
158
|
+
return register()
|
|
159
|
+
} catch (error) {
|
|
160
|
+
return () => {}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
disposers.push(safely(() => tabs.register({
|
|
142
164
|
id: SIDE_ID, // ui-copy-ignore
|
|
143
165
|
kind: SIDE_KIND, // ui-copy-ignore
|
|
144
166
|
keepMounted: true,
|
|
145
167
|
single: true,
|
|
146
168
|
priority: 'extension',
|
|
147
169
|
title: () => L('side.label'),
|
|
148
|
-
|
|
170
|
+
/* The tab strip's close control is not the pane's own button: a
|
|
171
|
+
tab closed there never told the host, the ask stayed alive, and
|
|
172
|
+
the keepalive put the mirror back a few seconds later — a tab
|
|
173
|
+
the person closed that would not stay closed. Every close path
|
|
174
|
+
ends here. */
|
|
175
|
+
onClose: (_tab, where) => closeHere(String(where?.sessionId ?? mountedOf())),
|
|
176
|
+
})))
|
|
149
177
|
disposers.push(scope.slots.inject(SLOT, () => scope.slots.register({
|
|
150
178
|
name: SLOT, // ui-copy-ignore
|
|
151
179
|
key: SIDE_ID, // ui-copy-ignore
|
|
@@ -177,7 +205,10 @@ window.__ModuleLoader__.load({
|
|
|
177
205
|
}
|
|
178
206
|
const syncAsk = () => {
|
|
179
207
|
const who = mountedOf()
|
|
180
|
-
|
|
208
|
+
/* A stream the browser gave up on keeps its address here, so the
|
|
209
|
+
next tick has to notice it is dead and open a fresh one, or this
|
|
210
|
+
chat never hears an ask again. */
|
|
211
|
+
if (who === askWho && askSource !== null && Number(askSource.readyState) !== 2) return
|
|
181
212
|
closeAsk()
|
|
182
213
|
if (who === '') return
|
|
183
214
|
try {
|
|
@@ -202,7 +233,10 @@ window.__ModuleLoader__.load({
|
|
|
202
233
|
const watchAsk = () => {
|
|
203
234
|
try {
|
|
204
235
|
if (typeof surface?.mounted?.subscribe === 'function') {
|
|
205
|
-
const off = surface.mounted.subscribe(
|
|
236
|
+
const off = surface.mounted.subscribe(() => {
|
|
237
|
+
syncAsk()
|
|
238
|
+
restore()
|
|
239
|
+
})
|
|
206
240
|
if (typeof off === 'function') return off
|
|
207
241
|
}
|
|
208
242
|
} catch {
|
|
@@ -211,10 +245,41 @@ window.__ModuleLoader__.load({
|
|
|
211
245
|
return null
|
|
212
246
|
}
|
|
213
247
|
syncAsk()
|
|
248
|
+
/* An app restart leaves the browser alive but no ask on the wire, so
|
|
249
|
+
the column would come back empty in a chat that never closed it.
|
|
250
|
+
This chat asking for the state of the browser it still owns is what
|
|
251
|
+
puts the mirror back without anyone pressing anything. */
|
|
252
|
+
const restoreTimers = []
|
|
253
|
+
const restore = (tries = 0) => {
|
|
254
|
+
const who = mountedOf()
|
|
255
|
+
if (who === '') return
|
|
256
|
+
const again = () => {
|
|
257
|
+
if (tries >= 2) return
|
|
258
|
+
restoreTimers.push(setTimeout(() => restore(tries + 1), 1500))
|
|
259
|
+
}
|
|
260
|
+
try {
|
|
261
|
+
fetch(`${ROUTE}/sidecar/state?session=${encodeURIComponent(who)}`) // ui-copy-ignore
|
|
262
|
+
.then((said) => (said.ok === true ? said.json() : null))
|
|
263
|
+
.then((state) => {
|
|
264
|
+
if (state?.owner !== who) return
|
|
265
|
+
const live = state?.alive === true || state?.chat?.alive === true
|
|
266
|
+
if (live !== true) {
|
|
267
|
+
again()
|
|
268
|
+
return
|
|
269
|
+
}
|
|
270
|
+
ask(who)
|
|
271
|
+
})
|
|
272
|
+
.catch(again)
|
|
273
|
+
} catch {
|
|
274
|
+
again()
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
restore()
|
|
214
278
|
const offAsk = watchAsk()
|
|
215
279
|
const pollAsk = setInterval(syncAsk, 1000)
|
|
216
280
|
const stopAsk = () => {
|
|
217
281
|
clearInterval(pollAsk)
|
|
282
|
+
for (const timer of restoreTimers) clearTimeout(timer)
|
|
218
283
|
if (typeof offAsk === 'function') {
|
|
219
284
|
try {
|
|
220
285
|
offAsk()
|
|
@@ -249,8 +314,37 @@ window.__ModuleLoader__.load({
|
|
|
249
314
|
function SidecarBody(props) {
|
|
250
315
|
const [frame, setFrame] = React.useState(null)
|
|
251
316
|
const [ptr, setPtr] = React.useState(null)
|
|
317
|
+
/* The page is laid out at the size of this column, so the picture it
|
|
318
|
+
sends already has the right shape; these are its pixels. */
|
|
319
|
+
const [view, setView] = React.useState({ w: SIDE_VIEW.width, h: SIDE_VIEW.height })
|
|
320
|
+
/* The person at the mirror drives with their own hand: while they are,
|
|
321
|
+
the agent's arrow stays out of the way, and it comes back by itself
|
|
322
|
+
once their hold runs out. */
|
|
323
|
+
const [person, setPerson] = React.useState(false)
|
|
324
|
+
const personTimer = React.useRef(0)
|
|
325
|
+
/* The person's own hand hides the arrow the moment it lands, without
|
|
326
|
+
waiting for the browser to report a hold: the arrow is a claim about
|
|
327
|
+
what the agent did, and a claim that outlives the agent's turn is a lie
|
|
328
|
+
the person has to look past. It returns when their hand stops. */
|
|
329
|
+
const [hand, setHand] = React.useState(false)
|
|
330
|
+
const handTimer = React.useRef(0)
|
|
331
|
+
const touched = () => {
|
|
332
|
+
setHand(true)
|
|
333
|
+
clearTimeout(handTimer.current)
|
|
334
|
+
handTimer.current = setTimeout(() => setHand(false), 1500)
|
|
335
|
+
}
|
|
336
|
+
const hearHold = (message) => {
|
|
337
|
+
const hold = Number(message?.hold ?? 0)
|
|
338
|
+
if (!Number.isFinite(hold) || hold <= 0) return
|
|
339
|
+
setPerson(true)
|
|
340
|
+
clearTimeout(personTimer.current)
|
|
341
|
+
personTimer.current = setTimeout(() => setPerson(false), hold + 120)
|
|
342
|
+
}
|
|
252
343
|
const [phase, setPhase] = React.useState('connecting')
|
|
253
344
|
const [note, setNote] = React.useState('')
|
|
345
|
+
/* The address this chat had before its tab went away: the pane reopens
|
|
346
|
+
that address, so a lost picture costs one click. */
|
|
347
|
+
const [back, setBack] = React.useState('')
|
|
254
348
|
const [url, setUrl] = React.useState('')
|
|
255
349
|
const [draft, setDraft] = React.useState('')
|
|
256
350
|
const [editing, setEditing] = React.useState(false)
|
|
@@ -263,13 +357,48 @@ window.__ModuleLoader__.load({
|
|
|
263
357
|
React.useEffect(() => {
|
|
264
358
|
let alive = true
|
|
265
359
|
let dropped = false
|
|
360
|
+
/* The tab is still in this chat's column, so this chat may say the
|
|
361
|
+
mirror is its own: a reload forgets the owner, and a pane that only
|
|
362
|
+
waited would stay dark until some tool ran. */
|
|
363
|
+
const reclaim = () => {
|
|
364
|
+
const id = String(props.sessionId ?? '')
|
|
365
|
+
if (id === '') return
|
|
366
|
+
void fetch(`${ROUTE}/sidecar/claim`, { // ui-copy-ignore
|
|
367
|
+
method: 'POST',
|
|
368
|
+
headers: { 'content-type': 'application/json' },
|
|
369
|
+
body: JSON.stringify({ session: id }),
|
|
370
|
+
})
|
|
371
|
+
.then((reply) => (reply.ok === true ? reply.json() : null))
|
|
372
|
+
.then((info) => {
|
|
373
|
+
if (alive !== true) return
|
|
374
|
+
if (info?.ok === true) {
|
|
375
|
+
setOwner(id)
|
|
376
|
+
/* The owner was just named, so the next poll may adopt the
|
|
377
|
+
browser: ask now instead of up to a poll later. */
|
|
378
|
+
setTimeout(read, 0)
|
|
379
|
+
}
|
|
380
|
+
})
|
|
381
|
+
.catch(() => {})
|
|
382
|
+
}
|
|
266
383
|
const read = () => {
|
|
267
|
-
|
|
384
|
+
const who = String(props.sessionId ?? '')
|
|
385
|
+
/* Naming the chat lets a browser that outlived a reload be picked
|
|
386
|
+
up for us; an anonymous poll cannot, and the column would sit on
|
|
387
|
+
an empty stage with the browser running behind it. */
|
|
388
|
+
void fetch(`${ROUTE}/sidecar/state?session=${encodeURIComponent(who)}`, { cache: 'no-store' }) // ui-copy-ignore
|
|
268
389
|
.then((reply) => (reply.ok === true ? reply.json() : null))
|
|
269
390
|
.then((info) => {
|
|
270
391
|
if (alive !== true) return
|
|
271
392
|
const next = String(info?.owner ?? '')
|
|
393
|
+
const asked = String(props.sessionId ?? '')
|
|
394
|
+
/* Nobody owns it, or this chat does but the note was lost: ask
|
|
395
|
+
back instead of showing a note for a browser this column shows. */
|
|
396
|
+
if (asked !== '' && next !== asked) reclaim()
|
|
272
397
|
setOwner((prev) => (prev === next ? prev : next))
|
|
398
|
+
/* A browser that is gone still remembers where this chat was:
|
|
399
|
+
keep the address so the pane can offer to reopen it. */
|
|
400
|
+
const was = String(info?.chat?.remembered ?? '')
|
|
401
|
+
if (was !== '') setBack((prev) => (prev === was ? prev : was))
|
|
273
402
|
/* Another chat holds the browser now: this tab goes away instead
|
|
274
403
|
of sitting here as a note. */
|
|
275
404
|
if (dropped !== true && next !== '' && next !== String(props.sessionId ?? '')) {
|
|
@@ -297,9 +426,15 @@ window.__ModuleLoader__.load({
|
|
|
297
426
|
this chat keeps only a note. // ui-copy-ignore */
|
|
298
427
|
const unknown = owner === null
|
|
299
428
|
const mine = unknown !== true && owner === asked
|
|
429
|
+
/* A picture that stops arriving has to come back by itself: the pane
|
|
430
|
+
re-opens its stream instead of sitting on an empty stage until the
|
|
431
|
+
person reloads the page. */
|
|
432
|
+
const [attempt, setAttempt] = React.useState(0)
|
|
433
|
+
const drawnAt = React.useRef(0)
|
|
300
434
|
|
|
301
435
|
React.useEffect(() => {
|
|
302
436
|
if (mine !== true) return undefined
|
|
437
|
+
drawnAt.current = Date.now()
|
|
303
438
|
const who = encodeURIComponent(String(props.sessionId ?? ''))
|
|
304
439
|
const source = new EventSource(`${ROUTE}/sidecar/frames?session=${who}`)
|
|
305
440
|
/* Bursts arrive faster than a picture can decode: only the newest frame
|
|
@@ -315,15 +450,25 @@ window.__ModuleLoader__.load({
|
|
|
315
450
|
pending = null
|
|
316
451
|
setFrame(message)
|
|
317
452
|
setPtr(message.pointer ?? null)
|
|
453
|
+
if (Number.isFinite(Number(message.w)) && Number.isFinite(Number(message.h))) {
|
|
454
|
+
setView({ w: Number(message.w), h: Number(message.h) })
|
|
455
|
+
}
|
|
318
456
|
setPhase('live')
|
|
457
|
+
drawnAt.current = Date.now()
|
|
319
458
|
if (typeof message.url === 'string' && message.url !== '') setUrl(message.url)
|
|
320
459
|
}
|
|
321
460
|
const draw = (message) => {
|
|
461
|
+
hearHold(message)
|
|
322
462
|
pending = message
|
|
323
463
|
if (raf !== 0) return
|
|
324
464
|
if (typeof requestAnimationFrame === 'function') raf = requestAnimationFrame(paint)
|
|
325
465
|
else paint()
|
|
326
466
|
}
|
|
467
|
+
const stuck = setInterval(() => {
|
|
468
|
+
/* Hello alone is not a picture: six quiet seconds means the stream is
|
|
469
|
+
dead, so ask for a new one. */
|
|
470
|
+
if (Date.now() - drawnAt.current > 6000) setAttempt((n) => n + 1)
|
|
471
|
+
}, 3000)
|
|
327
472
|
source.onmessage = (event) => {
|
|
328
473
|
let message
|
|
329
474
|
try {
|
|
@@ -333,13 +478,16 @@ window.__ModuleLoader__.load({
|
|
|
333
478
|
}
|
|
334
479
|
if (message.type === 'hello') {
|
|
335
480
|
setPhase('live')
|
|
481
|
+
drawnAt.current = Date.now()
|
|
336
482
|
setUrl(String(message.url ?? ''))
|
|
483
|
+
setBack(String(message.chat?.remembered ?? ''))
|
|
337
484
|
} else if (message.type === 'failed') {
|
|
338
485
|
setPhase('failed')
|
|
339
486
|
setNote(String(message.text ?? ''))
|
|
340
487
|
} else if (message.type === 'pointer') {
|
|
341
488
|
/* The arrow is drawn the moment the agent moves, not when the page
|
|
342
489
|
next repaints: it stays honest even on a page that is idle. */
|
|
490
|
+
hearHold(message)
|
|
343
491
|
setPtr(message.pointer ?? null)
|
|
344
492
|
} else if (typeof message.jpeg === 'string') {
|
|
345
493
|
draw(message)
|
|
@@ -347,10 +495,13 @@ window.__ModuleLoader__.load({
|
|
|
347
495
|
}
|
|
348
496
|
source.onerror = () => setPhase('lost')
|
|
349
497
|
return () => {
|
|
498
|
+
clearInterval(stuck)
|
|
499
|
+
clearTimeout(personTimer.current)
|
|
500
|
+
clearTimeout(handTimer.current)
|
|
350
501
|
if (raf !== 0 && typeof cancelAnimationFrame === 'function') cancelAnimationFrame(raf)
|
|
351
502
|
source.close()
|
|
352
503
|
}
|
|
353
|
-
}, [props.sessionId, mine])
|
|
504
|
+
}, [props.sessionId, mine, attempt])
|
|
354
505
|
|
|
355
506
|
/* The address field follows the page except while the person is editing it. */
|
|
356
507
|
React.useEffect(() => {
|
|
@@ -370,9 +521,49 @@ window.__ModuleLoader__.load({
|
|
|
370
521
|
return () => observer.disconnect()
|
|
371
522
|
}, [])
|
|
372
523
|
|
|
373
|
-
const scale = size.w > 0 && size.h > 0 ? Math.min(size.w /
|
|
374
|
-
const padX = (size.w -
|
|
375
|
-
const padY = (size.h -
|
|
524
|
+
const scale = size.w > 0 && size.h > 0 ? Math.min(size.w / view.w, size.h / view.h) : 0
|
|
525
|
+
const padX = (size.w - view.w * scale) / 2
|
|
526
|
+
const padY = (size.h - view.h * scale) / 2
|
|
527
|
+
|
|
528
|
+
/* The column tells the page how big it is, a beat after the resize stops:
|
|
529
|
+
a page that relayouts on every pointer step would never settle. */
|
|
530
|
+
React.useEffect(() => {
|
|
531
|
+
if (mine !== true || size.w < 200 || size.h < 200) return undefined
|
|
532
|
+
/* The page already has that shape: a resize that lands between two
|
|
533
|
+
viewport sizes would only cost a relayout and another frame. */
|
|
534
|
+
if (Math.abs(Number(view?.w ?? 0) - size.w) < 1 && Math.abs(Number(view?.h ?? 0) - size.h) < 1) return undefined
|
|
535
|
+
/* The page is told the column's size a beat after the resize stops, and
|
|
536
|
+
again until the browser says it took it: a request that arrives while
|
|
537
|
+
the page is still starting is refused, and a pane that gives up then
|
|
538
|
+
keeps a letterboxed picture for as long as it stays open. */
|
|
539
|
+
let live = true
|
|
540
|
+
let tries = 0
|
|
541
|
+
let timer = 0
|
|
542
|
+
const send = () => {
|
|
543
|
+
if (!live) return
|
|
544
|
+
fetch(`${ROUTE}/sidecar/viewport`, {
|
|
545
|
+
method: 'POST',
|
|
546
|
+
headers: { 'content-type': 'application/json' },
|
|
547
|
+
body: JSON.stringify({ session: String(props.sessionId ?? ''), w: Math.round(size.w), h: Math.round(size.h) }),
|
|
548
|
+
})
|
|
549
|
+
.then((answer) => (answer.ok === true ? answer.json() : { ok: false }))
|
|
550
|
+
.then((said) => {
|
|
551
|
+
if (!live || said?.ok === true || tries >= 5) return
|
|
552
|
+
tries += 1
|
|
553
|
+
timer = setTimeout(send, 400)
|
|
554
|
+
})
|
|
555
|
+
.catch(() => {
|
|
556
|
+
if (!live || tries >= 5) return
|
|
557
|
+
tries += 1
|
|
558
|
+
timer = setTimeout(send, 600)
|
|
559
|
+
})
|
|
560
|
+
}
|
|
561
|
+
timer = setTimeout(send, 250)
|
|
562
|
+
return () => {
|
|
563
|
+
live = false
|
|
564
|
+
clearTimeout(timer)
|
|
565
|
+
}
|
|
566
|
+
}, [mine, size.w, size.h, view?.w, view?.h])
|
|
376
567
|
|
|
377
568
|
const who = String(props.sessionId ?? '')
|
|
378
569
|
const post = (path, body) => {
|
|
@@ -409,7 +600,7 @@ window.__ModuleLoader__.load({
|
|
|
409
600
|
const box = el.getBoundingClientRect()
|
|
410
601
|
const x = (event.clientX - box.left - padX) / scale
|
|
411
602
|
const y = (event.clientY - box.top - padY) / scale
|
|
412
|
-
if (x < 0 || y < 0 || x >
|
|
603
|
+
if (x < 0 || y < 0 || x > view.w || y > view.h) return null
|
|
413
604
|
return { x: Math.round(x), y: Math.round(y) }
|
|
414
605
|
}
|
|
415
606
|
const ime = React.useRef(null)
|
|
@@ -425,6 +616,7 @@ window.__ModuleLoader__.load({
|
|
|
425
616
|
}
|
|
426
617
|
const stageRef = stage
|
|
427
618
|
const onPointerDown = (event) => {
|
|
619
|
+
touched()
|
|
428
620
|
const point = toView(event)
|
|
429
621
|
if (point === null) return
|
|
430
622
|
const field = ime.current
|
|
@@ -437,6 +629,7 @@ window.__ModuleLoader__.load({
|
|
|
437
629
|
}
|
|
438
630
|
const movedAt = React.useRef(0)
|
|
439
631
|
const onPointerMove = (event) => {
|
|
632
|
+
touched()
|
|
440
633
|
const point = toView(event)
|
|
441
634
|
if (point === null) return
|
|
442
635
|
/* Hover is part of a page: menus open under the hand, so the movement
|
|
@@ -448,24 +641,46 @@ window.__ModuleLoader__.load({
|
|
|
448
641
|
hover(point)
|
|
449
642
|
}
|
|
450
643
|
const onWheel = (event) => {
|
|
644
|
+
touched()
|
|
451
645
|
event.preventDefault()
|
|
452
|
-
const point = toView(event) ?? { x: Math.round(
|
|
646
|
+
const point = toView(event) ?? { x: Math.round(view.w / 2), y: Math.round(view.h / 2) }
|
|
453
647
|
post('/sidecar/input', { kind: 'wheel', x: point.x, y: point.y, dx: event.deltaX, dy: event.deltaY })
|
|
454
648
|
}
|
|
649
|
+
/* The page's keys are the person's keys: shift makes a capital, a
|
|
650
|
+
shortcut stays a shortcut so Ctrl+V pastes and Ctrl+A selects, and the
|
|
651
|
+
key that goes down also goes up so a game can hold it. */
|
|
652
|
+
const modsOf = (event) => (event.altKey === true ? 1 : 0)
|
|
653
|
+
| (event.ctrlKey === true ? 2 : 0)
|
|
654
|
+
| (event.metaKey === true ? 4 : 0)
|
|
655
|
+
| (event.shiftKey === true ? 8 : 0)
|
|
656
|
+
const MODIFIER_KEYS = ['Alt', 'Control', 'Meta', 'Shift', 'AltGraph', 'OS', 'Super', 'Hyper']
|
|
657
|
+
const keyBody = (event, down) => {
|
|
658
|
+
const modifiers = modsOf(event)
|
|
659
|
+
const shortcut = (modifiers & 7) !== 0
|
|
660
|
+
const typing = down === true && event.key.length === 1 && (modifiers & 7) === 0 && event.repeat !== true
|
|
661
|
+
return {
|
|
662
|
+
kind: 'key',
|
|
663
|
+
key: event.key,
|
|
664
|
+
code: event.code || event.key,
|
|
665
|
+
keyCode: event.keyCode || 0,
|
|
666
|
+
modifiers,
|
|
667
|
+
...(typing ? { text: event.key } : {}),
|
|
668
|
+
...(down === false ? { down: false } : {}),
|
|
669
|
+
}
|
|
670
|
+
}
|
|
455
671
|
const onKeyDown = (event) => {
|
|
672
|
+
touched()
|
|
456
673
|
/* A key still being composed belongs to the IME, not to the page. */
|
|
457
674
|
if (event.isComposing === true) return
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
if (event.
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
post('/sidecar/input', { kind: 'text', text: event.key })
|
|
468
|
-
}
|
|
675
|
+
if (event.key == null || MODIFIER_KEYS.includes(event.key) === true) return
|
|
676
|
+
event.preventDefault()
|
|
677
|
+
post('/sidecar/input', event.repeat === true ? { ...keyBody(event, true), repeat: true } : keyBody(event, true))
|
|
678
|
+
}
|
|
679
|
+
const onKeyUp = (event) => {
|
|
680
|
+
if (event.isComposing === true) return
|
|
681
|
+
if (event.key == null || MODIFIER_KEYS.includes(event.key) === true) return
|
|
682
|
+
event.preventDefault()
|
|
683
|
+
post('/sidecar/input', keyBody(event, false))
|
|
469
684
|
}
|
|
470
685
|
const goto = () => {
|
|
471
686
|
const address = String(draft ?? '').trim()
|
|
@@ -479,7 +694,7 @@ window.__ModuleLoader__.load({
|
|
|
479
694
|
return h('div', { className: 'dshib-side' },
|
|
480
695
|
h('div', { className: 'dshib-side-note' }, L('side.foreign')))
|
|
481
696
|
}
|
|
482
|
-
const pointer = ptr
|
|
697
|
+
const pointer = person === true || hand === true ? null : ptr
|
|
483
698
|
const noteText = phase === 'failed' ? note || L('side.failed') : phase === 'live' ? L('side.empty') : L('side.connecting')
|
|
484
699
|
return h('div', { className: 'dshib-side' },
|
|
485
700
|
h('div', { className: 'dshib-side-bar' },
|
|
@@ -509,8 +724,26 @@ window.__ModuleLoader__.load({
|
|
|
509
724
|
onPointerMove,
|
|
510
725
|
onWheel,
|
|
511
726
|
onKeyDown,
|
|
727
|
+
onKeyUp,
|
|
512
728
|
},
|
|
513
|
-
frame === null
|
|
729
|
+
frame === null && back !== ''
|
|
730
|
+
? h('button', {
|
|
731
|
+
className: 'dshib-side-back',
|
|
732
|
+
type: 'button', // ui-copy-ignore
|
|
733
|
+
onClick: () => { // ui-copy-ignore
|
|
734
|
+
void fetch(`${ROUTE}/sidecar/open`, { // ui-copy-ignore
|
|
735
|
+
method: 'POST',
|
|
736
|
+
headers: { 'content-type': 'application/json' },
|
|
737
|
+
body: JSON.stringify({ session: who, url: back }), // ui-copy-ignore
|
|
738
|
+
})
|
|
739
|
+
.then((reply) => (reply.ok === true ? reply.json() : null))
|
|
740
|
+
.then((info) => {
|
|
741
|
+
if (info?.ok === true) setAttempt((n) => n + 1)
|
|
742
|
+
})
|
|
743
|
+
.catch(() => {})
|
|
744
|
+
},
|
|
745
|
+
}, L('side.restore'))
|
|
746
|
+
: frame === null ? h('div', { className: 'dshib-side-note' }, noteText) : null,
|
|
514
747
|
h('textarea', {
|
|
515
748
|
className: 'dshib-side-ime',
|
|
516
749
|
ref: ime,
|
|
@@ -532,12 +765,13 @@ window.__ModuleLoader__.load({
|
|
|
532
765
|
clearIme()
|
|
533
766
|
},
|
|
534
767
|
onKeyDown,
|
|
768
|
+
onKeyUp,
|
|
535
769
|
}),
|
|
536
770
|
frame === null ? null : h('img', {
|
|
537
771
|
className: 'dshib-side-frame',
|
|
538
772
|
src: `data:image/jpeg;base64,${frame.jpeg}`,
|
|
539
|
-
width: Math.max(1, Math.round(
|
|
540
|
-
height: Math.max(1, Math.round(
|
|
773
|
+
width: Math.max(1, Math.round(view.w * scale)),
|
|
774
|
+
height: Math.max(1, Math.round(view.h * scale)),
|
|
541
775
|
alt: '', // ui-copy-ignore
|
|
542
776
|
draggable: false,
|
|
543
777
|
}),
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -4,7 +4,7 @@ Two halves in two processes, and a Chrome that neither of them is.
|
|
|
4
4
|
|
|
5
5
|
```
|
|
6
6
|
server side (index.js, in the DSH process)
|
|
7
|
-
|
|
7
|
+
browser_* tools ──> Sidecar ── CDP over a socket ──> headless Chrome
|
|
8
8
|
│
|
|
9
9
|
HTTP + SSE routes <────┘
|
|
10
10
|
│
|
|
@@ -41,7 +41,7 @@ Other launch flags:
|
|
|
41
41
|
| `--use-mock-keychain`, `--password-store=basic` | No keychain prompt on macOS. |
|
|
42
42
|
| `--remote-allow-origins=*` | The CDP client connects over a plain WebSocket. |
|
|
43
43
|
| `--disable-blink-features=AutomationControlled` | Sites that sniff for automation get the ordinary path. |
|
|
44
|
-
| `--window-size=1440,900`, `--lang=zh-CN` | The
|
|
44
|
+
| `--window-size=1440,900`, `--lang=zh-CN` | The window the browser starts in, and a Chinese page locale; the layout viewport is set to the column instead. |
|
|
45
45
|
|
|
46
46
|
The binary comes from a fixed list: Google Chrome, Chromium or Canary under `/Applications`,
|
|
47
47
|
then `/usr/bin/google-chrome` and `/usr/bin/chromium`. Nothing else is probed and there is no
|
|
@@ -58,7 +58,7 @@ The browser outlives the plugin:
|
|
|
58
58
|
`/json/new?about:blank` as a fallback) and a client connects to it.
|
|
59
59
|
|
|
60
60
|
One page per browser. `apply()` only constructs the `Sidecar`; no process exists until the
|
|
61
|
-
first tool call, and `
|
|
61
|
+
first tool call, and `browser_state` reports a stopped browser without starting one.
|
|
62
62
|
|
|
63
63
|
Shutdown asks Chrome to close (`Browser.close`), waits briefly, then sends `SIGTERM`, which is
|
|
64
64
|
the point where Chrome flushes cookies. An idle timer closes the browser after 15 minutes; a
|
|
@@ -85,13 +85,16 @@ the CDP socket stays in the server half.
|
|
|
85
85
|
## Mirror
|
|
86
86
|
|
|
87
87
|
The server half starts `Page.startScreencast` with JPEG quality 72, `maxWidth` and `maxHeight`
|
|
88
|
-
at
|
|
88
|
+
at the pane's size, `optimizeForSpeed`, `everyNthFrame: 1`, and acknowledges every frame it receives
|
|
89
89
|
so Chrome keeps sending. Frames are never queued: with two frames in flight, further frames
|
|
90
90
|
are dropped and the next one that fits takes their place. A slow reader loses frames, never
|
|
91
91
|
freshness.
|
|
92
92
|
|
|
93
|
-
The viewport
|
|
94
|
-
|
|
93
|
+
The layout viewport follows the column: the pane measures itself, posts the size to
|
|
94
|
+
`/sidecar/viewport` a beat after the resize settles, and the server pins
|
|
95
|
+
`Emulation.setDeviceMetricsOverride` to that size and restarts the screencast at it. So a frame
|
|
96
|
+
is the whole page as laid out at the shape it is shown in — no letterbox bands, no crop. Sizes
|
|
97
|
+
below 320 px or above 1920×1600 are ignored rather than applied as a sliver.
|
|
95
98
|
|
|
96
99
|
If `Browser.getVersion` reports `HeadlessChrome`, the user agent is overridden with an ordinary
|
|
97
100
|
Chrome string, `acceptLanguage: 'zh-CN,zh;q=0.9,en;q=0.8'` and `platform: 'MacIntel'`, so a site
|
|
@@ -133,7 +136,7 @@ browser without pulling the column away from the person reading it.
|
|
|
133
136
|
|
|
134
137
|
`sidecarTools()` in `tools.js` builds the 28 definitions and `index.js` registers them on the
|
|
135
138
|
plugin's tool scope. All of them return one text block. A page read numbers the controls and
|
|
136
|
-
keeps those numbers across later reads, so `
|
|
139
|
+
keeps those numbers across later reads, so `browser_click` by ref means the element the model
|
|
137
140
|
last saw; navigation drops them and the model gets `REF_GONE` rather than a wrong click.
|
|
138
141
|
|
|
139
142
|
Before a ref click or a ref focus, one page-side probe asks what is on top of that control.
|
|
@@ -145,7 +148,7 @@ reported, never accepted on the person's behalf. The whole exchange is one `Runt
|
|
|
145
148
|
and a possible `mousePressed`/`mouseReleased` pair on an already-open CDP connection, so an
|
|
146
149
|
uncovered click pays nothing.
|
|
147
150
|
|
|
148
|
-
`
|
|
151
|
+
`browser_do` runs the same probe before a step that names a control — by `ref`, by
|
|
149
152
|
text, or by field label — so a banner the first step clears is gone for the rest of
|
|
150
153
|
the batch, and the step's own line says it in one clause (`after Accept on a banner,`).
|
|
151
154
|
A batch therefore pays for one overlay, not one per step.
|
|
@@ -154,7 +157,7 @@ An action answers with one line — address, title, focused field — and reads
|
|
|
154
157
|
`read: true` is passed. That is what keeps a long batch cheap (see `BENCH.md`).
|
|
155
158
|
|
|
156
159
|
Calls that touch the browser are marked not concurrency-safe; reading state, console and failed
|
|
157
|
-
requests is. `
|
|
160
|
+
requests is. `browser_open`, `browser_view`, `browser_wait` and `browser_shot` carry longer
|
|
158
161
|
timeouts than the rest.
|
|
159
162
|
|
|
160
163
|
Sign-in state lives in the profile, and session cookies — the kind a sign-in usually sets,
|
|
@@ -169,7 +172,7 @@ without asking first when someone wants something done on a site.
|
|
|
169
172
|
`Network.enable` is already on for the mirror. A request is kept when it is
|
|
170
173
|
sent and settled when its response arrives, so the list holds method, status,
|
|
171
174
|
type and address of what the page actually got — 120 rows, oldest dropped.
|
|
172
|
-
`
|
|
175
|
+
`browser_network` reads that list without touching the page, which is why it
|
|
173
176
|
can answer while a dialog is open.
|
|
174
177
|
|
|
175
178
|
## Downloads
|
|
@@ -177,7 +180,7 @@ can answer while a dialog is open.
|
|
|
177
180
|
Chrome is pointed at `<profile>/downloads` when the page starts, so a click
|
|
178
181
|
that saves a file writes it there instead of nowhere. The folder belongs to
|
|
179
182
|
the profile: it is the same one on the next launch, and nothing outside the
|
|
180
|
-
profile is touched. `
|
|
183
|
+
profile is touched. `browser_downloads` lists what is on disk, newest first;
|
|
181
184
|
a `.crdownload` still in flight is not listed.
|
|
182
185
|
|
|
183
186
|
## Frames and popups
|
|
@@ -215,4 +218,4 @@ and there is still one page to drive.
|
|
|
215
218
|
| `locale/*.json` | Page: pane labels. |
|
|
216
219
|
|
|
217
220
|
Data on disk is confined to `~/.dsh/inner-browser-use/`: `chrome/<profile>` for the browser,
|
|
218
|
-
`screenshots/` for `
|
|
221
|
+
`screenshots/` for `browser_shot`.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# 文件落在哪里
|
|
2
|
+
|
|
3
|
+
浏览器产生的东西——截图、抓下来的图、打印的 PDF——不落在项目根目录,也不散在 home 下。固定规则:
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
~/.dsh-inner-browser-use/artifacts/<项目>/<会话>/<种类>/
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
- **项目**:会话工作目录的文件夹名,加路径哈希前 6 位,例如 `dsh-inner-browser-use-a1b2c3`。同名文件夹、不同路径不会混。
|
|
10
|
+
- **会话**:会话 id 去掉 `session-` 后的前 8 位,例如 `72557d6e`。同一项目里两个会话各存各的。
|
|
11
|
+
- **种类**:`shots/` 是 `browser_shot` 的截图,`pictures/` 是 `browser_grab` 抓的图,`pdf/` 是 `browser_pdf` 打印的文档。
|
|
12
|
+
|
|
13
|
+
文件名默认带时间戳(`shot-20261028-124500.png`),给了 `name` 就用它;名字里的分隔符、连续点、开头点都会被去掉,跳不出这个目录。
|
|
14
|
+
|
|
15
|
+
每个种类保留最新 40 个文件,旧的自动删,免得跑一天的会话留下一个没人翻得动的文件夹。开着的文件不会被删掉(尽力删除,失败就留着)。
|
|
16
|
+
|
|
17
|
+
调用环境里没有项目或会话信息时,落到 `no-project/no-chat/<种类>/`,不猜。
|
|
18
|
+
|
|
19
|
+
测试里用 `DSH_INNER_BROWSER_ARTIFACTS` 指向临时目录,不写真实 home。
|