dsh-sidebar-control 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +23 -0
- package/README.md +226 -2
- package/client.js +1147 -0
- package/cordis.patch.yml +3 -0
- package/icon.svg +8 -0
- package/index.js +528 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +73 -4
package/client.js
ADDED
|
@@ -0,0 +1,1147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client half of the Sidebar Browser Control bundle.
|
|
3
|
+
*
|
|
4
|
+
* It runs in the Harness page, long-polls the Host half over the private
|
|
5
|
+
* Connection channel, and executes each command against the right-Sidebar
|
|
6
|
+
* Browser frame. On Desktop that frame is an Electron <webview>, so page
|
|
7
|
+
* scripts can be evaluated inside the guest and the visible page can be
|
|
8
|
+
* captured — the Host never gets DOM access itself.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
window.__ModuleLoader__.load({
|
|
12
|
+
id: 'dsh-sidebar-control',
|
|
13
|
+
factory(require) {
|
|
14
|
+
const POLL_WAIT_MS = 25000
|
|
15
|
+
const FRAME_SELECTOR = 'webview[data-sidebar-browser-frame]'
|
|
16
|
+
const LIST_LIMIT = 120
|
|
17
|
+
/* Actions a batch may contain. A batch is one tool call for several steps,
|
|
18
|
+
so it saves the model a round trip per step; it may not nest or open tabs. */
|
|
19
|
+
const BATCH_ACTIONS = new Set(['navigate', 'click', 'type', 'press', 'select', 'scroll', 'wait', 'eval', 'read', 'diff', 'snapshot'])
|
|
20
|
+
|
|
21
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
22
|
+
|
|
23
|
+
function safe(read) {
|
|
24
|
+
try {
|
|
25
|
+
return read()
|
|
26
|
+
} catch {
|
|
27
|
+
return undefined
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
//#region Guest page helpers — stringified and evaluated inside the Browser frame.
|
|
32
|
+
function pageSnapshot(options) {
|
|
33
|
+
const maxRefs = Number(options.maxRefs) > 0 ? Number(options.maxRefs) : 80
|
|
34
|
+
const maxTextChars = Number(options.maxTextChars) > 0 ? Number(options.maxTextChars) : 12000
|
|
35
|
+
const selector = [
|
|
36
|
+
'a[href]',
|
|
37
|
+
'button',
|
|
38
|
+
'input:not([type="hidden"])',
|
|
39
|
+
'textarea',
|
|
40
|
+
'select',
|
|
41
|
+
'summary',
|
|
42
|
+
'[role="button"]',
|
|
43
|
+
'[role="link"]',
|
|
44
|
+
'[role="checkbox"]',
|
|
45
|
+
'[role="radio"]',
|
|
46
|
+
'[role="tab"]',
|
|
47
|
+
'[role="menuitem"]',
|
|
48
|
+
'[role="option"]',
|
|
49
|
+
'[role="switch"]',
|
|
50
|
+
'[contenteditable="true"]',
|
|
51
|
+
].join(',')
|
|
52
|
+
const isVisible = (element) => {
|
|
53
|
+
const rect = element.getBoundingClientRect()
|
|
54
|
+
if (rect.width < 1 || rect.height < 1) return false
|
|
55
|
+
const style = getComputedStyle(element)
|
|
56
|
+
return style.visibility !== 'hidden' && style.display !== 'none' && Number(style.opacity) !== 0
|
|
57
|
+
}
|
|
58
|
+
const labelOf = (element) => {
|
|
59
|
+
const raw = element.getAttribute('aria-label')
|
|
60
|
+
|| element.getAttribute('title')
|
|
61
|
+
|| element.getAttribute('placeholder')
|
|
62
|
+
|| element.getAttribute('alt')
|
|
63
|
+
|| element.innerText
|
|
64
|
+
|| element.value
|
|
65
|
+
|| element.getAttribute('name')
|
|
66
|
+
|| ''
|
|
67
|
+
return String(raw).replace(/\s+/g, ' ').trim().slice(0, 160)
|
|
68
|
+
}
|
|
69
|
+
const refs = []
|
|
70
|
+
let refsTruncated = false
|
|
71
|
+
const candidates = document.querySelectorAll(selector)
|
|
72
|
+
for (const element of candidates) {
|
|
73
|
+
if (refs.length >= maxRefs) {
|
|
74
|
+
refsTruncated = true
|
|
75
|
+
break
|
|
76
|
+
}
|
|
77
|
+
if (!isVisible(element) || element.disabled === true) continue
|
|
78
|
+
const ref = refs.length + 1
|
|
79
|
+
element.setAttribute('data-dsh-bc-ref', String(ref))
|
|
80
|
+
const tag = element.tagName.toLowerCase()
|
|
81
|
+
const entry = { ref, tag }
|
|
82
|
+
const role = element.getAttribute('role')
|
|
83
|
+
if (role !== null) entry.role = role
|
|
84
|
+
const type = element.getAttribute('type')
|
|
85
|
+
if (type !== null) entry.type = type
|
|
86
|
+
const label = labelOf(element)
|
|
87
|
+
if (label.length > 0) entry.label = label
|
|
88
|
+
if (tag === 'a') {
|
|
89
|
+
const href = element.getAttribute('href')
|
|
90
|
+
if (href !== null) entry.href = String(href).slice(0, 500)
|
|
91
|
+
if (label.length === 0) {
|
|
92
|
+
const image = element.querySelector('img[alt]')
|
|
93
|
+
const alt = image === null ? '' : String(image.getAttribute('alt') || '').replace(/\s+/g, ' ').trim()
|
|
94
|
+
if (alt.length === 0) continue
|
|
95
|
+
entry.label = alt.slice(0, 160)
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if (tag === 'input' || tag === 'textarea' || tag === 'select') {
|
|
99
|
+
if (element.value !== undefined && element.value !== '') entry.value = String(element.value).slice(0, 300)
|
|
100
|
+
if (element.type === 'checkbox' || element.type === 'radio') entry.checked = element.checked === true
|
|
101
|
+
if (tag === 'select') {
|
|
102
|
+
entry.options = Array.from(element.options).slice(0, 40).map((option) => ({
|
|
103
|
+
value: option.value,
|
|
104
|
+
label: String(option.label || option.text || '').slice(0, 120),
|
|
105
|
+
selected: option.selected === true,
|
|
106
|
+
}))
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
refs.push(entry)
|
|
110
|
+
}
|
|
111
|
+
const includeText = options.includeText !== false
|
|
112
|
+
const rawText = includeText && document.body !== null ? String(document.body.innerText || '') : ''
|
|
113
|
+
const text = rawText.replace(/\n[ \t]+\n/g, '\n\n').replace(/\n{3,}/g, '\n\n').trim()
|
|
114
|
+
return {
|
|
115
|
+
url: String(location.href),
|
|
116
|
+
title: String(document.title || ''),
|
|
117
|
+
readyState: String(document.readyState),
|
|
118
|
+
scroll: {
|
|
119
|
+
x: Math.round(window.scrollX),
|
|
120
|
+
y: Math.round(window.scrollY),
|
|
121
|
+
height: document.documentElement === null ? 0 : document.documentElement.scrollHeight,
|
|
122
|
+
},
|
|
123
|
+
refs,
|
|
124
|
+
refsTruncated,
|
|
125
|
+
text: text.slice(0, maxTextChars),
|
|
126
|
+
textTruncated: text.length > maxTextChars,
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function pageFind(options) {
|
|
131
|
+
if (options.ref !== undefined && options.ref !== null) {
|
|
132
|
+
return document.querySelector('[data-dsh-bc-ref="' + String(options.ref) + '"]')
|
|
133
|
+
}
|
|
134
|
+
if (typeof options.selector === 'string' && options.selector.length > 0) {
|
|
135
|
+
return document.querySelector(options.selector)
|
|
136
|
+
}
|
|
137
|
+
return null
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function pageClick(options) {
|
|
141
|
+
const element = pageFind(options)
|
|
142
|
+
if (element === null) {
|
|
143
|
+
throw new Error(options.ref !== undefined
|
|
144
|
+
? 'No element has ref ' + String(options.ref) + ' on this page; take a new snapshot.'
|
|
145
|
+
: 'No element matches selector ' + String(options.selector) + '.')
|
|
146
|
+
}
|
|
147
|
+
const label = String(element.innerText || element.getAttribute('aria-label') || element.getAttribute('title') || '')
|
|
148
|
+
.replace(/\s+/g, ' ')
|
|
149
|
+
.trim()
|
|
150
|
+
.slice(0, 120)
|
|
151
|
+
element.scrollIntoView({ block: 'center', inline: 'center' })
|
|
152
|
+
const target = element.closest('a,button,label,summary,[role="button"],[role="link"]') || element
|
|
153
|
+
target.click()
|
|
154
|
+
return { clicked: { tag: target.tagName.toLowerCase(), label }, url: String(location.href) }
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function pageType(options) {
|
|
158
|
+
const element = pageFind(options)
|
|
159
|
+
if (element === null) {
|
|
160
|
+
throw new Error(options.ref !== undefined
|
|
161
|
+
? 'No element has ref ' + String(options.ref) + ' on this page; take a new snapshot.'
|
|
162
|
+
: 'No element matches selector ' + String(options.selector) + '.')
|
|
163
|
+
}
|
|
164
|
+
const text = String(options.text)
|
|
165
|
+
element.scrollIntoView({ block: 'center' })
|
|
166
|
+
if (typeof element.focus === 'function') element.focus()
|
|
167
|
+
if (element.isContentEditable === true) {
|
|
168
|
+
element.textContent = text
|
|
169
|
+
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }))
|
|
170
|
+
} else {
|
|
171
|
+
const prototype = element.tagName === 'TEXTAREA' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype
|
|
172
|
+
const setter = Object.getOwnPropertyDescriptor(prototype, 'value')
|
|
173
|
+
if (setter !== undefined && typeof setter.set === 'function') setter.set.call(element, text)
|
|
174
|
+
else element.value = text
|
|
175
|
+
element.dispatchEvent(new Event('input', { bubbles: true }))
|
|
176
|
+
element.dispatchEvent(new Event('change', { bubbles: true }))
|
|
177
|
+
}
|
|
178
|
+
if (options.submit === true) {
|
|
179
|
+
if (element.form !== null && element.form !== undefined && typeof element.form.requestSubmit === 'function') element.form.requestSubmit()
|
|
180
|
+
else element.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true, cancelable: true }))
|
|
181
|
+
}
|
|
182
|
+
return { typed: text.length, target: element.tagName.toLowerCase(), url: String(location.href) }
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function pagePress(options) {
|
|
186
|
+
const key = String(options.key)
|
|
187
|
+
const target = document.activeElement || document.body
|
|
188
|
+
const keyCode = key === 'Enter' ? 13
|
|
189
|
+
: key === 'Tab' ? 9
|
|
190
|
+
: key === 'Escape' ? 27
|
|
191
|
+
: key === 'ArrowDown' ? 40
|
|
192
|
+
: key === 'ArrowUp' ? 38
|
|
193
|
+
: key === 'ArrowLeft' ? 37
|
|
194
|
+
: key === 'ArrowRight' ? 39
|
|
195
|
+
: key === ' ' || key === 'Space' ? 32
|
|
196
|
+
: 0
|
|
197
|
+
const init = { key, code: key === ' ' ? 'Space' : key, keyCode, which: keyCode, bubbles: true, cancelable: true }
|
|
198
|
+
target.dispatchEvent(new KeyboardEvent('keydown', init))
|
|
199
|
+
target.dispatchEvent(new KeyboardEvent('keyup', init))
|
|
200
|
+
if (key === 'Enter' && target.form !== null && target.form !== undefined && (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA')) {
|
|
201
|
+
try {
|
|
202
|
+
target.form.requestSubmit()
|
|
203
|
+
} catch {
|
|
204
|
+
// A form without a submitter still keeps the dispatched key event.
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return { pressed: key, active: target === null ? 'body' : target.tagName.toLowerCase() }
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function pageSelect(options) {
|
|
211
|
+
const element = pageFind(options)
|
|
212
|
+
if (element === null) throw new Error('No select element matches the given ref or selector.')
|
|
213
|
+
if (element.tagName !== 'SELECT') throw new Error('The target element is <' + element.tagName.toLowerCase() + '>, not a <select>.')
|
|
214
|
+
const value = String(options.value)
|
|
215
|
+
element.value = value
|
|
216
|
+
if (element.value !== value) {
|
|
217
|
+
const option = Array.from(element.options).find((candidate) => candidate.text === value || candidate.label === value)
|
|
218
|
+
if (option !== undefined) element.value = option.value
|
|
219
|
+
}
|
|
220
|
+
element.dispatchEvent(new Event('input', { bubbles: true }))
|
|
221
|
+
element.dispatchEvent(new Event('change', { bubbles: true }))
|
|
222
|
+
return { value: element.value, selectedIndex: element.selectedIndex }
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function pageScroll(options) {
|
|
226
|
+
if (options.ref !== undefined && options.ref !== null) {
|
|
227
|
+
const element = pageFind(options)
|
|
228
|
+
if (element === null) throw new Error('No element has ref ' + String(options.ref) + '; take a new snapshot.')
|
|
229
|
+
element.scrollIntoView({ block: 'center' })
|
|
230
|
+
return { scrolledTo: { ref: options.ref }, y: Math.round(window.scrollY) }
|
|
231
|
+
}
|
|
232
|
+
const amount = Number(options.amount) > 0 ? Number(options.amount) : 600
|
|
233
|
+
const direction = String(options.direction || 'down')
|
|
234
|
+
const dx = direction === 'left' ? -amount : direction === 'right' ? amount : 0
|
|
235
|
+
const dy = direction === 'up' ? -amount : amount
|
|
236
|
+
window.scrollBy({ left: dx, top: dy, behavior: 'instant' })
|
|
237
|
+
return { x: Math.round(window.scrollX), y: Math.round(window.scrollY) }
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async function pageWait(options) {
|
|
241
|
+
const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : 5000
|
|
242
|
+
const deadline = Date.now() + timeoutMs
|
|
243
|
+
while (Date.now() < deadline) {
|
|
244
|
+
if (typeof options.selector === 'string' && options.selector.length > 0) {
|
|
245
|
+
if (document.querySelector(options.selector) !== null) return { found: true, kind: 'selector', selector: options.selector }
|
|
246
|
+
} else if (typeof options.text === 'string' && options.text.length > 0) {
|
|
247
|
+
const bodyText = document.body === null ? '' : String(document.body.innerText || '')
|
|
248
|
+
if (bodyText.includes(options.text)) return { found: true, kind: 'text', text: options.text }
|
|
249
|
+
}
|
|
250
|
+
await new Promise((resolve) => setTimeout(resolve, 150))
|
|
251
|
+
}
|
|
252
|
+
throw new Error('Timed out after ' + String(timeoutMs) + 'ms waiting for '
|
|
253
|
+
+ (typeof options.selector === 'string' && options.selector.length > 0 ? 'selector ' + options.selector : 'text ' + String(options.text)) + '.')
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Read the page the way a person answers a question: split it into
|
|
258
|
+
* paragraphs and return only the ones that match the query (BM25 over
|
|
259
|
+
* Latin words and CJK bigrams). On a long article this is a fraction of a
|
|
260
|
+
* full snapshot; with no query it is the plain text.
|
|
261
|
+
*/
|
|
262
|
+
function pageRead(options) {
|
|
263
|
+
const tokenize = (value) => {
|
|
264
|
+
const text = String(value || '').toLowerCase()
|
|
265
|
+
const out = []
|
|
266
|
+
const latin = text.match(/[a-z0-9_]+/g) || []
|
|
267
|
+
for (const word of latin) out.push(word)
|
|
268
|
+
const cjk = text.match(/[\u4e00-\u9fff]+/g) || []
|
|
269
|
+
for (const run of cjk) {
|
|
270
|
+
if (run.length === 1) out.push(run)
|
|
271
|
+
for (let i = 0; i + 1 < run.length; i += 1) out.push(run.slice(i, i + 2))
|
|
272
|
+
}
|
|
273
|
+
return out
|
|
274
|
+
}
|
|
275
|
+
const query = String(options.query || '').trim()
|
|
276
|
+
const budget = Number(options.maxChars) > 0 ? Math.min(Number(options.maxChars), 20000) : 4000
|
|
277
|
+
const raw = document.body === null ? '' : String(document.body.innerText || '')
|
|
278
|
+
const paragraphs = raw.split(/\n+/)
|
|
279
|
+
.map((part) => part.replace(/[ \t]+/g, ' ').trim())
|
|
280
|
+
.filter((part) => part.length > 0)
|
|
281
|
+
.slice(0, 800)
|
|
282
|
+
const whole = paragraphs.join('\n\n')
|
|
283
|
+
if (query === '') {
|
|
284
|
+
return {
|
|
285
|
+
url: String(location.href),
|
|
286
|
+
title: String(document.title || ''),
|
|
287
|
+
mode: 'text',
|
|
288
|
+
text: whole.slice(0, budget),
|
|
289
|
+
truncated: whole.length > budget,
|
|
290
|
+
matched: null,
|
|
291
|
+
candidates: paragraphs.length,
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
const terms = tokenize(query)
|
|
295
|
+
const docs = paragraphs.map((part) => tokenize(part))
|
|
296
|
+
const df = new Map()
|
|
297
|
+
for (const doc of docs) for (const term of new Set(doc)) df.set(term, (df.get(term) || 0) + 1)
|
|
298
|
+
const count = docs.length || 1
|
|
299
|
+
const avg = docs.reduce((sum, doc) => sum + doc.length, 0) / count || 1
|
|
300
|
+
const scored = []
|
|
301
|
+
for (let i = 0; i < docs.length; i += 1) {
|
|
302
|
+
const doc = docs[i]
|
|
303
|
+
if (doc.length === 0) continue
|
|
304
|
+
const tf = new Map()
|
|
305
|
+
for (const term of doc) tf.set(term, (tf.get(term) || 0) + 1)
|
|
306
|
+
let score = 0
|
|
307
|
+
for (const term of terms) {
|
|
308
|
+
const frequency = tf.get(term) || 0
|
|
309
|
+
if (frequency === 0) continue
|
|
310
|
+
const idf = Math.log(1 + (count - (df.get(term) || 0) + 0.5) / ((df.get(term) || 0) + 0.5))
|
|
311
|
+
score += idf * (frequency * 2.2) / (frequency + 1.2 * (0.25 + 0.75 * doc.length / avg))
|
|
312
|
+
}
|
|
313
|
+
if (score > 0) scored.push({ index: i, score })
|
|
314
|
+
}
|
|
315
|
+
scored.sort((left, right) => right.score - left.score)
|
|
316
|
+
const picked = []
|
|
317
|
+
let used = 0
|
|
318
|
+
for (const hit of scored) {
|
|
319
|
+
const paragraph = paragraphs[hit.index]
|
|
320
|
+
if (used + paragraph.length > budget && picked.length > 0) continue
|
|
321
|
+
picked.push(hit.index)
|
|
322
|
+
used += paragraph.length
|
|
323
|
+
if (used >= budget) break
|
|
324
|
+
}
|
|
325
|
+
picked.sort((left, right) => left - right)
|
|
326
|
+
return {
|
|
327
|
+
url: String(location.href),
|
|
328
|
+
title: String(document.title || ''),
|
|
329
|
+
mode: 'query',
|
|
330
|
+
query,
|
|
331
|
+
matched: picked.length,
|
|
332
|
+
candidates: scored.length,
|
|
333
|
+
text: picked.map((index) => paragraphs[index]).join('\n\n').slice(0, budget),
|
|
334
|
+
truncated: scored.length > picked.length,
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/**
|
|
339
|
+
* Compare the page with the previous diff (paragraph sets): what appeared
|
|
340
|
+
* and what disappeared since the last call, instead of the whole page.
|
|
341
|
+
*/
|
|
342
|
+
function pageDiff(options) {
|
|
343
|
+
const budget = Number(options.maxChars) > 0 ? Math.min(Number(options.maxChars), 20000) : 4000
|
|
344
|
+
const raw = document.body === null ? '' : String(document.body.innerText || '')
|
|
345
|
+
const paragraphs = raw.split(/\n+/)
|
|
346
|
+
.map((part) => part.replace(/[ \t]+/g, ' ').trim())
|
|
347
|
+
.filter((part) => part.length > 0)
|
|
348
|
+
.slice(0, 800)
|
|
349
|
+
const previous = Array.isArray(window.__dshScParagraphs) ? window.__dshScParagraphs : []
|
|
350
|
+
const prevSet = new Set(previous)
|
|
351
|
+
const nextSet = new Set(paragraphs)
|
|
352
|
+
const added = []
|
|
353
|
+
const removed = []
|
|
354
|
+
for (const part of paragraphs) {
|
|
355
|
+
if (prevSet.has(part) !== true) added.push(part.slice(0, 300))
|
|
356
|
+
if (added.length >= 40) break
|
|
357
|
+
}
|
|
358
|
+
for (const part of previous) {
|
|
359
|
+
if (nextSet.has(part) !== true) removed.push(part.slice(0, 300))
|
|
360
|
+
if (removed.length >= 40) break
|
|
361
|
+
}
|
|
362
|
+
window.__dshScParagraphs = paragraphs
|
|
363
|
+
let text = ''
|
|
364
|
+
if (added.length > 0) text += 'added:\n+ ' + added.join('\n+ ') + '\n'
|
|
365
|
+
if (removed.length > 0) text += (text === '' ? '' : '\n') + 'removed:\n- ' + removed.join('\n- ') + '\n'
|
|
366
|
+
return {
|
|
367
|
+
url: String(location.href),
|
|
368
|
+
title: String(document.title || ''),
|
|
369
|
+
first: previous.length === 0,
|
|
370
|
+
changed: added.length > 0 || removed.length > 0,
|
|
371
|
+
added,
|
|
372
|
+
removed,
|
|
373
|
+
text: text.slice(0, budget),
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
//#endregion
|
|
377
|
+
|
|
378
|
+
function frameIsVisible(element) {
|
|
379
|
+
try {
|
|
380
|
+
if (element.isConnected !== true) return false
|
|
381
|
+
if (element.closest('[hidden], [aria-hidden="true"], [inert]') !== null) return false
|
|
382
|
+
if (element.getClientRects().length === 0) return false
|
|
383
|
+
const rect = element.getBoundingClientRect()
|
|
384
|
+
if (rect.width < 2 || rect.height < 2) return false
|
|
385
|
+
const style = window.getComputedStyle(element)
|
|
386
|
+
return style.display !== 'none' && style.visibility !== 'hidden' && Number(style.opacity) > 0
|
|
387
|
+
} catch {
|
|
388
|
+
return false
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
function browserFrames() {
|
|
393
|
+
return Array.from(document.querySelectorAll(FRAME_SELECTOR))
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
function activeBrowserTab(ctx) {
|
|
397
|
+
const sidebarRight = sidebarRightOf(ctx)
|
|
398
|
+
if (sidebarRight === undefined || sidebarRight === null || typeof sidebarRight.active !== 'function') return undefined
|
|
399
|
+
const tab = safe(() => sidebarRight.active())
|
|
400
|
+
if (tab === undefined || tab === null || tab.kind !== 'browser') return undefined
|
|
401
|
+
return tab
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function frameForTab(tab) {
|
|
405
|
+
if (tab === undefined || tab === null || tab.id === undefined || tab.id === null) return undefined
|
|
406
|
+
const escape = typeof CSS !== 'undefined' && typeof CSS.escape === 'function' ? CSS.escape(String(tab.id)) : String(tab.id)
|
|
407
|
+
return document.querySelector('div[data-sidebar-right-tab="' + escape + '"] ' + FRAME_SELECTOR)
|
|
408
|
+
|| undefined
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
function pickFrame(state, ctx) {
|
|
412
|
+
const tab = activeBrowserTab(ctx)
|
|
413
|
+
if (tab !== undefined) {
|
|
414
|
+
const exact = frameForTab(tab)
|
|
415
|
+
if (exact !== undefined) return exact
|
|
416
|
+
}
|
|
417
|
+
const frames = browserFrames()
|
|
418
|
+
if (frames.length === 0) return undefined
|
|
419
|
+
const visibleFrames = frames.filter(frameIsVisible)
|
|
420
|
+
if (state.last !== undefined && state.last.isConnected === true && visibleFrames.includes(state.last)) return state.last
|
|
421
|
+
if (visibleFrames.length > 0) return visibleFrames[visibleFrames.length - 1]
|
|
422
|
+
if (state.last !== undefined && state.last.isConnected === true) return state.last
|
|
423
|
+
return frames[frames.length - 1]
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
async function frameReady(frame, timeoutMs) {
|
|
427
|
+
const deadline = Date.now() + timeoutMs
|
|
428
|
+
let lastError
|
|
429
|
+
while (Date.now() < deadline) {
|
|
430
|
+
try {
|
|
431
|
+
if (frame.isConnected === false) throw new Error('the Browser tab was closed')
|
|
432
|
+
const ping = await frame.executeJavaScript('1', true)
|
|
433
|
+
if (ping === 1) return true
|
|
434
|
+
} catch (error) {
|
|
435
|
+
lastError = error
|
|
436
|
+
}
|
|
437
|
+
await sleep(150)
|
|
438
|
+
}
|
|
439
|
+
throw new Error('The Browser frame is not ready'
|
|
440
|
+
+ (lastError instanceof Error && lastError.message ? ': ' + lastError.message : '. Open or reload the Browser tab and retry.'))
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function sidebarRightOf(ctx) {
|
|
444
|
+
if (typeof ctx.get === 'function') return ctx.get('sidebarRight')
|
|
445
|
+
return ctx.sidebarRight
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
async function openBrowserTab(ctx, url) {
|
|
449
|
+
const sidebarRight = sidebarRightOf(ctx)
|
|
450
|
+
if (sidebarRight === undefined || sidebarRight === null) {
|
|
451
|
+
throw new Error('The right Sidebar is unavailable in this Harness page, so a Browser tab cannot be opened (Desktop app required for Browser control).')
|
|
452
|
+
}
|
|
453
|
+
try {
|
|
454
|
+
sidebarRight.openTab('browser', { params: { url } })
|
|
455
|
+
} catch (error) {
|
|
456
|
+
throw new Error('Could not open the right-Sidebar Browser: ' + (error instanceof Error ? error.message : String(error)))
|
|
457
|
+
}
|
|
458
|
+
const deadline = Date.now() + 20000
|
|
459
|
+
while (Date.now() < deadline) {
|
|
460
|
+
const tab = activeBrowserTab(ctx)
|
|
461
|
+
const frame = (tab === undefined ? undefined : frameForTab(tab))
|
|
462
|
+
|| browserFrames().filter(frameIsVisible)[0]
|
|
463
|
+
|| browserFrames()[0]
|
|
464
|
+
if (frame !== undefined) return frame
|
|
465
|
+
await sleep(150)
|
|
466
|
+
}
|
|
467
|
+
throw new Error('The Browser tab was requested but no Browser frame appeared. Check that the right Sidebar Browser is enabled in this Harness build.')
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
async function settleActiveFrame(state, ctx, fallback) {
|
|
471
|
+
let frame = pickFrame(state, ctx) || fallback
|
|
472
|
+
if (frame === undefined) return undefined
|
|
473
|
+
await frameReady(frame, 15000)
|
|
474
|
+
await waitForHttpPage(frame, 30000)
|
|
475
|
+
const latest = pickFrame(state, ctx)
|
|
476
|
+
if (latest !== undefined && latest !== frame) {
|
|
477
|
+
// The dock re-created the tab body while loading (new occurrence/lease).
|
|
478
|
+
frame = latest
|
|
479
|
+
await frameReady(frame, 15000)
|
|
480
|
+
await waitForHttpPage(frame, 30000)
|
|
481
|
+
}
|
|
482
|
+
state.last = frame
|
|
483
|
+
return frame
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
async function activeFrame(state, ctx, options) {
|
|
487
|
+
let frame = pickFrame(state, ctx)
|
|
488
|
+
if (frame === undefined && options.openUrl !== undefined) frame = await openBrowserTab(ctx, options.openUrl)
|
|
489
|
+
if (frame === undefined) {
|
|
490
|
+
throw new Error('No Browser tab is open in the right Sidebar. Call the browser tool with action "open" and a url first.')
|
|
491
|
+
}
|
|
492
|
+
await frameReady(frame, options.readyTimeoutMs === undefined ? 15000 : options.readyTimeoutMs)
|
|
493
|
+
state.last = frame
|
|
494
|
+
return frame
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
function tabIdOfFrame(frame) {
|
|
498
|
+
const holder = frame.closest('div[data-sidebar-right-tab]')
|
|
499
|
+
if (holder === null) return undefined
|
|
500
|
+
return holder.getAttribute('data-sidebar-right-tab') || undefined
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
function frameTabInfo(ctx, frame) {
|
|
504
|
+
const tabId = tabIdOfFrame(frame)
|
|
505
|
+
const holder = frame.closest('[data-sidebar-right-session]')
|
|
506
|
+
const frameSession = holder === null ? undefined : holder.getAttribute('data-sidebar-right-session') || undefined
|
|
507
|
+
const fallback = { same: true, tabId, browserTab: undefined, current: undefined, frameSession }
|
|
508
|
+
const sidebarRight = sidebarRightOf(ctx)
|
|
509
|
+
if (sidebarRight === undefined || sidebarRight === null) return fallback
|
|
510
|
+
const mounted = safe(() => sidebarRight.mounted?.getSnapshot())
|
|
511
|
+
const current = safe(() => sidebarRight.focusedTarget?.())
|
|
512
|
+
const tabs = safe(() => sidebarRight.openTabs?.getSnapshot())
|
|
513
|
+
const currentSession = (current === undefined || current === null ? undefined : current.sessionId) ?? mounted
|
|
514
|
+
const same = currentSession === undefined || frameSession === undefined ? true : frameSession === currentSession
|
|
515
|
+
let browserTab
|
|
516
|
+
if (Array.isArray(tabs)) {
|
|
517
|
+
const candidates = tabs.filter((tab) => tab.kind === 'browser' && tab.tabId === tabId)
|
|
518
|
+
browserTab = candidates.find((tab) => tab.sessionId === currentSession) || candidates[candidates.length - 1]
|
|
519
|
+
}
|
|
520
|
+
return { same, tabId, browserTab, current, frameSession }
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/** Bring the frame into the visible, expanded right Sidebar so capturePage is not blank. */
|
|
524
|
+
async function revealFrame(ctx, frame) {
|
|
525
|
+
if (frameIsVisible(frame)) return
|
|
526
|
+
const sidebarRight = sidebarRightOf(ctx)
|
|
527
|
+
const info = frameTabInfo(ctx, frame)
|
|
528
|
+
if (!info.same) {
|
|
529
|
+
throw new Error('The Browser page belongs to another Session than the one on screen, so it cannot be shown or captured here. '
|
|
530
|
+
+ 'Bring that Session on screen, or call action "open" to open the page in the current Session.')
|
|
531
|
+
}
|
|
532
|
+
const tabId = info.tabId
|
|
533
|
+
if (sidebarRight !== undefined && sidebarRight !== null && tabId !== undefined && typeof sidebarRight.focus === 'function') {
|
|
534
|
+
try {
|
|
535
|
+
sidebarRight.focus(tabId)
|
|
536
|
+
} catch {
|
|
537
|
+
// Focus is best-effort; expansion below still runs.
|
|
538
|
+
}
|
|
539
|
+
const deadline = Date.now() + 2000
|
|
540
|
+
while (Date.now() < deadline && !frameIsVisible(frame)) await sleep(100)
|
|
541
|
+
}
|
|
542
|
+
if (frameIsVisible(frame)) return
|
|
543
|
+
if (sidebarRight !== undefined && sidebarRight !== null
|
|
544
|
+
&& safe(() => sidebarRight.isExpanded()) !== true
|
|
545
|
+
&& typeof sidebarRight.toggleExpanded === 'function') {
|
|
546
|
+
try {
|
|
547
|
+
sidebarRight.toggleExpanded()
|
|
548
|
+
} catch (error) {
|
|
549
|
+
throw new Error('The Browser tab is hidden and the right Sidebar could not be expanded: '
|
|
550
|
+
+ (error instanceof Error ? error.message : String(error)))
|
|
551
|
+
}
|
|
552
|
+
const deadline = Date.now() + 3000
|
|
553
|
+
while (Date.now() < deadline && !frameIsVisible(frame)) await sleep(100)
|
|
554
|
+
}
|
|
555
|
+
if (!frameIsVisible(frame)) {
|
|
556
|
+
const active = safe(() => sidebarRight?.active())
|
|
557
|
+
throw new Error('The Browser tab is hidden (collapsed, inert, or another tab is in front), so a screenshot would be blank.'
|
|
558
|
+
+ (active === undefined ? '' : ' Active right-Sidebar tab: ' + String(active.kind) + ' ' + String(active.id) + '.')
|
|
559
|
+
+ ' Open the Browser panel visibly, then retry.')
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
/**
|
|
564
|
+
* Make a frame capturable. When the frame is hidden behind another right-Sidebar
|
|
565
|
+
* tab (or belongs to a Session that is no longer on screen), reopen the same page
|
|
566
|
+
* in a fresh Browser tab of the on-screen Session so the capture is never blank.
|
|
567
|
+
*/
|
|
568
|
+
async function ensureScreenshotFrame(state, ctx, frame) {
|
|
569
|
+
try {
|
|
570
|
+
await revealFrame(ctx, frame)
|
|
571
|
+
return frame
|
|
572
|
+
} catch (error) {
|
|
573
|
+
const url = String(safe(() => frame.getURL()) || '')
|
|
574
|
+
if (!/^https?:/i.test(url)) throw error
|
|
575
|
+
let fresh
|
|
576
|
+
try {
|
|
577
|
+
await openBrowserTab(ctx, url)
|
|
578
|
+
fresh = await settleActiveFrame(state, ctx, undefined)
|
|
579
|
+
} catch {
|
|
580
|
+
throw error
|
|
581
|
+
}
|
|
582
|
+
if (fresh === undefined) throw error
|
|
583
|
+
await revealFrame(ctx, fresh)
|
|
584
|
+
return fresh
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
function runPage(frame, fn, args) {
|
|
589
|
+
const code = '(async () => { try {'
|
|
590
|
+
+ ' const pageFind = ' + pageFind.toString() + ';'
|
|
591
|
+
+ ' const value = await (' + fn.toString() + ')(' + JSON.stringify(args === undefined ? {} : args) + ');'
|
|
592
|
+
+ ' return value === undefined ? null : value;'
|
|
593
|
+
+ ' } catch (error) { return { __dshError: error && error.message ? String(error.message) : String(error) } } })()'
|
|
594
|
+
return frame.executeJavaScript(code, true).then((value) => {
|
|
595
|
+
if (value !== null && typeof value === 'object' && typeof value.__dshError === 'string') {
|
|
596
|
+
throw new Error(value.__dshError)
|
|
597
|
+
}
|
|
598
|
+
return value
|
|
599
|
+
})
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
async function waitForHttpPage(frame, timeoutMs) {
|
|
603
|
+
const deadline = Date.now() + timeoutMs
|
|
604
|
+
while (Date.now() < deadline) {
|
|
605
|
+
const url = safe(() => frame.getURL())
|
|
606
|
+
if (typeof url === 'string' && /^https?:/i.test(url) && safe(() => frame.isLoading()) !== true) {
|
|
607
|
+
const state = await safe(() => frame.executeJavaScript('document.readyState', true))
|
|
608
|
+
if (state === 'complete' || state === 'interactive') return url
|
|
609
|
+
}
|
|
610
|
+
await sleep(150)
|
|
611
|
+
}
|
|
612
|
+
return safe(() => frame.getURL())
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
async function isLoading(frame) {
|
|
616
|
+
try {
|
|
617
|
+
return frame.isLoading() === true
|
|
618
|
+
} catch {
|
|
619
|
+
return false
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function sendKey(frame, key) {
|
|
624
|
+
try {
|
|
625
|
+
frame.focus()
|
|
626
|
+
frame.sendInputEvent({ type: 'keyDown', keyCode: key })
|
|
627
|
+
if (key.length === 1) frame.sendInputEvent({ type: 'char', keyCode: key })
|
|
628
|
+
frame.sendInputEvent({ type: 'keyUp', keyCode: key })
|
|
629
|
+
return 'native'
|
|
630
|
+
} catch {
|
|
631
|
+
return 'dom'
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
async function settle(frame, quietMs, waitStartMs) {
|
|
636
|
+
const start = Date.now()
|
|
637
|
+
await sleep(quietMs)
|
|
638
|
+
const startWindow = waitStartMs === undefined ? 1500 : waitStartMs
|
|
639
|
+
let sawLoading = await isLoading(frame)
|
|
640
|
+
while (Date.now() - start < 15000) {
|
|
641
|
+
const loading = await isLoading(frame)
|
|
642
|
+
if (loading) sawLoading = true
|
|
643
|
+
else if (sawLoading || Date.now() - start > quietMs + startWindow) break
|
|
644
|
+
await sleep(120)
|
|
645
|
+
}
|
|
646
|
+
await sleep(150)
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Return as soon as the page stops changing instead of sleeping a fixed
|
|
651
|
+
* interval: a click that only focuses a field settles in about a tenth of a
|
|
652
|
+
* second, and a page that navigates settles when the address and the
|
|
653
|
+
* document stop moving. This is the latency difference a person feels when
|
|
654
|
+
* clicking through a form.
|
|
655
|
+
*/
|
|
656
|
+
async function settleFast(frame, options) {
|
|
657
|
+
const budget = Number(options?.timeoutMs) > 0 ? Number(options.timeoutMs) : 2500
|
|
658
|
+
const quiet = Number(options?.quietMs) > 0 ? Number(options.quietMs) : 180
|
|
659
|
+
const floor = Number(options?.minMs) > 0 ? Number(options.minMs) : 80
|
|
660
|
+
const start = Date.now()
|
|
661
|
+
try {
|
|
662
|
+
await frame.executeJavaScript('(() => { if (!window.__dshScObs) { window.__dshScMutations = 0; window.__dshScObs = new MutationObserver(() => { window.__dshScMutations += 1 }); window.__dshScObs.observe(document.documentElement, { subtree: true, childList: true, characterData: true, attributes: true }) } return 1 })()', true)
|
|
663
|
+
} catch {
|
|
664
|
+
/* a page between documents keeps the polling fallback below */
|
|
665
|
+
}
|
|
666
|
+
let last = ''
|
|
667
|
+
let changedAt = start
|
|
668
|
+
while (Date.now() - start < budget) {
|
|
669
|
+
let probe = null
|
|
670
|
+
try {
|
|
671
|
+
probe = await frame.executeJavaScript('({ url: location.href, ready: document.readyState, n: window.__dshScMutations || 0, len: document.body ? document.body.innerText.length : 0 })', true)
|
|
672
|
+
} catch {
|
|
673
|
+
/* navigation in flight: poll again */
|
|
674
|
+
}
|
|
675
|
+
if (probe !== null && typeof probe === 'object') {
|
|
676
|
+
const signature = String(probe.url) + '|' + String(probe.ready) + '|' + String(probe.n) + '|' + String(probe.len)
|
|
677
|
+
if (signature !== last) {
|
|
678
|
+
last = signature
|
|
679
|
+
changedAt = Date.now()
|
|
680
|
+
}
|
|
681
|
+
if (probe.ready !== 'loading' && Date.now() - changedAt >= quiet && Date.now() - start >= floor) return
|
|
682
|
+
}
|
|
683
|
+
await sleep(60)
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
async function navigateTo(frame, url) {
|
|
688
|
+
try {
|
|
689
|
+
await frame.loadURL(url)
|
|
690
|
+
} catch (error) {
|
|
691
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
692
|
+
if (!/ERR_ABORTED/.test(message)) throw new Error('Navigation to ' + url + ' failed: ' + message)
|
|
693
|
+
}
|
|
694
|
+
await settle(frame, 150)
|
|
695
|
+
await frameReady(frame, 10000)
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
function formatSnapshot(snapshot) {
|
|
699
|
+
const lines = []
|
|
700
|
+
lines.push('url: ' + snapshot.url)
|
|
701
|
+
lines.push('title: ' + (snapshot.title.length > 0 ? snapshot.title : '(untitled)'))
|
|
702
|
+
lines.push('scroll: y=' + String(snapshot.scroll.y) + ' pageHeight=' + String(snapshot.scroll.height))
|
|
703
|
+
lines.push('refs: ' + String(snapshot.refs.length) + (snapshot.refsTruncated ? ' (truncated; refine with eval or selector)' : ''))
|
|
704
|
+
for (const ref of snapshot.refs) {
|
|
705
|
+
const details = []
|
|
706
|
+
if (ref.role !== undefined && ref.role !== '') details.push('role=' + ref.role)
|
|
707
|
+
if (ref.type !== undefined && ref.type !== '') details.push('type=' + ref.type)
|
|
708
|
+
if (ref.value !== undefined && ref.value !== '') details.push('value=' + JSON.stringify(ref.value))
|
|
709
|
+
if (ref.checked !== undefined) details.push('checked=' + String(ref.checked))
|
|
710
|
+
if (ref.label !== undefined && ref.label !== '') details.push(JSON.stringify(ref.label))
|
|
711
|
+
if (ref.href !== undefined) details.push('-> ' + ref.href)
|
|
712
|
+
if (ref.options !== undefined) {
|
|
713
|
+
const shown = ref.options.slice(0, 8).map((option) => option.value + (option.selected ? '*' : ''))
|
|
714
|
+
details.push('options[' + String(ref.options.length) + ']=' + shown.join('|') + (ref.options.length > 8 ? '|...' : ''))
|
|
715
|
+
}
|
|
716
|
+
lines.push('#' + String(ref.ref) + ' <' + ref.tag + '> ' + details.join(' '))
|
|
717
|
+
}
|
|
718
|
+
if (snapshot.refs.length === 0) lines.push('(no visible interactive elements)')
|
|
719
|
+
lines.push('')
|
|
720
|
+
lines.push('text:')
|
|
721
|
+
lines.push(snapshot.text.length > 0 ? snapshot.text : '(empty page text)')
|
|
722
|
+
if (snapshot.textTruncated) lines.push('(text truncated; use eval to read a specific region)')
|
|
723
|
+
return lines.join('\n')
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
function statusText(ctx) {
|
|
727
|
+
const frames = browserFrames()
|
|
728
|
+
const sidebarRight = sidebarRightOf(ctx)
|
|
729
|
+
const lines = []
|
|
730
|
+
lines.push('platform: ' + (globalThis.dshDesktop === undefined ? 'web' : 'desktop'))
|
|
731
|
+
lines.push('browserFrames: ' + String(frames.length))
|
|
732
|
+
let activeTab
|
|
733
|
+
if (sidebarRight !== undefined && sidebarRight !== null && typeof sidebarRight.active === 'function') {
|
|
734
|
+
const tab = safe(() => sidebarRight.active())
|
|
735
|
+
if (tab !== undefined && tab !== null) {
|
|
736
|
+
activeTab = { id: tab.id, kind: tab.kind, title: tab.title }
|
|
737
|
+
lines.push('activeTab: ' + JSON.stringify(activeTab))
|
|
738
|
+
lines.push('rightbarExpanded: ' + String(safe(() => sidebarRight.isExpanded()) === true))
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
frames.forEach((frame, index) => {
|
|
742
|
+
const info = frameTabInfo(ctx, frame)
|
|
743
|
+
lines.push('frame[' + String(index) + '] visible=' + String(frameIsVisible(frame))
|
|
744
|
+
+ ' url=' + String(safe(() => frame.getURL()) || '(unknown)')
|
|
745
|
+
+ ' title=' + JSON.stringify(String(safe(() => frame.getTitle()) || ''))
|
|
746
|
+
+ ' loading=' + String(safe(() => frame.isLoading()) === true)
|
|
747
|
+
+ (info.same ? '' : ' (Session ' + String(info.frameSession) + ' is not the one on screen)'))
|
|
748
|
+
})
|
|
749
|
+
if (frames.length === 0) lines.push('No Browser frame is mounted. Use action "open" with a url to create the tab.')
|
|
750
|
+
return lines.join('\n')
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
async function capture(frame) {
|
|
754
|
+
if (typeof frame.capturePage !== 'function') throw new Error('This Harness build does not expose capturePage on the Browser frame.')
|
|
755
|
+
/* capturePage can never settle for an occluded guest; a deadline turns a
|
|
756
|
+
wedged call into an answer instead of a bridge stuck for minutes. */
|
|
757
|
+
let timer
|
|
758
|
+
const image = await Promise.race([
|
|
759
|
+
frame.capturePage(),
|
|
760
|
+
new Promise((_, reject) => {
|
|
761
|
+
timer = setTimeout(() => reject(new Error('the screenshot did not finish within 10s — is the Harness window visible?')), 10000)
|
|
762
|
+
}),
|
|
763
|
+
]).finally(() => clearTimeout(timer))
|
|
764
|
+
if (image === undefined || image === null) throw new Error('The Browser frame returned no screenshot.')
|
|
765
|
+
if (typeof image.isEmpty === 'function' && image.isEmpty()) throw new Error('The Browser frame returned an empty screenshot.')
|
|
766
|
+
const dataUrl = String(image.toDataURL())
|
|
767
|
+
const prefix = 'data:image/png;base64,'
|
|
768
|
+
if (!dataUrl.startsWith(prefix)) throw new Error('The screenshot used an unexpected encoding.')
|
|
769
|
+
const size = typeof image.getSize === 'function' ? image.getSize() : { width: 0, height: 0 }
|
|
770
|
+
return { base64: dataUrl.slice(prefix.length), width: size.width, height: size.height }
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
async function executeCommand(state, command) {
|
|
774
|
+
const action = String(command.action)
|
|
775
|
+
const args = command.args !== null && typeof command.args === 'object' ? command.args : {}
|
|
776
|
+
const ctx = state.ctx
|
|
777
|
+
|
|
778
|
+
if (action === 'status') return { text: statusText(ctx) }
|
|
779
|
+
|
|
780
|
+
if (action === 'open') {
|
|
781
|
+
const url = String(args.url || '')
|
|
782
|
+
const side = sidebarRightOf(ctx)
|
|
783
|
+
if (side === undefined || side === null) throw new Error('The right Sidebar is unavailable in this Harness page, so a Browser tab cannot be opened (Desktop app required).')
|
|
784
|
+
const existing = pickFrame(state, ctx)
|
|
785
|
+
/* A Browser tab already on screen is reused and navigated. A tab that is
|
|
786
|
+
hidden, or belongs to another Session, is not navigated: open a tab in
|
|
787
|
+
the current Session instead, which is what the person will see. */
|
|
788
|
+
if (existing !== undefined && frameIsVisible(existing)) {
|
|
789
|
+
await navigateTo(existing, url)
|
|
790
|
+
state.last = existing
|
|
791
|
+
return { text: 'Opened ' + url + '\n' + statusText(ctx) }
|
|
792
|
+
}
|
|
793
|
+
let opened
|
|
794
|
+
try {
|
|
795
|
+
opened = side.openTab('browser', { params: { url } })
|
|
796
|
+
} catch (error) {
|
|
797
|
+
throw new Error('Could not open the Browser tab: ' + (error instanceof Error ? error.message : String(error)))
|
|
798
|
+
}
|
|
799
|
+
if (opened !== undefined && opened !== null && typeof opened.then === 'function') {
|
|
800
|
+
await opened.catch(() => {})
|
|
801
|
+
}
|
|
802
|
+
/* Do not wait out a chain of long timeouts: a frame that appears does so
|
|
803
|
+
within a few seconds, and a missing one is reported instead of hanging
|
|
804
|
+
the tool call for a minute and a half. */
|
|
805
|
+
const deadline = Date.now() + 8000
|
|
806
|
+
while (Date.now() < deadline) {
|
|
807
|
+
const frame = pickFrame(state, ctx)
|
|
808
|
+
if (frame !== undefined) {
|
|
809
|
+
state.last = frame
|
|
810
|
+
await frameReady(frame, 5000).catch(() => {})
|
|
811
|
+
return { text: 'Opened ' + url + '\n' + statusText(ctx) }
|
|
812
|
+
}
|
|
813
|
+
await sleep(150)
|
|
814
|
+
}
|
|
815
|
+
return {
|
|
816
|
+
text: 'Requested a Browser tab at ' + url + ', but no frame appeared within 8s.\n'
|
|
817
|
+
+ 'The tab may still be loading; call action "status" in a moment.\n' + statusText(ctx),
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
const frame = await activeFrame(state, ctx, {})
|
|
822
|
+
|
|
823
|
+
switch (action) {
|
|
824
|
+
case 'navigate': {
|
|
825
|
+
const url = String(args.url || '')
|
|
826
|
+
await navigateTo(frame, url)
|
|
827
|
+
return { text: 'Navigated to ' + url + '\n' + statusText(ctx) }
|
|
828
|
+
}
|
|
829
|
+
case 'back': {
|
|
830
|
+
if (safe(() => frame.canGoBack()) !== true) throw new Error('The Browser tab has no back history.')
|
|
831
|
+
safe(() => frame.goBack())
|
|
832
|
+
await settle(frame, 200)
|
|
833
|
+
await frameReady(frame, 10000)
|
|
834
|
+
return { text: 'Went back.\n' + statusText(ctx) }
|
|
835
|
+
}
|
|
836
|
+
case 'forward': {
|
|
837
|
+
if (safe(() => frame.canGoForward()) !== true) throw new Error('The Browser tab has no forward history.')
|
|
838
|
+
safe(() => frame.goForward())
|
|
839
|
+
await settle(frame, 200)
|
|
840
|
+
await frameReady(frame, 10000)
|
|
841
|
+
return { text: 'Went forward.\n' + statusText(ctx) }
|
|
842
|
+
}
|
|
843
|
+
case 'reload': {
|
|
844
|
+
safe(() => frame.reload())
|
|
845
|
+
await settle(frame, 250)
|
|
846
|
+
await frameReady(frame, 15000)
|
|
847
|
+
return { text: 'Reloaded the page.\n' + statusText(ctx) }
|
|
848
|
+
}
|
|
849
|
+
case 'snapshot': {
|
|
850
|
+
const snapshot = await runPage(frame, pageSnapshot, {
|
|
851
|
+
maxRefs: Number(args.max_refs) > 0 ? Math.min(Number(args.max_refs), LIST_LIMIT) : 80,
|
|
852
|
+
maxTextChars: Number(args.max_text_chars) > 0 ? Number(args.max_text_chars) : 12000,
|
|
853
|
+
includeText: args.include_text !== false,
|
|
854
|
+
})
|
|
855
|
+
return { text: formatSnapshot(snapshot) }
|
|
856
|
+
}
|
|
857
|
+
case 'read': {
|
|
858
|
+
const result = await runPage(frame, pageRead, {
|
|
859
|
+
query: args.query,
|
|
860
|
+
maxChars: Number(args.max_chars) > 0 ? Number(args.max_chars) : 4000,
|
|
861
|
+
})
|
|
862
|
+
const header = result.mode === 'query'
|
|
863
|
+
? 'Read ' + String(result.matched) + ' of ' + String(result.candidates) + ' matching passage(s) for ' + JSON.stringify(String(result.query)) + '.'
|
|
864
|
+
: 'Read the visible text.'
|
|
865
|
+
return {
|
|
866
|
+
text: header + '\n' + String(result.url) + '\n# ' + String(result.title)
|
|
867
|
+
+ '\n\n' + String(result.text) + (result.truncated === true ? '\n…(truncated)' : ''),
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
case 'diff': {
|
|
871
|
+
const result = await runPage(frame, pageDiff, { maxChars: Number(args.max_chars) > 0 ? Number(args.max_chars) : 4000 })
|
|
872
|
+
if (result.first === true) {
|
|
873
|
+
return { text: 'First diff: this page is the baseline now. Run "diff" again after an action to see what changed.\n' + String(result.url) }
|
|
874
|
+
}
|
|
875
|
+
return {
|
|
876
|
+
text: (result.changed === true ? 'Changed since the previous diff.' : 'No visible change since the previous diff.')
|
|
877
|
+
+ '\n' + String(result.url) + '\n# ' + String(result.title)
|
|
878
|
+
+ (String(result.text).length > 0 ? '\n\n' + String(result.text) : ''),
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
case 'tabs': {
|
|
882
|
+
const side = sidebarRightOf(ctx)
|
|
883
|
+
if (side === undefined || side === null) throw new Error('The right Sidebar is unavailable in this Harness page.')
|
|
884
|
+
const open = safe(() => side.openTabs?.getSnapshot())
|
|
885
|
+
const active = safe(() => side.active())
|
|
886
|
+
const rows = (Array.isArray(open) ? open : []).filter((tab) => tab.kind === 'browser')
|
|
887
|
+
if (rows.length === 0) return { text: 'No Browser tab is open. Use action "open" with a url.' }
|
|
888
|
+
const activeId = active === undefined || active === null ? undefined : String(active.id)
|
|
889
|
+
const lines = rows.map((tab) => {
|
|
890
|
+
const id = String(tab.tabId ?? tab.id ?? '')
|
|
891
|
+
const holder = id === '' ? undefined : frameForTab({ id })
|
|
892
|
+
const url = String(tab.url ?? tab.address ?? (holder === undefined ? '' : safe(() => holder.getURL())) ?? '')
|
|
893
|
+
const title = String(tab.title ?? tab.label ?? (holder === undefined ? '' : safe(() => holder.getTitle())) ?? '')
|
|
894
|
+
const mark = activeId !== undefined && activeId === id ? '*' : ' '
|
|
895
|
+
return mark + ' ' + id + ' ' + title + ' ' + url
|
|
896
|
+
})
|
|
897
|
+
return { text: 'Browser tabs (' + String(rows.length) + ', * = active):\n' + lines.join('\n') }
|
|
898
|
+
}
|
|
899
|
+
case 'focus': {
|
|
900
|
+
const side = sidebarRightOf(ctx)
|
|
901
|
+
if (side === undefined || side === null) throw new Error('The right Sidebar is unavailable in this Harness page.')
|
|
902
|
+
side.focus(String(args.tab_id))
|
|
903
|
+
await sleep(250)
|
|
904
|
+
return { text: 'Focused Browser tab ' + String(args.tab_id) + '.\n' + statusText(ctx) }
|
|
905
|
+
}
|
|
906
|
+
case 'close_tab': {
|
|
907
|
+
const side = sidebarRightOf(ctx)
|
|
908
|
+
if (side === undefined || side === null) throw new Error('The right Sidebar is unavailable in this Harness page.')
|
|
909
|
+
side.close(String(args.tab_id))
|
|
910
|
+
state.last = undefined
|
|
911
|
+
return { text: 'Closed Browser tab ' + String(args.tab_id) + '.' }
|
|
912
|
+
}
|
|
913
|
+
case 'click': {
|
|
914
|
+
const outcome = await runPage(frame, pageClick, { ref: args.ref, selector: args.selector })
|
|
915
|
+
await settleFast(frame)
|
|
916
|
+
return {
|
|
917
|
+
text: 'Clicked <' + outcome.clicked.tag + '> ' + JSON.stringify(outcome.clicked.label)
|
|
918
|
+
+ '\nurl: ' + String(safe(() => frame.getURL()) || outcome.url)
|
|
919
|
+
+ '\ntitle: ' + JSON.stringify(String(safe(() => frame.getTitle()) || ''))
|
|
920
|
+
+ '\nTake a new snapshot before the next ref-based action.',
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
case 'type': {
|
|
924
|
+
const outcome = await runPage(frame, pageType, { ref: args.ref, selector: args.selector, text: args.text, submit: args.submit === true })
|
|
925
|
+
await settleFast(frame, { quietMs: 150 })
|
|
926
|
+
return {
|
|
927
|
+
text: 'Typed ' + String(outcome.typed) + ' characters into <' + outcome.target + '>.'
|
|
928
|
+
+ (args.submit === true ? ' Submitted the form.' : '')
|
|
929
|
+
+ '\nurl: ' + String(safe(() => frame.getURL()) || outcome.url)
|
|
930
|
+
+ '\ntitle: ' + JSON.stringify(String(safe(() => frame.getTitle()) || '')),
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
case 'press': {
|
|
934
|
+
const key = String(args.key)
|
|
935
|
+
if (sendKey(frame, key) === 'dom') {
|
|
936
|
+
const outcome = await runPage(frame, pagePress, { key })
|
|
937
|
+
await settleFast(frame, { quietMs: 120 })
|
|
938
|
+
return { text: 'Pressed ' + JSON.stringify(outcome.pressed) + ' on <' + outcome.active + '> (DOM event fallback).' }
|
|
939
|
+
}
|
|
940
|
+
if (key === 'Enter' || key === ' ' || key === 'Space') {
|
|
941
|
+
// Native key events drive page handlers; the DOM pass also submits a
|
|
942
|
+
// focused form when the guest swallowed the native default action.
|
|
943
|
+
try {
|
|
944
|
+
await runPage(frame, pagePress, { key })
|
|
945
|
+
} catch {
|
|
946
|
+
// The guest may already be navigating.
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
await settleFast(frame, { quietMs: 120 })
|
|
950
|
+
return {
|
|
951
|
+
text: 'Pressed ' + JSON.stringify(key) + '.'
|
|
952
|
+
+ '\nurl: ' + String(safe(() => frame.getURL()) || '')
|
|
953
|
+
+ '\ntitle: ' + JSON.stringify(String(safe(() => frame.getTitle()) || '')),
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
case 'select': {
|
|
957
|
+
const outcome = await runPage(frame, pageSelect, { ref: args.ref, selector: args.selector, value: args.value })
|
|
958
|
+
await settleFast(frame, { quietMs: 120 })
|
|
959
|
+
return { text: 'Selected value ' + JSON.stringify(outcome.value) + ' (index ' + String(outcome.selectedIndex) + ').'
|
|
960
|
+
+ '\ntitle: ' + JSON.stringify(String(safe(() => frame.getTitle()) || '')) }
|
|
961
|
+
}
|
|
962
|
+
case 'scroll': {
|
|
963
|
+
const outcome = await runPage(frame, pageScroll, {
|
|
964
|
+
ref: args.ref,
|
|
965
|
+
selector: args.selector,
|
|
966
|
+
direction: args.direction,
|
|
967
|
+
amount: args.amount,
|
|
968
|
+
})
|
|
969
|
+
return { text: 'Scrolled to y=' + String(outcome.y === undefined ? '?' : outcome.y) }
|
|
970
|
+
}
|
|
971
|
+
case 'wait': {
|
|
972
|
+
const outcome = await runPage(frame, pageWait, {
|
|
973
|
+
selector: args.selector,
|
|
974
|
+
text: args.text,
|
|
975
|
+
timeoutMs: Number(args.timeout_ms) > 0 ? Number(args.timeout_ms) : 5000,
|
|
976
|
+
})
|
|
977
|
+
return { text: 'Appeared: ' + JSON.stringify(outcome) }
|
|
978
|
+
}
|
|
979
|
+
case 'eval': {
|
|
980
|
+
const script = String(args.script)
|
|
981
|
+
const code = '(async () => { const value = (' + script + '); const resolved = await value;'
|
|
982
|
+
+ ' try { return JSON.parse(JSON.stringify(resolved === undefined ? null : resolved)) } catch { return String(resolved) } })()'
|
|
983
|
+
const value = await frame.executeJavaScript(code, true)
|
|
984
|
+
const text = JSON.stringify(value, null, 2)
|
|
985
|
+
return { text: text.length > 20000 ? text.slice(0, 20000) + '\n(truncated)' : text }
|
|
986
|
+
}
|
|
987
|
+
case 'screenshot': {
|
|
988
|
+
/* One attempt, fail fast: reopening a tab to chase a capturable frame
|
|
989
|
+
can cost a minute and is worse than telling the person to bring the
|
|
990
|
+
Browser panel to the front. */
|
|
991
|
+
await revealFrame(ctx, frame)
|
|
992
|
+
const image = await capture(frame)
|
|
993
|
+
const text = 'Captured the visible Browser page.\n' + statusText(ctx)
|
|
994
|
+
return { text, png: image }
|
|
995
|
+
}
|
|
996
|
+
case 'close': {
|
|
997
|
+
const sidebarRight = sidebarRightOf(ctx)
|
|
998
|
+
if (sidebarRight === undefined || sidebarRight === null) throw new Error('The right Sidebar is unavailable in this Harness page.')
|
|
999
|
+
const tab = safe(() => sidebarRight.active())
|
|
1000
|
+
if (tab !== undefined && tab !== null && tab.kind === 'browser' && tab.id !== undefined && tab.id !== null) {
|
|
1001
|
+
sidebarRight.close(tab.id)
|
|
1002
|
+
state.last = undefined
|
|
1003
|
+
return { text: 'Closed Browser tab ' + String(tab.id) + '.' }
|
|
1004
|
+
}
|
|
1005
|
+
// The Browser tab may sit behind another tab or in a Session that is no longer
|
|
1006
|
+
// on screen; close the tab that owns our frame instead of the visible one.
|
|
1007
|
+
const frame = pickFrame(state, ctx)
|
|
1008
|
+
const info = frame === undefined ? undefined : frameTabInfo(ctx, frame)
|
|
1009
|
+
if (info !== undefined && info.frameSession !== undefined && info.tabId !== undefined
|
|
1010
|
+
&& typeof sidebarRight.closeIn === 'function') {
|
|
1011
|
+
try {
|
|
1012
|
+
sidebarRight.closeIn(info.frameSession, info.tabId)
|
|
1013
|
+
state.last = undefined
|
|
1014
|
+
return { text: 'Closed Browser tab ' + String(info.tabId) + '.' }
|
|
1015
|
+
} catch (error) {
|
|
1016
|
+
throw new Error('Could not close the Browser tab: ' + (error instanceof Error ? error.message : String(error)))
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
if (tab !== undefined && tab !== null) throw new Error('The active right-Sidebar tab is "' + String(tab.kind) + '", not a Browser tab.')
|
|
1020
|
+
throw new Error('There is no active right-Sidebar tab to close.')
|
|
1021
|
+
}
|
|
1022
|
+
case 'batch': {
|
|
1023
|
+
const steps = Array.isArray(args.steps) ? args.steps.slice(0, 12) : []
|
|
1024
|
+
if (steps.length === 0) throw new Error('"batch" needs a non-empty steps array')
|
|
1025
|
+
const started = Date.now()
|
|
1026
|
+
const lines = []
|
|
1027
|
+
let failed = ''
|
|
1028
|
+
for (let index = 0; index < steps.length; index += 1) {
|
|
1029
|
+
const step = steps[index] !== null && typeof steps[index] === 'object' ? steps[index] : {}
|
|
1030
|
+
const sub = String(step.action || '')
|
|
1031
|
+
if (BATCH_ACTIONS.has(sub) !== true) {
|
|
1032
|
+
failed = 'step ' + String(index + 1) + ': ' + JSON.stringify(sub) + ' is not allowed inside batch'
|
|
1033
|
+
break
|
|
1034
|
+
}
|
|
1035
|
+
try {
|
|
1036
|
+
const result = await executeCommand(state, { action: sub, args: step })
|
|
1037
|
+
const summary = String(result?.text || '').split('\n').filter((line) => line !== '').slice(0, 3).join(' · ')
|
|
1038
|
+
lines.push(String(index + 1) + '. ' + sub + ': ' + summary.slice(0, 300))
|
|
1039
|
+
} catch (error) {
|
|
1040
|
+
failed = 'step ' + String(index + 1) + ' (' + sub + '): ' + (error instanceof Error ? error.message : String(error))
|
|
1041
|
+
break
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
return {
|
|
1045
|
+
text: 'Batch ' + String(steps.length) + ' step(s) in ' + String(Date.now() - started) + ' ms.'
|
|
1046
|
+
+ (failed === '' ? '' : '\nfailed: ' + failed)
|
|
1047
|
+
+ '\n' + lines.join('\n'),
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
default:
|
|
1051
|
+
throw new Error('Unsupported browser action ' + JSON.stringify(action) + '.')
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
function randomInstanceId() {
|
|
1056
|
+
try {
|
|
1057
|
+
return 'page-' + crypto.randomUUID()
|
|
1058
|
+
} catch {
|
|
1059
|
+
return 'page-' + String(Date.now()) + '-' + String(Math.random()).slice(2)
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
/** Run a promise with a deadline, so one wedged action never blocks the bridge. */
|
|
1064
|
+
async function withDeadline(promise, ms, label) {
|
|
1065
|
+
let timer
|
|
1066
|
+
try {
|
|
1067
|
+
return await Promise.race([
|
|
1068
|
+
promise,
|
|
1069
|
+
new Promise((_, reject) => {
|
|
1070
|
+
timer = setTimeout(() => reject(new Error(label + ' did not finish within ' + String(Math.round(ms / 1000)) + 's')), ms)
|
|
1071
|
+
}),
|
|
1072
|
+
])
|
|
1073
|
+
} finally {
|
|
1074
|
+
clearTimeout(timer)
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
function apply(ctx) {
|
|
1079
|
+
const state = { ctx, last: undefined, running: true }
|
|
1080
|
+
/* A plugin reload can leave the previous poll loop alive for a moment. The
|
|
1081
|
+
newest instance owns the bridge; older ones stop before taking a command
|
|
1082
|
+
they would execute with stale code. */
|
|
1083
|
+
const generation = (globalThis.__dshScGeneration = (globalThis.__dshScGeneration || 0) + 1)
|
|
1084
|
+
|
|
1085
|
+
async function post(name, payload, signal) {
|
|
1086
|
+
const url = new URL('sidebar-control/' + name, document.baseURI)
|
|
1087
|
+
const response = await fetch(url, {
|
|
1088
|
+
method: 'POST',
|
|
1089
|
+
headers: { 'content-type': 'application/json' },
|
|
1090
|
+
body: JSON.stringify(payload === undefined ? {} : payload),
|
|
1091
|
+
credentials: 'same-origin',
|
|
1092
|
+
signal,
|
|
1093
|
+
})
|
|
1094
|
+
if (response.ok !== true) throw new Error('Sidebar Browser Control request failed: HTTP ' + String(response.status))
|
|
1095
|
+
return await response.json()
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
async function loop(signal) {
|
|
1099
|
+
const instance = randomInstanceId()
|
|
1100
|
+
while (state.running === true && signal.aborted === false) {
|
|
1101
|
+
if (globalThis.__dshScGeneration !== generation) return
|
|
1102
|
+
try {
|
|
1103
|
+
const outcome = await post('poll', { instance, waitMs: POLL_WAIT_MS }, signal)
|
|
1104
|
+
if (signal.aborted || state.running === false) return
|
|
1105
|
+
const commands = Array.isArray(outcome && outcome.commands) ? outcome.commands : []
|
|
1106
|
+
for (const command of commands) {
|
|
1107
|
+
if (signal.aborted || state.running === false) return
|
|
1108
|
+
let message
|
|
1109
|
+
try {
|
|
1110
|
+
message = { ok: true, value: await withDeadline(executeCommand(state, command), 45000, 'The action "' + String(command.action) + '"') }
|
|
1111
|
+
} catch (error) {
|
|
1112
|
+
message = {
|
|
1113
|
+
ok: false,
|
|
1114
|
+
error: { message: error instanceof Error ? error.message : String(error) },
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
try {
|
|
1118
|
+
await post('result', { instance, commandId: command.id, ...message }, signal)
|
|
1119
|
+
} catch {
|
|
1120
|
+
// The Host may have timed the command out; the next poll recovers.
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
} catch (error) {
|
|
1124
|
+
if (signal.aborted || state.running === false) return
|
|
1125
|
+
await sleep(900)
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
ctx.effect(() => {
|
|
1131
|
+
const controller = new AbortController()
|
|
1132
|
+
loop(controller.signal).catch((error) => {
|
|
1133
|
+
console.error('[sidebar-control] poll loop stopped', error)
|
|
1134
|
+
})
|
|
1135
|
+
return () => {
|
|
1136
|
+
state.running = false
|
|
1137
|
+
controller.abort()
|
|
1138
|
+
}
|
|
1139
|
+
}, 'sidebar-browser-control: command poller')
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
return {
|
|
1143
|
+
inject: [],
|
|
1144
|
+
apply,
|
|
1145
|
+
}
|
|
1146
|
+
},
|
|
1147
|
+
})
|