dsh-sidebar-control 0.1.3 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/CHANGELOG.md +80 -0
  2. package/README.md +291 -61
  3. package/client.js +487 -214
  4. package/index.js +97 -14
  5. package/package.json +2 -1
package/client.js CHANGED
@@ -14,6 +14,9 @@ window.__ModuleLoader__.load({
14
14
  const POLL_WAIT_MS = 25000
15
15
  const FRAME_SELECTOR = 'webview[data-sidebar-browser-frame]'
16
16
  const LIST_LIMIT = 120
17
+ const VERSION = '0.2.1'
18
+ const DEFAULT_ACTION_BUDGET_MS = 85000
19
+ const MAX_ACTION_BUDGET_MS = 295000
17
20
  /* Actions a batch may contain. A batch is one tool call for several steps,
18
21
  so it saves the model a round trip per step; it may not nest or open tabs. */
19
22
  const BATCH_ACTIONS = new Set(['navigate', 'click', 'type', 'press', 'select', 'scroll', 'wait', 'eval', 'read', 'diff', 'snapshot'])
@@ -50,24 +53,33 @@ window.__ModuleLoader__.load({
50
53
  '[contenteditable="true"]',
51
54
  ].join(',')
52
55
  const isVisible = (element) => {
56
+ if (element.closest('[hidden], [inert]') !== null) return false
57
+ /* Chromium still lays out content of a closed <details> here, so its
58
+ rect is non-zero even though the person cannot see it. */
59
+ const details = element.closest('details:not([open])')
60
+ if (details !== null && element.tagName !== 'SUMMARY') return false
53
61
  const rect = element.getBoundingClientRect()
54
62
  if (rect.width < 1 || rect.height < 1) return false
55
63
  const style = getComputedStyle(element)
56
64
  return style.visibility !== 'hidden' && style.display !== 'none' && Number(style.opacity) !== 0
57
65
  }
58
66
  const labelOf = (element) => {
59
- const raw = element.getAttribute('aria-label')
60
- || element.getAttribute('title')
61
- || element.getAttribute('placeholder')
62
- || element.getAttribute('alt')
63
- || element.innerText
64
- || element.value
65
- || element.getAttribute('name')
66
- || ''
67
+ let raw = element.getAttribute('aria-label') || element.getAttribute('title') || ''
68
+ if (raw === '' && element.labels !== undefined && element.labels !== null && element.labels.length > 0) {
69
+ raw = element.labels[0].innerText
70
+ }
71
+ if (raw === '' && (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA')) raw = element.getAttribute('placeholder') || ''
72
+ if (raw === '') raw = element.innerText || element.getAttribute('alt') || ''
73
+ if (raw === '' && (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA')) raw = element.value || ''
74
+ if (raw === '') raw = element.getAttribute('name') || ''
67
75
  return String(raw).replace(/\s+/g, ' ').trim().slice(0, 160)
68
76
  }
69
77
  const refs = []
70
78
  let refsTruncated = false
79
+ /* Snapshot refs are positional. Clear numbers from an older snapshot first,
80
+ otherwise an element skipped this time keeps a number that a fresh ref
81
+ may now own, and a stale ref could hit the wrong element. */
82
+ for (const stale of document.querySelectorAll('[data-dsh-bc-ref]')) stale.removeAttribute('data-dsh-bc-ref')
71
83
  const candidates = document.querySelectorAll(selector)
72
84
  for (const element of candidates) {
73
85
  if (refs.length >= maxRefs) {
@@ -87,7 +99,10 @@ window.__ModuleLoader__.load({
87
99
  if (label.length > 0) entry.label = label
88
100
  if (tag === 'a') {
89
101
  const href = element.getAttribute('href')
90
- if (href !== null) entry.href = String(href).slice(0, 500)
102
+ if (href !== null) {
103
+ const text = String(href)
104
+ entry.href = text.length > 240 ? text.slice(0, 240) + '…' : text
105
+ }
91
106
  if (label.length === 0) {
92
107
  const image = element.querySelector('img[alt]')
93
108
  const alt = image === null ? '' : String(image.getAttribute('alt') || '').replace(/\s+/g, ' ').trim()
@@ -122,6 +137,8 @@ window.__ModuleLoader__.load({
122
137
  },
123
138
  refs,
124
139
  refsTruncated,
140
+ iframes: document.querySelectorAll('iframe').length,
141
+ includeText,
125
142
  text: text.slice(0, maxTextChars),
126
143
  textTruncated: text.length > maxTextChars,
127
144
  }
@@ -140,61 +157,122 @@ window.__ModuleLoader__.load({
140
157
  function pageClick(options) {
141
158
  const element = pageFind(options)
142
159
  if (element === null) {
143
- throw new Error(options.ref !== undefined
160
+ const iframes = document.querySelectorAll('iframe').length
161
+ const hint = iframes > 0
162
+ ? ' (the page has ' + String(iframes) + ' iframe(s); refs and selectors only see the top document)'
163
+ : ''
164
+ throw new Error((options.ref !== undefined
144
165
  ? 'No element has ref ' + String(options.ref) + ' on this page; take a new snapshot.'
145
- : 'No element matches selector ' + String(options.selector) + '.')
166
+ : 'No element matches selector ' + String(options.selector) + '.') + hint)
167
+ }
168
+ const describe = (node) => {
169
+ const label = node.getAttribute('aria-label')
170
+ || node.getAttribute('title')
171
+ || (node.labels !== undefined && node.labels !== null && node.labels.length > 0 ? node.labels[0].innerText : '')
172
+ || node.innerText
173
+ || node.getAttribute('placeholder')
174
+ || node.getAttribute('alt')
175
+ || node.value
176
+ || ''
177
+ return { tag: node.tagName.toLowerCase(), label: String(label).replace(/\s+/g, ' ').trim().slice(0, 120) }
146
178
  }
147
- const label = String(element.innerText || element.getAttribute('aria-label') || element.getAttribute('title') || '')
148
- .replace(/\s+/g, ' ')
149
- .trim()
150
- .slice(0, 120)
151
179
  element.scrollIntoView({ block: 'center', inline: 'center' })
152
180
  const target = element.closest('a,button,label,summary,[role="button"],[role="link"]') || element
153
181
  target.click()
154
- return { clicked: { tag: target.tagName.toLowerCase(), label }, url: String(location.href) }
182
+ /* element.click() on an input does not focus it, so a following press would
183
+ land on <body>. Clicking with a mouse focuses text fields; do the same. */
184
+ if (element.matches('input,textarea,select,[contenteditable="true"]') && typeof element.focus === 'function') {
185
+ try {
186
+ element.focus({ preventScroll: true })
187
+ } catch {
188
+ element.focus()
189
+ }
190
+ }
191
+ const info = describe(target)
192
+ const href = target.tagName === 'A' ? String(target.getAttribute('href') || '') : ''
193
+ return { clicked: { tag: info.tag, label: info.label, ...(href === '' ? {} : { href }) }, url: String(location.href) }
155
194
  }
156
195
 
157
196
  function pageType(options) {
197
+ const unsupported = new Set(['file', 'checkbox', 'radio', 'button', 'submit', 'reset', 'image'])
158
198
  const element = pageFind(options)
159
199
  if (element === null) {
160
- throw new Error(options.ref !== undefined
200
+ const iframes = document.querySelectorAll('iframe').length
201
+ const hint = iframes > 0
202
+ ? ' (the page has ' + String(iframes) + ' iframe(s); refs and selectors only see the top document)'
203
+ : ''
204
+ throw new Error((options.ref !== undefined
161
205
  ? 'No element has ref ' + String(options.ref) + ' on this page; take a new snapshot.'
162
- : 'No element matches selector ' + String(options.selector) + '.')
206
+ : 'No element matches selector ' + String(options.selector) + '.') + hint)
207
+ }
208
+ const tag = element.tagName
209
+ const editable = element.isContentEditable === true
210
+ if (tag !== 'INPUT' && tag !== 'TEXTAREA' && editable !== true) {
211
+ throw new Error('Cannot type into <' + tag.toLowerCase() + '>; use a text input, textarea or contenteditable element (for a <select> use action "select").')
212
+ }
213
+ const type = tag === 'INPUT' ? String(element.type || '').toLowerCase() : ''
214
+ if (unsupported.has(type)) {
215
+ throw new Error('Cannot type into <input type="' + type + '">; use action "click" to toggle it (for a <select> use action "select").')
163
216
  }
164
217
  const text = String(options.text)
165
218
  element.scrollIntoView({ block: 'center' })
166
219
  if (typeof element.focus === 'function') element.focus()
167
- if (element.isContentEditable === true) {
220
+ if (editable === true) {
168
221
  element.textContent = text
169
222
  element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }))
223
+ element.dispatchEvent(new Event('change', { bubbles: true }))
170
224
  } else {
171
- const prototype = element.tagName === 'TEXTAREA' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype
225
+ const prototype = tag === 'TEXTAREA' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype
172
226
  const setter = Object.getOwnPropertyDescriptor(prototype, 'value')
173
227
  if (setter !== undefined && typeof setter.set === 'function') setter.set.call(element, text)
174
228
  else element.value = text
175
229
  element.dispatchEvent(new Event('input', { bubbles: true }))
176
230
  element.dispatchEvent(new Event('change', { bubbles: true }))
231
+ /* A number/date/maxlength field silently drops or truncates text. Report
232
+ that instead of claiming success. */
233
+ const applied = String(element.value ?? '')
234
+ if (applied !== text) {
235
+ const shown = applied.length > 120 ? applied.slice(0, 120) + '…' : applied
236
+ throw new Error('<' + tag.toLowerCase() + (type === '' ? '' : ' type="' + type + '"') + '> did not keep the text: value is ' + JSON.stringify(shown) + '. The field may reject, mask or truncate this value.')
237
+ }
177
238
  }
239
+ let submitted = ''
178
240
  if (options.submit === true) {
179
- if (element.form !== null && element.form !== undefined && typeof element.form.requestSubmit === 'function') element.form.requestSubmit()
180
- else element.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true, cancelable: true }))
241
+ if (element.form !== null && element.form !== undefined && typeof element.form.requestSubmit === 'function') {
242
+ element.form.requestSubmit()
243
+ submitted = 'form'
244
+ } else {
245
+ submitted = 'enter'
246
+ }
181
247
  }
182
- return { typed: text.length, target: element.tagName.toLowerCase(), url: String(location.href) }
248
+ return { typed: text.length, target: tag.toLowerCase(), submitted, url: String(location.href) }
183
249
  }
184
250
 
185
251
  function pagePress(options) {
252
+ const KEY_CODES = {
253
+ Enter: 13, Tab: 9, Escape: 27, Esc: 27, Backspace: 8, Delete: 46, Home: 36, End: 35,
254
+ PageUp: 33, PageDown: 34, ArrowUp: 38, ArrowDown: 40, ArrowLeft: 37, ArrowRight: 39,
255
+ ' ': 32, Space: 32,
256
+ }
186
257
  const key = String(options.key)
258
+ const mods = Array.isArray(options.modifiers) ? options.modifiers : []
187
259
  const target = document.activeElement || document.body
188
- const keyCode = key === 'Enter' ? 13
189
- : key === 'Tab' ? 9
190
- : key === 'Escape' ? 27
191
- : key === 'ArrowDown' ? 40
192
- : key === 'ArrowUp' ? 38
193
- : key === 'ArrowLeft' ? 37
194
- : key === 'ArrowRight' ? 39
195
- : key === ' ' || key === 'Space' ? 32
196
- : 0
197
- const init = { key, code: key === ' ' ? 'Space' : key, keyCode, which: keyCode, bubbles: true, cancelable: true }
260
+ const named = KEY_CODES[key] || 0
261
+ const upper = key.length === 1 ? key.toUpperCase() : key
262
+ const keyCode = named || (key.length === 1 ? upper.charCodeAt(0) : 0)
263
+ const code = key.length === 1 && /[a-z0-9]/i.test(key) ? 'Key' + upper : (key === ' ' || key === 'Space' ? 'Space' : key)
264
+ const init = {
265
+ key,
266
+ code,
267
+ keyCode,
268
+ which: keyCode,
269
+ bubbles: true,
270
+ cancelable: true,
271
+ ctrlKey: mods.includes('ctrl'),
272
+ altKey: mods.includes('alt'),
273
+ shiftKey: mods.includes('shift'),
274
+ metaKey: mods.includes('meta'),
275
+ }
198
276
  target.dispatchEvent(new KeyboardEvent('keydown', init))
199
277
  target.dispatchEvent(new KeyboardEvent('keyup', init))
200
278
  if (key === 'Enter' && target.form !== null && target.form !== undefined && (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA')) {
@@ -211,46 +289,54 @@ window.__ModuleLoader__.load({
211
289
  const element = pageFind(options)
212
290
  if (element === null) throw new Error('No select element matches the given ref or selector.')
213
291
  if (element.tagName !== 'SELECT') throw new Error('The target element is <' + element.tagName.toLowerCase() + '>, not a <select>.')
214
- const value = String(options.value)
215
- element.value = value
216
- if (element.value !== value) {
217
- const option = Array.from(element.options).find((candidate) => candidate.text === value || candidate.label === value)
218
- if (option !== undefined) element.value = option.value
292
+ const want = String(options.value)
293
+ const normalize = (value) => String(value === undefined || value === null ? '' : value).trim()
294
+ const list = Array.from(element.options)
295
+ let index = list.findIndex((option) => option.value === want)
296
+ if (index < 0) index = list.findIndex((option) => normalize(option.label || option.text) === want)
297
+ if (index < 0) index = list.findIndex((option) => normalize(option.label || option.text).toLowerCase() === want.toLowerCase())
298
+ if (index < 0) {
299
+ const shown = list.slice(0, 40).map((option) => JSON.stringify(option.value) + '=' + JSON.stringify(normalize(option.label || option.text)))
300
+ throw new Error('No option matches ' + JSON.stringify(want) + ' in this <select>; available: '
301
+ + shown.join(', ') + (list.length > 40 ? ', …' : '') + '.')
219
302
  }
303
+ element.selectedIndex = index
220
304
  element.dispatchEvent(new Event('input', { bubbles: true }))
221
305
  element.dispatchEvent(new Event('change', { bubbles: true }))
222
- return { value: element.value, selectedIndex: element.selectedIndex }
306
+ return { value: element.value, label: normalize(list[index].label || list[index].text), selectedIndex: index }
223
307
  }
224
308
 
225
309
  function pageScroll(options) {
310
+ const direction = String(options.direction || 'down')
311
+ const amount = Number(options.amount) > 0 ? Number(options.amount) : 600
226
312
  if (options.ref !== undefined && options.ref !== null) {
227
313
  const element = pageFind(options)
228
314
  if (element === null) throw new Error('No element has ref ' + String(options.ref) + '; take a new snapshot.')
229
315
  element.scrollIntoView({ block: 'center' })
230
- return { scrolledTo: { ref: options.ref }, y: Math.round(window.scrollY) }
316
+ const rect = element.getBoundingClientRect()
317
+ return {
318
+ x: Math.round(window.scrollX), y: Math.round(window.scrollY),
319
+ target: { ref: options.ref, tag: element.tagName.toLowerCase() }, rectTop: Math.round(rect.top),
320
+ }
231
321
  }
232
- const amount = Number(options.amount) > 0 ? Number(options.amount) : 600
233
- const direction = String(options.direction || 'down')
234
- const dx = direction === 'left' ? -amount : direction === 'right' ? amount : 0
235
- const dy = direction === 'up' ? -amount : amount
236
- window.scrollBy({ left: dx, top: dy, behavior: 'instant' })
237
- return { x: Math.round(window.scrollX), y: Math.round(window.scrollY) }
238
- }
239
-
240
- async function pageWait(options) {
241
- const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : 5000
242
- const deadline = Date.now() + timeoutMs
243
- while (Date.now() < deadline) {
244
- if (typeof options.selector === 'string' && options.selector.length > 0) {
245
- if (document.querySelector(options.selector) !== null) return { found: true, kind: 'selector', selector: options.selector }
246
- } else if (typeof options.text === 'string' && options.text.length > 0) {
247
- const bodyText = document.body === null ? '' : String(document.body.innerText || '')
248
- if (bodyText.includes(options.text)) return { found: true, kind: 'text', text: options.text }
322
+ if (typeof options.selector === 'string' && options.selector.length > 0) {
323
+ const element = pageFind({ selector: options.selector })
324
+ if (element === null) throw new Error('No element matches selector ' + options.selector + '.')
325
+ element.scrollIntoView({ block: 'center' })
326
+ const rect = element.getBoundingClientRect()
327
+ return {
328
+ x: Math.round(window.scrollX), y: Math.round(window.scrollY),
329
+ target: { selector: options.selector, tag: element.tagName.toLowerCase() }, rectTop: Math.round(rect.top),
249
330
  }
250
- await new Promise((resolve) => setTimeout(resolve, 150))
251
331
  }
252
- throw new Error('Timed out after ' + String(timeoutMs) + 'ms waiting for '
253
- + (typeof options.selector === 'string' && options.selector.length > 0 ? 'selector ' + options.selector : 'text ' + String(options.text)) + '.')
332
+ const dx = direction === 'left' ? -amount : direction === 'right' ? amount : 0
333
+ const dy = direction === 'up' ? -amount : direction === 'down' ? amount : 0
334
+ const beforeX = Math.round(window.scrollX)
335
+ const beforeY = Math.round(window.scrollY)
336
+ window.scrollBy({ left: dx, top: dy, behavior: 'instant' })
337
+ const x = Math.round(window.scrollX)
338
+ const y = Math.round(window.scrollY)
339
+ return { x, y, moved: x !== beforeX || y !== beforeY }
254
340
  }
255
341
 
256
342
  /**
@@ -416,11 +502,18 @@ window.__ModuleLoader__.load({
416
502
  }
417
503
  const frames = browserFrames()
418
504
  if (frames.length === 0) return undefined
419
- const visibleFrames = frames.filter(frameIsVisible)
420
- if (state.last !== undefined && state.last.isConnected === true && visibleFrames.includes(state.last)) return state.last
505
+ /* Never adopt a frame that belongs to another Session: driving it would
506
+ navigate/click a page the person is not looking at. */
507
+ const own = frames.filter((frame) => frameTabInfo(ctx, frame).same !== false)
508
+ if (own.length === 0) return undefined
509
+ const visibleFrames = own.filter(frameIsVisible)
510
+ /* A frame proven ours by an earlier command wins; otherwise a visible one;
511
+ a single candidate is unambiguous; several without an active tab is not,
512
+ and guessing could drive another Session's page. */
513
+ if (state.last !== undefined && state.last.isConnected === true && own.includes(state.last)) return state.last
421
514
  if (visibleFrames.length > 0) return visibleFrames[visibleFrames.length - 1]
422
- if (state.last !== undefined && state.last.isConnected === true) return state.last
423
- return frames[frames.length - 1]
515
+ if (own.length === 1) return own[0]
516
+ return undefined
424
517
  }
425
518
 
426
519
  async function frameReady(frame, timeoutMs) {
@@ -440,12 +533,39 @@ window.__ModuleLoader__.load({
440
533
  + (lastError instanceof Error && lastError.message ? ': ' + lastError.message : '. Open or reload the Browser tab and retry.'))
441
534
  }
442
535
 
536
+ /** A freshly mounted webview answers frameReady before it has an address, and
537
+ starts on about:blank while the requested URL is still loading. The dock
538
+ may also replace the tab body while it loads, so re-pick every tick. */
539
+ async function waitPageReady(state, ctx, frame, timeoutMs, allowBlank) {
540
+ const deadline = Date.now() + timeoutMs
541
+ let current = frame
542
+ let seen = ''
543
+ while (Date.now() < deadline) {
544
+ const picked = pickFrame(state, ctx)
545
+ if (picked !== undefined) current = picked
546
+ const url = String(safe(() => current.getURL()) || '')
547
+ if (url !== '' && (allowBlank === true || url !== 'about:blank')) {
548
+ if (seen === '') seen = url
549
+ if (safe(() => current.isLoading()) !== true) {
550
+ const ready = await safe(() => current.executeJavaScript('document.readyState', true))
551
+ if (ready === 'complete' || ready === 'interactive') return { frame: current, url }
552
+ }
553
+ }
554
+ await sleep(150)
555
+ }
556
+ return { frame: current, url: seen }
557
+ }
558
+
443
559
  function sidebarRightOf(ctx) {
444
560
  if (typeof ctx.get === 'function') return ctx.get('sidebarRight')
445
561
  return ctx.sidebarRight
446
562
  }
447
563
 
448
- async function openBrowserTab(ctx, url) {
564
+ /**
565
+ * Ask the right Sidebar for a Browser tab and wait until *its* frame exists.
566
+ * Foreign Session frames are ignored: they are not the tab we just asked for.
567
+ */
568
+ async function openBrowserTab(ctx, url, previousTabId) {
449
569
  const sidebarRight = sidebarRightOf(ctx)
450
570
  if (sidebarRight === undefined || sidebarRight === null) {
451
571
  throw new Error('The right Sidebar is unavailable in this Harness page, so a Browser tab cannot be opened (Desktop app required for Browser control).')
@@ -458,36 +578,27 @@ window.__ModuleLoader__.load({
458
578
  const deadline = Date.now() + 20000
459
579
  while (Date.now() < deadline) {
460
580
  const tab = activeBrowserTab(ctx)
461
- const frame = (tab === undefined ? undefined : frameForTab(tab))
462
- || browserFrames().filter(frameIsVisible)[0]
463
- || browserFrames()[0]
464
- if (frame !== undefined) return frame
581
+ const tabId = tab === undefined || tab.id === undefined || tab.id === null ? undefined : String(tab.id)
582
+ const own = browserFrames().filter((frame) => frameTabInfo(ctx, frame).same !== false)
583
+ const exact = tab === undefined ? undefined : frameForTab(tab)
584
+ const fresh = previousTabId === undefined || tabId === undefined || tabId !== previousTabId
585
+ if (exact !== undefined && fresh) return { frame: exact, tabId }
586
+ const visible = own.filter(frameIsVisible)
587
+ if (visible.length > 0 && fresh) return { frame: visible[visible.length - 1], tabId }
465
588
  await sleep(150)
466
589
  }
467
- throw new Error('The Browser tab was requested but no Browser frame appeared. Check that the right Sidebar Browser is enabled in this Harness build.')
468
- }
469
-
470
- async function settleActiveFrame(state, ctx, fallback) {
471
- let frame = pickFrame(state, ctx) || fallback
472
- if (frame === undefined) return undefined
473
- await frameReady(frame, 15000)
474
- await waitForHttpPage(frame, 30000)
475
- const latest = pickFrame(state, ctx)
476
- if (latest !== undefined && latest !== frame) {
477
- // The dock re-created the tab body while loading (new occurrence/lease).
478
- frame = latest
479
- await frameReady(frame, 15000)
480
- await waitForHttpPage(frame, 30000)
481
- }
482
- state.last = frame
483
- return frame
590
+ throw new Error('A Browser tab was requested for ' + url + ' but no Browser frame appeared within 20s. '
591
+ + 'The tab may still be loading; call action "status" in a moment.')
484
592
  }
485
593
 
486
594
  async function activeFrame(state, ctx, options) {
487
- let frame = pickFrame(state, ctx)
488
- if (frame === undefined && options.openUrl !== undefined) frame = await openBrowserTab(ctx, options.openUrl)
595
+ const frame = pickFrame(state, ctx)
489
596
  if (frame === undefined) {
490
- throw new Error('No Browser tab is open in the right Sidebar. Call the browser tool with action "open" and a url first.')
597
+ const total = browserFrames().length
598
+ const foreign = browserFrames().filter((candidate) => frameTabInfo(ctx, candidate).same === false).length
599
+ throw new Error('No Browser tab is available for this Session'
600
+ + (foreign > 0 ? ' (' + String(foreign) + ' Browser tab(s) belong to other Sessions and are left alone)' : '')
601
+ + (total > 0 ? '. ' + String(total) + ' Browser frame(s) exist but none can be proven to belong here; pass a url to action "open" to establish one.' : '. Call action "open" with a url first.'))
491
602
  }
492
603
  await frameReady(frame, options.readyTimeoutMs === undefined ? 15000 : options.readyTimeoutMs)
493
604
  state.last = frame
@@ -560,31 +671,6 @@ window.__ModuleLoader__.load({
560
671
  }
561
672
  }
562
673
 
563
- /**
564
- * Make a frame capturable. When the frame is hidden behind another right-Sidebar
565
- * tab (or belongs to a Session that is no longer on screen), reopen the same page
566
- * in a fresh Browser tab of the on-screen Session so the capture is never blank.
567
- */
568
- async function ensureScreenshotFrame(state, ctx, frame) {
569
- try {
570
- await revealFrame(ctx, frame)
571
- return frame
572
- } catch (error) {
573
- const url = String(safe(() => frame.getURL()) || '')
574
- if (!/^https?:/i.test(url)) throw error
575
- let fresh
576
- try {
577
- await openBrowserTab(ctx, url)
578
- fresh = await settleActiveFrame(state, ctx, undefined)
579
- } catch {
580
- throw error
581
- }
582
- if (fresh === undefined) throw error
583
- await revealFrame(ctx, fresh)
584
- return fresh
585
- }
586
- }
587
-
588
674
  function runPage(frame, fn, args) {
589
675
  const code = '(async () => { try {'
590
676
  + ' const pageFind = ' + pageFind.toString() + ';'
@@ -599,19 +685,6 @@ window.__ModuleLoader__.load({
599
685
  })
600
686
  }
601
687
 
602
- async function waitForHttpPage(frame, timeoutMs) {
603
- const deadline = Date.now() + timeoutMs
604
- while (Date.now() < deadline) {
605
- const url = safe(() => frame.getURL())
606
- if (typeof url === 'string' && /^https?:/i.test(url) && safe(() => frame.isLoading()) !== true) {
607
- const state = await safe(() => frame.executeJavaScript('document.readyState', true))
608
- if (state === 'complete' || state === 'interactive') return url
609
- }
610
- await sleep(150)
611
- }
612
- return safe(() => frame.getURL())
613
- }
614
-
615
688
  async function isLoading(frame) {
616
689
  try {
617
690
  return frame.isLoading() === true
@@ -620,12 +693,33 @@ window.__ModuleLoader__.load({
620
693
  }
621
694
  }
622
695
 
623
- function sendKey(frame, key) {
696
+ /* Electron's accelerator names differ from DOM key names. */
697
+ const ACCEL_KEY = new Map([
698
+ ['ArrowUp', 'Up'], ['ArrowDown', 'Down'], ['ArrowLeft', 'Left'], ['ArrowRight', 'Right'],
699
+ ['Escape', 'Esc'], [' ', 'Space'], ['Space', 'Space'], ['Spacebar', 'Space'],
700
+ ])
701
+ const MODIFIER_NAMES = { ctrl: 'control', alt: 'alt', shift: 'shift', meta: 'meta' }
702
+
703
+ function modifierList(modifiers) {
704
+ const list = []
705
+ for (const name of Array.isArray(modifiers) ? modifiers : []) {
706
+ const mapped = MODIFIER_NAMES[name]
707
+ if (mapped !== undefined && list.includes(mapped) !== true) list.push(mapped)
708
+ }
709
+ return list
710
+ }
711
+
712
+ function sendKey(frame, key, modifiers) {
713
+ const accel = ACCEL_KEY.get(key) || key
714
+ const mods = modifierList(modifiers)
624
715
  try {
625
716
  frame.focus()
626
- frame.sendInputEvent({ type: 'keyDown', keyCode: key })
627
- if (key.length === 1) frame.sendInputEvent({ type: 'char', keyCode: key })
628
- frame.sendInputEvent({ type: 'keyUp', keyCode: key })
717
+ /* Electron expects modifiers as a string[] such as ['control','shift'],
718
+ not a bitmask. */
719
+ frame.sendInputEvent({ type: 'keyDown', keyCode: accel, modifiers: mods })
720
+ const plain = /^.$/.test(key) && mods.length === 0
721
+ if (plain) frame.sendInputEvent({ type: 'char', keyCode: key })
722
+ frame.sendInputEvent({ type: 'keyUp', keyCode: accel, modifiers: mods })
629
723
  return 'native'
630
724
  } catch {
631
725
  return 'dom'
@@ -689,18 +783,44 @@ window.__ModuleLoader__.load({
689
783
  await frame.loadURL(url)
690
784
  } catch (error) {
691
785
  const message = error instanceof Error ? error.message : String(error)
692
- if (!/ERR_ABORTED/.test(message)) throw new Error('Navigation to ' + url + ' failed: ' + message)
786
+ if (!/ERR_ABORTED/.test(message)) {
787
+ if (/^file:/i.test(url) && /ERR_FAILED|ERR_ACCESS_DENIED|ERR_FILE_NOT_FOUND|ERR_UNKNOWN_URL_SCHEME/i.test(message)) {
788
+ throw new Error('This Browser build refuses to load local files (' + message + '). '
789
+ + 'Serve the file over http — for example "python -m http.server 8000" — and open http://127.0.0.1:8000/… instead.')
790
+ }
791
+ throw new Error('Navigation to ' + url + ' failed: ' + message)
792
+ }
693
793
  }
694
794
  await settle(frame, 150)
695
795
  await frameReady(frame, 10000)
696
796
  }
697
797
 
798
+ function estimateTokens(text) {
799
+ const value = String(text || '')
800
+ let latin = 0
801
+ let cjk = 0
802
+ let other = 0
803
+ for (const ch of value) {
804
+ const code = ch.codePointAt(0)
805
+ if (code >= 0x4e00 && code <= 0x9fff) cjk += 1
806
+ else if (code < 0x80) latin += 1
807
+ else other += 1
808
+ }
809
+ return Math.ceil(latin / 4 + cjk / 1.5 + other / 2)
810
+ }
811
+
812
+ function sizeNote(text) {
813
+ const value = String(text || '')
814
+ return value.length + ' chars ≈ ' + estimateTokens(value) + ' tokens'
815
+ }
816
+
698
817
  function formatSnapshot(snapshot) {
699
818
  const lines = []
700
819
  lines.push('url: ' + snapshot.url)
701
820
  lines.push('title: ' + (snapshot.title.length > 0 ? snapshot.title : '(untitled)'))
702
821
  lines.push('scroll: y=' + String(snapshot.scroll.y) + ' pageHeight=' + String(snapshot.scroll.height))
703
822
  lines.push('refs: ' + String(snapshot.refs.length) + (snapshot.refsTruncated ? ' (truncated; refine with eval or selector)' : ''))
823
+ if (snapshot.includeText !== false) lines.push('size: ' + sizeNote(snapshot.text) + (snapshot.textTruncated ? ' (truncated)' : ''))
704
824
  for (const ref of snapshot.refs) {
705
825
  const details = []
706
826
  if (ref.role !== undefined && ref.role !== '') details.push('role=' + ref.role)
@@ -716,10 +836,15 @@ window.__ModuleLoader__.load({
716
836
  lines.push('#' + String(ref.ref) + ' <' + ref.tag + '> ' + details.join(' '))
717
837
  }
718
838
  if (snapshot.refs.length === 0) lines.push('(no visible interactive elements)')
719
- lines.push('')
720
- lines.push('text:')
721
- lines.push(snapshot.text.length > 0 ? snapshot.text : '(empty page text)')
722
- if (snapshot.textTruncated) lines.push('(text truncated; use eval to read a specific region)')
839
+ if (snapshot.iframes > 0) {
840
+ lines.push('(note: ' + String(snapshot.iframes) + ' iframe(s) on the page are not traversed by refs or text)')
841
+ }
842
+ if (snapshot.includeText !== false) {
843
+ lines.push('')
844
+ lines.push('text:')
845
+ lines.push(snapshot.text.length > 0 ? snapshot.text : '(empty page text)')
846
+ if (snapshot.textTruncated) lines.push('(text truncated; use eval to read a specific region)')
847
+ }
723
848
  return lines.join('\n')
724
849
  }
725
850
 
@@ -727,6 +852,7 @@ window.__ModuleLoader__.load({
727
852
  const frames = browserFrames()
728
853
  const sidebarRight = sidebarRightOf(ctx)
729
854
  const lines = []
855
+ lines.push('plugin: ' + VERSION)
730
856
  lines.push('platform: ' + (globalThis.dshDesktop === undefined ? 'web' : 'desktop'))
731
857
  lines.push('browserFrames: ' + String(frames.length))
732
858
  let activeTab
@@ -750,12 +876,53 @@ window.__ModuleLoader__.load({
750
876
  return lines.join('\n')
751
877
  }
752
878
 
879
+ /**
880
+ * Wait from the client: each probe is a fresh injection, so a navigation that
881
+ * replaces the document does not tear the wait down. A selector must be
882
+ * visible (a hidden dialog is not "appeared"); a text wait compares visible
883
+ * text. With both, both must match.
884
+ */
885
+ async function waitFor(frame, options, timeoutMs) {
886
+ const hasSelector = typeof options.selector === 'string' && options.selector.length > 0
887
+ const hasText = typeof options.text === 'string' && options.text.length > 0
888
+ const deadline = Date.now() + timeoutMs
889
+ const wanted = hasSelector && hasText
890
+ ? 'selector ' + options.selector + ' and text ' + JSON.stringify(String(options.text))
891
+ : hasSelector ? 'selector ' + options.selector : 'text ' + JSON.stringify(String(options.text))
892
+ while (Date.now() < deadline) {
893
+ let probe = null
894
+ try {
895
+ probe = await frame.executeJavaScript('(() => {'
896
+ + ' const selector = ' + JSON.stringify(hasSelector ? options.selector : '') + ';'
897
+ + ' const text = ' + JSON.stringify(hasText ? options.text : '') + ';'
898
+ + ' const visible = (element) => { if (element === null || element === undefined) return false;'
899
+ + ' if (element.closest("[hidden], [inert]") !== null) return false;'
900
+ + ' const closed = element.closest("details:not([open])");'
901
+ + ' if (closed !== null && element.tagName !== "SUMMARY") return false;'
902
+ + ' const rect = element.getBoundingClientRect();'
903
+ + ' if (rect.width < 1 || rect.height < 1) return false;'
904
+ + ' const style = getComputedStyle(element);'
905
+ + ' return style.display !== "none" && style.visibility !== "hidden" && Number(style.opacity) !== 0 };'
906
+ + ' const found = selector === "" ? document.body !== null : visible(document.querySelector(selector));'
907
+ + ' const matched = text === "" ? true : (document.body !== null && String(document.body.innerText || "").includes(text));'
908
+ + ' return { found, matched }; })()', true)
909
+ } catch {
910
+ probe = null
911
+ }
912
+ if (probe !== null && probe !== undefined && probe.found === true && probe.matched === true) {
913
+ return { found: true, ...(hasSelector ? { selector: options.selector } : {}), ...(hasText ? { text: options.text } : {}) }
914
+ }
915
+ await sleep(150)
916
+ }
917
+ throw new Error('Timed out after ' + String(timeoutMs) + 'ms waiting for ' + wanted + '.')
918
+ }
919
+
753
920
  async function capture(frame) {
754
921
  if (typeof frame.capturePage !== 'function') throw new Error('This Harness build does not expose capturePage on the Browser frame.')
755
922
  /* capturePage can never settle for an occluded guest; a deadline turns a
756
923
  wedged call into an answer instead of a bridge stuck for minutes. */
757
924
  let timer
758
- const image = await Promise.race([
925
+ let image = await Promise.race([
759
926
  frame.capturePage(),
760
927
  new Promise((_, reject) => {
761
928
  timer = setTimeout(() => reject(new Error('the screenshot did not finish within 10s — is the Harness window visible?')), 10000)
@@ -763,13 +930,62 @@ window.__ModuleLoader__.load({
763
930
  ]).finally(() => clearTimeout(timer))
764
931
  if (image === undefined || image === null) throw new Error('The Browser frame returned no screenshot.')
765
932
  if (typeof image.isEmpty === 'function' && image.isEmpty()) throw new Error('The Browser frame returned an empty screenshot.')
766
- const dataUrl = String(image.toDataURL())
767
933
  const prefix = 'data:image/png;base64,'
934
+ let dataUrl = String(image.toDataURL())
768
935
  if (!dataUrl.startsWith(prefix)) throw new Error('The screenshot used an unexpected encoding.')
769
- const size = typeof image.getSize === 'function' ? image.getSize() : { width: 0, height: 0 }
936
+ let size = typeof image.getSize === 'function' ? image.getSize() : { width: 0, height: 0 }
937
+ /* A 4K viewport can exceed the bridge's JSON body limit; downscale before
938
+ sending rather than letting the result POST fail silently. */
939
+ if (dataUrl.length > 6000000 && typeof image.resize === 'function') {
940
+ const scaled = image.resize({ width: 1600, quality: 'good' })
941
+ if (scaled !== undefined && scaled !== null && typeof scaled.toDataURL === 'function') {
942
+ const scaledUrl = String(scaled.toDataURL())
943
+ if (scaledUrl.startsWith(prefix)) {
944
+ image = scaled
945
+ dataUrl = scaledUrl
946
+ size = typeof scaled.getSize === 'function' ? scaled.getSize() : size
947
+ }
948
+ }
949
+ }
770
950
  return { base64: dataUrl.slice(prefix.length), width: size.width, height: size.height }
771
951
  }
772
952
 
953
+ function currentSessionId(ctx) {
954
+ const sidebarRight = sidebarRightOf(ctx)
955
+ if (sidebarRight === undefined || sidebarRight === null) return undefined
956
+ const mounted = safe(() => sidebarRight.mounted?.getSnapshot())
957
+ const current = safe(() => sidebarRight.focusedTarget?.())
958
+ const active = safe(() => sidebarRight.active())
959
+ return (current === undefined || current === null ? undefined : current.sessionId)
960
+ ?? mounted
961
+ ?? (active === undefined || active === null ? undefined : active.sessionId)
962
+ }
963
+
964
+ /** Browser tabs split into "this Session" and "all" so tab ids with the same
965
+ value in another Session are never silently focused or closed. */
966
+ function browserTabs(ctx) {
967
+ const side = sidebarRightOf(ctx)
968
+ const open = side === undefined || side === null ? undefined : safe(() => side.openTabs?.getSnapshot())
969
+ const current = currentSessionId(ctx)
970
+ const all = (Array.isArray(open) ? open : []).filter((tab) => tab.kind === 'browser')
971
+ const rows = all.filter((tab) => current === undefined || tab.sessionId === undefined || tab.sessionId === current)
972
+ return { rows, all }
973
+ }
974
+
975
+ function findBrowserTab(ctx, rawId) {
976
+ const id = String(rawId)
977
+ const { rows, all } = browserTabs(ctx)
978
+ const hit = rows.find((tab) => String(tab.tabId ?? tab.id ?? '') === id)
979
+ if (hit !== undefined) return hit
980
+ const foreign = all.find((tab) => String(tab.tabId ?? tab.id ?? '') === id)
981
+ if (foreign !== undefined) {
982
+ throw new Error('Browser tab ' + JSON.stringify(id) + ' belongs to another Session than the one on screen; bring that Session on screen first.')
983
+ }
984
+ const available = rows.map((tab) => String(tab.tabId ?? tab.id ?? '')).filter((value) => value !== '')
985
+ throw new Error('No Browser tab ' + JSON.stringify(id) + ' is open for this Session'
986
+ + (available.length > 0 ? '; open tabs: ' + available.join(', ') : '. Use action "open" to create one.'))
987
+ }
988
+
773
989
  async function executeCommand(state, command) {
774
990
  const action = String(command.action)
775
991
  const args = command.args !== null && typeof command.args === 'object' ? command.args : {}
@@ -782,43 +998,31 @@ window.__ModuleLoader__.load({
782
998
  const side = sidebarRightOf(ctx)
783
999
  if (side === undefined || side === null) throw new Error('The right Sidebar is unavailable in this Harness page, so a Browser tab cannot be opened (Desktop app required).')
784
1000
  const existing = pickFrame(state, ctx)
785
- /* A Browser tab already on screen is reused and navigated. A tab that is
786
- hidden, or belongs to another Session, is not navigated: open a tab in
787
- the current Session instead, which is what the person will see. */
788
- if (existing !== undefined && frameIsVisible(existing)) {
1001
+ const existingInfo = existing === undefined ? undefined : frameTabInfo(ctx, existing)
1002
+ /* Reuse only a visible tab that belongs to this Session: navigating a
1003
+ hidden or foreign tab would change a page the person cannot see. */
1004
+ if (args.new_tab !== true && existing !== undefined && frameIsVisible(existing) && existingInfo.same !== false) {
789
1005
  await navigateTo(existing, url)
790
1006
  state.last = existing
791
1007
  return { text: 'Opened ' + url + '\n' + statusText(ctx) }
792
1008
  }
793
- let opened
794
- try {
795
- opened = side.openTab('browser', { params: { url } })
796
- } catch (error) {
797
- throw new Error('Could not open the Browser tab: ' + (error instanceof Error ? error.message : String(error)))
798
- }
799
- if (opened !== undefined && opened !== null && typeof opened.then === 'function') {
800
- await opened.catch(() => {})
801
- }
802
- /* Do not wait out a chain of long timeouts: a frame that appears does so
803
- within a few seconds, and a missing one is reported instead of hanging
804
- the tool call for a minute and a half. */
805
- const deadline = Date.now() + 8000
806
- while (Date.now() < deadline) {
807
- const frame = pickFrame(state, ctx)
808
- if (frame !== undefined) {
809
- state.last = frame
810
- await frameReady(frame, 5000).catch(() => {})
811
- return { text: 'Opened ' + url + '\n' + statusText(ctx) }
812
- }
813
- await sleep(150)
814
- }
1009
+ const previousTab = activeBrowserTab(ctx)
1010
+ const previousTabId = previousTab === undefined || previousTab.id === undefined || previousTab.id === null
1011
+ ? undefined
1012
+ : String(previousTab.id)
1013
+ const opened = await openBrowserTab(ctx, url, args.new_tab === true ? previousTabId : undefined)
1014
+ await frameReady(opened.frame, 8000).catch(() => {})
1015
+ const ready = await waitPageReady(state, ctx, opened.frame, 15000, /^about:/i.test(url))
1016
+ state.last = ready.frame
815
1017
  return {
816
- text: 'Requested a Browser tab at ' + url + ', but no frame appeared within 8s.\n'
817
- + 'The tab may still be loading; call action "status" in a moment.\n' + statusText(ctx),
1018
+ text: 'Opened ' + url + (opened.tabId === undefined ? '' : ' in Browser tab ' + opened.tabId) + '\n' + statusText(ctx),
818
1019
  }
819
1020
  }
820
1021
 
821
- const frame = await activeFrame(state, ctx, {})
1022
+ /* Sidebar-only actions must work even when the visible right-Sidebar tab is
1023
+ an editor/terminal and no single Browser frame can be chosen. */
1024
+ const needsFrame = ['navigate', 'back', 'forward', 'reload', 'snapshot', 'read', 'diff', 'click', 'type', 'press', 'select', 'scroll', 'wait', 'eval', 'screenshot'].includes(action)
1025
+ const frame = needsFrame ? await activeFrame(state, ctx, {}) : undefined
822
1026
 
823
1027
  switch (action) {
824
1028
  case 'navigate': {
@@ -863,7 +1067,7 @@ window.__ModuleLoader__.load({
863
1067
  ? 'Read ' + String(result.matched) + ' of ' + String(result.candidates) + ' matching passage(s) for ' + JSON.stringify(String(result.query)) + '.'
864
1068
  : 'Read the visible text.'
865
1069
  return {
866
- text: header + '\n' + String(result.url) + '\n# ' + String(result.title)
1070
+ text: header + ' · ' + sizeNote(result.text) + '\n' + String(result.url) + '\n# ' + String(result.title)
867
1071
  + '\n\n' + String(result.text) + (result.truncated === true ? '\n…(truncated)' : ''),
868
1072
  }
869
1073
  }
@@ -874,6 +1078,7 @@ window.__ModuleLoader__.load({
874
1078
  }
875
1079
  return {
876
1080
  text: (result.changed === true ? 'Changed since the previous diff.' : 'No visible change since the previous diff.')
1081
+ + ' · ' + sizeNote(result.text)
877
1082
  + '\n' + String(result.url) + '\n# ' + String(result.title)
878
1083
  + (String(result.text).length > 0 ? '\n\n' + String(result.text) : ''),
879
1084
  }
@@ -881,74 +1086,91 @@ window.__ModuleLoader__.load({
881
1086
  case 'tabs': {
882
1087
  const side = sidebarRightOf(ctx)
883
1088
  if (side === undefined || side === null) throw new Error('The right Sidebar is unavailable in this Harness page.')
884
- const open = safe(() => side.openTabs?.getSnapshot())
885
1089
  const active = safe(() => side.active())
886
- const rows = (Array.isArray(open) ? open : []).filter((tab) => tab.kind === 'browser')
887
- if (rows.length === 0) return { text: 'No Browser tab is open. Use action "open" with a url.' }
1090
+ const { rows, all } = browserTabs(ctx)
1091
+ if (rows.length === 0) {
1092
+ return { text: 'No Browser tab is open for this Session. Use action "open" with a url.'
1093
+ + (all.length > 0 ? ' (' + String(all.length) + ' Browser tab(s) belong to other Sessions.)' : '') }
1094
+ }
888
1095
  const activeId = active === undefined || active === null ? undefined : String(active.id)
889
1096
  const lines = rows.map((tab) => {
890
1097
  const id = String(tab.tabId ?? tab.id ?? '')
891
1098
  const holder = id === '' ? undefined : frameForTab({ id })
892
1099
  const url = String(tab.url ?? tab.address ?? (holder === undefined ? '' : safe(() => holder.getURL())) ?? '')
893
1100
  const title = String(tab.title ?? tab.label ?? (holder === undefined ? '' : safe(() => holder.getTitle())) ?? '')
894
- const mark = activeId !== undefined && activeId === id ? '*' : ' '
1101
+ const mark = activeId !== undefined && (activeId === id || activeId === String(tab.id ?? '')) ? '*' : ' '
895
1102
  return mark + ' ' + id + ' ' + title + ' ' + url
896
1103
  })
897
- return { text: 'Browser tabs (' + String(rows.length) + ', * = active):\n' + lines.join('\n') }
1104
+ const hidden = all.length - rows.length
1105
+ return {
1106
+ text: 'Browser tabs (' + String(rows.length) + ', * = active):\n' + lines.join('\n')
1107
+ + (hidden > 0 ? '\n(' + String(hidden) + ' more belong to other Sessions and are hidden here.)' : ''),
1108
+ }
898
1109
  }
899
1110
  case 'focus': {
900
1111
  const side = sidebarRightOf(ctx)
901
1112
  if (side === undefined || side === null) throw new Error('The right Sidebar is unavailable in this Harness page.')
902
- side.focus(String(args.tab_id))
1113
+ const target = findBrowserTab(ctx, args.tab_id)
1114
+ side.focus(target.tabId ?? target.id)
903
1115
  await sleep(250)
904
- return { text: 'Focused Browser tab ' + String(args.tab_id) + '.\n' + statusText(ctx) }
1116
+ return { text: 'Focused Browser tab ' + String(target.tabId ?? target.id) + '.\n' + statusText(ctx) }
905
1117
  }
906
1118
  case 'close_tab': {
907
1119
  const side = sidebarRightOf(ctx)
908
1120
  if (side === undefined || side === null) throw new Error('The right Sidebar is unavailable in this Harness page.')
909
- side.close(String(args.tab_id))
1121
+ const target = findBrowserTab(ctx, args.tab_id)
1122
+ side.close(target.tabId ?? target.id)
910
1123
  state.last = undefined
911
- return { text: 'Closed Browser tab ' + String(args.tab_id) + '.' }
1124
+ return { text: 'Closed Browser tab ' + String(target.tabId ?? target.id) + '.' }
912
1125
  }
913
1126
  case 'click': {
1127
+ const before = activeBrowserTab(ctx)
1128
+ const beforeId = before === undefined || before.id === undefined || before.id === null ? undefined : String(before.id)
914
1129
  const outcome = await runPage(frame, pageClick, { ref: args.ref, selector: args.selector })
915
1130
  await settleFast(frame)
1131
+ const after = activeBrowserTab(ctx)
1132
+ const afterId = after === undefined || after.id === undefined || after.id === null ? undefined : String(after.id)
1133
+ const openedTab = afterId !== undefined && afterId !== beforeId ? afterId : ''
1134
+ const active = openedTab === '' ? frame : (pickFrame(state, ctx) || frame)
1135
+ state.last = active
916
1136
  return {
917
1137
  text: 'Clicked <' + outcome.clicked.tag + '> ' + JSON.stringify(outcome.clicked.label)
918
- + '\nurl: ' + String(safe(() => frame.getURL()) || outcome.url)
919
- + '\ntitle: ' + JSON.stringify(String(safe(() => frame.getTitle()) || ''))
1138
+ + (openedTab === '' ? '' : '\nA new Browser tab opened: ' + openedTab)
1139
+ + '\nurl: ' + String(safe(() => active.getURL()) || outcome.url)
1140
+ + '\ntitle: ' + JSON.stringify(String(safe(() => active.getTitle()) || ''))
920
1141
  + '\nTake a new snapshot before the next ref-based action.',
921
1142
  }
922
1143
  }
923
1144
  case 'type': {
924
1145
  const outcome = await runPage(frame, pageType, { ref: args.ref, selector: args.selector, text: args.text, submit: args.submit === true })
1146
+ let submitted = ''
1147
+ if (args.submit === true) {
1148
+ if (outcome.submitted === 'enter') {
1149
+ sendKey(frame, 'Enter')
1150
+ submitted = ' Pressed Enter (the field is not inside a form).'
1151
+ } else if (outcome.submitted === 'form') {
1152
+ submitted = ' Submitted the form.'
1153
+ }
1154
+ }
925
1155
  await settleFast(frame, { quietMs: 150 })
926
1156
  return {
927
- text: 'Typed ' + String(outcome.typed) + ' characters into <' + outcome.target + '>.'
928
- + (args.submit === true ? ' Submitted the form.' : '')
1157
+ text: 'Typed ' + String(outcome.typed) + ' characters into <' + outcome.target + '>.' + submitted
929
1158
  + '\nurl: ' + String(safe(() => frame.getURL()) || outcome.url)
930
1159
  + '\ntitle: ' + JSON.stringify(String(safe(() => frame.getTitle()) || '')),
931
1160
  }
932
1161
  }
933
1162
  case 'press': {
934
1163
  const key = String(args.key)
935
- if (sendKey(frame, key) === 'dom') {
936
- const outcome = await runPage(frame, pagePress, { key })
1164
+ const modifiers = Array.isArray(args.modifiers) ? args.modifiers : []
1165
+ const label = (modifiers.length > 0 ? modifiers.join('+') + '+' : '') + key
1166
+ if (sendKey(frame, key, modifiers) === 'dom') {
1167
+ const outcome = await runPage(frame, pagePress, { key, modifiers })
937
1168
  await settleFast(frame, { quietMs: 120 })
938
- return { text: 'Pressed ' + JSON.stringify(outcome.pressed) + ' on <' + outcome.active + '> (DOM event fallback).' }
939
- }
940
- if (key === 'Enter' || key === ' ' || key === 'Space') {
941
- // Native key events drive page handlers; the DOM pass also submits a
942
- // focused form when the guest swallowed the native default action.
943
- try {
944
- await runPage(frame, pagePress, { key })
945
- } catch {
946
- // The guest may already be navigating.
947
- }
1169
+ return { text: 'Pressed ' + JSON.stringify(label) + ' on <' + outcome.active + '> (DOM event fallback).' }
948
1170
  }
949
1171
  await settleFast(frame, { quietMs: 120 })
950
1172
  return {
951
- text: 'Pressed ' + JSON.stringify(key) + '.'
1173
+ text: 'Pressed ' + JSON.stringify(label) + '.'
952
1174
  + '\nurl: ' + String(safe(() => frame.getURL()) || '')
953
1175
  + '\ntitle: ' + JSON.stringify(String(safe(() => frame.getTitle()) || '')),
954
1176
  }
@@ -956,8 +1178,10 @@ window.__ModuleLoader__.load({
956
1178
  case 'select': {
957
1179
  const outcome = await runPage(frame, pageSelect, { ref: args.ref, selector: args.selector, value: args.value })
958
1180
  await settleFast(frame, { quietMs: 120 })
959
- return { text: 'Selected value ' + JSON.stringify(outcome.value) + ' (index ' + String(outcome.selectedIndex) + ').'
960
- + '\ntitle: ' + JSON.stringify(String(safe(() => frame.getTitle()) || '')) }
1181
+ return {
1182
+ text: 'Selected ' + JSON.stringify(outcome.label) + ' (' + JSON.stringify(outcome.value) + ', index ' + String(outcome.selectedIndex) + ').'
1183
+ + '\nurl: ' + String(safe(() => frame.getURL()) || ''),
1184
+ }
961
1185
  }
962
1186
  case 'scroll': {
963
1187
  const outcome = await runPage(frame, pageScroll, {
@@ -966,22 +1190,58 @@ window.__ModuleLoader__.load({
966
1190
  direction: args.direction,
967
1191
  amount: args.amount,
968
1192
  })
969
- return { text: 'Scrolled to y=' + String(outcome.y === undefined ? '?' : outcome.y) }
1193
+ const parts = ['Scrolled to y=' + String(outcome.y === undefined ? '?' : outcome.y)]
1194
+ if (outcome.x !== undefined && outcome.x !== 0) parts.push('x=' + String(outcome.x))
1195
+ if (outcome.target !== undefined) parts.push('target <' + outcome.target.tag + '>')
1196
+ if (outcome.rectTop !== undefined) parts.push('viewport top=' + String(outcome.rectTop))
1197
+ if (outcome.moved === false) parts.push('(already at that position)')
1198
+ return { text: parts.join(' ') }
970
1199
  }
971
1200
  case 'wait': {
972
- const outcome = await runPage(frame, pageWait, {
973
- selector: args.selector,
974
- text: args.text,
975
- timeoutMs: Number(args.timeout_ms) > 0 ? Number(args.timeout_ms) : 5000,
976
- })
1201
+ const timeoutMs = Number(args.timeout_ms) > 0 ? Number(args.timeout_ms) : 5000
1202
+ const outcome = await waitFor(frame, { selector: args.selector, text: args.text }, timeoutMs)
977
1203
  return { text: 'Appeared: ' + JSON.stringify(outcome) }
978
1204
  }
979
1205
  case 'eval': {
980
1206
  const script = String(args.script)
981
- const code = '(async () => { const value = (' + script + '); const resolved = await value;'
982
- + ' try { return JSON.parse(JSON.stringify(resolved === undefined ? null : resolved)) } catch { return String(resolved) } })()'
983
- const value = await frame.executeJavaScript(code, true)
984
- const text = JSON.stringify(value, null, 2)
1207
+ const serialize = '(value) => {'
1208
+ + ' if (value === undefined) return null;'
1209
+ + ' if (value !== null && typeof value === "object" && typeof value.nodeType === "number") {'
1210
+ + ' return { node: String(value.nodeName || ""), text: String(value.innerText !== undefined && value.innerText !== null ? value.innerText : (value.textContent || "")).slice(0, 4000), ...(typeof value.value === "string" ? { value: value.value.slice(0, 500) } : {}) };'
1211
+ + ' }'
1212
+ + ' if (typeof value === "function") return String(value);'
1213
+ + ' if (typeof value === "bigint" || typeof value === "symbol") return String(value);'
1214
+ + ' const seen = new WeakSet();'
1215
+ + ' try { const json = JSON.stringify(value, (key, item) => { if (item !== null && typeof item === "object") { if (seen.has(item)) return "[circular]"; seen.add(item) } return item }); return json === undefined ? String(value) : JSON.parse(json) } catch { return String(value) }'
1216
+ + ' }'
1217
+ const wrap = (body) => '(async () => { try { const __v = await (' + body + '); return { ok: true, value: (' + serialize + ')(__v) } }'
1218
+ + ' catch (error) { return { ok: false, name: error && error.name ? String(error.name) : "Error", message: error && error.message ? String(error.message) : String(error) } } })()'
1219
+ /* eval() returns the completion value, so expressions and statement
1220
+ blocks both work; a page CSP may forbid it, in which case a plain
1221
+ expression is still evaluated. */
1222
+ let outcome = await frame.executeJavaScript(wrap('(0, eval)(' + JSON.stringify(script) + ')'), true)
1223
+ const failed = outcome === null || typeof outcome !== 'object' || outcome.ok !== true
1224
+ if (failed) {
1225
+ const name = outcome !== null && typeof outcome === 'object' ? String(outcome.name || '') : ''
1226
+ const message = outcome !== null && typeof outcome === 'object' ? String(outcome.message || '') : 'the page returned no value'
1227
+ if (/EvalError|Content Security Policy|unsafe-eval/i.test(name + ' ' + message)) {
1228
+ try {
1229
+ outcome = await frame.executeJavaScript(wrap('(' + script + ')'), true)
1230
+ } catch {
1231
+ throw new Error('This page\'s Content Security Policy blocks eval(), so only a single expression can be evaluated here. '
1232
+ + 'Pass an expression, or use read/snapshot instead.')
1233
+ }
1234
+ } else if (/SyntaxError/i.test(name) || /SyntaxError/i.test(message)) {
1235
+ throw new Error('The script could not be parsed: ' + message + '. Pass an expression or a statement block.')
1236
+ } else {
1237
+ throw new Error(message)
1238
+ }
1239
+ }
1240
+ if (outcome === null || typeof outcome !== 'object' || outcome.ok !== true) {
1241
+ const message = outcome !== null && typeof outcome === 'object' ? String(outcome.message || '') : 'the page returned no value'
1242
+ throw new Error(message)
1243
+ }
1244
+ const text = JSON.stringify(outcome.value, null, 2)
985
1245
  return { text: text.length > 20000 ? text.slice(0, 20000) + '\n(truncated)' : text }
986
1246
  }
987
1247
  case 'screenshot': {
@@ -990,7 +1250,8 @@ window.__ModuleLoader__.load({
990
1250
  Browser panel to the front. */
991
1251
  await revealFrame(ctx, frame)
992
1252
  const image = await capture(frame)
993
- const text = 'Captured the visible Browser page.\n' + statusText(ctx)
1253
+ const note = args.full_page === true ? 'Full-page capture is not supported yet; captured the visible viewport.\n' : ''
1254
+ const text = note + 'Captured the visible Browser page.\n' + statusText(ctx)
994
1255
  return { text, png: image }
995
1256
  }
996
1257
  case 'close': {
@@ -1025,6 +1286,7 @@ window.__ModuleLoader__.load({
1025
1286
  const started = Date.now()
1026
1287
  const lines = []
1027
1288
  let failed = ''
1289
+ let ran = 0
1028
1290
  for (let index = 0; index < steps.length; index += 1) {
1029
1291
  const step = steps[index] !== null && typeof steps[index] === 'object' ? steps[index] : {}
1030
1292
  const sub = String(step.action || '')
@@ -1034,16 +1296,22 @@ window.__ModuleLoader__.load({
1034
1296
  }
1035
1297
  try {
1036
1298
  const result = await executeCommand(state, { action: sub, args: step })
1037
- const summary = String(result?.text || '').split('\n').filter((line) => line !== '').slice(0, 3).join(' · ')
1038
- lines.push(String(index + 1) + '. ' + sub + ': ' + summary.slice(0, 300))
1299
+ ran += 1
1300
+ const raw = String(result?.text || '')
1301
+ /* A read/diff/snapshot in a batch is usually the point of the
1302
+ batch, so keep its payload; other steps only need a summary. */
1303
+ const text = ['read', 'diff', 'snapshot'].includes(sub)
1304
+ ? raw.slice(0, 3000) + (raw.length > 3000 ? '…' : '')
1305
+ : raw.split('\n').filter((line) => line !== '').slice(0, 3).join(' · ').slice(0, 300)
1306
+ lines.push(String(index + 1) + '. ' + sub + ': ' + text)
1039
1307
  } catch (error) {
1040
1308
  failed = 'step ' + String(index + 1) + ' (' + sub + '): ' + (error instanceof Error ? error.message : String(error))
1041
1309
  break
1042
1310
  }
1043
1311
  }
1044
1312
  return {
1045
- text: 'Batch ' + String(steps.length) + ' step(s) in ' + String(Date.now() - started) + ' ms.'
1046
- + (failed === '' ? '' : '\nfailed: ' + failed)
1313
+ text: 'Batch: ' + String(ran) + ' of ' + String(steps.length) + ' step(s) ran in ' + String(Date.now() - started) + ' ms.'
1314
+ + (failed === '' ? '' : '\nfailed: ' + failed + ' (remaining steps not run)')
1047
1315
  + '\n' + lines.join('\n'),
1048
1316
  }
1049
1317
  }
@@ -1107,7 +1375,11 @@ window.__ModuleLoader__.load({
1107
1375
  if (signal.aborted || state.running === false) return
1108
1376
  let message
1109
1377
  try {
1110
- message = { ok: true, value: await withDeadline(executeCommand(state, command), 45000, 'The action "' + String(command.action) + '"') }
1378
+ /* Give up a few seconds before the host deadline so the failure
1379
+ is a readable client message instead of a host timeout. */
1380
+ const budget = Number(command.deadlineMs) > 0 ? Number(command.deadlineMs) : DEFAULT_ACTION_BUDGET_MS
1381
+ const closed = Math.max(5000, Math.min(budget - 5000, MAX_ACTION_BUDGET_MS))
1382
+ message = { ok: true, value: await withDeadline(executeCommand(state, command), closed, 'The action "' + String(command.action) + '"') }
1111
1383
  } catch (error) {
1112
1384
  message = {
1113
1385
  ok: false,
@@ -1116,8 +1388,9 @@ window.__ModuleLoader__.load({
1116
1388
  }
1117
1389
  try {
1118
1390
  await post('result', { instance, commandId: command.id, ...message }, signal)
1119
- } catch {
1391
+ } catch (error) {
1120
1392
  // The Host may have timed the command out; the next poll recovers.
1393
+ console.warn('[sidebar-control] the result could not be delivered', error)
1121
1394
  }
1122
1395
  }
1123
1396
  } catch (error) {