dsh-inner-browser-use 0.35.1 → 0.35.3

package/README.zh.md

@@ -18,7 +18,7 @@ profile，不是产品自带的那个浏览页签：右栏放的是它的画面
 | `browser_type` | 往焦点输入框输入文字，也可以先点一个编号控件。中文整段送进去。`submit: true` 回车提交。 |
 | `browser_key` | 按一个键：Enter、Tab、Escape、Backspace、Delete、上下左右、Home、End、PageUp、PageDown。 |
 | `browser_scroll` | 在页面中心滚轮，也可以指定 `x`/`y`。`dy: 600` 约一屏。 |
-| `browser_shot` | 存一张原尺寸 PNG 到 `~/.dsh-inner-browser-use/screenshots`，返回路径。 |
+| `browser_shot` | 存一张 PNG 到 `~/.dsh/inner-browser-use/screenshots`，返回路径；`region: [x, y, w, h]` 只截一块（游戏、图表、看局部）。 |
 | `browser_state` | 报浏览器在不在跑、用的哪个 profile、地址、标题，以及有几个右栏在看它。 |
 
 点、输入、按键之后的那一行会多一句结果：`· navigated to …`（换页了）、`· content changed`（页内有变化）、`· no visible change`（点了没动）、Flash 页则是 `· screen updated` / `· no frame change`；`read: true` 时不再多说。

package/index.js

@@ -117,6 +117,11 @@ export class Bridge {
        is not on screen has no stream, so a push at ask time is lost; remembering
        the ask is what makes the mirror appear when the person returns. */
     this.wanted = { session: '', at: 0 }
+    /* Frame streams by chat. A close can arrive while a stream is still
+       attaching, so closing the mirror hangs up what that chat started: a
+       browser nobody watches is what lets it go to rest. */
+    /** @type {Map<string, Set<() => void>>} */
+    this.viewers = new Map()
     /* A running plugin keeps the note; a bare Bridge in a test does not. */
     this.persist = options.persist === true
     const kept = this.persist ? readMirror() : { owner: '', session: '', at: 0 }
@@ -208,6 +213,42 @@ export class Bridge {
     return [...this.seats.keys()]
   }
 
+  /**
+   * Remember one open frame stream so the chat can hang it up later.
+   * @param {string} sessionId - chat drawing the mirror.
+   * @param {() => void} off - ends that one stream.
+   */
+  watch(sessionId, off) {
+    const id = this.canon(sessionId)
+    if (id === '' || typeof off !== 'function') return
+    const set = this.viewers.get(id) ?? new Set()
+    set.add(off)
+    /* A tab that reloads faster than its old socket closes must not pile up
+       streams: the oldest goes, which is the one no page is reading. */
+    while (set.size > 4) {
+      const oldest = set.values().next().value
+      set.delete(oldest)
+      try { oldest() } catch { /* a stream already gone is already gone */ }
+    }
+    this.viewers.set(id, set)
+  }
+
+  /**
+   * Hang up every frame stream one chat started.
+   * @param {string} sessionId - chat whose streams go away.
+   * @returns {number} how many streams were hanging up.
+   */
+  unwatch(sessionId) {
+    const id = this.canon(sessionId)
+    const set = this.viewers.get(id)
+    if (set === undefined) return 0
+    this.viewers.delete(id)
+    for (const off of set) {
+      try { off() } catch { /* a stream already gone is already gone */ }
+    }
+    return set.size
+  }
+
   /**
    * Give the mirror up: the person closed the column, so the chat stops owning
    * it and no keepalive may ask for it again. Ownership is what the column
@@ -219,6 +260,9 @@ export class Bridge {
     const id = this.canon(sessionId)
     if (id === '') return false
     const held = this.owner === id || this.wanted.session === id
+    /* The picture goes with the ask: a stream left attached after the column
+       closed keeps the browser awake with nobody looking at it. */
+    this.unwatch(id)
     this.wanted = { session: '', at: 0 }
     if (this.owner === id) this.owner = ''
     if (this.persist === true) writeMirror({ owner: this.owner, session: '', at: 0 })
@@ -229,6 +273,7 @@ export class Bridge {
   dispose() {
     for (const res of this.seats.values()) res.end()
     this.seats.clear()
+    for (const id of [...this.viewers.keys()]) this.unwatch(id)
   }
 }
 
@@ -447,17 +492,33 @@ export function apply(ctx) {
       res.socket?.setNoDelay?.(true)
       res.write('retry: 2000\n\n')
       /** @type {() => void} */
-      let detach = () => {}
-      const drop = () => detach()
+      let off = () => {}
+      /* The stream can end while `watch` is still waiting for a browser to come
+         up: that close has to be remembered, or the sink added a second later is
+         never taken away and the browser is kept awake with nobody looking. */
+      let shut = false
+      let done = false
+      const hang = () => {
+        if (done) return
+        done = true
+        try { off() } catch { /* a stream already gone is already gone */ }
+        try { res.end() } catch { /* the socket may be the thing that closed */ }
+      }
+      const drop = () => {
+        shut = true
+        hang()
+      }
       req.on('close', drop)
       res.on('error', drop)
       try {
-        detach = await sidecar.watch((frame) => {
+        off = await sidecar.watch((frame) => {
           /* A viewer that cannot keep up loses frames instead of making the
              page wait: the newest picture is the only one worth sending. */
           if (res.writableLength > 3 * 1024 * 1024) return
           res.write(`data: ${JSON.stringify(frame)}\n\n`)
         }, streaming)
+        if (shut) hang()
+        else if (watching !== '') bridge.watch(watching, hang)
         const info = await sidecar.state()
         /* The pane needs one address to offer back when this chat's own page
            went away; the state route answers the same question for the tab
@@ -725,6 +786,13 @@ export function apply(ctx) {
         } else {
           await sidecar.useTab(at, actor)
         }
+        /* Closing the last page is this chat saying it is done with the mirror.
+           An owner left behind is what the next poll reads as "put my tab back":
+           the pane reappeared right after the person closed it. */
+        if (body.op === 'close') {
+          const left = await sidecar.chatState(actor)
+          if (left?.alive !== true && bridge.forget(actor) && sidecar.watched() === 0) sidecar.touch(REST_MS)
+        }
         const tabs = await sidecar.windowsFor(actor)
         json(200, { ok: true, tabs })
       } catch (error) {
@@ -876,6 +944,7 @@ export function apply(ctx) {
         '先接着用已有的页：`browser_view`（拿编号控件与可见文字）→ `browser_click` / `browser_type` → 再 `browser_view` 直到出现要的结果；页面要等就重复读，不 sleep。`browser_open` 只用于没有页、要换新地址、或要明确刷新；同一个地址不要反复重开。',
         '页面很长、只想确认某句话/按钮在哪时用 `browser_find`，别整页读。要读长文（文章、文档、issue、论坛帖）用 `browser_read`：它回 Markdown，只花几十分之一的 token，链接可点；要动手点、填表才用 `browser_view` 拿编号。',
         '要保存页面样子用 `browser_shot`，它会给出 PNG 绝对路径；要交给人看的文件，把绝对路径写进回复。',
+        '游戏、canvas、Flash：画面自己画的地方，`browser_view` 和 `browser_read` 读不出里面是什么。先 `browser_prepare` 拿画面位置、帧率、按键是否到达和下一步；之后看 `browser_shot` 的图定位置，按坐标点、按住、按组合键，再拍一张看画面变了没有——变了就是这一步生效了。当年用 Flash 做的游戏用 `browser_open` 带 `flash:true` 在栏内跑，不装播放器、不开窗口。',
       ].join('\n'),
     }), 'dsh-inner-browser-use: system prompt section')
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-inner-browser-use",
-  "version": "0.35.1",
+  "version": "0.35.3",
   "description": "An agent-owned Chromium sidecar mirrored into the DeepSeek Harness right column: the agent drives it, the person watches and can take the mouse, and logins survive restarts.",
   "type": "module",
   "main": "index.js",

package/read.js

@@ -432,6 +432,66 @@ export function resolve(ref) {
 })()`
 }
 
+/**
+ * The first of several controls that still says these words, found in one
+ * evaluate. A caller hunts through a dozen numbers after a redraw: one page
+ * round trip answers "which of them is still there", instead of a dozen.
+ * @param {number[]} refs - numbers to try, in the order a person would.
+ * @param {string} wanted - the words, compared with the spaces taken out.
+ * @returns {string} an expression that answers with the first that is live.
+ */
+export function resolveFirst(refs, wanted) {
+  const order = refs.map((ref) => Number(ref)).filter((ref) => Number.isFinite(ref))
+  return `(() => {
+  const reg = window.${REGISTRY}
+  if (!reg) return JSON.stringify({ gone: true })
+  const squeeze = (text) => String(text).toLowerCase().replace(/\\s+/g, '')
+  const want = ${JSON.stringify(String(wanted))}
+  const spotOf = (el) => {
+    const b = el.getBoundingClientRect()
+    if (b.width < 2 || b.height < 2) return null
+    const clipped = () => {
+      const me = el.getBoundingClientRect()
+      for (let up = el.parentElement; up; up = up.parentElement) {
+        const css = getComputedStyle(up)
+        if (!/auto|scroll|hidden|clip/.test(css.overflowX + ' ' + css.overflowY)) continue
+        const box = up.getBoundingClientRect()
+        if (me.top < box.top - 1 || me.bottom > box.bottom + 1 || me.left < box.left - 1 || me.right > box.right + 1) return true
+      }
+      return false
+    }
+    if (b.top < 0 || b.left < 0 || b.bottom > innerHeight || b.right > innerWidth || clipped()) el.scrollIntoView({ block: 'center', inline: 'center' })
+    const after = el.getBoundingClientRect()
+    return { x: Math.round(after.left + after.width / 2), y: Math.round(after.top + after.height / 2) }
+  }
+  for (const ref of ${JSON.stringify(order)}) {
+    const hit = reg.find(ref)
+    if (!hit) continue
+    const el = hit.el
+    const name = String(el.innerText ?? el.value ?? el.getAttribute?.('aria-label') ?? '').trim().slice(0, 80)
+    if (want !== '' && !squeeze(name).includes(want)) continue
+    const spot = spotOf(el)
+    if (spot === null) continue
+    return JSON.stringify({ gone: false, ref, name, x: spot.x, y: spot.y })
+  }
+  return JSON.stringify({ gone: true })
+})()`
+}
+
+/**
+ * Ask the page which of these numbers still carries the words.
+ * @param {any} page - sidecar page.
+ * @param {number[]} refs - numbers to try, in order.
+ * @param {string} wanted - the words said by the caller.
+ * @returns {Promise<{gone: boolean, ref?: number, name?: string, x?: number, y?: number}>} the first live one.
+ */
+export async function locateFirst(page, refs, wanted) {
+  const said = await evaluateIn(page, resolveFirst(refs, wanted)).catch(() => undefined)
+  let spot = { gone: true }
+  try { spot = JSON.parse(String(said ?? '{"gone":true}')) } catch (error) { spot = { gone: true } }
+  return spot
+}
+
 /**
  * Click the control the page itself registered, instead of moving the mouse to
  * it. A page whose own handlers want a click, or a control an overlay covers,
@@ -685,7 +745,7 @@ export function covering(ref) {
 }
 
 /** Where refs of controls inside iframes start. Main-frame refs stay small. */
-const FRAME_BASE = 900_000
+export const FRAME_BASE = 900_000
 
 /** Room for the refs of one frame between FRAME_BASE steps. */
 const FRAME_SLOT = 100_000

package/tools.js

@@ -15,7 +15,7 @@ import { homedir } from 'node:os'
 import { basename, join } from 'node:path'
 import { mkdir, readdir, stat, unlink, writeFile } from 'node:fs/promises'
 import { VIEW } from './sidecar.js?v=1'
-import { readPage, renderPage, readArticle, readRelevant, findOnPage, locate, covering, contextFor, identityOf, sameThing, differs, labelSpot, selectSpot, bounds, pictures, clickInDom, writeInDom, REF_GONE } from './read.js?v=1'
+import { readPage, renderPage, readArticle, readRelevant, findOnPage, locate, locateFirst, FRAME_BASE, covering, contextFor, identityOf, sameThing, differs, labelSpot, selectSpot, bounds, pictures, clickInDom, writeInDom, REF_GONE } from './read.js?v=1'
 import { brief as askBrief, recordRound, lastRoundNote, lastRound, markRound, MARK_STATUS } from './review.js?v=1'
 
 /** Which chat the running call belongs to, so each gets its own tab. */
@@ -1050,6 +1050,49 @@ async function until(page, expression, timeout) {
  * @returns {Promise<string>} a compact report.
  */
 /** The first control whose words match, scrolled into view. Text can beat a number: it costs no read. */
+/** Words a label can lose without changing what it asks for. */
+const SMALL_WORDS = new Set(['a', 'an', 'the', 'to', 'of', 'and', 'or', 'in', 'on', 'at', 'for', 'with', 'my', 'your', 'please', 'click', 'tap', 'button', 'btn', 'link', 'now'])
+
+/**
+ * The significant words of a label, lowercased.
+ * @param {unknown} text - a label.
+ * @returns {string[]} words worth comparing.
+ */
+export function tokensOf(text) {
+  return String(text ?? '').toLowerCase().split(/[^a-z0-9\u4e00-\u9fff]+/).filter((word) => word !== '' && !SMALL_WORDS.has(word))
+}
+
+/**
+ * The one label that means what the caller said when no label repeats it.
+ * Only an unambiguous winner is offered, so a paraphrase cannot silently click
+ * a different control than the one asked for.
+ * @param {Array<{ref: number, name?: string, view?: boolean}>} controls - controls from a read.
+ * @param {string} words - what the caller asked for.
+ * @returns {any} the best control, or null when nothing stands out.
+ */
+export function nearestByWords(controls, words) {
+  const want = tokensOf(words)
+  if (want.length === 0 || !Array.isArray(controls)) return null
+  const scored = []
+  for (const control of controls) {
+    if (control?.view === false) continue
+    const have = new Set(tokensOf(control?.name))
+    if (have.size === 0) continue
+    let shared = 0
+    for (const word of want) if (have.has(word)) shared += 1
+    const cover = shared / want.length
+    if (shared === 0 || cover < 0.5) continue
+    scored.push({ control, score: cover * 0.7 + (shared / have.size) * 0.3 })
+  }
+  if (scored.length === 0) return null
+  scored.sort((a, b) => b.score - a.score)
+  const best = scored[0]
+  const second = scored[1]
+  if (second !== undefined && second.score > best.score - 0.05) return null
+  if (String(best.control.name ?? '').length > String(words).length * 3 + 12) return null
+  return { ...best.control, ref: Number(best.control.ref) }
+}
+
 export async function findWords(page, words, known) {
   /* The words a caller used usually came from a read that already happened, so
      look among those numbers first: one position check is a single evaluate,
@@ -1066,7 +1109,19 @@ export async function findWords(page, words, known) {
      end is only the fallback. */
   const firstInOrder = remembered.filter((control) => control.view !== false).sort((a, b) => Number(a.ref) - Number(b.ref))
   const nearEnd = remembered.slice(-8).reverse()
-  for (const control of [...firstInOrder.slice(0, 4), ...nearEnd]) {
+  const order = [...firstInOrder.slice(0, 4), ...nearEnd]
+  /* One round trip answers which of these the page still has: a dozen numbers
+     left over from a read that a redraw invalidated used to cost a dozen
+     checks. Numbers inside a frame keep the per-number route below. */
+  const plain = order.map((control) => Number(control.ref)).filter((ref) => Number.isFinite(ref) && ref < FRAME_BASE)
+  if (plain.length > 0) {
+    const any = await locateFirst(page, plain, wantedFirst)
+    if (any?.gone !== true && Number.isFinite(Number(any?.x))) {
+      const byRef = order.find((control) => Number(control.ref) === Number(any.ref)) ?? { ref: Number(any.ref) }
+      return { ...byRef, ref: Number(any.ref), name: String(any.name ?? byRef.name ?? ''), x: Number(any.x), y: Number(any.y) }
+    }
+  }
+  for (const control of order.filter((control) => Number(control.ref) >= FRAME_BASE)) {
     const spot = await locate(page, Number(control.ref))
     if (spot.gone === true) continue
     const name = String(spot.name ?? control.name ?? '')
@@ -1080,7 +1135,17 @@ export async function findWords(page, words, known) {
   const squeeze = (text) => String(text).toLowerCase().replace(/\s+/g, '')
   const wanted = squeeze(words)
   const found = shown.find((control) => squeeze(control.name) === wanted) ?? shown.find((control) => squeeze(control.name).includes(wanted))
-  if (found === undefined) return null
+  if (found === undefined) {
+    /* A button the page redrew often only swaps one word, and a person says
+       "submit order" for "Place order". Accept only an unambiguous winner. */
+    const near = nearestByWords(shown, words)
+    if (near !== null) {
+      const spot = await locate(page, near.ref)
+      if (spot.gone === true) return null
+      return { ...near, x: Number(spot.x), y: Number(spot.y) }
+    }
+    return null
+  }
   /* When only the squeezed form matched, say the words the caller used: the
      page spread them over nodes and "Signin" is not what anyone wrote. */
   const hit = squeeze(found.name) === wanted && found.name.toLowerCase() !== words.toLowerCase() ? { ...found, name: words } : found
@@ -2738,13 +2803,14 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
     },
     {
       name: 'browser_shot', // ui-copy-ignore
-      description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path; when the current model accepts images the picture comes back in the same call, so a captcha, a chart or a broken layout can be looked at instead of guessed. Default is the visible screen; full takes the whole page, ref takes one control.',
+      description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path; when the current model accepts images the picture comes back in the same call, so a captcha, a chart or a broken layout can be looked at instead of guessed. Default is the visible screen; full takes the whole page, ref takes one control, region takes one box of it.',
       parameters: {
         type: 'object', // ui-copy-ignore
         properties: {
           name: { type: 'string', description: 'File name without extension.' },
           full: { type: 'boolean', description: 'The whole page, not just the screen.' },
           ref: { type: 'integer', description: 'One control from browser_view.' },
+          region: { type: 'array', items: { type: 'number' }, description: 'One box instead of the whole screen: [x, y, width, height] in page pixels, the same numbers browser_prepare and browser_find report.' },
           scale: { type: 'integer', description: 'Pixel ratio. Default 1; 2 is sharper and heavier.' },
         },
         additionalProperties: false,
@@ -2766,6 +2832,17 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
         const said = await withRead(async (page) => {
         const scale = Math.max(1, Math.min(3, Number(args?.scale ?? 1)))
         let clip = undefined
+        /* A game or a chart does not need the whole screen: one box costs a
+           fraction of the picture, and a picture is billed by area. */
+        if (args?.ref === undefined && Array.isArray(args?.region) && args.region.length === 4) {
+          const box = args.region.map((part) => Number(part))
+          clip = {
+            x: Math.max(0, Math.round(Number.isFinite(box[0]) ? box[0] : 0)),
+            y: Math.max(0, Math.round(Number.isFinite(box[1]) ? box[1] : 0)),
+            width: Math.max(8, Math.min(4000, Math.round(Number.isFinite(box[2]) ? box[2] : 0) || 8)),
+            height: Math.max(8, Math.min(4000, Math.round(Number.isFinite(box[3]) ? box[3] : 0) || 8)),
+          }
+        }
         if (args?.ref !== undefined) {
           const spot = await findAgain(page, Number(args.ref))
           if (spot.gone === true) return spot.why ?? REF_GONE