dsh-inner-browser-use 0.35.0 → 0.35.2

package/README.zh.md

@@ -18,7 +18,7 @@ profile，不是产品自带的那个浏览页签：右栏放的是它的画面
 | `browser_type` | 往焦点输入框输入文字，也可以先点一个编号控件。中文整段送进去。`submit: true` 回车提交。 |
 | `browser_key` | 按一个键：Enter、Tab、Escape、Backspace、Delete、上下左右、Home、End、PageUp、PageDown。 |
 | `browser_scroll` | 在页面中心滚轮，也可以指定 `x`/`y`。`dy: 600` 约一屏。 |
-| `browser_shot` | 存一张原尺寸 PNG 到 `~/.dsh-inner-browser-use/screenshots`，返回路径。 |
+| `browser_shot` | 存一张 PNG 到 `~/.dsh/inner-browser-use/screenshots`，返回路径；`region: [x, y, w, h]` 只截一块（游戏、图表、看局部）。 |
 | `browser_state` | 报浏览器在不在跑、用的哪个 profile、地址、标题，以及有几个右栏在看它。 |
 
 点、输入、按键之后的那一行会多一句结果：`· navigated to …`（换页了）、`· content changed`（页内有变化）、`· no visible change`（点了没动）、Flash 页则是 `· screen updated` / `· no frame change`；`read: true` 时不再多说。

package/index.js

@@ -117,6 +117,11 @@ export class Bridge {
        is not on screen has no stream, so a push at ask time is lost; remembering
        the ask is what makes the mirror appear when the person returns. */
     this.wanted = { session: '', at: 0 }
+    /* Frame streams by chat. A close can arrive while a stream is still
+       attaching, so closing the mirror hangs up what that chat started: a
+       browser nobody watches is what lets it go to rest. */
+    /** @type {Map<string, Set<() => void>>} */
+    this.viewers = new Map()
     /* A running plugin keeps the note; a bare Bridge in a test does not. */
     this.persist = options.persist === true
     const kept = this.persist ? readMirror() : { owner: '', session: '', at: 0 }
@@ -208,6 +213,42 @@ export class Bridge {
     return [...this.seats.keys()]
   }
 
+  /**
+   * Remember one open frame stream so the chat can hang it up later.
+   * @param {string} sessionId - chat drawing the mirror.
+   * @param {() => void} off - ends that one stream.
+   */
+  watch(sessionId, off) {
+    const id = this.canon(sessionId)
+    if (id === '' || typeof off !== 'function') return
+    const set = this.viewers.get(id) ?? new Set()
+    set.add(off)
+    /* A tab that reloads faster than its old socket closes must not pile up
+       streams: the oldest goes, which is the one no page is reading. */
+    while (set.size > 4) {
+      const oldest = set.values().next().value
+      set.delete(oldest)
+      try { oldest() } catch { /* a stream already gone is already gone */ }
+    }
+    this.viewers.set(id, set)
+  }
+
+  /**
+   * Hang up every frame stream one chat started.
+   * @param {string} sessionId - chat whose streams go away.
+   * @returns {number} how many streams were hanging up.
+   */
+  unwatch(sessionId) {
+    const id = this.canon(sessionId)
+    const set = this.viewers.get(id)
+    if (set === undefined) return 0
+    this.viewers.delete(id)
+    for (const off of set) {
+      try { off() } catch { /* a stream already gone is already gone */ }
+    }
+    return set.size
+  }
+
   /**
    * Give the mirror up: the person closed the column, so the chat stops owning
    * it and no keepalive may ask for it again. Ownership is what the column
@@ -219,6 +260,9 @@ export class Bridge {
     const id = this.canon(sessionId)
     if (id === '') return false
     const held = this.owner === id || this.wanted.session === id
+    /* The picture goes with the ask: a stream left attached after the column
+       closed keeps the browser awake with nobody looking at it. */
+    this.unwatch(id)
     this.wanted = { session: '', at: 0 }
     if (this.owner === id) this.owner = ''
     if (this.persist === true) writeMirror({ owner: this.owner, session: '', at: 0 })
@@ -229,6 +273,7 @@ export class Bridge {
   dispose() {
     for (const res of this.seats.values()) res.end()
     this.seats.clear()
+    for (const id of [...this.viewers.keys()]) this.unwatch(id)
   }
 }
 
@@ -447,17 +492,33 @@ export function apply(ctx) {
       res.socket?.setNoDelay?.(true)
       res.write('retry: 2000\n\n')
       /** @type {() => void} */
-      let detach = () => {}
-      const drop = () => detach()
+      let off = () => {}
+      /* The stream can end while `watch` is still waiting for a browser to come
+         up: that close has to be remembered, or the sink added a second later is
+         never taken away and the browser is kept awake with nobody looking. */
+      let shut = false
+      let done = false
+      const hang = () => {
+        if (done) return
+        done = true
+        try { off() } catch { /* a stream already gone is already gone */ }
+        try { res.end() } catch { /* the socket may be the thing that closed */ }
+      }
+      const drop = () => {
+        shut = true
+        hang()
+      }
       req.on('close', drop)
       res.on('error', drop)
       try {
-        detach = await sidecar.watch((frame) => {
+        off = await sidecar.watch((frame) => {
           /* A viewer that cannot keep up loses frames instead of making the
              page wait: the newest picture is the only one worth sending. */
           if (res.writableLength > 3 * 1024 * 1024) return
           res.write(`data: ${JSON.stringify(frame)}\n\n`)
         }, streaming)
+        if (shut) hang()
+        else if (watching !== '') bridge.watch(watching, hang)
         const info = await sidecar.state()
         /* The pane needs one address to offer back when this chat's own page
            went away; the state route answers the same question for the tab
@@ -725,6 +786,13 @@ export function apply(ctx) {
         } else {
           await sidecar.useTab(at, actor)
         }
+        /* Closing the last page is this chat saying it is done with the mirror.
+           An owner left behind is what the next poll reads as "put my tab back":
+           the pane reappeared right after the person closed it. */
+        if (body.op === 'close') {
+          const left = await sidecar.chatState(actor)
+          if (left?.alive !== true && bridge.forget(actor) && sidecar.watched() === 0) sidecar.touch(REST_MS)
+        }
         const tabs = await sidecar.windowsFor(actor)
         json(200, { ok: true, tabs })
       } catch (error) {
@@ -876,6 +944,7 @@ export function apply(ctx) {
         '先接着用已有的页：`browser_view`（拿编号控件与可见文字）→ `browser_click` / `browser_type` → 再 `browser_view` 直到出现要的结果；页面要等就重复读，不 sleep。`browser_open` 只用于没有页、要换新地址、或要明确刷新；同一个地址不要反复重开。',
         '页面很长、只想确认某句话/按钮在哪时用 `browser_find`，别整页读。要读长文（文章、文档、issue、论坛帖）用 `browser_read`：它回 Markdown，只花几十分之一的 token，链接可点；要动手点、填表才用 `browser_view` 拿编号。',
         '要保存页面样子用 `browser_shot`，它会给出 PNG 绝对路径；要交给人看的文件，把绝对路径写进回复。',
+        '游戏、canvas、Flash：画面自己画的地方，`browser_view` 和 `browser_read` 读不出里面是什么。先 `browser_prepare` 拿画面位置、帧率、按键是否到达和下一步；之后看 `browser_shot` 的图定位置，按坐标点、按住、按组合键，再拍一张看画面变了没有——变了就是这一步生效了。当年用 Flash 做的游戏用 `browser_open` 带 `flash:true` 在栏内跑，不装播放器、不开窗口。',
       ].join('\n'),
     }), 'dsh-inner-browser-use: system prompt section')
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-inner-browser-use",
-  "version": "0.35.0",
+  "version": "0.35.2",
   "description": "An agent-owned Chromium sidecar mirrored into the DeepSeek Harness right column: the agent drives it, the person watches and can take the mouse, and logins survive restarts.",
   "type": "module",
   "main": "index.js",

package/sidecar.js

@@ -826,18 +826,17 @@ export class SidecarPage {
        probe is only for the case where the click never made one, so it gets a
        single short look instead of the old half-second of polling. */
     if (target === null) {
-      /* Look, wait for the announcement, look again: a new tab is announced
-         before the browser lists it, so one look is not enough. */
-      const deadline = Date.now() + POPUP_GRACE_MS
-      while (target === null && Date.now() < deadline) {
-        const found = await this.findPopup(foreign)
-        if (found !== null) {
-          target = found
-          break
+      /* One look at the tab list answers the common case — a click that opened
+         nothing. When it says nothing, the announcement is the cheaper door:
+         a tab exists a moment before the browser lists it, so the wait ends on
+         the event instead of running out the window. */
+      target = await this.findPopup(foreign)
+      if (target === null) {
+        await this.waitPopup(Math.min(8, POPUP_GRACE_MS))
+        if (this.opened !== null && this.opened !== undefined && Date.now() - Number(this.opened.at ?? 0) < 2000) {
+          target = { targetId: String(this.opened.targetId), url: String(this.opened.url) }
         }
-        const left = deadline - Date.now()
-        if (left <= 0) break
-        await this.waitPopup(Math.min(8, left))
+        this.opened = null
       }
     }
     if (target === null) {

package/tools.js

@@ -310,7 +310,17 @@ export async function outcome(page, before) {
   const [wasTitle, wasUrl, wasText, wasNodes] = before.split('|')
   const [nowTitle, nowUrl, nowText, nowNodes] = after.split('|')
   if (after === before) return ' · no visible change'
-  if (nowUrl !== wasUrl) return ` · navigated to ${String(nowUrl).slice(0, 300)}`
+  if (nowUrl !== wasUrl) {
+    /* The brief on the same line already names the address this step went to.
+       What it cannot say is whether the page moved within itself, so a jump to
+       another spot on the same address is said as the spot alone. */
+    const was = String(wasUrl ?? '')
+    const now = String(nowUrl ?? '')
+    const base = (value) => (value.includes('#') ? value.slice(0, value.indexOf('#')) : value)
+    const spot = now.includes('#') ? now.slice(now.indexOf('#')) : ''
+    if (spot !== '' && base(was) === base(now)) return ` · ${spot.slice(0, 120)}`
+    return ' · new page'
+  }
   const words = Math.abs(Number(nowText ?? 0) - Number(wasText ?? 0))
   const nodes = Math.abs(Number(nowNodes ?? 0) - Number(wasNodes ?? 0))
   if (nowTitle !== wasTitle) return ' · page moved on'
@@ -352,9 +362,27 @@ export function withoutSeenAddress(page, url, text) {
   return body.slice(first.length + 1)
 }
 
+/** The focus line repeats the control the call named: say the name once. */
+export function dropSameFocus(line, args) {
+  const squeezed = (value) => String(value ?? '').toLowerCase().replace(/\s+/g, '')
+  const named = squeezed(args?.text ?? args?.name ?? args?.ref ?? '')
+  if (named === '') return line
+  const at = line.indexOf(' · focus ')
+  if (at < 0) return line
+  const head = line.slice(0, at)
+  const parts = line.slice(at + ' · focus '.length).split(' · ')
+  const said = String(parts[0] ?? '')
+  /* A field carries a value the words do not: that part is news, so it stays. */
+  if (said.includes(' = ')) return line
+  const label = squeezed(said)
+  if (label !== named && !label.includes(named) && !named.includes(label)) return line
+  const rest = parts.slice(1)
+  return rest.length === 0 ? head : `${head} · ${rest.join(' · ')}`
+}
+
 async function acted(page, args, look, label) {
   if (args?.read === true) return await look()
-  return `ok ${label} · ${oncePerPage(page, await brief(page))}`
+  return `ok ${label} · ${dropSameFocus(oncePerPage(page, await brief(page)), args)}`
 }
 
 /**
@@ -2710,13 +2738,14 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
     },
     {
       name: 'browser_shot', // ui-copy-ignore
-      description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path; when the current model accepts images the picture comes back in the same call, so a captcha, a chart or a broken layout can be looked at instead of guessed. Default is the visible screen; full takes the whole page, ref takes one control.',
+      description: 'Save a PNG of the sidecar browser into the session folder and return its absolute path; when the current model accepts images the picture comes back in the same call, so a captcha, a chart or a broken layout can be looked at instead of guessed. Default is the visible screen; full takes the whole page, ref takes one control, region takes one box of it.',
       parameters: {
         type: 'object', // ui-copy-ignore
         properties: {
           name: { type: 'string', description: 'File name without extension.' },
           full: { type: 'boolean', description: 'The whole page, not just the screen.' },
           ref: { type: 'integer', description: 'One control from browser_view.' },
+          region: { type: 'array', items: { type: 'number' }, description: 'One box instead of the whole screen: [x, y, width, height] in page pixels, the same numbers browser_prepare and browser_find report.' },
           scale: { type: 'integer', description: 'Pixel ratio. Default 1; 2 is sharper and heavier.' },
         },
         additionalProperties: false,
@@ -2738,6 +2767,17 @@ export function sidecarTools(sidecar, refs, host = {}) { // ui-copy-ignore
         const said = await withRead(async (page) => {
         const scale = Math.max(1, Math.min(3, Number(args?.scale ?? 1)))
         let clip = undefined
+        /* A game or a chart does not need the whole screen: one box costs a
+           fraction of the picture, and a picture is billed by area. */
+        if (args?.ref === undefined && Array.isArray(args?.region) && args.region.length === 4) {
+          const box = args.region.map((part) => Number(part))
+          clip = {
+            x: Math.max(0, Math.round(Number.isFinite(box[0]) ? box[0] : 0)),
+            y: Math.max(0, Math.round(Number.isFinite(box[1]) ? box[1] : 0)),
+            width: Math.max(8, Math.min(4000, Math.round(Number.isFinite(box[2]) ? box[2] : 0) || 8)),
+            height: Math.max(8, Math.min(4000, Math.round(Number.isFinite(box[3]) ? box[3] : 0) || 8)),
+          }
+        }
         if (args?.ref !== undefined) {
           const spot = await findAgain(page, Number(args.ref))
           if (spot.gone === true) return spot.why ?? REF_GONE