dsh-browser-plus 0.5.0 → 0.5.1

package/scripts/test-orb-drag.mjs

@@ -0,0 +1,83 @@
+/**
+ * 真正驱动指针事件的悬浮球拖动测试（审查要的那条）。
+ *
+ * 单元测试只能断言「生成的脚本文本里有某个公式」，抓不到 H2 那类错误 ——
+ * 慢速拖动曾经因为阈值拿「上一帧已应用的位置」当基准，永远不算拖动：
+ * 位置不落盘、松手还会误触 click。这里用离屏 Electron 跑真脚本、派发真指针事件。
+ *
+ *   node scripts/test-orb-drag.mjs
+ */
+import { spawnSync } from 'node:child_process'
+import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { createRequire } from 'node:module'
+
+const require = createRequire(import.meta.url)
+const chrome = require('../lib/browser-electron/page-chrome.js')
+const dir = mkdtempSync(join(tmpdir(), 'orb-drag-'))
+const W = 620
+const H = 480
+const CENTER = { x: W - 18 - 22, y: H - 18 - 22 }
+
+const probe = [
+  'window.__moves = [];',
+  'window.__seen = [];',
+  'window.addEventListener("pointerdown", e => window.__seen.push(["down", e.buttons, Math.round(e.clientX)]), true);',
+  'window.addEventListener("pointermove", e => window.__seen.push(["move", e.buttons, Math.round(e.clientX)]), true);',
+  'window.__dshChromeToken = "t";',
+  'window.__dshChromeTaskKey = "default";',
+  'window.__dshBrowserTaskAction = function (payload) { try { const a = JSON.parse(payload); if (a && a.type === "orb-move") window.__moves.push(a) } catch {} };',
+  'window.__dshChromeBootstrap = ' + JSON.stringify({
+    kind: 'bootstrap', epoch: 1, revision: 1, selectedTaskKey: 'default',
+    panels: { tasks: false, trail: false },
+    tasks: [{ key: 'default', label: '', active: true, tabs: 1, status: 'running', updatedAt: Date.now() }],
+    tabs: [], trail: [], bookmarks: [], bookmarkBar: false,
+  }) + ';',
+].join('')
+
+const html = '<!doctype html><html><head><meta charset="utf-8"><style>html,body{margin:0;height:100%;background:#202124}</style></head><body>'
+  + '<scr' + 'ipt>' + probe + '</scr' + 'ipt>'
+  + '<scr' + 'ipt>' + chrome.buildPageChromeScript('t', 'page') + '</scr' + 'ipt>'
+  + '</body></html>'
+
+const driver = `
+const { app, BrowserWindow } = require('electron')
+const fs = require('fs')
+const file = process.argv[2]
+const out = process.argv[3]
+const center = JSON.parse(process.argv[4])
+app.whenReady().then(async () => {
+  const win = new BrowserWindow({ width: ${W}, height: ${H}, show: false, webPreferences: { offscreen: true } })
+  await win.loadFile(file)
+  await new Promise(r => setTimeout(r, 700))
+  const send = (type, x, y, buttons) => win.webContents.sendInputEvent({ type, x: Math.round(x), y: Math.round(y), button: 'left', buttons, clickCount: 1 })
+  send('mouseDown', center.x, center.y, 1)
+  await new Promise(r => setTimeout(r, 80))
+  // 每步之间等一帧多一点：Chromium 会把挨得太近的 pointermove 合并掉，
+  // 合并之后「每帧位移」就超过阈值了，测不出 H2 那种慢速拖动。
+  for (let i = 1; i <= 60; i++) { send('mouseMove', center.x - i * 2, center.y - i, 1); await new Promise(r => setTimeout(r, 25)) }
+  send('mouseUp', center.x - 120, center.y - 60, 0)
+  await new Promise(r => setTimeout(r, 250))
+  fs.writeFileSync(out, JSON.stringify({
+    moves: await win.webContents.executeJavaScript('window.__moves'),
+    seen: await win.webContents.executeJavaScript('window.__seen.slice(0, 3)'),
+    total: await win.webContents.executeJavaScript('window.__seen.length'),
+  }))
+  console.log('DRIVER_DONE')
+  app.quit()
+})
+`
+writeFileSync(join(dir, 'page.html'), html)
+writeFileSync(join(dir, 'drive.cjs'), driver)
+const out = join(dir, 'result.json')
+const run = spawnSync(process.execPath, [require.resolve('electron/cli.js'), join(dir, 'drive.cjs'), join(dir, 'page.html'), out, JSON.stringify(CENTER)], { encoding: 'utf8' })
+const text = String(run.stdout ?? '') + String(run.stderr ?? '')
+if (!text.includes('DRIVER_DONE')) { console.error('driver did not finish:\n' + text.slice(-900)); process.exit(1) }
+const result = JSON.parse(readFileSync(out, 'utf8'))
+console.log('page saw ' + result.total + ' pointer events; first 3: ' + JSON.stringify(result.seen))
+const moves = result.moves
+if (moves.length === 0) { console.error('FAIL: a slow drag produced no orb-move — the position would not be persisted'); process.exit(1) }
+const last = moves[moves.length - 1]
+if (!(last.x < CENTER.x) || !(last.y < CENTER.y)) { console.error('FAIL: the orb did not move up-left: ' + JSON.stringify(last)); process.exit(1) }
+console.log('OK: slow drag persisted ' + moves.length + ' move(s), last ' + last.x + ',' + last.y)

package/src/browser/runtime.ts

@@ -47,9 +47,14 @@ import type {
   BrowserTypeRequest,
   BrowserUploadFileRequest,
   BrowserUploadFileResult,
+  BrowserPdfRequest,
+  BrowserPdfResult,
+  BrowserHighlightRequest,
+  BrowserHighlightResult,
   BrowserWaitForRequest,
   BrowserWaitForResult,
   BrowserChallenge,
+  BrowserTaskTodo,
   ExportedCookie,
 } from './types.ts'
 import { BrowserError } from './types.ts'
@@ -334,6 +339,14 @@ export class BrowserRuntime extends Service {
   }
 
   /** Wait for an element through the selected provider (bounded polling). */
+  async highlight(session: BrowserSessionId, request: BrowserHighlightRequest, signal?: AbortSignal): Promise<BrowserHighlightResult> {
+    return this.resolveProvider().highlight(session, request, signal)
+  }
+
+  async pdf(session: BrowserSessionId, request: BrowserPdfRequest, signal?: AbortSignal): Promise<BrowserPdfResult> {
+    return this.resolveProvider().pdf(session, request, signal)
+  }
+
   async waitForElement(session: BrowserSessionId, request: BrowserWaitForRequest, signal?: AbortSignal): Promise<BrowserWaitForResult> {
     return this.resolveProvider().waitForElement(session, request, signal)
   }
@@ -448,6 +461,29 @@ export class BrowserRuntime extends Service {
     return this.resolveProvider().setHandoff(session, state)
   }
 
+  /**
+   * Bring the shared browser window to the front through the selected
+   * provider, opening it when nothing is open yet.
+   */
+  async ensureWindowVisible(): Promise<void> {
+    const provider = this.resolveProvider()
+    if (provider.ensureWindowVisible === undefined) {
+      throw new BrowserError('browser: this provider has no window to open', 'BROWSER_WINDOW_UNSUPPORTED')
+    }
+    return provider.ensureWindowVisible()
+  }
+
+  /**
+   * Mirror one task's Agent todo list into the shared window (the floating orb
+   * reads it there). A provider without a window has nowhere to put it, so this
+   * is a no-op rather than an error.
+   */
+  async pushTaskTodos(taskKey: string, todos: readonly BrowserTaskTodo[]): Promise<void> {
+    const provider = this.resolveProvider()
+    if (provider.pushTaskTodos === undefined) return
+    return provider.pushTaskTodos(taskKey, todos)
+  }
+
   /** Close the session through the selected provider. Idempotent; a missing
    *  provider is treated as already-closed so teardown paths stay no-ops. */
   async close(session: BrowserSessionId): Promise<void> {

package/src/browser/types.ts

@@ -147,26 +147,81 @@ export interface BrowserUploadFileResult {
   readonly path: string
 }
 
-/** Wait for an element matching a CSS selector to appear (optionally visible). */
+/** Which state a wait is watching for. */
+export type BrowserWaitForState = 'visible' | 'attached' | 'hidden' | 'detached'
+
+/** Wait for a selector, some text, or both to reach a state. */
 export interface BrowserWaitForRequest {
-  /** CSS selector to wait for. */
-  readonly selector: string
+  /** CSS selector to wait for. Omit to watch the document's own text (give `text`). */
+  readonly selector?: string
+  /** Text that must be present: inside the matched element when a selector is given, otherwise anywhere in the document. */
+  readonly text?: string
+  /**
+   * The state to wait for. `visible` (the default) needs a matching element at least 4x4 px and not
+   * visibility:hidden / display:none; `attached` only needs it to exist; `hidden` and `detached`
+   * wait for the opposite, so they are how you wait for something to go away.
+   */
+  readonly state?: BrowserWaitForState
   /** Total budget in ms. Default 15000. */
   readonly timeoutMs?: number
-  /** Wait for the element to be visible (at least 4x4 px and not visibility:hidden or display:none). Default true. */
+  /** Back-compat for `state: 'attached'` (`visible: false`). Prefer `state`. */
   readonly visible?: boolean
 }
 
 /** Outcome of a successful wait. */
 export interface BrowserWaitForResult {
+  /** The awaited condition is now true. */
   readonly found: true
+  /** Which state was awaited. */
+  readonly state: BrowserWaitForState
+  /** The selector waited on, or `''` for a text-only wait. */
   readonly selector: string
-  /** Matching element's tag name. */
+  /** Matched element's tag name, or `''` when nothing matched (a `detached` wait). */
   readonly tag: string
-  /** Matching element's visible text (first 200 chars). */
+  /** Matched element's visible text (first 200 chars), or the document text that satisfied a text wait. */
   readonly text: string
 }
 
+/** Print the active tab to a PDF file. */
+export interface BrowserPdfRequest {
+  /** Absolute path of the .pdf to write. Must be inside the write roots. */
+  readonly savePath: string
+  /** Landscape orientation. Default portrait. */
+  readonly landscape?: boolean
+  /** Include background colours and images. Default true. */
+  readonly printBackground?: boolean
+  /** Paper width in inches. CDP default is 8.5. */
+  readonly paperWidth?: number
+  /** Paper height in inches. CDP default is 11. */
+  readonly paperHeight?: number
+}
+
+/** Where the PDF landed. */
+export interface BrowserPdfResult {
+  readonly path: string
+  readonly bytes: number
+}
+
+/** Draw or clear the DevTools-style highlight box over a selector's first match. */
+export interface BrowserHighlightRequest {
+  /** CSS selector whose first match to highlight. Ignored when clear is true. */
+  readonly selector?: string
+  /** Remove any existing highlight instead of drawing one. */
+  readonly clear?: boolean
+}
+
+/** What the highlight call found. */
+export interface BrowserHighlightResult {
+  /** A match was found and highlighted. */
+  readonly matched: boolean
+  /** The highlight was cleared. */
+  readonly cleared: boolean
+  /** CDP node id of the match, when there was one. */
+  readonly nodeId?: number
+  /** Content box in CSS pixels, when the element has one. */
+  readonly box?: { readonly x: number; readonly y: number; readonly width: number; readonly height: number }
+}
+
 /** One field of a batch form fill. Match by selector, or by name/label/placeholder. */
 export interface BrowserFillField {
   /** CSS selector; when present, candidates are scoped to it. */
@@ -354,6 +409,18 @@ export interface BrowserSnapshotElement {
   /** Viewport-relative center, for coordinate fallbacks. */
   readonly x: number
   readonly y: number
+  /** Index into `frames` when the element lives inside an iframe; absent at the top level. */
+  readonly frame?: number
+}
+
+/** One iframe found on the page. */
+export interface BrowserSnapshotFrame {
+  /** Index into the page's iframe list; matches `BrowserSnapshotElement.frame`. */
+  readonly index: number
+  /** The frame's src, or its document URL when it has one. */
+  readonly url: string
+  /** False when the frame is cross-origin, so its contents cannot be read. */
+  readonly readable: boolean
 }
 
 /**
@@ -371,6 +438,8 @@ export interface BrowserSnapshotResult {
   readonly elements: readonly BrowserSnapshotElement[]
   /** True when the snapshot was truncated (element cap reached). */
   readonly truncated: boolean
+  /** Every iframe on the page, including the cross-origin ones that could not be read. */
+  readonly frames?: readonly BrowserSnapshotFrame[]
   /** Human-verification challenge blocking the page, when one is detected. */
   readonly challenge?: BrowserChallenge
   /** True when a human interacted with the page within the last minute. */
@@ -571,6 +640,10 @@ export interface BrowserProvider {
   uploadFile(session: BrowserSessionId, request: BrowserUploadFileRequest, signal?: AbortSignal): Promise<BrowserUploadFileResult>
   /** Wait until an element matching the selector exists (and optionally is visible). Honor `signal` for cancellation. */
   waitForElement(session: BrowserSessionId, request: BrowserWaitForRequest, signal?: AbortSignal): Promise<BrowserWaitForResult>
+  /** Print the active tab to a PDF file. */
+  pdf(session: BrowserSessionId, request: BrowserPdfRequest, signal?: AbortSignal): Promise<BrowserPdfResult>
+  /** Draw or clear the DevTools-style highlight box over a selector's first match. */
+  highlight(session: BrowserSessionId, request: BrowserHighlightRequest, signal?: AbortSignal): Promise<BrowserHighlightResult>
   /** Fill a form's fields in one batch. Honor `signal` for cancellation. */
   fillForm(session: BrowserSessionId, request: BrowserFillRequest, signal?: AbortSignal): Promise<BrowserFillResult>
   /** Capture the current page. Honor `signal` for cancellation. */
@@ -615,6 +688,25 @@ export interface BrowserProvider {
   setHandoff(session: BrowserSessionId, state: BrowserHandoffState): Promise<BrowserTaskInfo>
   /** Close the session and destroy its backing surface. Idempotent. */
   close(session: BrowserSessionId): Promise<void>
+  /**
+   * Bring the shared browser window to the front, creating it (and a default
+   * session) when nothing is open yet. Optional: a provider whose browser has
+   * no window of its own omits it, and callers must report that as unsupported
+   * rather than pretending a window was raised.
+   */
+  ensureWindowVisible?(): Promise<void>
+  /**
+   * Mirror one task's Agent todo list into the shared window, where the floating
+   * orb renders it. Optional: a provider without a window of its own omits it.
+   * `taskKey` is the same task key the tools use (the calling Agent's id).
+   */
+  pushTaskTodos?(taskKey: string, todos: readonly BrowserTaskTodo[]): Promise<void>
+}
+
+/** One entry of an Agent's todo list, as the floating orb shows it. */
+export interface BrowserTaskTodo {
+  readonly content: string
+  readonly status: 'pending' | 'in_progress' | 'completed'
 }
 
 /** One recorded browser operation, in chronological order (seq 1, 2, 3…). */

package/src/browser-electron/chrome-state.ts

@@ -125,17 +125,35 @@ export interface ChromeWorkspaceState {
    * rounds before this made it visible.
    */
   readonly windowProbe?: string
+  /** Where the human dragged the floating orb, in page CSS pixels. */
+  readonly orbPosition?: { readonly x: number; readonly y: number }
 }
 
 export interface ChromeBootstrapMessage extends ChromeWorkspaceState {
   readonly kind: 'bootstrap'
 }
 
+/**
+ * One entry of the Agent's todo list (`todo_write`) as the floating orb renders
+ * it: the same three states the model writes, so the orb spins on
+ * `in_progress` and ticks `completed` without translating anything.
+ */
+export interface ChromeTaskTodo {
+  readonly content: string
+  readonly status: 'pending' | 'in_progress' | 'completed'
+}
+
 export type ChromePatchOperation =
   | { readonly op: 'task.upsert'; readonly task: ChromeTaskSummary }
   | { readonly op: 'task.remove'; readonly key: string }
   | { readonly op: 'task.active'; readonly key?: string }
   | { readonly op: 'task.thumbnail'; readonly key: string; readonly version: number; readonly dataUrl?: string }
+  /**
+   * The Agent's todo list for one task. Like the thumbnail it is NOT part of a
+   * summary — summaries are injected into every visited page's main world, and
+   * this is the Agent's own plan — so it travels only as a targeted patch.
+   */
+  | { readonly op: 'task.todos'; readonly key: string; readonly todos: readonly ChromeTaskTodo[] }
   | { readonly op: 'trail.append'; readonly taskKey: string; readonly entry: ChromeTrailEntry }
   | { readonly op: 'trail.replace'; readonly taskKey?: string; readonly entries: readonly ChromeTrailEntry[] }
   | { readonly op: 'panels.set'; readonly panels: ChromePanels }

package/src/browser-electron/entry.ts

@@ -49,11 +49,18 @@ export interface Config {
    */
   readonly readRoots?: string[]
   /**
-   * Which JavaScript world the injected page chrome lives in. `main` (default)
-   * is the proven path; `isolated` keeps the chrome's task state and its
-   * binding token out of the page's own context, at the cost of an extra CDP
-   * context per document. Opt in only after confirming the toolbar in a real
-   * window (see docs/SOAK-CHECKLIST.md).
+   * Which JavaScript world the injected page chrome lives in.
+   *
+   * `isolated` (default) keeps the chrome's task state — including the Agent's
+   * plan, which the floating orb renders — and its binding token out of the
+   * page's own context: the page's scripts see `undefined` for every `__dsh*`
+   * global instead of a readable copy. It costs one extra CDP context per
+   * document, and the chrome reads the DOM through that context (the DOM itself
+   * is shared, so element lookups and layout still work).
+   *
+   * `main` puts everything in the page's world: the proven-against-everything
+   * path, but a page can read the plan by hooking `Map.prototype.set` or
+   * `document.createElement`. Use it only to bisect an isolated-world bug.
    */
   readonly chromeWorld?: 'main' | 'isolated'
   /**
@@ -82,7 +89,7 @@ export const Config: z<Config> = z.object({
   // meaning "the documented defaults" while an explicit [] still denies all.
   writeRoots: z.array(z.string()).default(defaultWriteRoots()),
   readRoots: z.array(z.string()).default(defaultWriteRoots()),
-  chromeWorld: z.union(['main', 'isolated'] as const).default('main'),
+  chromeWorld: z.union(['main', 'isolated'] as const).default('isolated'),
   userAgent: z.string(),
   maskAutomation: z.boolean().default(true),
 })
@@ -93,7 +100,10 @@ export function apply(ctx: Context & { browser: BrowserRuntime }, config: Config
   // child is disposed with the fiber, mirroring the shell's lifetime.
   const host: ElectronBrowserViewHost = config.viewHost
     ?? new RemoteElectronViewHost(defaultHostMainPath(), {
-      chromeWorld: config.chromeWorld,
+      // `?? 'isolated'` 不只是为了类型：入口可能拿到**未经 schema 解析**的配置
+      // （测试探针、其它调用方直接调 apply），那时缺省值不会自动出现，
+      // 而 remote-host 把 undefined 当成「不加 --chrome-world」→ 子进程退回 main ✗。
+      chromeWorld: config.chromeWorld ?? 'isolated',
       maskAutomation: config.maskAutomation,
       ...config.userAgent === undefined ? {} : { userAgent: config.userAgent },
     })