dsh-builtin-browser 0.1.23 → 0.3.0

package/lib/tool-browser/index.js

@@ -63,8 +63,13 @@ function taskKey(exec) {
  */
 async function ensureSession(browser, state, key, agent) {
     const existing = state.sessionsByTask.get(key);
-    if (existing !== undefined)
-        return existing;
+    if (existing !== undefined) {
+        // The human closing a browser window ends that session, so a cached id can
+        // be stale: never hand a dead session back to a tool call.
+        if (browser.exists(existing))
+            return existing;
+        state.sessionsByTask.delete(key);
+    }
     const pending = state.pendingOpens.get(key);
     if (pending !== undefined)
         return pending;
@@ -117,9 +122,20 @@ function parseFillValue(v) {
         return Number(v);
     return v ?? '';
 }
-/** Format a snapshot element list for the model. */
-function formatSnapshot(snapshot) {
-    const lines = snapshot.elements.map(el => `[${el.ref}] ${el.kind}: ${el.label}${el.frame === true ? ' (iframe)' : ''} (${el.x},${el.y})`);
+/**
+ * Format a snapshot element list for the model.
+ *
+ * Coordinates are opt-in. They are only meaningful to a caller that intends to
+ * click a pixel position, which means a vision pass; for the far more common case
+ * of locating an element by role/label and passing a semantic target, they are the
+ * bulkiest and least useful part of every line. Omitting them keeps the listing
+ * readable — which is exactly what a model without image input has to work from.
+ * @param snapshot - the snapshot payload.
+ * @param options - `coords: true` to include each element's viewport position.
+ */
+function formatSnapshot(snapshot, options = {}) {
+    const showCoords = options.coords === true;
+    const lines = snapshot.elements.map(el => `[${el.ref}] ${el.kind}: ${el.label}${el.frame === true ? ' (iframe)' : ''}${showCoords ? ` (${el.x},${el.y})` : ''}`);
     const header = `URL: ${snapshot.url}${snapshot.title !== undefined ? `\nTitle: ${snapshot.title}` : ''}`;
     const body = lines.length > 0 ? lines.join('\n') : '(no interactive elements found)';
     const tail = snapshot.truncated === true ? '\n(snapshot truncated)' : '';
@@ -188,7 +204,10 @@ export function apply(ctx, config = {}) {
                     },
                 },
             },
-            render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
+            // browser_open is an overview of what just loaded, not a locator: it has no
+            // `coords` parameter, so coordinates stay off here (browser_snapshot owns
+            // that switch for callers that actually target pixels).
+            render: (args, value) => [{ type: 'text', text: formatSnapshot(value, { coords: args.coords === true }) }],
         },
         timeoutMs,
         isConcurrencySafe: () => false, // opens tabs / navigates; exclusive within a task
@@ -214,8 +233,10 @@ export function apply(ctx, config = {}) {
     }));
     ctx.tools.register(defineTool({
         name: 'browser_snapshot',
-        description: 'Return an AI-friendly snapshot of the current shared-browser page: numbered interactive elements (inputs, buttons, links) the model can cite. Use this to understand an interactive page before driving it.',
-        parameters: {},
+        description: 'Return an AI-friendly snapshot of the current shared-browser page: numbered interactive elements (inputs, buttons, links) the model can cite. This is the primary way to understand a page, and it needs no image input — every listed element can afterwards be addressed by a semantic target (browser_click, browser_type, browser_check, …) rather than by coordinates.',
+        parameters: {
+            coords: { type: 'boolean', description: 'Also print each element\'s viewport coordinates. Off by default; only useful when a vision pass is going to click a pixel position.' },
+        },
         output: {
             schema: {
                 type: 'object',
@@ -251,7 +272,7 @@ export function apply(ctx, config = {}) {
                     },
                 },
             },
-            render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
+            render: (args, value) => [{ type: 'text', text: formatSnapshot(value, { coords: args.coords === true }) }],
         },
         timeoutMs,
         isConcurrencySafe: () => true,
@@ -276,6 +297,7 @@ export function apply(ctx, config = {}) {
         parameters: {
             includeHidden: { type: 'boolean', description: 'Include hidden elements (default false).' },
             maxNodes: { type: 'number', description: 'Maximum nodes (default 500, range 10-5000).' },
+            coords: { type: 'boolean', description: 'Include each element\'s viewport coordinates. Off by default: a semantic target needs no coordinates, and they are pure noise unless a vision pass is going to click a pixel position.' },
         },
         output: {
             schema: {
@@ -308,14 +330,24 @@ export function apply(ctx, config = {}) {
                     },
                 },
             },
-            render: (_args, value) => {
+            render: (args, value) => {
                 const nodes = value.nodes;
+                // Built for reading, not for imaging: containment is shown as indentation
+                // (the depth was already collected and simply thrown away), an empty state
+                // list adds nothing, and coordinates appear only when asked for — a model
+                // working from the DOM never needs them, and they are the bulkiest part of
+                // every line.
+                const showCoords = args.coords === true;
                 const lines = nodes.map(n => {
                     const valuePart = n.value !== undefined && n.value !== null ? ` value="${String(n.value).slice(0, 60)}"` : '';
-                    return `[${n.ref}] ${n.role} "${n.name}"${valuePart} (${n.x},${n.y}) states=[${n.states.join(',')}]${n.frame === true ? ' (iframe)' : ''}`;
+                    const indent = '  '.repeat(Math.max(0, Math.min(8, Number(n.depth) || 0)));
+                    const coordsPart = showCoords ? ` (${n.x},${n.y})` : '';
+                    const statePart = Array.isArray(n.states) && n.states.length > 0 ? ` states=[${n.states.join(',')}]` : '';
+                    return `${indent}[${n.ref}] ${n.role} "${n.name}"${valuePart}${coordsPart}${statePart}${n.frame === true ? ' (iframe)' : ''}`;
                 });
                 const header = `URL: ${value.url}${value.title !== undefined ? `\nTitle: ${value.title}` : ''}`;
-                return [{ type: 'text', text: `${header}\n\n${lines.length > 0 ? lines.join('\n') : '(no accessible interactive nodes)'}${value.truncated === true ? '\n(truncated)' : ''}` }];
+                const hint = showCoords ? '' : '\n(coordinates omitted — pass coords: true to include them, or click by semantic target)';
+                return [{ type: 'text', text: `${header}\n\n${lines.length > 0 ? lines.join('\n') : '(no accessible interactive nodes)'}${value.truncated === true ? '\n(truncated)' : ''}${hint}` }];
             },
         },
         timeoutMs,
@@ -548,7 +580,7 @@ export function apply(ctx, config = {}) {
     }));
     ctx.tools.register(defineTool({
         name: 'browser_click',
-        description: 'Click in the shared browser. Two ways: (1) semantic target — pass target {by: css|text|xpath, value, index?} and the element is located, scrolled into view and clicked at its center; (2) viewport coordinates (x/y) — use with browser_screenshot when a vision model locates an element on the screenshot (covers icons, image buttons, and canvas that DOM locators cannot target). Provide exactly one of target or x/y.',
+        description: 'Click in the shared browser. Prefer the semantic target: pass target {by: css|text|xpath, value, index?} and the element is located from the DOM, scrolled into view and clicked at its centre — this needs no image input and is the only form that works without vision. The alternative is viewport coordinates (x/y), which are meaningful only after a vision model has located the element on a browser_screenshot; coordinate clicks are refused outright under the non-visual strategy. Provide exactly one of target or x/y.',
         parameters: {
             target: {
                 type: 'object',
@@ -560,8 +592,8 @@ export function apply(ctx, config = {}) {
                 },
                 description: 'Locate the element semantically and click it (css/text/xpath).',
             },
-            x: { type: 'number', description: 'Viewport x coordinate (CSS px), e.g. from a vision model reading the screenshot.' },
-            y: { type: 'number', description: 'Viewport y coordinate (CSS px).' },
+            x: { type: 'number', description: 'Viewport x coordinate (CSS px), taken from a browser_screenshot read by a vision model. Prefer a semantic target — it works without image input.' },
+            y: { type: 'number', description: 'Viewport y coordinate (CSS px), taken from the same screenshot as x.' },
         },
         output: {
             schema: { type: 'object', additionalProperties: false, properties: { clicked: { type: 'boolean', required: true } } },
@@ -1039,7 +1071,7 @@ export function apply(ctx, config = {}) {
     }));
     ctx.tools.register(defineTool({
         name: 'browser_screenshot',
-        description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). Use for visual confirmation of layout, charts, designs, or CAPTCHAs, or to feed a vision tool (read_image) that locates elements visually. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG is only available on the self-hosted native path; the desktop-shell path returns PNG.',
+        description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it — browser_snapshot, browser_a11y and browser_content carry the same page as text, and browser_scrape extracts structured data. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG is only available on the self-hosted native path; the desktop-shell path returns PNG.',
         parameters: {
             fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of the viewport (default false).' },
             savePath: { type: 'string', description: 'Absolute file path to also save the image to (e.g. for read_image vision location). Must resolve inside the configured downloadDir (default: the system Downloads folder, localized names such as ~/下载 included); an existing file is never overwritten.' },
@@ -1285,6 +1317,80 @@ export function apply(ctx, config = {}) {
             return { entries: rendered };
         },
     }));
+    ctx.tools.register(defineTool({
+        name: 'browser_visited',
+        description: 'List pages the shared browser has visited, newest first, from the PERSISTENT browsing history: it survives closing the browser and restarting DSH, so it is how you (or the human) find a page again afterwards. Reopen one by passing its url to browser_open. Not to be confused with browser_history, which is only the current session\'s operation log (navigate/click/type) and disappears with the session.',
+        parameters: {
+            limit: { type: 'number', description: 'Maximum entries to return (default 30, capped at 200).' },
+            domain: { type: 'string', description: 'Only visits whose hostname contains this text (case-insensitive).' },
+            query: { type: 'string', description: 'Only visits whose URL or title contains this text (case-insensitive).' },
+            session: { type: 'string', description: 'Only visits recorded by this task/session label — use it to separate your own pages from another session\'s or the human\'s.' },
+        },
+        output: {
+            schema: {
+                type: 'object',
+                additionalProperties: false,
+                properties: {
+                    count: { type: 'number', required: true },
+                    entries: {
+                        type: 'array',
+                        required: true,
+                        items: {
+                            type: 'object',
+                            additionalProperties: false,
+                            properties: {
+                                at: { type: 'number', required: true },
+                                url: { type: 'string', required: true },
+                                title: { type: 'string' },
+                                session: { type: 'string' },
+                            },
+                        },
+                    },
+                },
+            },
+            render: (_args, value) => {
+                const entries = value.entries;
+                if (entries.length === 0) {
+                    return [{ type: 'text', text: '(no recorded visits — nothing has been browsed yet, or history recording is turned off in settings)' }];
+                }
+                return [{
+                        type: 'text',
+                        text: entries.map(entry => {
+                            const when = new Date(entry.at).toISOString().replace('T', ' ').slice(0, 19);
+                            const title = entry.title !== undefined ? `${entry.title} — ` : '';
+                            return `${when}  ${title}${entry.url}${entry.session !== undefined ? `  [${entry.session}]` : ''}`;
+                        }).join('\n'),
+                    }];
+            },
+        },
+        timeoutMs,
+        isConcurrencySafe: () => true,
+        async execute(args, _exec) {
+            const browser = ctx.get('browser');
+            if (browser === undefined)
+                throw new Error('tool-browser: browser service unavailable');
+            const rawLimit = typeof args.limit === 'number' && Number.isFinite(args.limit) ? args.limit : 30;
+            const limit = Math.max(1, Math.min(200, Math.floor(rawLimit)));
+            const domain = typeof args.domain === 'string' && args.domain.trim() !== '' ? args.domain.trim() : undefined;
+            const query = typeof args.query === 'string' && args.query.trim() !== '' ? args.query.trim() : undefined;
+            const session = typeof args.session === 'string' && args.session.trim() !== '' ? args.session.trim() : undefined;
+            const entries = browser.visited({
+                limit,
+                ...domain !== undefined ? { domain } : {},
+                ...query !== undefined ? { query } : {},
+                ...session !== undefined ? { session } : {},
+            });
+            return {
+                count: entries.length,
+                entries: entries.map(entry => ({
+                    at: entry.at,
+                    url: entry.url,
+                    ...entry.title !== undefined ? { title: entry.title } : {},
+                    ...entry.session !== undefined ? { session: entry.session } : {},
+                })),
+            };
+        },
+    }));
     ctx.tools.register(defineTool({
         name: 'browser_replay',
         description: 'Replay one recorded browser operation by its history sequence number (from browser_history). Navigate/click/type are re-issued against the current page; execute re-runs its script. The replayed step is appended to history as a new entry.',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-builtin-browser",
-  "version": "0.1.23",
+  "version": "0.3.0",
   "description": "Shared real browser plugin for DeepSeek Harness: install-and-use — a visible native browser the human can take over, driven by the agent over CDP. DOM-level interaction (React/Vue safe), per-task session isolation, cookie persistence (browser_auth), CAPTCHA detection, batch form fill (browser_fill), authenticated download, history/replay, action restriction",
   "type": "module",
   "main": "lib/index.js",
@@ -25,6 +25,9 @@
     "./host-main": {
       "default": "./lib/browser-electron/host-main.js"
     },
+    "./client": {
+      "default": "./client.js"
+    },
     "./cordis.patch.yml": "./cordis.patch.yml",
     "./package.json": "./package.json"
   },
@@ -32,6 +35,8 @@
     "lib",
     "src",
     "docs",
+    "desktop-bridge",
+    "client.js",
     "cordis.patch.yml",
     "README.md",
     "README.en.md",
@@ -78,6 +83,14 @@
     "bundle": {
       "patch": "./cordis.patch.yml"
     },
+    "client": {
+      "platform": "web",
+      "inject": [
+        "@deepseek-ai/dsh-client-locale",
+        "@deepseek-ai/dsh-client-ui-slots",
+        "@deepseek-ai/dsh-client-ui-settings"
+      ]
+    },
     "compatibility": {
       "dsh": ">=0.1.1-rc.1 <0.3.0",
       "profiles": [

package/src/browser/runtime.ts

@@ -47,6 +47,7 @@ import type {
   BrowserKeyRequest,
   BrowserChallenge,
   ExportedCookie,
+  VisitedPage,
 } from './types.js'
 import { BrowserError } from './types.js'
 
@@ -345,6 +346,27 @@ export class BrowserRuntime extends Service {
     return this.resolveProvider().history(session)
   }
 
+  /**
+   * Persisted visits, newest first — the record that outlives the browser
+   * process, as opposed to {@link history}'s session-scoped operation log.
+   * @param options - result cap and an optional hostname filter.
+   * @returns the retained visits (empty when history recording is disabled).
+   */
+  visited(options?: { readonly limit?: number; readonly domain?: string }): readonly VisitedPage[] {
+    return this.resolveProvider().visited(options)
+  }
+
+  /**
+   * Whether a session is still live. Sessions can end outside the tool layer
+   * (the human closed its window), so a cached id must be re-checked before
+   * reuse; a provider that does not implement the check counts as live.
+   * @param session - the session id to test.
+   */
+  exists(session: BrowserSessionId): boolean {
+    const provider = this.resolveProvider()
+    return provider.exists === undefined ? true : provider.exists(session)
+  }
+
   /** Replay one recorded operation by sequence number through the provider. */
   async replay(session: BrowserSessionId, seq: number): Promise<void> {
     return this.resolveProvider().replay(session, seq)

package/src/browser/types.ts

@@ -573,12 +573,43 @@ export interface BrowserProvider {
   restoreAuth(session: BrowserSessionId, cookies: readonly ExportedCookie[]): Promise<number>
   /** Return the session's chronological operation log. */
   history(session: BrowserSessionId): Promise<readonly BrowserHistoryEntry[]>
+  /**
+   * Persisted visits, newest first: what survives the browser process, so a
+   * human or an Agent can find a page again after the interface was closed.
+   * @param options - result cap and an optional hostname filter.
+   * @returns the retained visits (empty when history is disabled).
+   */
+  visited(options?: { readonly limit?: number; readonly domain?: string }): readonly VisitedPage[]
+  /**
+   * Optional: whether a session is still live. A session can end outside the
+   * tool layer — the human closed the browser window — so a cached id must be
+   * checked before it is reused. Providers that never end a session on their own
+   * can omit this, and callers treat absence as "still live".
+   * @param session - the session id to test.
+   */
+  exists?(session: BrowserSessionId): boolean
   /** Replay one recorded operation by sequence number. */
   replay(session: BrowserSessionId, seq: number): Promise<void>
   /** Close the session and destroy its backing surface. Idempotent. */
   close(session: BrowserSessionId): Promise<void>
 }
 
+/**
+ * One visited page in the persistent browsing history — the durable record that
+ * outlives sessions, unlike {@link BrowserHistoryEntry}, which is a
+ * session-scoped operation log (navigate/click/type/…).
+ */
+export interface VisitedPage {
+  /** Epoch milliseconds when the visit was recorded. */
+  readonly at: number
+  /** Absolute URL of the visited document. */
+  readonly url: string
+  /** Document title at visit time, when the page exposed one. */
+  readonly title?: string
+  /** Session/task label that drove the visit, when the provider knows it. */
+  readonly session?: string
+}
+
 /** One recorded browser operation, in chronological order (seq 1, 2, 3…). */
 export interface BrowserHistoryEntry {
   /** Monotonic sequence number within the session. */

package/src/browser-electron/bridge-connection.ts

@@ -0,0 +1,145 @@
+/**
+ * Transport for the desktop shell's browser bridge: one long-lived connection to a
+ * loopback TCP service.
+ *
+ * WHY THIS IS ITS OWN MODULE
+ * The bridge used to be reached with a fresh connection per request, which measured
+ * at ~25ms per call (TCP handshake + token exchange) against ~0.2ms when the socket
+ * is reused. A single browser action issues several CDP commands, so that cost was
+ * multiplied on every tool call — the difference between a snappy agent and a
+ * visibly sluggish one. Reuse is therefore a property of the transport, and keeping
+ * it here means the view host above does not have to think about sockets at all.
+ *
+ * PROTOCOL
+ * The first line on a connection carries the token; the service marks the socket
+ * authenticated and ignores that line as a command. Requests and answers are then
+ * newline-delimited JSON, one answer per request, in order.
+ *
+ * @module dsh-browser/browser-electron/bridge-connection
+ */
+
+import { connect, type Socket } from 'node:net'
+
+/** Where the shell publishes its bridge endpoint. */
+export interface BridgeEndpoint {
+  readonly port: number
+  readonly token: string
+  readonly pid: number
+  readonly updatedAt?: string
+}
+
+/** One queued request, waiting for its answer. */
+interface PendingCall {
+  readonly resolve: (answer: Record<string, unknown>) => void
+  readonly reject: (error: Error) => void
+}
+
+/**
+ * A reusable connection to the bridge.
+ *
+ * Requests are serialised by construction: one socket, one in-flight request at a
+ * time, answered in order. A broken socket is discarded so the next call dials
+ * again — a dead connection must never become a dead plugin.
+ */
+export class BridgeConnection {
+  private socket: Socket | undefined
+  private buffer = ''
+  private readonly queue: PendingCall[] = []
+
+  /**
+   * @param endpoint - the shell's published bridge endpoint.
+   */
+  constructor(private readonly endpoint: BridgeEndpoint) {}
+
+  /**
+   * Send one request, reusing the socket if it is still healthy.
+   * @param request - the request body (the token is added here).
+   * @param timeoutMs - how long to wait for the answer before dropping the socket.
+   * @returns the bridge's answer.
+   */
+  async call(request: Record<string, unknown>, timeoutMs = 20_000): Promise<Record<string, unknown>> {
+    const socket = this.ensureSocket()
+    return await new Promise<Record<string, unknown>>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        const index = this.queue.findIndex(entry => entry.resolve === resolve)
+        if (index !== -1) this.queue.splice(index, 1)
+        const error = new Error(`dsh-builtin-browser: bridge timed out after ${timeoutMs}ms`)
+        this.reset(error)
+        reject(error)
+      }, timeoutMs)
+      this.queue.push({
+        resolve: answer => { clearTimeout(timer); resolve(answer) },
+        reject: error => { clearTimeout(timer); reject(error) },
+      })
+      try {
+        socket.write(JSON.stringify({ token: this.endpoint.token, ...request }) + '\n')
+      } catch (error) {
+        clearTimeout(timer)
+        this.queue.pop()
+        const failure = error instanceof Error ? error : new Error(String(error))
+        this.reset(failure)
+        reject(failure)
+      }
+    })
+  }
+
+  /** Close the socket and fail anything still waiting on it. */
+  close(): void {
+    this.reset(new Error('dsh-builtin-browser: bridge closed'))
+  }
+
+  /** The live socket, connecting and authenticating it on first use. */
+  private ensureSocket(): Socket {
+    if (this.socket !== undefined) return this.socket
+    const socket = connect({ host: '127.0.0.1', port: this.endpoint.port })
+    socket.setEncoding('utf8')
+    socket.on('data', chunk => this.onData(String(chunk)))
+    socket.on('error', error => this.reset(error))
+    socket.on('close', () => this.reset(new Error('dsh-builtin-browser: bridge connection closed')))
+    this.buffer = ''
+    this.socket = socket
+    socket.write(JSON.stringify({ token: this.endpoint.token }) + '\n')
+    return socket
+  }
+
+  /** Resolve queued requests in order as answers arrive. */
+  private onData(chunk: string): void {
+    this.buffer += chunk
+    let newline = this.buffer.indexOf('\n')
+    while (newline !== -1) {
+      const line = this.buffer.slice(0, newline).trim()
+      this.buffer = this.buffer.slice(newline + 1)
+      newline = this.buffer.indexOf('\n')
+      if (line === '') continue
+      const entry = this.queue.shift()
+      if (entry === undefined) continue
+      try {
+        const answer = JSON.parse(line) as Record<string, unknown>
+        // A rejected token means the shell restarted with a new one, so this socket
+        // can never succeed again: answer (the caller needs the error) and drop it,
+        // so the next call dials afresh instead of retrying a doomed credential.
+        if (answer.ok === false && answer.error === 'bad token') {
+          entry.resolve(answer)
+          this.reset(new Error('dsh-builtin-browser: the bridge rejected our token'))
+          continue
+        }
+        entry.resolve(answer)
+      } catch (error) {
+        entry.reject(new Error(`dsh-builtin-browser: malformed bridge answer (${String(error)})`))
+      }
+    }
+  }
+
+  /** Tear the socket down, failing pending work. */
+  private reset(error: Error): void {
+    if (this.socket !== undefined) {
+      const socket = this.socket
+      this.socket = undefined
+      socket.removeAllListeners()
+      socket.destroy()
+    }
+    this.buffer = ''
+    const pending = this.queue.splice(0, this.queue.length)
+    for (const entry of pending) entry.reject(error)
+  }
+}