dsh-builtin-browser 0.1.23 → 0.3.0

package/src/browser-electron/virtual-cursor.ts

@@ -0,0 +1,152 @@
+/**
+ * Synthetic pointer feedback for a human watching the tab: an overlay drawn
+ * INSIDE the page, never the real system pointer. Its presence is the "the agent
+ * has taken over this tab" signal, so it also covers DOM-level operations that
+ * move no pointer at all.
+ *
+ * Implementation notes that matter:
+ * - The overlay is `position: fixed` and positioned from viewport coordinates,
+ *   which is exactly what the element locator already computes and what
+ *   `Input.dispatchMouseEvent` consumes.
+ * - All styling is inline and the ripple uses the Web Animations API: a
+ *   `<style>` element would be subject to the page's `style-src` CSP, an
+ *   injected script is not. Injected page feedback must never be silently
+ *   dropped by a strict CSP.
+ * - Every call is best-effort: feedback is never allowed to fail an operation.
+ * @module dsh-browser/browser-electron/virtual-cursor
+ */
+
+/** What the pointer is doing at the reported point. */
+export type CursorAction = 'move' | 'click'
+
+/** Element id of the overlay root, stable so repeat calls update one node. */
+const CURSOR_ID = '__dsh_agent_cursor__'
+
+/**
+ * Where the overlay was last placed, so an unchanged position is not repainted.
+ *
+ * A single tool call issues several CDP commands (locate, click, cursor), and the
+ * cursor is repainted for each one that moves it. Repainting to the same pixel is a
+ * wasted round-trip *and* a visible flicker: the transition restarts from the same
+ * place, which reads as a stutter rather than as movement.
+ */
+const lastPlaced = new WeakMap<object, { x: number; y: number; action: CursorAction }>()
+
+/**
+ * Whether a paint at this position would change anything on screen.
+ * @param handle - the view the overlay belongs to.
+ * @param x - viewport x in CSS pixels.
+ * @param y - viewport y in CSS pixels.
+ * @param action - the action being reported.
+ * @param force - paint even when unchanged (a click ripple must always fire).
+ * @returns true when the overlay should be painted.
+ */
+export function cursorNeedsPaint<TView>(handle: TView, x: number, y: number, action: CursorAction, force = false): boolean {
+  if (force) return true
+  const key = handle as unknown as object
+  if (key === null || typeof key !== 'object') return true
+  const previous = lastPlaced.get(key)
+  return previous === undefined || previous.x !== x || previous.y !== y || previous.action !== action
+}
+
+/**
+ * The in-page expression that materializes or moves the overlay. Idempotent: the
+ * first call creates the nodes, later calls only move them (and pulse).
+ * @param x - viewport x in CSS pixels.
+ * @param y - viewport y in CSS pixels.
+ * @param action - whether to pulse a click ripple at the point.
+ * @param label - optional short description of the operation, shown beside the
+ *   pointer. The overlay is the only "the agent has taken over this tab" signal, so
+ *   saying *what* it is doing is what turns it from a decoration into an account of
+ *   the agent's actions that a human can follow without reading the tool log.
+ * @returns a self-contained expression that evaluates to `true`.
+ */
+export function cursorScript(x: number, y: number, action: CursorAction, label?: string): string {
+  return `(() => {
+  const id = ${JSON.stringify(CURSOR_ID)}
+  let root = document.getElementById(id)
+  if (!root) {
+    root = document.createElement('div')
+    root.id = id
+    root.setAttribute('aria-hidden', 'true')
+    root.style.cssText = 'position:fixed;left:0;top:0;width:0;height:0;z-index:2147483647;pointer-events:none;transition:transform 190ms cubic-bezier(.22,.85,.24,1);will-change:transform'
+    const svg = '<svg width="22" height="22" viewBox="0 0 22 22" style="position:absolute;left:-2px;top:-2px;filter:drop-shadow(0 1px 2px rgba(0,0,0,.45))">'
+      + '<path d="M4 2 L4 18 L8.4 13.8 L11.2 19.6 L13.8 18.4 L11 12.8 L17 12.6 Z" fill="#ffffff" stroke="#111111" stroke-width="1.4" stroke-linejoin="round"/></svg>'
+    const ripple = '<span data-dsh-role="ripple" style="position:absolute;left:0;top:0;width:34px;height:34px;margin:-17px 0 0 -17px;border-radius:50%;border:2px solid rgba(64,160,255,.9);opacity:0"></span>'
+    // The bubble sits below-right of the pointer tip and is capped in width so a long
+    // label cannot cover the page it is describing.
+    const bubble = '<span data-dsh-role="bubble" style="position:absolute;left:18px;top:16px;max-width:280px;padding:3px 8px;border-radius:5px;background:rgba(17,17,17,.92);color:#fff;font:12px/1.45 -apple-system,Segoe UI,Roboto,sans-serif;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;opacity:0;box-shadow:0 1px 3px rgba(0,0,0,.4)"></span>'
+    root.innerHTML = svg + ripple + bubble
+    ;(document.body || document.documentElement).appendChild(root)
+  }
+  root.style.transform = 'translate(${x}px, ${y}px)'
+  ${action === 'click' ? `const ripple = root.querySelector('[data-dsh-role="ripple"]')
+  if (ripple && typeof ripple.animate === 'function') {
+    ripple.animate(
+      [{ transform: 'scale(.25)', opacity: 0.95 }, { transform: 'scale(1.6)', opacity: 0 }],
+      { duration: 550, easing: 'ease-out' },
+    )
+  }` : ''}
+  ${label !== undefined && label !== '' ? `const bubble = root.querySelector('[data-dsh-role="bubble"]')
+  if (bubble) {
+    bubble.textContent = ${JSON.stringify(label)}
+    if (typeof bubble.animate === 'function') {
+      // Fade in, hold, fade out: the label is a narration of the action, not a
+      // permanent caption, so a stale one never lingers over later work.
+      bubble.animate(
+        [{ opacity: 0, transform: 'translateY(-3px)' }, { opacity: 1, transform: 'translateY(0)' }, { opacity: 1, offset: 0.6 }, { opacity: 0 }],
+        { duration: 2200, easing: 'ease-out', fill: 'forwards' },
+      )
+    } else {
+      bubble.style.opacity = '1'
+    }
+  }` : ''}
+  return true
+})()`
+}
+
+/**
+ * Forget the remembered position for a view.
+ *
+ * Called when the document is replaced: the overlay lived in the old document and
+ * went with it, so the cache would otherwise claim the pointer is already where it
+ * needs to be and skip every later paint — the pointer would simply never return
+ * after a navigation.
+ * @param handle - the view whose cache should be dropped.
+ */
+export function forgetCursor<TView>(handle: TView): void {
+  const key = handle as unknown as object
+  if (key !== null && typeof key === 'object') lastPlaced.delete(key)
+}
+
+/**
+ * Draw (or move) the pointer overlay on one view's page.
+ *
+ * Fire-and-forget: a page that is mid-navigation, cross-origin-blocked, or
+ * otherwise unable to answer must not turn a successful operation into a
+ * failure, and the navigation that just committed may still be replacing the
+ * document this call would have painted into.
+ * @param handle - the view to paint into.
+ * @param x - viewport x in CSS pixels.
+ * @param y - viewport y in CSS pixels.
+ * @param action - whether to pulse a click ripple.
+ * @param evaluate - the provider's evaluate bridge (keeps the CDP coupling in one place).
+ * @param label - optional short description of the operation, shown beside the pointer.
+ */
+export function paintCursor<TView>(
+  handle: TView,
+  x: number,
+  y: number,
+  action: CursorAction,
+  evaluate: (handle: TView, expression: string) => Promise<unknown>,
+  label?: string,
+  force = false,
+): void {
+  if (!Number.isFinite(x) || !Number.isFinite(y)) return
+  const roundedX = Math.round(x)
+  const roundedY = Math.round(y)
+  if (!cursorNeedsPaint(handle, roundedX, roundedY, action, force)) return
+  const key = handle as unknown as object
+  if (key !== null && typeof key === 'object') lastPlaced.set(key, { x: roundedX, y: roundedY, action })
+  void evaluate(handle, cursorScript(roundedX, roundedY, action, label)).catch(() => {})
+}

package/src/index.ts

@@ -4,11 +4,29 @@
  *   - `dsh-builtin-browser/browser`          -> the ctx.browser seam (Service)
  *   - `dsh-builtin-browser/browser-electron` -> the Electron CDP provider
  *   - `dsh-builtin-browser/tool-browser`     -> the model-facing browser_* tools
- * This root entry only re-exports for programmatic use; the loader rows are
- * the composition surface.
+ * The capability itself is composed by those rows; this root entry exists for
+ * programmatic imports AND because the host's client-module scan identifies a
+ * package's browser half through a loader row whose specifier resolves to the
+ * PACKAGE ROOT. A row named after a subpath (`dsh-builtin-browser/browser`) is
+ * not a package specifier and is skipped by design, which is why the patch also
+ * registers a row under the bare package name — see cordis.patch.yml.
  * @module dsh-builtin-browser
  */
 
+/** Plugin identity for the root row (the composition surface is the subpath rows). */
+export const name = 'dsh-builtin-browser'
+
+/**
+ * Inert application for the root row: the browser seam, the Electron provider and
+ * the tools are mounted by the subpath rows, so this one only exists to give the
+ * package a loader row named after the package itself. Without it the host's
+ * client scan finds no row to resolve `dsh.client` from, and the settings panel
+ * never reaches the browser (the plugin works, its client half is invisible).
+ */
+export function apply(): void {
+  // Intentionally empty: see the module comment above.
+}
+
 export { BrowserError } from './browser/types.js'
 export type {
   BrowserA11yNode,

package/src/tool-browser/index.ts

@@ -105,7 +105,12 @@ function taskKey(exec: { agent?: { id?: string } } | undefined): string {
  */
 async function ensureSession(browser: NonNullable<Context['browser']>, state: ToolBrowserState, key: string, agent?: ExecAgent): Promise<BrowserSessionId> {
   const existing = state.sessionsByTask.get(key)
-  if (existing !== undefined) return existing
+  if (existing !== undefined) {
+    // The human closing a browser window ends that session, so a cached id can
+    // be stale: never hand a dead session back to a tool call.
+    if (browser.exists(existing)) return existing
+    state.sessionsByTask.delete(key)
+  }
   const pending = state.pendingOpens.get(key)
   if (pending !== undefined) return pending
   // The task key rides along as the session label so the window title shows
@@ -156,15 +161,26 @@ function parseFillValue(v: string | undefined): string | number | boolean {
   return v ?? ''
 }
 
-/** Format a snapshot element list for the model. */
+/**
+ * Format a snapshot element list for the model.
+ *
+ * Coordinates are opt-in. They are only meaningful to a caller that intends to
+ * click a pixel position, which means a vision pass; for the far more common case
+ * of locating an element by role/label and passing a semantic target, they are the
+ * bulkiest and least useful part of every line. Omitting them keeps the listing
+ * readable — which is exactly what a model without image input has to work from.
+ * @param snapshot - the snapshot payload.
+ * @param options - `coords: true` to include each element's viewport position.
+ */
 function formatSnapshot(snapshot: {
   url: string
   title?: string
   elements: readonly { ref: number; kind: string; label: string; x: number; y: number; frame?: boolean }[]
   truncated?: boolean
   challenge?: { blocked: boolean; kind?: string; reason?: string }
-}): string {
-  const lines = snapshot.elements.map(el => `[${el.ref}] ${el.kind}: ${el.label}${el.frame === true ? ' (iframe)' : ''} (${el.x},${el.y})`)
+}, options: { coords?: boolean } = {}): string {
+  const showCoords = options.coords === true
+  const lines = snapshot.elements.map(el => `[${el.ref}] ${el.kind}: ${el.label}${el.frame === true ? ' (iframe)' : ''}${showCoords ? ` (${el.x},${el.y})` : ''}`)
   const header = `URL: ${snapshot.url}${snapshot.title !== undefined ? `\nTitle: ${snapshot.title}` : ''}`
   const body = lines.length > 0 ? lines.join('\n') : '(no interactive elements found)'
   const tail = snapshot.truncated === true ? '\n(snapshot truncated)' : ''
@@ -236,7 +252,10 @@ export function apply(ctx: Context, config: Config = {}): void {
           },
         },
       },
-      render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
+      // browser_open is an overview of what just loaded, not a locator: it has no
+      // `coords` parameter, so coordinates stay off here (browser_snapshot owns
+      // that switch for callers that actually target pixels).
+      render: (args, value) => [{ type: 'text', text: formatSnapshot(value, { coords: (args as { coords?: boolean }).coords === true }) }],
     },
     timeoutMs,
     isConcurrencySafe: () => false, // opens tabs / navigates; exclusive within a task
@@ -262,8 +281,10 @@ export function apply(ctx: Context, config: Config = {}): void {
 
   ctx.tools.register(defineTool({
     name: 'browser_snapshot',
-    description: 'Return an AI-friendly snapshot of the current shared-browser page: numbered interactive elements (inputs, buttons, links) the model can cite. Use this to understand an interactive page before driving it.',
-    parameters: {},
+    description: 'Return an AI-friendly snapshot of the current shared-browser page: numbered interactive elements (inputs, buttons, links) the model can cite. This is the primary way to understand a page, and it needs no image input — every listed element can afterwards be addressed by a semantic target (browser_click, browser_type, browser_check, …) rather than by coordinates.',
+    parameters: {
+      coords: { type: 'boolean', description: 'Also print each element\'s viewport coordinates. Off by default; only useful when a vision pass is going to click a pixel position.' },
+    },
     output: {
       schema: {
         type: 'object',
@@ -299,7 +320,7 @@ export function apply(ctx: Context, config: Config = {}): void {
           },
         },
       },
-      render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
+      render: (args, value) => [{ type: 'text', text: formatSnapshot(value, { coords: args.coords === true }) }],
     },
     timeoutMs,
     isConcurrencySafe: () => true,
@@ -324,6 +345,7 @@ export function apply(ctx: Context, config: Config = {}): void {
     parameters: {
       includeHidden: { type: 'boolean', description: 'Include hidden elements (default false).' },
       maxNodes: { type: 'number', description: 'Maximum nodes (default 500, range 10-5000).' },
+      coords: { type: 'boolean', description: 'Include each element\'s viewport coordinates. Off by default: a semantic target needs no coordinates, and they are pure noise unless a vision pass is going to click a pixel position.' },
     },
     output: {
       schema: {
@@ -356,14 +378,24 @@ export function apply(ctx: Context, config: Config = {}): void {
           },
         },
       },
-      render: (_args, value) => {
+      render: (args, value) => {
         const nodes = value.nodes as Array<{ ref: number; role: string; name: string; value?: string | null; states: string[]; depth: number; tag: string; x: number; y: number; frame?: boolean }>
+        // Built for reading, not for imaging: containment is shown as indentation
+        // (the depth was already collected and simply thrown away), an empty state
+        // list adds nothing, and coordinates appear only when asked for — a model
+        // working from the DOM never needs them, and they are the bulkiest part of
+        // every line.
+        const showCoords = args.coords === true
         const lines = nodes.map(n => {
           const valuePart = n.value !== undefined && n.value !== null ? ` value="${String(n.value).slice(0, 60)}"` : ''
-          return `[${n.ref}] ${n.role} "${n.name}"${valuePart} (${n.x},${n.y}) states=[${n.states.join(',')}]${n.frame === true ? ' (iframe)' : ''}`
+          const indent = '  '.repeat(Math.max(0, Math.min(8, Number(n.depth) || 0)))
+          const coordsPart = showCoords ? ` (${n.x},${n.y})` : ''
+          const statePart = Array.isArray(n.states) && n.states.length > 0 ? ` states=[${n.states.join(',')}]` : ''
+          return `${indent}[${n.ref}] ${n.role} "${n.name}"${valuePart}${coordsPart}${statePart}${n.frame === true ? ' (iframe)' : ''}`
         })
         const header = `URL: ${value.url}${value.title !== undefined ? `\nTitle: ${value.title}` : ''}`
-        return [{ type: 'text', text: `${header}\n\n${lines.length > 0 ? lines.join('\n') : '(no accessible interactive nodes)'}${value.truncated === true ? '\n(truncated)' : ''}` }]
+        const hint = showCoords ? '' : '\n(coordinates omitted — pass coords: true to include them, or click by semantic target)'
+        return [{ type: 'text', text: `${header}\n\n${lines.length > 0 ? lines.join('\n') : '(no accessible interactive nodes)'}${value.truncated === true ? '\n(truncated)' : ''}${hint}` }]
       },
     },
     timeoutMs,
@@ -596,7 +628,7 @@ export function apply(ctx: Context, config: Config = {}): void {
 
   ctx.tools.register(defineTool({
     name: 'browser_click',
-    description: 'Click in the shared browser. Two ways: (1) semantic target — pass target {by: css|text|xpath, value, index?} and the element is located, scrolled into view and clicked at its center; (2) viewport coordinates (x/y) — use with browser_screenshot when a vision model locates an element on the screenshot (covers icons, image buttons, and canvas that DOM locators cannot target). Provide exactly one of target or x/y.',
+    description: 'Click in the shared browser. Prefer the semantic target: pass target {by: css|text|xpath, value, index?} and the element is located from the DOM, scrolled into view and clicked at its centre — this needs no image input and is the only form that works without vision. The alternative is viewport coordinates (x/y), which are meaningful only after a vision model has located the element on a browser_screenshot; coordinate clicks are refused outright under the non-visual strategy. Provide exactly one of target or x/y.',
     parameters: {
       target: {
         type: 'object',
@@ -608,8 +640,8 @@ export function apply(ctx: Context, config: Config = {}): void {
         },
         description: 'Locate the element semantically and click it (css/text/xpath).',
       },
-      x: { type: 'number', description: 'Viewport x coordinate (CSS px), e.g. from a vision model reading the screenshot.' },
-      y: { type: 'number', description: 'Viewport y coordinate (CSS px).' },
+      x: { type: 'number', description: 'Viewport x coordinate (CSS px), taken from a browser_screenshot read by a vision model. Prefer a semantic target — it works without image input.' },
+      y: { type: 'number', description: 'Viewport y coordinate (CSS px), taken from the same screenshot as x.' },
     },
     output: {
       schema: { type: 'object', additionalProperties: false, properties: { clicked: { type: 'boolean', required: true } } },
@@ -1084,7 +1116,7 @@ export function apply(ctx: Context, config: Config = {}): void {
 
   ctx.tools.register(defineTool({
     name: 'browser_screenshot',
-    description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). Use for visual confirmation of layout, charts, designs, or CAPTCHAs, or to feed a vision tool (read_image) that locates elements visually. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG is only available on the self-hosted native path; the desktop-shell path returns PNG.',
+    description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it — browser_snapshot, browser_a11y and browser_content carry the same page as text, and browser_scrape extracts structured data. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG is only available on the self-hosted native path; the desktop-shell path returns PNG.',
     parameters: {
       fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of the viewport (default false).' },
       savePath: { type: 'string', description: 'Absolute file path to also save the image to (e.g. for read_image vision location). Must resolve inside the configured downloadDir (default: the system Downloads folder, localized names such as ~/下载 included); an existing file is never overwritten.' },
@@ -1327,6 +1359,80 @@ export function apply(ctx: Context, config: Config = {}): void {
     },
   }))
 
+  ctx.tools.register(defineTool({
+    name: 'browser_visited',
+    description: 'List pages the shared browser has visited, newest first, from the PERSISTENT browsing history: it survives closing the browser and restarting DSH, so it is how you (or the human) find a page again afterwards. Reopen one by passing its url to browser_open. Not to be confused with browser_history, which is only the current session\'s operation log (navigate/click/type) and disappears with the session.',
+    parameters: {
+      limit: { type: 'number', description: 'Maximum entries to return (default 30, capped at 200).' },
+      domain: { type: 'string', description: 'Only visits whose hostname contains this text (case-insensitive).' },
+      query: { type: 'string', description: 'Only visits whose URL or title contains this text (case-insensitive).' },
+      session: { type: 'string', description: 'Only visits recorded by this task/session label — use it to separate your own pages from another session\'s or the human\'s.' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          count: { type: 'number', required: true },
+          entries: {
+            type: 'array',
+            required: true,
+            items: {
+              type: 'object',
+              additionalProperties: false,
+              properties: {
+                at: { type: 'number', required: true },
+                url: { type: 'string', required: true },
+                title: { type: 'string' },
+                session: { type: 'string' },
+              },
+            },
+          },
+        },
+      },
+      render: (_args, value) => {
+        const entries = value.entries as Array<{ at: number; url: string; title?: string; session?: string }>
+        if (entries.length === 0) {
+          return [{ type: 'text', text: '(no recorded visits — nothing has been browsed yet, or history recording is turned off in settings)' }]
+        }
+        return [{
+          type: 'text',
+          text: entries.map(entry => {
+            const when = new Date(entry.at).toISOString().replace('T', ' ').slice(0, 19)
+            const title = entry.title !== undefined ? `${entry.title} — ` : ''
+            return `${when}  ${title}${entry.url}${entry.session !== undefined ? `  [${entry.session}]` : ''}`
+          }).join('\n'),
+        }]
+      },
+    },
+    timeoutMs,
+    isConcurrencySafe: () => true,
+    async execute(args, _exec) {
+      const browser = ctx.get('browser')
+      if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
+      const rawLimit = typeof args.limit === 'number' && Number.isFinite(args.limit) ? args.limit : 30
+      const limit = Math.max(1, Math.min(200, Math.floor(rawLimit)))
+      const domain = typeof args.domain === 'string' && args.domain.trim() !== '' ? args.domain.trim() : undefined
+      const query = typeof args.query === 'string' && args.query.trim() !== '' ? args.query.trim() : undefined
+      const session = typeof args.session === 'string' && args.session.trim() !== '' ? args.session.trim() : undefined
+      const entries = browser.visited({
+        limit,
+        ...domain !== undefined ? { domain } : {},
+        ...query !== undefined ? { query } : {},
+        ...session !== undefined ? { session } : {},
+      })
+      return {
+        count: entries.length,
+        entries: entries.map(entry => ({
+          at: entry.at,
+          url: entry.url,
+          ...entry.title !== undefined ? { title: entry.title } : {},
+          ...entry.session !== undefined ? { session: entry.session } : {},
+        })),
+      }
+    },
+  }))
+
   ctx.tools.register(defineTool({
     name: 'browser_replay',
     description: 'Replay one recorded browser operation by its history sequence number (from browser_history). Navigate/click/type are re-issued against the current page; execute re-runs its script. The replayed step is appended to history as a new entry.',