dsh-browser-plus 0.5.0 → 0.5.1

package/src/tool-browser/index.ts

@@ -17,7 +17,7 @@
 import type { Context } from '@deepseek-ai/cordis'
 import { defineTool } from '@deepseek-ai/dsh-tools'
 import type {} from '@deepseek-ai/dsh-system-prompt'
-import type { BrowserSessionId } from '../browser/types.ts'
+import type { BrowserSessionId, BrowserSnapshotResult } from '../browser/types.ts'
 
 /** Plugin name used by loader diagnostics. */
 export const name = 'tool-browser'
@@ -280,6 +280,88 @@ function formatSnapshot(snapshot: {
   return `${header}\n\n${body}${tail}${banner}${user}`
 }
 
+/**
+ * The snapshot shape both `browser_open` and `browser_snapshot` return.
+ *
+ * Shared on purpose: a tool output schema rejects unknown properties, so a
+ * field declared on only one of the two schemas is silently dropped before the
+ * model ever sees it — which is exactly what happened to `frames` and the
+ * element `frame` index when iframe support landed (declared on browser_open,
+ * missing from browser_snapshot, and dropped from browser_open's own mapper).
+ */
+const SNAPSHOT_OUTPUT_SCHEMA = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    snapshotId: { type: 'string', required: true },
+    url: { type: 'string', required: true },
+    title: { type: 'string' },
+    truncated: { type: 'boolean' },
+    frames: {
+      type: 'array',
+      description: 'Every iframe on the page; readable:false marks the cross-origin ones whose contents cannot be read.',
+      items: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          index: { type: 'number', required: true },
+          url: { type: 'string', required: true },
+          readable: { type: 'boolean', required: true },
+        },
+      },
+    },
+    elements: {
+      type: 'array',
+      required: true,
+      items: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          ref: { type: 'number', required: true },
+          kind: { type: 'string', required: true },
+          label: { type: 'string', required: true },
+          x: { type: 'number', required: true },
+          y: { type: 'number', required: true },
+          loc: { type: 'string', required: true },
+          frame: { type: 'number', description: 'Index into frames when this element lives inside an iframe.' },
+        },
+      },
+    },
+    challenge: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        blocked: { type: 'boolean', required: true },
+        kind: { type: 'string' },
+        reason: { type: 'string' },
+      },
+    },
+    userControlling: { type: 'boolean' },
+  },
+} as const
+
+/** Project a provider snapshot onto that shared model-facing shape. */
+function snapshotOutput(snapshot: BrowserSnapshotResult) {
+  return {
+    snapshotId: snapshot.snapshotId,
+    url: snapshot.url,
+    ...snapshot.title !== undefined ? { title: snapshot.title } : {},
+    ...snapshot.frames !== undefined ? { frames: snapshot.frames.map(frame => ({ index: frame.index, url: frame.url, readable: frame.readable })) } : {},
+    elements: snapshot.elements.map(el => ({
+      ref: el.ref,
+      kind: el.kind,
+      label: el.label,
+      x: el.x,
+      y: el.y,
+      loc: el.loc,
+      ...el.frame !== undefined ? { frame: el.frame } : {},
+    })),
+    truncated: snapshot.truncated,
+    ...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
+    ...snapshot.userControlling !== undefined ? { userControlling: snapshot.userControlling } : {},
+  }
+}
+
 /** Register all browser tools with `ctx.tools`. */
 export function apply(ctx: Context, config: Config = {}): void {
   const timeoutMs = config.timeoutMs ?? 60_000
@@ -311,42 +393,7 @@ export function apply(ctx: Context, config: Config = {}): void {
       space: { type: 'string', description: 'Optional browser-task label shown in the task manager and active window title.' },
     },
     output: {
-      schema: {
-        type: 'object',
-        additionalProperties: false,
-        properties: {
-          snapshotId: { type: 'string', required: true },
-          url: { type: 'string', required: true },
-          title: { type: 'string' },
-          truncated: { type: 'boolean' },
-          elements: {
-            type: 'array',
-            required: true,
-            items: {
-              type: 'object',
-              additionalProperties: false,
-              properties: {
-                ref: { type: 'number', required: true },
-                kind: { type: 'string', required: true },
-                label: { type: 'string', required: true },
-                x: { type: 'number', required: true },
-                y: { type: 'number', required: true },
-                loc: { type: 'string', required: true },
-              },
-            },
-          },
-          challenge: {
-            type: 'object',
-            additionalProperties: false,
-            properties: {
-              blocked: { type: 'boolean', required: true },
-              kind: { type: 'string' },
-              reason: { type: 'string' },
-            },
-          },
-          userControlling: { type: 'boolean' },
-        },
-      },
+      schema: SNAPSHOT_OUTPUT_SCHEMA,
       render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
     },
     timeoutMs,
@@ -362,15 +409,7 @@ export function apply(ctx: Context, config: Config = {}): void {
           ...args.newTab === true ? { newTab: true } : {},
         }, exec.signal)
         const snapshot = await browser.snapshot(session, {}, exec.signal)
-        return {
-          snapshotId: snapshot.snapshotId,
-          url: snapshot.url,
-          ...snapshot.title !== undefined ? { title: snapshot.title } : {},
-          elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, x: el.x, y: el.y, loc: el.loc })),
-          truncated: snapshot.truncated,
-          ...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
-          ...snapshot.userControlling !== undefined ? { userControlling: snapshot.userControlling } : {},
-        }
+        return snapshotOutput(snapshot)
       }, args.space)
     },
   }))
@@ -597,42 +636,7 @@ export function apply(ctx: Context, config: Config = {}): void {
       limit: { type: 'number', description: 'Maximum number of elements to return (1-1000; default 60).' },
     },
     output: {
-      schema: {
-        type: 'object',
-        additionalProperties: false,
-        properties: {
-          snapshotId: { type: 'string', required: true },
-          url: { type: 'string', required: true },
-          title: { type: 'string' },
-          truncated: { type: 'boolean' },
-          elements: {
-            type: 'array',
-            required: true,
-            items: {
-              type: 'object',
-              additionalProperties: false,
-              properties: {
-                ref: { type: 'number', required: true },
-                kind: { type: 'string', required: true },
-                label: { type: 'string', required: true },
-                x: { type: 'number', required: true },
-                y: { type: 'number', required: true },
-                loc: { type: 'string', required: true },
-              },
-            },
-          },
-          challenge: {
-            type: 'object',
-            additionalProperties: false,
-            properties: {
-              blocked: { type: 'boolean', required: true },
-              kind: { type: 'string' },
-              reason: { type: 'string' },
-            },
-          },
-          userControlling: { type: 'boolean' },
-        },
-      },
+      schema: SNAPSHOT_OUTPUT_SCHEMA,
       render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
     },
     timeoutMs,
@@ -645,15 +649,7 @@ export function apply(ctx: Context, config: Config = {}): void {
         ...args.query !== undefined ? { query: args.query } : {},
         ...args.limit !== undefined ? { limit: args.limit } : {},
       }, exec.signal))
-      return {
-        snapshotId: snapshot.snapshotId,
-        url: snapshot.url,
-        ...snapshot.title !== undefined ? { title: snapshot.title } : {},
-        elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, x: el.x, y: el.y, loc: el.loc })),
-        truncated: snapshot.truncated,
-        ...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
-        ...snapshot.userControlling !== undefined ? { userControlling: snapshot.userControlling } : {},
-      }
+      return snapshotOutput(snapshot)
     },
   }))
 
@@ -1224,9 +1220,11 @@ export function apply(ctx: Context, config: Config = {}): void {
 
   ctx.tools.register(defineTool({
     name: 'browser_wait_for',
-    description: 'Wait until an element matching a CSS selector appears (and is visible), polling every 250ms. Use before interacting with dynamically-loaded content (SPA views, toasts, menus).',
+    description: 'Wait until a selector, some text, or both reaches a state, polling every 250ms. `state` is visible (default), attached, hidden or detached — hidden/detached are how you wait for something to go away. Use before clicking or reading, instead of sleeping.',
     parameters: {
-      selector: { type: 'string', required: true, description: 'CSS selector to wait for.' },
+      selector: { type: 'string', description: 'CSS selector to wait for. Omit to watch the document text (give `text`).' },
+      text: { type: 'string', description: 'Text that must be present: inside the matched element when a selector is given, otherwise anywhere in the document.' },
+      state: { type: 'string', enum: ['visible', 'attached', 'hidden', 'detached'], description: 'Which state to wait for. visible (default) needs a 4x4 px element that is not display:none / visibility:hidden; attached only needs it to exist; hidden and detached wait for it to go away.' },
       timeoutMs: { type: 'number', description: `Total budget in ms (default 15000), capped below this tool's ${String(timeoutMs)}ms budget.` },
       visible: { type: 'boolean', description: 'Require visibility (>4x4 px, not display:none). Default true.' },
     },
@@ -1236,12 +1234,18 @@ export function apply(ctx: Context, config: Config = {}): void {
         additionalProperties: false,
         properties: {
           found: { type: 'boolean', required: true },
+          state: { type: 'string', required: true },
           selector: { type: 'string', required: true },
           tag: { type: 'string', required: true },
           text: { type: 'string' },
         },
       },
-      render: (_args, value) => [{ type: 'text', text: `Found <${value.tag}> ${value.selector}${value.text !== undefined ? ` — "${value.text.slice(0, 80)}"` : ''}.` }],
+      render: (_args, value) => [{
+        type: 'text',
+        text: value.selector === ''
+          ? `Found text "${(value.text ?? '').slice(0, 80)}" (state "${value.state}").`
+          : `Found <${value.tag}> ${value.selector}${value.text !== undefined ? ` — "${value.text.slice(0, 80)}"` : ''}.`,
+      }],
     },
     timeoutMs,
     isConcurrencySafe: () => true,
@@ -1250,11 +1254,13 @@ export function apply(ctx: Context, config: Config = {}): void {
       const browser = ctx.get('browser')
       if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
       const result = await withTaskAction(browser, taskKey(exec), 'wait for element', exec, session => browser.waitForElement(session, {
-        selector: args.selector,
+        ...args.selector !== undefined ? { selector: args.selector } : {},
+        ...args.text !== undefined ? { text: args.text } : {},
+        ...args.state !== undefined ? { state: args.state as 'visible' | 'attached' | 'hidden' | 'detached' } : {},
         timeoutMs: withinToolBudget(args.timeoutMs, 15_000),
         ...args.visible !== undefined ? { visible: args.visible } : {},
       }, exec.signal))
-      return { found: true, selector: result.selector, tag: result.tag, text: result.text }
+      return { found: true, state: result.state, selector: result.selector, tag: result.tag, text: result.text }
     },
   }))
 
@@ -1492,6 +1498,97 @@ export function apply(ctx: Context, config: Config = {}): void {
     },
   }))
 
+  ctx.tools.register(defineTool({
+    name: 'browser_pdf',
+    description: 'Print the active tab to a PDF file — Chrome\'s "Save as PDF". The path must be inside the browser write roots. Returns the path and byte size.',
+    parameters: {
+      savePath: { type: 'string', required: true, description: 'Absolute path of the .pdf to write. Must be inside the browser write roots.' },
+      landscape: { type: 'boolean', description: 'Landscape orientation. Default portrait.' },
+      printBackground: { type: 'boolean', description: 'Include background colours and images. Default true — dark pages would otherwise print as white paper.' },
+      paperWidth: { type: 'number', description: 'Paper width in inches. Default 8.5.' },
+      paperHeight: { type: 'number', description: 'Paper height in inches. Default 11.' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          path: { type: 'string', required: true },
+          bytes: { type: 'number', required: true },
+        },
+      },
+      render: (_args, value) => [{ type: 'text', text: `Wrote ${value.bytes} bytes to ${value.path}` }],
+    },
+    timeoutMs,
+    isConcurrencySafe: () => true,
+    async execute(args, exec) {
+      assertAllowed('browser_pdf', exec)
+      const browser = ctx.get('browser')
+      if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
+      const result = await withTaskAction(browser, taskKey(exec), 'print to pdf', exec, session => browser.pdf(session, {
+        savePath: args.savePath,
+        ...args.landscape !== undefined ? { landscape: args.landscape } : {},
+        ...args.printBackground !== undefined ? { printBackground: args.printBackground } : {},
+        ...args.paperWidth !== undefined ? { paperWidth: args.paperWidth } : {},
+        ...args.paperHeight !== undefined ? { paperHeight: args.paperHeight } : {},
+      }, exec.signal))
+      return { path: result.path, bytes: result.bytes }
+    },
+  }))
+
+  ctx.tools.register(defineTool({
+    name: 'browser_highlight',
+    description: 'Draw the DevTools-style highlight box over the first element matching a CSS selector, so a human watching the window can see what the Agent is about to touch. It goes through the CDP overlay, so the page DOM is never modified. Pass clear: true to remove it.',
+    parameters: {
+      selector: { type: 'string', description: 'CSS selector whose first match to highlight. Required unless clear is true.' },
+      clear: { type: 'boolean', description: 'Remove the highlight instead of drawing one.' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          matched: { type: 'boolean', required: true },
+          cleared: { type: 'boolean', required: true },
+          nodeId: { type: 'number' },
+          box: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+              x: { type: 'number', required: true },
+              y: { type: 'number', required: true },
+              width: { type: 'number', required: true },
+              height: { type: 'number', required: true },
+            },
+          },
+        },
+      },
+      render: (_args, value) => [{
+        type: 'text',
+        text: value.matched
+          ? `Highlighted node ${String(value.nodeId)} at ${Math.round(value.box?.x ?? 0)},${Math.round(value.box?.y ?? 0)} (${Math.round(value.box?.width ?? 0)}x${Math.round(value.box?.height ?? 0)})`
+          : value.cleared ? 'Highlight cleared' : 'No element matched that selector',
+      }],
+    },
+    timeoutMs,
+    isConcurrencySafe: () => true,
+    async execute(args, exec) {
+      assertAllowed('browser_highlight', exec)
+      const browser = ctx.get('browser')
+      if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
+      const result = await withTaskAction(browser, taskKey(exec), 'highlight element', exec, session => browser.highlight(session, {
+        ...args.selector !== undefined ? { selector: args.selector } : {},
+        ...args.clear !== undefined ? { clear: args.clear } : {},
+      }, exec.signal))
+      return {
+        matched: result.matched,
+        cleared: result.cleared,
+        ...result.nodeId !== undefined ? { nodeId: result.nodeId } : {},
+        ...result.box !== undefined ? { box: result.box } : {},
+      }
+    },
+  }))
+
   if (config.tabTools !== false) {
     ctx.tools.register(defineTool({
       name: 'browser_list_tabs',