@vellumai/assistant 0.3.0

package/src/tools/browser/browser-screencast.ts

@@ -0,0 +1,217 @@
+import { v4 as uuid } from 'uuid';
+import { browserManager } from './browser-manager.js';
+import type { BrowserViewSurfaceData, BrowserFrame, ServerMessage } from '../../daemon/ipc-contract.js';
+
+// Track active screencast sessions
+const activeScreencasts = new Map<string, { surfaceId: string }>();
+
+// Registry of sendToClient callbacks per session
+const sessionSenders = new Map<string, (msg: ServerMessage) => void>();
+
+/**
+ * Register a sendToClient callback for a session.
+ * Called from session-tool-setup when the session is created.
+ */
+export function registerSessionSender(sessionId: string, sendToClient: (msg: ServerMessage) => void): void {
+  sessionSenders.set(sessionId, sendToClient);
+  browserManager.registerSender(sessionId, sendToClient as (msg: { type: string; sessionId: string }) => void);
+}
+
+/**
+ * Unregister the sendToClient callback for a session.
+ */
+export function unregisterSessionSender(sessionId: string): void {
+  sessionSenders.delete(sessionId);
+  browserManager.unregisterSender(sessionId);
+}
+
+function getSender(sessionId: string): ((msg: ServerMessage) => void) | undefined {
+  return sessionSenders.get(sessionId);
+}
+
+export async function ensureScreencast(
+  sessionId: string,
+  sendToClient: (msg: ServerMessage) => void,
+): Promise<void> {
+  if (activeScreencasts.has(sessionId)) return;
+
+  const surfaceId = uuid();
+  activeScreencasts.set(sessionId, { surfaceId });
+
+  try {
+    // Get current page info
+    const page = await browserManager.getOrCreateSessionPage(sessionId);
+    const currentUrl = page.url();
+    const title = await page.title();
+
+    // Send surface show
+    sendToClient({
+      type: 'ui_surface_show',
+      sessionId,
+      surfaceId,
+      surfaceType: 'browser_view',
+      title: 'Browser',
+      data: {
+        sessionId,
+        currentUrl: currentUrl || 'about:blank',
+        status: 'idle',
+        pages: [{ id: sessionId, title: title || 'New Tab', url: currentUrl || 'about:blank', active: true }],
+      } satisfies BrowserViewSurfaceData,
+      display: 'panel',
+    });
+
+    // Start CDP screencast
+    await browserManager.startScreencast(sessionId, (frame) => {
+      sendToClient({
+        type: 'browser_frame',
+        sessionId,
+        surfaceId,
+        frame: frame.data,
+        metadata: frame.metadata,
+      } satisfies BrowserFrame);
+    });
+  } catch (err) {
+    // Dismiss the surface we already showed so the client doesn't have an orphaned panel
+    sendToClient({
+      type: 'ui_surface_dismiss',
+      sessionId,
+      surfaceId,
+    });
+    // Roll back so future calls can retry
+    activeScreencasts.delete(sessionId);
+    throw err;
+  }
+}
+
+export function updateBrowserStatus(
+  sessionId: string,
+  sendToClient: (msg: ServerMessage) => void,
+  status: 'navigating' | 'idle' | 'interacting',
+  actionText?: string,
+  currentUrl?: string,
+): void {
+  const state = activeScreencasts.get(sessionId);
+  if (!state) return;
+
+  const update: Record<string, unknown> = { status };
+  if (actionText !== undefined) update.actionText = actionText;
+  if (currentUrl !== undefined) update.currentUrl = currentUrl;
+
+  sendToClient({
+    type: 'ui_surface_update',
+    sessionId,
+    surfaceId: state.surfaceId,
+    data: update,
+  });
+}
+
+export async function updatePagesList(
+  sessionId: string,
+  sendToClient: (msg: ServerMessage) => void,
+): Promise<void> {
+  const state = activeScreencasts.get(sessionId);
+  if (!state) return;
+
+  const page = await browserManager.getOrCreateSessionPage(sessionId);
+  const currentUrl = page.url();
+  const title = await page.title();
+
+  sendToClient({
+    type: 'ui_surface_update',
+    sessionId,
+    surfaceId: state.surfaceId,
+    data: {
+      currentUrl,
+      pages: [{ id: sessionId, title: title || 'Untitled', url: currentUrl, active: true }],
+    },
+  });
+}
+
+export async function stopBrowserScreencast(
+  sessionId: string,
+  sendToClient: (msg: ServerMessage) => void,
+): Promise<void> {
+  const state = activeScreencasts.get(sessionId);
+  if (!state) return;
+
+  await browserManager.stopScreencast(sessionId);
+
+  sendToClient({
+    type: 'ui_surface_dismiss',
+    sessionId,
+    surfaceId: state.surfaceId,
+  });
+
+  activeScreencasts.delete(sessionId);
+}
+
+export async function getElementBounds(
+  sessionId: string,
+  selector: string,
+): Promise<{ x: number; y: number; w: number; h: number } | null> {
+  try {
+    const page = await browserManager.getOrCreateSessionPage(sessionId);
+    const result = await page.evaluate(`
+      (() => {
+        const el = document.querySelector(${JSON.stringify(selector)});
+        if (!el) return null;
+        const rect = el.getBoundingClientRect();
+        return { x: rect.x, y: rect.y, w: rect.width, h: rect.height, vw: window.innerWidth, vh: window.innerHeight };
+      })()
+    `) as { x: number; y: number; w: number; h: number; vw: number; vh: number } | null;
+    if (!result) return null;
+    const scale = Math.min(1280 / result.vw, 960 / result.vh);
+    return {
+      x: result.x * scale,
+      y: result.y * scale,
+      w: result.w * scale,
+      h: result.h * scale,
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function updateHighlights(
+  sessionId: string,
+  sendToClient: (msg: ServerMessage) => void,
+  highlights: Array<{ x: number; y: number; w: number; h: number; label: string }>,
+): void {
+  const state = activeScreencasts.get(sessionId);
+  if (!state) return;
+  sendToClient({
+    type: 'ui_surface_update',
+    sessionId,
+    surfaceId: state.surfaceId,
+    data: { highlights },
+  });
+}
+
+export async function stopAllScreencasts(): Promise<void> {
+  const entries = Array.from(activeScreencasts.entries());
+  for (const [sessionId, state] of entries) {
+    try {
+      await browserManager.stopScreencast(sessionId);
+    } catch { /* best-effort */ }
+    const sender = sessionSenders.get(sessionId);
+    if (sender) {
+      sender({
+        type: 'ui_surface_dismiss',
+        sessionId,
+        surfaceId: state.surfaceId,
+      });
+    }
+  }
+  activeScreencasts.clear();
+}
+
+export function isScreencastActive(sessionId: string): boolean {
+  return activeScreencasts.has(sessionId);
+}
+
+export { getSender };
+
+export function getScreencastSurfaceId(sessionId: string): string | null {
+  const state = activeScreencasts.get(sessionId);
+  return state?.surfaceId ?? null;
+}

package/src/tools/browser/headless-browser.ts

@@ -0,0 +1,450 @@
+import { RiskLevel } from '../../permissions/types.js';
+import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+import { registerTool } from '../registry.js';
+import {
+  executeBrowserNavigate,
+  executeBrowserSnapshot,
+  executeBrowserScreenshot,
+  executeBrowserClose,
+  executeBrowserClick,
+  executeBrowserType,
+  executeBrowserPressKey,
+  executeBrowserScroll,
+  executeBrowserWaitFor,
+  executeBrowserExtract,
+  executeBrowserFillCredential,
+} from './browser-execution.js';
+
+// ── browser_navigate ─────────────────────────────────────────────────
+
+class BrowserNavigateTool implements Tool {
+  name = 'browser_navigate';
+  description = 'Navigate a headless browser to a URL and return the page title and status. Use this to load web pages for inspection or interaction.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          url: {
+            type: 'string',
+            description: 'The URL to navigate to. If scheme is missing, https:// is assumed.',
+          },
+          allow_private_network: {
+            type: 'boolean',
+            description: 'If true, allows navigation to localhost/private-network hosts. Disabled by default for SSRF safety.',
+          },
+        },
+        required: ['url'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserNavigate(input, context);
+  }
+}
+
+registerTool(new BrowserNavigateTool());
+
+// ── browser_snapshot ─────────────────────────────────────────────────
+
+class BrowserSnapshotTool implements Tool {
+  name = 'browser_snapshot';
+  description = 'List interactive elements on the current page. Returns elements with unique IDs (e.g. "e1", "e5") that MUST be used with browser_click, browser_type, and browser_press_key. Always run this before interacting with elements — never guess or fabricate selectors.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {},
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserSnapshot(input, context);
+  }
+}
+
+registerTool(new BrowserSnapshotTool());
+
+// ── browser_screenshot ───────────────────────────────────────────────
+
+class BrowserScreenshotTool implements Tool {
+  name = 'browser_screenshot';
+  description = 'Take a visual screenshot of the current page. Returns a JPEG image.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          full_page: {
+            type: 'boolean',
+            description: 'Capture the full scrollable page instead of just the viewport.',
+          },
+        },
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserScreenshot(input, context);
+  }
+}
+
+registerTool(new BrowserScreenshotTool());
+
+// ── browser_close ────────────────────────────────────────────────────
+
+class BrowserCloseTool implements Tool {
+  name = 'browser_close';
+  description = 'Close the browser page for the current session, or all pages if close_all_pages is true.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          close_all_pages: {
+            type: 'boolean',
+            description: 'If true, close all browser pages and the browser context. Default: false (close only the current session page).',
+          },
+        },
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserClose(input, context);
+  }
+}
+
+registerTool(new BrowserCloseTool());
+
+// ── browser_click ────────────────────────────────────────────────────
+
+class BrowserClickTool implements Tool {
+  name = 'browser_click';
+  description = 'Click an element on the page. Always use element_id from browser_snapshot — do not fabricate CSS selectors. For autocomplete dropdowns, search suggestion lists, or address pickers, prefer browser_press_key with ArrowDown/ArrowUp + Enter instead of clicking dropdown items.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          element_id: {
+            type: 'string',
+            description: 'The element ID from a previous browser_snapshot result (e.g. "e1").',
+          },
+          selector: {
+            type: 'string',
+            description: 'A CSS selector to target. Used as fallback when element_id is not available.',
+          },
+          timeout: {
+            type: 'number',
+            description: 'Max time in ms to wait for the element to be clickable (default: 10000).',
+          },
+        },
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserClick(input, context);
+  }
+}
+
+registerTool(new BrowserClickTool());
+
+// ── browser_type ─────────────────────────────────────────────────────
+
+class BrowserTypeTool implements Tool {
+  name = 'browser_type';
+  description = 'Type text into an input element. Target by element_id (from browser_snapshot) or CSS selector.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          element_id: {
+            type: 'string',
+            description: 'The element ID from a previous browser_snapshot result (e.g. "e3").',
+          },
+          selector: {
+            type: 'string',
+            description: 'A CSS selector to target. Used as fallback when element_id is not available.',
+          },
+          text: {
+            type: 'string',
+            description: 'The text to type into the element.',
+          },
+          clear_first: {
+            type: 'boolean',
+            description: 'If true (default), clear existing content before typing. Set to false to append.',
+          },
+          press_enter: {
+            type: 'boolean',
+            description: 'If true, press Enter after typing the text.',
+          },
+        },
+        required: ['text'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserType(input, context);
+  }
+}
+
+registerTool(new BrowserTypeTool());
+
+// ── browser_press_key ────────────────────────────────────────────────
+
+class BrowserPressKeyTool implements Tool {
+  name = 'browser_press_key';
+  description = 'Press a keyboard key, optionally targeting a specific element. Use for Enter, Escape, Tab, arrow keys, etc. Preferred method for navigating autocomplete dropdowns and search suggestion lists: use ArrowDown/ArrowUp to move through options, then Enter to select.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          key: {
+            type: 'string',
+            description: 'The key to press (e.g. "Enter", "Escape", "Tab", "ArrowDown", "a").',
+          },
+          element_id: {
+            type: 'string',
+            description: 'Optional element ID from browser_snapshot to target.',
+          },
+          selector: {
+            type: 'string',
+            description: 'Optional CSS selector to target.',
+          },
+        },
+        required: ['key'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserPressKey(input, context);
+  }
+}
+
+registerTool(new BrowserPressKeyTool());
+
+// ── browser_scroll ───────────────────────────────────────────────────
+
+class BrowserScrollTool implements Tool {
+  name = 'browser_scroll';
+  description = 'Scroll the page or a specific element in a given direction. Useful for viewing content below the fold on long pages.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          direction: {
+            type: 'string',
+            enum: ['up', 'down', 'left', 'right'],
+            description: 'The direction to scroll.',
+          },
+          amount: {
+            type: 'number',
+            description: 'The number of pixels to scroll (default: 500).',
+          },
+          element_id: {
+            type: 'string',
+            description: 'Optional element ID from browser_snapshot to scroll within.',
+          },
+          selector: {
+            type: 'string',
+            description: 'Optional CSS selector of element to scroll within.',
+          },
+        },
+        required: ['direction'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserScroll(input, context);
+  }
+}
+
+registerTool(new BrowserScrollTool());
+
+// ── browser_wait_for ─────────────────────────────────────────────────
+
+class BrowserWaitForTool implements Tool {
+  name = 'browser_wait_for';
+  description = 'Wait for a condition: a CSS selector to appear, text to appear on the page, or a fixed duration in milliseconds. Provide exactly one mode.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          selector: {
+            type: 'string',
+            description: 'Wait for an element matching this CSS selector to appear.',
+          },
+          text: {
+            type: 'string',
+            description: 'Wait for this text to appear on the page.',
+          },
+          duration: {
+            type: 'number',
+            description: 'Wait for this many milliseconds.',
+          },
+          timeout: {
+            type: 'number',
+            description: 'Maximum wait time in milliseconds (default and max: 30000).',
+          },
+        },
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserWaitFor(input, context);
+  }
+}
+
+registerTool(new BrowserWaitForTool());
+
+// ── browser_extract ──────────────────────────────────────────────────
+
+class BrowserExtractTool implements Tool {
+  name = 'browser_extract';
+  description = 'Extract the text content of the current page. Optionally include links. Output is capped to prevent excessive token usage.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          include_links: {
+            type: 'boolean',
+            description: 'If true, include a list of links found on the page (up to 200).',
+          },
+        },
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserExtract(input, context);
+  }
+}
+
+registerTool(new BrowserExtractTool());
+
+// ── browser_fill_credential ──────────────────────────────────────────
+
+class BrowserFillCredentialTool implements Tool {
+  name = 'browser_fill_credential';
+  description = 'Fill a stored credential into a form field without exposing the value. Target by element_id (from browser_snapshot) or CSS selector.';
+  category = 'browser';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          service: {
+            type: 'string',
+            description: 'Credential vault service name (e.g. gmail)',
+          },
+          field: {
+            type: 'string',
+            description: 'Credential vault field name (e.g. password)',
+          },
+          element_id: {
+            type: 'string',
+            description: 'Element ID from browser_snapshot',
+          },
+          selector: {
+            type: 'string',
+            description: 'CSS selector for target element',
+          },
+          press_enter: {
+            type: 'boolean',
+            description: 'Press Enter after filling',
+          },
+        },
+        required: ['service', 'field'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    return executeBrowserFillCredential(input, context);
+  }
+}
+
+registerTool(new BrowserFillCredentialTool());
+
+// Re-export execution functions for backward compatibility
+export {
+  executeBrowserNavigate,
+  executeBrowserSnapshot,
+  executeBrowserScreenshot,
+  executeBrowserClose,
+  executeBrowserClick,
+  executeBrowserType,
+  executeBrowserPressKey,
+  executeBrowserScroll,
+  executeBrowserWaitFor,
+  executeBrowserExtract,
+  executeBrowserFillCredential,
+} from './browser-execution.js';