illuma-agents 1.0.18 → 1.0.19

package/src/tools/BrowserTools.ts

@@ -0,0 +1,582 @@
+/**
+ * Browser Automation Tools for Ranger Browser Extension
+ * 
+ * These tools allow the LLM to interact with the browser through the 
+ * ranger-browser extension. They generate structured actions that are
+ * sent to the extension via SSE streaming for execution.
+ * 
+ * The extension handles:
+ * - DOM extraction with element indexing
+ * - Click, type, hover, scroll actions
+ * - Navigation and page context
+ * - Visual element highlighting
+ */
+
+import { z } from 'zod';
+import { tool, DynamicStructuredTool } from '@langchain/core/tools';
+
+// ============================================
+// Tool Schemas
+// ============================================
+
+/**
+ * Enhanced click schema that supports both index-based and coordinate-based clicking
+ */
+const BrowserClickSchema = z.object({
+  index: z.number().optional().describe(
+    'The index of the element to click, as shown in the page context (e.g., [0], [1], [2]). ' +
+    'Use the element index from the interactive elements list provided in the page context. ' +
+    'Either index OR coordinates must be provided.'
+  ),
+  coordinates: z.object({
+    x: z.number().describe('X coordinate in viewport pixels'),
+    y: z.number().describe('Y coordinate in viewport pixels'),
+  }).optional().describe(
+    'Coordinates for clicking elements that lack semantic info (marked with ⚠️). ' +
+    'The coordinates are provided in the element listing as coords:(x,y). ' +
+    'Either index OR coordinates must be provided.'
+  ),
+  visualDescription: z.string().optional().describe(
+    'Description of what the element looks like visually. Used when clicking by appearance ' +
+    '(e.g., "blue button in top right corner", "hamburger menu icon")'
+  ),
+  reason: z.string().optional().describe(
+    'Brief explanation of why you are clicking this element (for user transparency)'
+  ),
+});
+
+const BrowserTypeSchema = z.object({
+  index: z.number().describe(
+    'The index of the input element to type into, as shown in the page context'
+  ),
+  text: z.string().describe(
+    'The text to type into the input field'
+  ),
+  clear: z.boolean().optional().describe(
+    'Whether to clear the existing content before typing (default: false)'
+  ),
+  pressEnter: z.boolean().optional().describe(
+    'Whether to press Enter after typing (useful for search fields, default: false)'
+  ),
+});
+
+const BrowserNavigateSchema = z.object({
+  url: z.string().describe(
+    'The URL to navigate to. Can be a full URL or a relative path.'
+  ),
+  reason: z.string().optional().describe(
+    'Brief explanation of why you are navigating to this URL'
+  ),
+});
+
+const BrowserScrollSchema = z.object({
+  direction: z.enum(['up', 'down', 'left', 'right']).describe(
+    'The direction to scroll'
+  ),
+  amount: z.number().optional().describe(
+    'The amount to scroll in pixels (default: 500)'
+  ),
+});
+
+const BrowserExtractSchema = z.object({
+  query: z.string().optional().describe(
+    'Optional query to filter extracted content. If provided, only content related to the query will be extracted.'
+  ),
+  selector: z.string().optional().describe(
+    'Optional CSS selector to extract content from a specific element'
+  ),
+});
+
+const BrowserHoverSchema = z.object({
+  index: z.number().describe(
+    'The index of the element to hover over, as shown in the page context'
+  ),
+});
+
+const BrowserWaitSchema = z.object({
+  duration: z.number().optional().describe(
+    'Duration to wait in milliseconds (default: 1000)'
+  ),
+  reason: z.string().optional().describe(
+    'Why we are waiting (e.g., "for page to load", "for animation to complete")'
+  ),
+});
+
+const BrowserGoBackSchema = z.object({
+  reason: z.string().optional().describe(
+    'Brief explanation of why you are going back'
+  ),
+});
+
+const BrowserScreenshotSchema = z.object({
+  fullPage: z.boolean().optional().describe(
+    'Whether to capture the full page or just the viewport (default: viewport only)'
+  ),
+  reason: z.string().optional().describe(
+    'Why you need a screenshot (e.g., "to identify visual elements", "to analyze page layout")'
+  ),
+});
+
+// ============================================
+// Tool Implementations
+// ============================================
+
+/**
+ * Browser click tool - clicks an element by index or coordinates
+ * Supports both semantic (index-based) and vision (coordinate-based) clicking
+ */
+export function createBrowserClickTool(): DynamicStructuredTool<typeof BrowserClickSchema> {
+  return tool<typeof BrowserClickSchema>(
+    async ({ index, coordinates, visualDescription, reason }) => {
+      // Validate that at least one targeting method is provided
+      if (index === undefined && !coordinates) {
+        return JSON.stringify({
+          type: 'error',
+          error: 'Either index or coordinates must be provided to click an element',
+        });
+      }
+
+      // Return a structured action for the extension to execute
+      // The actual execution happens in the browser extension
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'click',
+          ...(index !== undefined && { index }),
+          ...(coordinates && { coordinates }),
+          ...(visualDescription && { visualDescription }),
+          reason,
+        },
+        // Signal that this requires browser execution
+        requiresBrowserExecution: true,
+      });
+    },
+    {
+      name: EBrowserTools.CLICK,
+      description: `Click an interactive element on the current page.
+
+**Two ways to target elements:**
+
+1. **By index (preferred)**: Use the element's index number from the interactive elements list
+   - Format: [index] {semantic role} <tag>text</tag>
+   - Example: browser_click({ index: 5 }) to click element [5]
+
+2. **By coordinates (vision fallback)**: For elements marked with ⚠️ that lack semantic info
+   - Use the coords:(x,y) shown after the ⚠️ marker
+   - Example: browser_click({ coordinates: { x: 150, y: 200 } })
+
+**When to use coordinates:**
+- Elements marked with ⚠️ have poor semantic understanding
+- Icon-only buttons without labels
+- Custom canvas/SVG elements
+- When you identify an element visually in a screenshot
+
+Example: If element shows \`[12] {button} <div>⚠️ [left side, small, clickable] coords:(45,120)\`
+Use either: browser_click({ index: 12 }) or browser_click({ coordinates: { x: 45, y: 120 } })`,
+      schema: BrowserClickSchema,
+    }
+  );
+}
+
+/**
+ * Browser type tool - types text into an input field
+ */
+export function createBrowserTypeTool(): DynamicStructuredTool<typeof BrowserTypeSchema> {
+  return tool<typeof BrowserTypeSchema>(
+    async ({ index, text, clear, pressEnter }) => {
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'type',
+          index,
+          text,
+          clear: clear ?? false,
+          pressEnter: pressEnter ?? false,
+        },
+        requiresBrowserExecution: true,
+      });
+    },
+    {
+      name: EBrowserTools.TYPE,
+      description: `Type text into an input field on the current page.
+
+Use this tool when you need to:
+- Fill in a text input or textarea
+- Enter a search query
+- Fill out form fields
+
+The element index comes from the page context's interactive elements list.
+Set 'clear: true' to clear existing content before typing.
+Set 'pressEnter: true' to submit after typing (useful for search fields).
+
+Example: To type "hello world" into a search field shown as "[2]<input>Search...</input>",
+use index: 2, text: "hello world"`,
+      schema: BrowserTypeSchema,
+    }
+  );
+}
+
+/**
+ * Browser navigate tool - navigates to a URL
+ */
+export function createBrowserNavigateTool(): DynamicStructuredTool<typeof BrowserNavigateSchema> {
+  return tool<typeof BrowserNavigateSchema>(
+    async ({ url, reason }) => {
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'navigate',
+          url,
+          reason,
+        },
+        requiresBrowserExecution: true,
+      });
+    },
+    {
+      name: EBrowserTools.NAVIGATE,
+      description: `Navigate to a specific URL in the browser.
+
+Use this tool when you need to:
+- Go to a specific website
+- Navigate to a different page
+- Open a new URL
+
+Provide the full URL including the protocol (https://).
+
+Example: browser_navigate({ url: "https://www.google.com" })`,
+      schema: BrowserNavigateSchema,
+    }
+  );
+}
+
+/**
+ * Browser scroll tool - scrolls the page
+ */
+export function createBrowserScrollTool(): DynamicStructuredTool<typeof BrowserScrollSchema> {
+  return tool<typeof BrowserScrollSchema>(
+    async ({ direction, amount }) => {
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'scroll',
+          scroll: {
+            direction,
+            amount: amount ?? 500,
+          },
+        },
+        requiresBrowserExecution: true,
+      });
+    },
+    {
+      name: EBrowserTools.SCROLL,
+      description: `Scroll the current page in a specified direction.
+
+Use this tool when you need to:
+- See more content on the page
+- Scroll to find elements not currently visible
+- Navigate long pages
+
+Default scroll amount is 500 pixels. Adjust as needed.
+
+Example: browser_scroll({ direction: "down", amount: 800 })`,
+      schema: BrowserScrollSchema,
+    }
+  );
+}
+
+/**
+ * Browser extract tool - extracts content from the page
+ */
+export function createBrowserExtractTool(): DynamicStructuredTool<typeof BrowserExtractSchema> {
+  return tool<typeof BrowserExtractSchema>(
+    async ({ query, selector }) => {
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'extract',
+          query,
+          selector,
+        },
+        requiresBrowserExecution: true,
+      });
+    },
+    {
+      name: EBrowserTools.EXTRACT,
+      description: `Extract text content from the current page.
+
+Use this tool when you need to:
+- Get specific information from the page
+- Extract text that matches a query
+- Read content from a specific element
+
+If no query or selector is provided, extracts the main page content.
+Use a CSS selector to extract from a specific element.
+Use a query to filter for relevant content.
+
+Example: browser_extract({ query: "price" }) - extracts content related to pricing`,
+      schema: BrowserExtractSchema,
+    }
+  );
+}
+
+/**
+ * Browser hover tool - hovers over an element
+ */
+export function createBrowserHoverTool(): DynamicStructuredTool<typeof BrowserHoverSchema> {
+  return tool<typeof BrowserHoverSchema>(
+    async ({ index }) => {
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'hover',
+          index,
+        },
+        requiresBrowserExecution: true,
+      });
+    },
+    {
+      name: EBrowserTools.HOVER,
+      description: `Hover over an element to reveal tooltips or dropdown menus.
+
+Use this tool when you need to:
+- Reveal a dropdown menu
+- Show a tooltip
+- Trigger hover effects
+
+Example: browser_hover({ index: 3 }) - hovers over element at index 3`,
+      schema: BrowserHoverSchema,
+    }
+  );
+}
+
+/**
+ * Browser wait tool - waits for a specified duration
+ */
+export function createBrowserWaitTool(): DynamicStructuredTool<typeof BrowserWaitSchema> {
+  return tool<typeof BrowserWaitSchema>(
+    async ({ duration, reason }) => {
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'wait',
+          duration: duration ?? 1000,
+          reason,
+        },
+        requiresBrowserExecution: true,
+      });
+    },
+    {
+      name: EBrowserTools.WAIT,
+      description: `Wait for a specified duration before the next action.
+
+Use this tool when you need to:
+- Wait for a page to load
+- Wait for an animation to complete
+- Add delay between actions
+
+Default wait time is 1000ms (1 second).
+
+Example: browser_wait({ duration: 2000, reason: "waiting for page to load" })`,
+      schema: BrowserWaitSchema,
+    }
+  );
+}
+
+/**
+ * Browser go back tool - navigates back in history
+ */
+export function createBrowserGoBackTool(): DynamicStructuredTool<typeof BrowserGoBackSchema> {
+  return tool<typeof BrowserGoBackSchema>(
+    async ({ reason }) => {
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'back',
+          reason,
+        },
+        requiresBrowserExecution: true,
+      });
+    },
+    {
+      name: EBrowserTools.BACK,
+      description: `Navigate back to the previous page in browser history.
+
+Use this tool when you need to:
+- Return to a previous page
+- Undo a navigation
+
+Example: browser_back({ reason: "returning to search results" })`,
+      schema: BrowserGoBackSchema,
+    }
+  );
+}
+
+/**
+ * Browser screenshot tool - captures a screenshot
+ */
+export function createBrowserScreenshotTool(): DynamicStructuredTool<typeof BrowserScreenshotSchema> {
+  return tool<typeof BrowserScreenshotSchema>(
+    async ({ fullPage }) => {
+      return JSON.stringify({
+        type: 'browser_action',
+        action: {
+          type: 'screenshot',
+          fullPage: fullPage ?? false,
+        },
+        requiresBrowserExecution: true,
+      });
+    },
+    {
+      name: EBrowserTools.SCREENSHOT,
+      description: `Capture a screenshot of the current page.
+
+Use this tool when you need to:
+- Capture the current state of a page
+- Document visual elements
+- Verify page appearance
+
+Set fullPage: true to capture the entire page (may be large).
+Default captures only the visible viewport.
+
+Example: browser_screenshot({ fullPage: false })`,
+      schema: BrowserScreenshotSchema,
+    }
+  );
+}
+
+// ============================================
+// Tool Collection
+// ============================================
+
+export type BrowserToolsConfig = {
+  /** Enable click tool */
+  enableClick?: boolean;
+  /** Enable type tool */
+  enableType?: boolean;
+  /** Enable navigate tool */
+  enableNavigate?: boolean;
+  /** Enable scroll tool */
+  enableScroll?: boolean;
+  /** Enable extract tool */
+  enableExtract?: boolean;
+  /** Enable hover tool */
+  enableHover?: boolean;
+  /** Enable wait tool */
+  enableWait?: boolean;
+  /** Enable back tool */
+  enableBack?: boolean;
+  /** Enable screenshot tool */
+  enableScreenshot?: boolean;
+};
+
+/**
+ * Create all browser automation tools
+ * 
+ * IMPORTANT: These tools should ONLY be registered when:
+ * 1. The request comes from a browser extension that can execute them
+ * 2. The client has indicated browser capability (e.g., via header or parameter)
+ * 
+ * DO NOT register these for normal web UI users - they cannot execute browser actions.
+ * 
+ * Detection in Ranger API:
+ * - Check for `X-Ranger-Browser-Extension: true` header
+ * - Or check for `browserCapable: true` in request body
+ * - Or check user agent for extension identifier
+ * 
+ * @example
+ * // In Ranger API endpoint:
+ * const hasBrowserExtension = req.headers['x-ranger-browser-extension'] === 'true';
+ * const tools = hasBrowserExtension 
+ *   ? [...normalTools, ...createBrowserTools()]
+ *   : normalTools;
+ */
+export function createBrowserTools(config: BrowserToolsConfig = {}): DynamicStructuredTool[] {
+  const tools: DynamicStructuredTool[] = [];
+  
+  // Enable all by default
+  const {
+    enableClick = true,
+    enableType = true,
+    enableNavigate = true,
+    enableScroll = true,
+    enableExtract = true,
+    enableHover = true,
+    enableWait = true,
+    enableBack = true,
+    enableScreenshot = true,
+  } = config;
+  
+  if (enableClick) tools.push(createBrowserClickTool());
+  if (enableType) tools.push(createBrowserTypeTool());
+  if (enableNavigate) tools.push(createBrowserNavigateTool());
+  if (enableScroll) tools.push(createBrowserScrollTool());
+  if (enableExtract) tools.push(createBrowserExtractTool());
+  if (enableHover) tools.push(createBrowserHoverTool());
+  if (enableWait) tools.push(createBrowserWaitTool());
+  if (enableBack) tools.push(createBrowserGoBackTool());
+  if (enableScreenshot) tools.push(createBrowserScreenshotTool());
+  
+  return tools;
+}
+
+/**
+ * Browser tool name constants
+ * Use these instead of magic strings
+ */
+export const EBrowserTools = {
+  CLICK: 'browser_click',
+  TYPE: 'browser_type',
+  NAVIGATE: 'browser_navigate',
+  SCROLL: 'browser_scroll',
+  EXTRACT: 'browser_extract',
+  HOVER: 'browser_hover',
+  WAIT: 'browser_wait',
+  BACK: 'browser_back',
+  SCREENSHOT: 'browser_screenshot',
+} as const;
+
+/**
+ * Get browser tool names for filtering/identification
+ */
+export const BROWSER_TOOL_NAMES = [
+  EBrowserTools.CLICK,
+  EBrowserTools.TYPE,
+  EBrowserTools.NAVIGATE,
+  EBrowserTools.SCROLL,
+  EBrowserTools.EXTRACT,
+  EBrowserTools.HOVER,
+  EBrowserTools.WAIT,
+  EBrowserTools.BACK,
+  EBrowserTools.SCREENSHOT,
+] as const;
+
+export type BrowserToolName = typeof BROWSER_TOOL_NAMES[number];
+
+/**
+ * Check if a tool call is a browser action
+ */
+export function isBrowserToolCall(toolName: string): toolName is BrowserToolName {
+  return BROWSER_TOOL_NAMES.includes(toolName as BrowserToolName);
+}
+
+/**
+ * Check if request indicates browser extension capability
+ * Use this to conditionally register browser tools
+ * 
+ * @example
+ * // In Express middleware or endpoint:
+ * if (hasBrowserCapability(req.headers)) {
+ *   tools.push(...createBrowserTools());
+ * }
+ */
+export function hasBrowserCapability(headers: Record<string, string | string[] | undefined>): boolean {
+  const extensionHeader = headers['x-ranger-browser-extension'];
+  const capableHeader = headers['x-ranger-browser-capable'];
+  
+  return (
+    extensionHeader === 'true' || 
+    capableHeader === 'true' ||
+    (Array.isArray(extensionHeader) && extensionHeader.includes('true')) ||
+    (Array.isArray(capableHeader) && capableHeader.includes('true'))
+  );
+}