illuma-agents 1.0.23 → 1.0.25

package/src/tools/BrowserTools.ts

@@ -1,650 +0,0 @@
-/**
- * Browser Automation Tools for Ranger Browser Extension
- * 
- * These tools allow the LLM to interact with the browser through the 
- * ranger-browser extension. They generate structured actions that are
- * sent to the extension via SSE streaming for execution.
- * 
- * The extension handles:
- * - DOM extraction with element indexing
- * - Click, type, hover, scroll actions
- * - Navigation and page context
- * - Visual element highlighting
- */
-
-import { z } from 'zod';
-import { tool, DynamicStructuredTool } from '@langchain/core/tools';
-
-// ============================================
-// Tool Schemas
-// ============================================
-
-/**
- * Enhanced click schema that supports both index-based and coordinate-based clicking
- */
-const BrowserClickSchema = z.object({
-  index: z.number().optional().describe(
-    'The index of the element to click, as shown in the page context (e.g., [0], [1], [2]). ' +
-    'Use the element index from the interactive elements list provided in the page context. ' +
-    'Either index OR coordinates must be provided.'
-  ),
-  coordinates: z.object({
-    x: z.number().describe('X coordinate in viewport pixels'),
-    y: z.number().describe('Y coordinate in viewport pixels'),
-  }).optional().describe(
-    'Coordinates for clicking elements that lack semantic info (marked with ⚠️). ' +
-    'The coordinates are provided in the element listing as coords:(x,y). ' +
-    'Either index OR coordinates must be provided.'
-  ),
-  visualDescription: z.string().optional().describe(
-    'Description of what the element looks like visually. Used when clicking by appearance ' +
-    '(e.g., "blue button in top right corner", "hamburger menu icon")'
-  ),
-  reason: z.string().optional().describe(
-    'Brief explanation of why you are clicking this element (for user transparency)'
-  ),
-});
-
-const BrowserTypeSchema = z.object({
-  index: z.number().describe(
-    'The index of the input element to type into, as shown in the page context'
-  ),
-  text: z.string().describe(
-    'The text to type into the input field'
-  ),
-  clear: z.boolean().optional().describe(
-    'Whether to clear the existing content before typing (default: false)'
-  ),
-  pressEnter: z.boolean().optional().describe(
-    'Whether to press Enter after typing (useful for search fields, default: false)'
-  ),
-});
-
-const BrowserNavigateSchema = z.object({
-  url: z.string().describe(
-    'The URL to navigate to. Can be a full URL or a relative path.'
-  ),
-  reason: z.string().optional().describe(
-    'Brief explanation of why you are navigating to this URL'
-  ),
-});
-
-const BrowserScrollSchema = z.object({
-  direction: z.enum(['up', 'down', 'left', 'right']).describe(
-    'The direction to scroll'
-  ),
-  amount: z.number().optional().describe(
-    'The amount to scroll in pixels (default: 500)'
-  ),
-});
-
-const BrowserExtractSchema = z.object({
-  query: z.string().optional().describe(
-    'Optional query to filter extracted content. If provided, only content related to the query will be extracted.'
-  ),
-  selector: z.string().optional().describe(
-    'Optional CSS selector to extract content from a specific element'
-  ),
-});
-
-const BrowserHoverSchema = z.object({
-  index: z.number().describe(
-    'The index of the element to hover over, as shown in the page context'
-  ),
-});
-
-const BrowserWaitSchema = z.object({
-  duration: z.number().optional().describe(
-    'Duration to wait in milliseconds (default: 1000)'
-  ),
-  reason: z.string().optional().describe(
-    'Why we are waiting (e.g., "for page to load", "for animation to complete")'
-  ),
-});
-
-const BrowserGoBackSchema = z.object({
-  reason: z.string().optional().describe(
-    'Brief explanation of why you are going back'
-  ),
-});
-
-const BrowserScreenshotSchema = z.object({
-  fullPage: z.boolean().optional().describe(
-    'Whether to capture the full page or just the viewport (default: viewport only)'
-  ),
-  reason: z.string().optional().describe(
-    'Why you need a screenshot (e.g., "to identify visual elements", "to analyze page layout")'
-  ),
-});
-
-const BrowserGetPageStateSchema = z.object({
-  reason: z.string().optional().describe(
-    'Why you need fresh page state (e.g., "after navigation", "to see updated elements")'
-  ),
-});
-
-// ============================================
-// Tool Implementations
-// ============================================
-
-/**
- * Browser click tool - clicks an element by index or coordinates
- * Supports both semantic (index-based) and vision (coordinate-based) clicking
- */
-export function createBrowserClickTool(): DynamicStructuredTool<typeof BrowserClickSchema> {
-  return tool<typeof BrowserClickSchema>(
-    async ({ index, coordinates, visualDescription, reason }) => {
-      // Validate that at least one targeting method is provided
-      if (index === undefined && !coordinates) {
-        return JSON.stringify({
-          type: 'error',
-          error: 'Either index or coordinates must be provided to click an element',
-        });
-      }
-
-      // Return a structured action for the extension to execute
-      // The actual execution happens in the browser extension
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'click',
-          ...(index !== undefined && { index }),
-          ...(coordinates && { coordinates }),
-          ...(visualDescription && { visualDescription }),
-          reason,
-        },
-        // Signal that this requires browser execution
-        requiresBrowserExecution: true,
-      });
-    },
-    {
-      name: EBrowserTools.CLICK,
-      description: `Click an interactive element on the current page.
-
-**Two ways to target elements:**
-
-1. **By index (preferred)**: Use the element's index number from the interactive elements list
-   - Format: [index] {semantic role} <tag>text</tag>
-   - Example: browser_click({ index: 5 }) to click element [5]
-
-2. **By coordinates (vision fallback)**: For elements marked with ⚠️ that lack semantic info
-   - Use the coords:(x,y) shown after the ⚠️ marker
-   - Example: browser_click({ coordinates: { x: 150, y: 200 } })
-
-**When to use coordinates:**
-- Elements marked with ⚠️ have poor semantic understanding
-- Icon-only buttons without labels
-- Custom canvas/SVG elements
-- When you identify an element visually in a screenshot
-
-Example: If element shows \`[12] {button} <div>⚠️ [left side, small, clickable] coords:(45,120)\`
-Use either: browser_click({ index: 12 }) or browser_click({ coordinates: { x: 45, y: 120 } })`,
-      schema: BrowserClickSchema,
-    }
-  );
-}
-
-/**
- * Browser type tool - types text into an input field
- */
-export function createBrowserTypeTool(): DynamicStructuredTool<typeof BrowserTypeSchema> {
-  return tool<typeof BrowserTypeSchema>(
-    async ({ index, text, clear, pressEnter }) => {
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'type',
-          index,
-          text,
-          clear: clear ?? false,
-          pressEnter: pressEnter ?? false,
-        },
-        requiresBrowserExecution: true,
-      });
-    },
-    {
-      name: EBrowserTools.TYPE,
-      description: `Type text into an input field on the current page.
-
-Use this tool when you need to:
-- Fill in a text input or textarea
-- Enter a search query
-- Fill out form fields
-
-The element index comes from the page context's interactive elements list.
-Set 'clear: true' to clear existing content before typing.
-Set 'pressEnter: true' to submit after typing (useful for search fields).
-
-Example: To type "hello world" into a search field shown as "[2]<input>Search...</input>",
-use index: 2, text: "hello world"`,
-      schema: BrowserTypeSchema,
-    }
-  );
-}
-
-/**
- * Browser navigate tool - navigates to a URL
- */
-export function createBrowserNavigateTool(): DynamicStructuredTool<typeof BrowserNavigateSchema> {
-  return tool<typeof BrowserNavigateSchema>(
-    async ({ url, reason }) => {
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'navigate',
-          url,
-          reason,
-        },
-        requiresBrowserExecution: true,
-      });
-    },
-    {
-      name: EBrowserTools.NAVIGATE,
-      description: `Navigate to a specific URL in the browser.
-
-Use this tool when you need to:
-- Go to a specific website
-- Navigate to a different page
-- Open a new URL
-
-**IMPORTANT**: After calling browser_navigate, you MUST call browser_get_page_state 
-before using browser_click or browser_type. This is because navigation changes the page,
-and you need to see the new page's elements before you can interact with them.
-
-Provide the full URL including the protocol (https://).
-
-**Correct workflow**:
-1. browser_navigate({ url: "https://www.amazon.com" })
-2. browser_get_page_state({ reason: "see elements on Amazon" })
-3. Now find the search input's [index] in the returned state
-4. browser_type({ index: <search_input_index>, text: "query", pressEnter: true })
-
-Example: browser_navigate({ url: "https://www.google.com" })`,
-      schema: BrowserNavigateSchema,
-    }
-  );
-}
-
-/**
- * Browser scroll tool - scrolls the page
- */
-export function createBrowserScrollTool(): DynamicStructuredTool<typeof BrowserScrollSchema> {
-  return tool<typeof BrowserScrollSchema>(
-    async ({ direction, amount }) => {
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'scroll',
-          scroll: {
-            direction,
-            amount: amount ?? 500,
-          },
-        },
-        requiresBrowserExecution: true,
-      });
-    },
-    {
-      name: EBrowserTools.SCROLL,
-      description: `Scroll the current page in a specified direction.
-
-Use this tool when you need to:
-- See more content on the page
-- Scroll to find elements not currently visible
-- Navigate long pages
-
-Default scroll amount is 500 pixels. Adjust as needed.
-
-Example: browser_scroll({ direction: "down", amount: 800 })`,
-      schema: BrowserScrollSchema,
-    }
-  );
-}
-
-/**
- * Browser extract tool - extracts content from the page
- */
-export function createBrowserExtractTool(): DynamicStructuredTool<typeof BrowserExtractSchema> {
-  return tool<typeof BrowserExtractSchema>(
-    async ({ query, selector }) => {
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'extract',
-          query,
-          selector,
-        },
-        requiresBrowserExecution: true,
-      });
-    },
-    {
-      name: EBrowserTools.EXTRACT,
-      description: `Extract text content from the current page.
-
-Use this tool when you need to:
-- Get specific information from the page
-- Extract text that matches a query
-- Read content from a specific element
-
-If no query or selector is provided, extracts the main page content.
-Use a CSS selector to extract from a specific element.
-Use a query to filter for relevant content.
-
-Example: browser_extract({ query: "price" }) - extracts content related to pricing`,
-      schema: BrowserExtractSchema,
-    }
-  );
-}
-
-/**
- * Browser hover tool - hovers over an element
- */
-export function createBrowserHoverTool(): DynamicStructuredTool<typeof BrowserHoverSchema> {
-  return tool<typeof BrowserHoverSchema>(
-    async ({ index }) => {
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'hover',
-          index,
-        },
-        requiresBrowserExecution: true,
-      });
-    },
-    {
-      name: EBrowserTools.HOVER,
-      description: `Hover over an element to reveal tooltips or dropdown menus.
-
-Use this tool when you need to:
-- Reveal a dropdown menu
-- Show a tooltip
-- Trigger hover effects
-
-Example: browser_hover({ index: 3 }) - hovers over element at index 3`,
-      schema: BrowserHoverSchema,
-    }
-  );
-}
-
-/**
- * Browser wait tool - waits for a specified duration
- */
-export function createBrowserWaitTool(): DynamicStructuredTool<typeof BrowserWaitSchema> {
-  return tool<typeof BrowserWaitSchema>(
-    async ({ duration, reason }) => {
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'wait',
-          duration: duration ?? 1000,
-          reason,
-        },
-        requiresBrowserExecution: true,
-      });
-    },
-    {
-      name: EBrowserTools.WAIT,
-      description: `Wait for a specified duration before the next action.
-
-Use this tool when you need to:
-- Wait for a page to load
-- Wait for an animation to complete
-- Add delay between actions
-
-Default wait time is 1000ms (1 second).
-
-Example: browser_wait({ duration: 2000, reason: "waiting for page to load" })`,
-      schema: BrowserWaitSchema,
-    }
-  );
-}
-
-/**
- * Browser go back tool - navigates back in history
- */
-export function createBrowserGoBackTool(): DynamicStructuredTool<typeof BrowserGoBackSchema> {
-  return tool<typeof BrowserGoBackSchema>(
-    async ({ reason }) => {
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'back',
-          reason,
-        },
-        requiresBrowserExecution: true,
-      });
-    },
-    {
-      name: EBrowserTools.BACK,
-      description: `Navigate back to the previous page in browser history.
-
-Use this tool when you need to:
-- Return to a previous page
-- Undo a navigation
-
-Example: browser_back({ reason: "returning to search results" })`,
-      schema: BrowserGoBackSchema,
-    }
-  );
-}
-
-/**
- * Browser screenshot tool - captures a screenshot
- */
-export function createBrowserScreenshotTool(): DynamicStructuredTool<typeof BrowserScreenshotSchema> {
-  return tool<typeof BrowserScreenshotSchema>(
-    async ({ fullPage }) => {
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'screenshot',
-          fullPage: fullPage ?? false,
-        },
-        requiresBrowserExecution: true,
-      });
-    },
-    {
-      name: EBrowserTools.SCREENSHOT,
-      description: `Capture a screenshot of the current page.
-
-Use this tool when you need to:
-- Capture the current state of a page
-- Document visual elements
-- Verify page appearance
-
-Set fullPage: true to capture the entire page (may be large).
-Default captures only the visible viewport.
-
-Example: browser_screenshot({ fullPage: false })`,
-      schema: BrowserScreenshotSchema,
-    }
-  );
-}
-
-/**
- * Browser get page state tool - gets fresh page context after navigation or actions
- * CRITICAL: Use this after browser_navigate or any action that changes the page
- */
-export function createBrowserGetPageStateTool(): DynamicStructuredTool<typeof BrowserGetPageStateSchema> {
-  return tool<typeof BrowserGetPageStateSchema>(
-    async ({ reason }) => {
-      return JSON.stringify({
-        type: 'browser_action',
-        action: {
-          type: 'get_page_state',
-          reason,
-        },
-        requiresBrowserExecution: true,
-        // Special flag: extension should inject fresh context into the conversation
-        requiresContextRefresh: true,
-        // IMPORTANT: Tell the agent to wait
-        message: 'Page state is being captured by the browser extension. The element list will be provided in the next message. DO NOT proceed with click or type actions until you receive the actual element list.',
-      });
-    },
-    {
-      name: EBrowserTools.GET_PAGE_STATE,
-      description: `Get fresh page state showing current interactive elements.
-
-**CRITICAL WORKFLOW**: After calling this tool, you MUST STOP and WAIT. The browser extension will capture the page state and return the element list. DO NOT plan any browser_click or browser_type actions in the same response - you don't have the element indices yet!
-
-**When to use**:
-- After browser_navigate (to see elements on the new page)
-- After browser_click (if it caused navigation or page changes)
-- Any time you need to see what elements are currently on the page
-
-**IMPORTANT**: This tool captures the page state asynchronously. The actual element list will be provided AFTER this tool completes. You should:
-1. Call this tool
-2. STOP and wait for the response with the element list
-3. In your NEXT response, use the element indices for click/type actions
-
-Example workflow:
-- Turn 1: browser_navigate to amazon.com, then browser_get_page_state
-- Turn 2: (After receiving element list) browser_type with the correct search input index
-
-Example: browser_get_page_state({ reason: "to see elements after navigation" })`,
-      schema: BrowserGetPageStateSchema,
-    }
-  );
-}
-
-// ============================================
-// Tool Collection
-// ============================================
-
-export type BrowserToolsConfig = {
-  /** Enable click tool */
-  enableClick?: boolean;
-  /** Enable type tool */
-  enableType?: boolean;
-  /** Enable navigate tool */
-  enableNavigate?: boolean;
-  /** Enable scroll tool */
-  enableScroll?: boolean;
-  /** Enable extract tool */
-  enableExtract?: boolean;
-  /** Enable hover tool */
-  enableHover?: boolean;
-  /** Enable wait tool */
-  enableWait?: boolean;
-  /** Enable back tool */
-  enableBack?: boolean;
-  /** Enable screenshot tool */
-  enableScreenshot?: boolean;
-  /** Enable get page state tool */
-  enableGetPageState?: boolean;
-};
-
-/**
- * Create all browser automation tools
- * 
- * IMPORTANT: These tools should ONLY be registered when:
- * 1. The request comes from a browser extension that can execute them
- * 2. The client has indicated browser capability (e.g., via header or parameter)
- * 
- * DO NOT register these for normal web UI users - they cannot execute browser actions.
- * 
- * Detection in Ranger API:
- * - Check for `X-Ranger-Browser-Extension: true` header
- * - Or check for `browserCapable: true` in request body
- * - Or check user agent for extension identifier
- * 
- * @example
- * // In Ranger API endpoint:
- * const hasBrowserExtension = req.headers['x-ranger-browser-extension'] === 'true';
- * const tools = hasBrowserExtension 
- *   ? [...normalTools, ...createBrowserTools()]
- *   : normalTools;
- */
-export function createBrowserTools(config: BrowserToolsConfig = {}): DynamicStructuredTool[] {
-  const tools: DynamicStructuredTool[] = [];
-  
-  // Enable all by default
-  const {
-    enableClick = true,
-    enableType = true,
-    enableNavigate = true,
-    enableScroll = true,
-    enableExtract = true,
-    enableHover = true,
-    enableWait = true,
-    enableBack = true,
-    enableScreenshot = true,
-    enableGetPageState = true,
-  } = config;
-  
-  if (enableClick) tools.push(createBrowserClickTool());
-  if (enableType) tools.push(createBrowserTypeTool());
-  if (enableNavigate) tools.push(createBrowserNavigateTool());
-  if (enableScroll) tools.push(createBrowserScrollTool());
-  if (enableExtract) tools.push(createBrowserExtractTool());
-  if (enableHover) tools.push(createBrowserHoverTool());
-  if (enableWait) tools.push(createBrowserWaitTool());
-  if (enableBack) tools.push(createBrowserGoBackTool());
-  if (enableScreenshot) tools.push(createBrowserScreenshotTool());
-  if (enableGetPageState) tools.push(createBrowserGetPageStateTool());
-  
-  return tools;
-}
-
-/**
- * Browser tool name constants
- * Use these instead of magic strings
- */
-export const EBrowserTools = {
-  CLICK: 'browser_click',
-  TYPE: 'browser_type',
-  NAVIGATE: 'browser_navigate',
-  SCROLL: 'browser_scroll',
-  EXTRACT: 'browser_extract',
-  HOVER: 'browser_hover',
-  WAIT: 'browser_wait',
-  BACK: 'browser_back',
-  SCREENSHOT: 'browser_screenshot',
-  GET_PAGE_STATE: 'browser_get_page_state',
-} as const;
-
-/**
- * Get browser tool names for filtering/identification
- */
-export const BROWSER_TOOL_NAMES = [
-  EBrowserTools.CLICK,
-  EBrowserTools.TYPE,
-  EBrowserTools.NAVIGATE,
-  EBrowserTools.SCROLL,
-  EBrowserTools.EXTRACT,
-  EBrowserTools.HOVER,
-  EBrowserTools.WAIT,
-  EBrowserTools.BACK,
-  EBrowserTools.SCREENSHOT,
-  EBrowserTools.GET_PAGE_STATE,
-] as const;
-
-export type BrowserToolName = typeof BROWSER_TOOL_NAMES[number];
-
-/**
- * Check if a tool call is a browser action
- */
-export function isBrowserToolCall(toolName: string): toolName is BrowserToolName {
-  return BROWSER_TOOL_NAMES.includes(toolName as BrowserToolName);
-}
-
-/**
- * Check if request indicates browser extension capability
- * Use this to conditionally register browser tools
- * 
- * @example
- * // In Express middleware or endpoint:
- * if (hasBrowserCapability(req.headers)) {
- *   tools.push(...createBrowserTools());
- * }
- */
-export function hasBrowserCapability(headers: Record<string, string | string[] | undefined>): boolean {
-  const extensionHeader = headers['x-ranger-browser-extension'];
-  const capableHeader = headers['x-ranger-browser-capable'];
-  
-  return (
-    extensionHeader === 'true' || 
-    capableHeader === 'true' ||
-    (Array.isArray(extensionHeader) && extensionHeader.includes('true')) ||
-    (Array.isArray(capableHeader) && capableHeader.includes('true'))
-  );
-}