illuma-agents 1.0.27 → 1.0.29

package/src/tools/BrowserTools.ts

@@ -1,350 +1,424 @@
-import { z } from 'zod';
-import { tool, DynamicStructuredTool } from '@langchain/core/tools';
-import type * as t from '@/types';
-
-/**
- * Browser tool names - keep in sync with ranger-browser extension
- * These tools execute locally in the browser extension, NOT on the server
- */
-export const EBrowserTools = {
-  CLICK: 'browser_click',
-  TYPE: 'browser_type',
-  NAVIGATE: 'browser_navigate',
-  SCROLL: 'browser_scroll',
-  EXTRACT: 'browser_extract',
-  HOVER: 'browser_hover',
-  WAIT: 'browser_wait',
-  BACK: 'browser_back',
-  SCREENSHOT: 'browser_screenshot',
-  GET_PAGE_STATE: 'browser_get_page_state',
-} as const;
-
-export type BrowserToolName = typeof EBrowserTools[keyof typeof EBrowserTools];
-
-/**
- * Callback function type for waiting on browser action results
- * This allows the server (Ranger) to provide a callback that waits for the extension
- * to POST results back to the server before returning to the LLM.
- * 
- * @param action - The browser action (click, type, navigate, etc.)
- * @param args - Arguments for the action
- * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)
- * @returns Promise that resolves with the actual browser result (page state, etc.)
- */
-export type BrowserToolCallback = (
-  action: string,
-  args: Record<string, unknown>,
-  toolCallId: string
-) => Promise<BrowserActionResult>;
-
-/**
- * Result returned from browser action execution
- */
-export interface BrowserActionResult {
-  success: boolean;
-  url?: string;
-  title?: string;
-  elementList?: string;  // Text-based element list
-  error?: string;
-  screenshot?: string;   // Base64 screenshot (if requested)
-}
-
-/**
- * Check if browser capability is available based on request headers or context
- * The browser extension sets these headers when connected:
- * - X-Ranger-Browser-Extension: true
- * - X-Ranger-Browser-Capable: true
- */
-export function hasBrowserCapability(req?: { headers?: Record<string, string | string[] | undefined> }): boolean {
-  if (!req?.headers) {
-    return false;
-  }
-  
-  const browserExtension = req.headers['x-ranger-browser-extension'];
-  const browserCapable = req.headers['x-ranger-browser-capable'];
-  
-  return browserExtension === 'true' || browserCapable === 'true';
-}
-
-// Tool schemas
-const BrowserClickSchema = z.object({
-  index: z.number().describe('The index number [0], [1], etc. of the element to click from the page state element list'),
-});
-
-const BrowserTypeSchema = z.object({
-  index: z.number().describe('The index number of the input element to type into'),
-  text: z.string().describe('The text to type into the element'),
-  pressEnter: z.boolean().optional().describe('Whether to press Enter after typing (useful for search forms)'),
-});
-
-const BrowserNavigateSchema = z.object({
-  url: z.string().describe('The full URL to navigate to (must include https://)'),
-});
-
-const BrowserScrollSchema = z.object({
-  direction: z.enum(['up', 'down', 'left', 'right']).describe('Direction to scroll'),
-  amount: z.number().optional().describe('Pixels to scroll (default: one viewport height)'),
-});
-
-const BrowserExtractSchema = z.object({
-  query: z.string().optional().describe('Optional: specific content to extract from the page'),
-});
-
-const BrowserHoverSchema = z.object({
-  index: z.number().describe('The index number of the element to hover over'),
-});
-
-const BrowserWaitSchema = z.object({
-  duration: z.number().optional().describe('Milliseconds to wait (default: 1000)'),
-});
-
-const BrowserBackSchema = z.object({});
-
-const BrowserScreenshotSchema = z.object({});
-
-const BrowserGetPageStateSchema = z.object({});
-
-/**
- * Browser tool response interface
- * This is what the extension returns after executing the action
- */
-export interface BrowserToolResponse {
-  requiresBrowserExecution: true;
-  action: string;
-  args: Record<string, unknown>;
-  toolCallId?: string;  // Added to help extension correlate with callback
-}
-
-/**
- * Options for creating browser tools
- */
-export interface CreateBrowserToolsOptions {
-  /**
-   * Optional callback that waits for browser action results.
-   * When provided, tools will await this callback to get actual results from the extension.
-   * When not provided, tools return markers immediately (for non-server contexts).
-   */
-  waitForResult?: BrowserToolCallback;
-}
-
-/**
- * Format browser action result for LLM consumption
- */
-function formatResultForLLM(result: BrowserActionResult, action: string): string {
-  if (!result.success && result.error) {
-    return `Browser action "${action}" failed: ${result.error}`;
-  }
-
-  const parts: string[] = [];
-  
-  if (result.url) {
-    parts.push(`**Current URL:** ${result.url}`);
-  }
-  if (result.title) {
-    parts.push(`**Page Title:** ${result.title}`);
-  }
-  if (result.elementList) {
-    parts.push(`\n**Interactive Elements:**\n${result.elementList}`);
-  }
-  if (result.screenshot) {
-    parts.push(`\n[Screenshot captured and displayed to user]`);
-  }
-  
-  if (parts.length === 0) {
-    return `Browser action "${action}" completed successfully.`;
-  }
-  
-  return parts.join('\n');
-}
-
-/**
- * Create browser tools with optional callback for waiting on results
- * 
- * When waitForResult callback is provided:
- * 1. Tool returns marker that triggers extension
- * 2. Tool then awaits callback to get actual results
- * 3. Returns real page state to LLM
- * 
- * When no callback:
- * 1. Tool returns marker only (for non-server contexts)
- * 
- * NOTE: These tools use TEXT-BASED element lists, NOT screenshots
- * Screenshots would be 100K+ tokens each - element lists are ~100 tokens
- */
-export function createBrowserTools(options?: CreateBrowserToolsOptions): DynamicStructuredTool[] {
-  const { waitForResult } = options || {};
-  const tools: DynamicStructuredTool[] = [];
-
-  /**
-   * Helper to create tool function that optionally waits for results
-   * The toolCallId is extracted from the RunnableConfig passed by LangChain
-   */
-  const createToolFunction = (action: string) => {
-    return async (args: Record<string, unknown>, config?: { toolCall?: { id?: string } }): Promise<string> => {
-      const toolCallId = config?.toolCall?.id || `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;
-      
-      // Create marker for extension
-      const marker: BrowserToolResponse = {
-        requiresBrowserExecution: true,
-        action,
-        args,
-        toolCallId,
-      };
-      
-      // If no callback, return marker immediately (extension handles via SSE interception)
-      if (!waitForResult) {
-        return JSON.stringify(marker);
-      }
-      
-      // With callback: wait for actual results from extension
-      // The marker is still returned initially via SSE, but we wait for the callback
-      try {
-        const result = await waitForResult(action, args, toolCallId);
-        return formatResultForLLM(result, action);
-      } catch (error) {
-        const errorMessage = error instanceof Error ? error.message : String(error);
-        return `Browser action "${action}" failed: ${errorMessage}`;
-      }
-    };
-  };
-
-  // browser_click
-  tools.push(
-    tool(
-      createToolFunction('click'),
-      {
-        name: EBrowserTools.CLICK,
-        description: `Click an element on the current web page by its index number.
-The element list shows clickable items like: [0]<button>Submit</button> [1]<a href="/home">Home</a>
-Use the index number in brackets to click that element.
-After clicking, you receive an updated element list showing the new page state.`,
-        schema: BrowserClickSchema,
-      }
-    )
-  );
-
-  // browser_type
-  tools.push(
-    tool(
-      createToolFunction('type'),
-      {
-        name: EBrowserTools.TYPE,
-        description: `Type text into an input element on the page.
-Find the input element in the list by its index (e.g., [5]<input placeholder="Search">).
-Set pressEnter: true to submit forms after typing.
-After typing, you receive an updated element list.`,
-        schema: BrowserTypeSchema,
-      }
-    )
-  );
-
-  // browser_navigate
-  tools.push(
-    tool(
-      createToolFunction('navigate'),
-      {
-        name: EBrowserTools.NAVIGATE,
-        description: `Navigate to a URL. Always include the full URL with https://.
-After navigation, you receive the new page's element list.`,
-        schema: BrowserNavigateSchema,
-      }
-    )
-  );
-
-  // browser_scroll
-  tools.push(
-    tool(
-      createToolFunction('scroll'),
-      {
-        name: EBrowserTools.SCROLL,
-        description: `Scroll the page to reveal more content.
-Use 'down' to scroll down, 'up' to scroll up.
-After scrolling, you receive an updated element list with newly visible elements.`,
-        schema: BrowserScrollSchema,
-      }
-    )
-  );
-
-  // browser_extract
-  tools.push(
-    tool(
-      createToolFunction('extract'),
-      {
-        name: EBrowserTools.EXTRACT,
-        description: `Extract content from the current page.
-Returns page URL, title, and element list.`,
-        schema: BrowserExtractSchema,
-      }
-    )
-  );
-
-  // browser_hover
-  tools.push(
-    tool(
-      createToolFunction('hover'),
-      {
-        name: EBrowserTools.HOVER,
-        description: `Hover over an element to reveal tooltips, dropdowns, or other hover-triggered content.
-After hovering, you receive an updated element list with any newly revealed elements.`,
-        schema: BrowserHoverSchema,
-      }
-    )
-  );
-
-  // browser_wait
-  tools.push(
-    tool(
-      createToolFunction('wait'),
-      {
-        name: EBrowserTools.WAIT,
-        description: `Wait for a specified duration for page content to load.
-Use this after actions that trigger async content loading.
-After waiting, you receive an updated element list.`,
-        schema: BrowserWaitSchema,
-      }
-    )
-  );
-
-  // browser_back
-  tools.push(
-    tool(
-      createToolFunction('back'),
-      {
-        name: EBrowserTools.BACK,
-        description: `Go back to the previous page in browser history.
-After going back, you receive the previous page's element list.`,
-        schema: BrowserBackSchema,
-      }
-    )
-  );
-
-  // browser_screenshot
-  tools.push(
-    tool(
-      createToolFunction('screenshot'),
-      {
-        name: EBrowserTools.SCREENSHOT,
-        description: `Capture a screenshot of the current page.
-Returns the page state with a note that screenshot was displayed to the user.
-Use browser_get_page_state to get the element list for automation.`,
-        schema: BrowserScreenshotSchema,
-      }
-    )
-  );
-
-  // browser_get_page_state
-  tools.push(
-    tool(
-      createToolFunction('get_page_state'),
-      {
-        name: EBrowserTools.GET_PAGE_STATE,
-        description: `Get the current page state including URL, title, and all interactive elements.
-Use this at the start of a task to see what elements are available.
-Returns a text list of elements with their index numbers for interaction.`,
-        schema: BrowserGetPageStateSchema,
-      }
-    )
-  );
-
-  return tools;
-}
+import { z } from 'zod';
+import { tool, DynamicStructuredTool } from '@langchain/core/tools';
+import type * as _t from '@/types';
+
+/**
+ * Browser tool names - keep in sync with ranger-browser extension
+ * These tools execute locally in the browser extension, NOT on the server
+ */
+export const EBrowserTools = {
+  CLICK: 'browser_click',
+  TYPE: 'browser_type',
+  NAVIGATE: 'browser_navigate',
+  SCROLL: 'browser_scroll',
+  EXTRACT: 'browser_extract',
+  HOVER: 'browser_hover',
+  WAIT: 'browser_wait',
+  BACK: 'browser_back',
+  SCREENSHOT: 'browser_screenshot',
+  GET_PAGE_STATE: 'browser_get_page_state',
+  // Skyvern-inspired additions for robust form handling
+  SELECT_OPTION: 'browser_select_option',
+  UPLOAD_FILE: 'browser_upload_file',
+  KEYPRESS: 'browser_keypress',
+} as const;
+
+export type BrowserToolName =
+  (typeof EBrowserTools)[keyof typeof EBrowserTools];
+
+/**
+ * Callback function type for waiting on browser action results
+ * This allows the server (Ranger) to provide a callback that waits for the extension
+ * to POST results back to the server before returning to the LLM.
+ *
+ * @param action - The browser action (click, type, navigate, etc.)
+ * @param args - Arguments for the action
+ * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)
+ * @returns Promise that resolves with the actual browser result (page state, etc.)
+ */
+export type BrowserToolCallback = (
+  action: string,
+  args: Record<string, unknown>,
+  toolCallId: string
+) => Promise<BrowserActionResult>;
+
+/**
+ * Result returned from browser action execution
+ */
+export interface BrowserActionResult {
+  success: boolean;
+  url?: string;
+  title?: string;
+  elementList?: string; // Text-based element list
+  error?: string;
+  screenshot?: string; // Base64 screenshot (if requested)
+}
+
+/**
+ * Check if browser capability is available based on request headers or context
+ * The browser extension sets these headers when connected:
+ * - X-Ranger-Browser-Extension: true
+ * - X-Ranger-Browser-Capable: true
+ */
+export function hasBrowserCapability(req?: {
+  headers?: Record<string, string | string[] | undefined>;
+}): boolean {
+  if (!req?.headers) {
+    return false;
+  }
+
+  const browserExtension = req.headers['x-ranger-browser-extension'];
+  const browserCapable = req.headers['x-ranger-browser-capable'];
+
+  return browserExtension === 'true' || browserCapable === 'true';
+}
+
+// Tool schemas
+const BrowserClickSchema = z.object({
+  index: z
+    .number()
+    .describe(
+      'The index number [0], [1], etc. of the element to click from the page state element list'
+    ),
+});
+
+const BrowserTypeSchema = z.object({
+  index: z
+    .number()
+    .describe('The index number of the input element to type into'),
+  text: z.string().describe('The text to type into the element'),
+  pressEnter: z
+    .boolean()
+    .optional()
+    .describe('Whether to press Enter after typing (useful for search forms)'),
+});
+
+const BrowserNavigateSchema = z.object({
+  url: z
+    .string()
+    .describe('The full URL to navigate to (must include https://)'),
+});
+
+const BrowserScrollSchema = z.object({
+  direction: z
+    .enum(['up', 'down', 'left', 'right'])
+    .describe('Direction to scroll'),
+  amount: z
+    .number()
+    .optional()
+    .describe('Pixels to scroll (default: one viewport height)'),
+});
+
+const BrowserExtractSchema = z.object({
+  query: z
+    .string()
+    .optional()
+    .describe('Optional: specific content to extract from the page'),
+});
+
+const BrowserHoverSchema = z.object({
+  index: z.number().describe('The index number of the element to hover over'),
+});
+
+const BrowserWaitSchema = z.object({
+  duration: z
+    .number()
+    .optional()
+    .describe('Milliseconds to wait (default: 1000)'),
+});
+
+const BrowserBackSchema = z.object({});
+
+const BrowserScreenshotSchema = z.object({});
+
+const BrowserGetPageStateSchema = z.object({});
+
+// Skyvern-inspired schemas for robust form handling
+const BrowserSelectOptionSchema = z.object({
+  index: z
+    .number()
+    .describe('The index number of the select/dropdown element'),
+  value: z
+    .string()
+    .optional()
+    .describe('The value or label of the option to select. For native <select>, use the option text. For custom dropdowns, this is the option label to click.'),
+});
+
+const BrowserUploadFileSchema = z.object({
+  index: z
+    .number()
+    .describe('The index number of the file input element'),
+  fileUrl: z
+    .string()
+    .describe('URL of the file to upload (the system will download and upload it)'),
+});
+
+const BrowserKeypressSchema = z.object({
+  keys: z
+    .string()
+    .describe('Key(s) to press. Single key: "Enter", "Escape", "Tab", "ArrowDown". Combo: "Control+A", "Shift+Enter"'),
+});
+
+/**
+ * Browser tool response interface
+ * This is what the extension returns after executing the action
+ */
+export interface BrowserToolResponse {
+  requiresBrowserExecution: true;
+  action: string;
+  args: Record<string, unknown>;
+  toolCallId?: string; // Added to help extension correlate with callback
+}
+
+/**
+ * Options for creating browser tools
+ */
+export interface CreateBrowserToolsOptions {
+  /**
+   * Optional callback that waits for browser action results.
+   * When provided, tools will await this callback to get actual results from the extension.
+   * When not provided, tools return markers immediately (for non-server contexts).
+   */
+  waitForResult?: BrowserToolCallback;
+}
+
+/**
+ * Format browser action result for LLM consumption
+ */
+function formatResultForLLM(
+  result: BrowserActionResult,
+  action: string
+): string {
+  if (!result.success && result.error) {
+    return `Browser action "${action}" failed: ${result.error}`;
+  }
+
+  const parts: string[] = [];
+
+  if (result.url != null && result.url !== '') {
+    parts.push(`**Current URL:** ${result.url}`);
+  }
+  if (result.title != null && result.title !== '') {
+    parts.push(`**Page Title:** ${result.title}`);
+  }
+  if (result.elementList != null && result.elementList !== '') {
+    parts.push(`\n**Interactive Elements:**\n${result.elementList}`);
+  }
+  if (result.screenshot != null && result.screenshot !== '') {
+    parts.push('\n[Screenshot captured and displayed to user]');
+  }
+
+  if (parts.length === 0) {
+    return `Browser action "${action}" completed successfully.`;
+  }
+
+  return parts.join('\n');
+}
+
+/**
+ * Create browser tools with optional callback for waiting on results
+ *
+ * When waitForResult callback is provided:
+ * 1. Tool returns marker that triggers extension
+ * 2. Tool then awaits callback to get actual results
+ * 3. Returns real page state to LLM
+ *
+ * When no callback:
+ * 1. Tool returns marker only (for non-server contexts)
+ *
+ * NOTE: These tools use TEXT-BASED element lists, NOT screenshots
+ * Screenshots would be 100K+ tokens each - element lists are ~100 tokens
+ */
+export function createBrowserTools(
+  options?: CreateBrowserToolsOptions
+): DynamicStructuredTool[] {
+  const { waitForResult } = options || {};
+  const tools: DynamicStructuredTool[] = [];
+
+  /**
+   * Helper to create tool function that optionally waits for results
+   * The toolCallId is extracted from the RunnableConfig passed by LangChain
+   */
+  const createToolFunction = (action: string) => {
+    return async (
+      args: Record<string, unknown>,
+      config?: { toolCall?: { id?: string } }
+    ): Promise<string> => {
+      const toolCallId =
+        config?.toolCall?.id ??
+        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+
+      // Create marker for extension
+      const marker: BrowserToolResponse = {
+        requiresBrowserExecution: true,
+        action,
+        args,
+        toolCallId,
+      };
+
+      // If no callback, return marker immediately (extension handles via SSE interception)
+      if (!waitForResult) {
+        return JSON.stringify(marker);
+      }
+
+      // With callback: wait for actual results from extension
+      // The marker is still returned initially via SSE, but we wait for the callback
+      try {
+        const result = await waitForResult(action, args, toolCallId);
+        return formatResultForLLM(result, action);
+      } catch (error) {
+        const errorMessage =
+          error instanceof Error ? error.message : String(error);
+        return `Browser action "${action}" failed: ${errorMessage}`;
+      }
+    };
+  };
+
+  // browser_click
+  tools.push(
+    tool(createToolFunction('click'), {
+      name: EBrowserTools.CLICK,
+      description: `Click an element on the current web page by its index number.
+The element list shows clickable items like: [0]<button>Submit</button> [1]<a href="/home">Home</a>
+Use the index number in brackets to click that element.
+After clicking, you receive an updated element list showing the new page state.`,
+      schema: BrowserClickSchema,
+    })
+  );
+
+  // browser_type
+  tools.push(
+    tool(createToolFunction('type'), {
+      name: EBrowserTools.TYPE,
+      description: `Type text into an input element on the page.
+Find the input element in the list by its index (e.g., [5]<input placeholder="Search">).
+Set pressEnter: true to submit forms after typing.
+After typing, you receive an updated element list.`,
+      schema: BrowserTypeSchema,
+    })
+  );
+
+  // browser_navigate
+  tools.push(
+    tool(createToolFunction('navigate'), {
+      name: EBrowserTools.NAVIGATE,
+      description: `Navigate to a URL. Always include the full URL with https://.
+After navigation, you receive the new page's element list.`,
+      schema: BrowserNavigateSchema,
+    })
+  );
+
+  // browser_scroll
+  tools.push(
+    tool(createToolFunction('scroll'), {
+      name: EBrowserTools.SCROLL,
+      description: `Scroll the page to reveal more content.
+Use 'down' to scroll down, 'up' to scroll up.
+After scrolling, you receive an updated element list with newly visible elements.`,
+      schema: BrowserScrollSchema,
+    })
+  );
+
+  // browser_extract
+  tools.push(
+    tool(createToolFunction('extract'), {
+      name: EBrowserTools.EXTRACT,
+      description: `Extract content from the current page.
+Returns page URL, title, and element list.`,
+      schema: BrowserExtractSchema,
+    })
+  );
+
+  // browser_hover
+  tools.push(
+    tool(createToolFunction('hover'), {
+      name: EBrowserTools.HOVER,
+      description: `Hover over an element to reveal tooltips, dropdowns, or other hover-triggered content.
+After hovering, you receive an updated element list with any newly revealed elements.`,
+      schema: BrowserHoverSchema,
+    })
+  );
+
+  // browser_wait
+  tools.push(
+    tool(createToolFunction('wait'), {
+      name: EBrowserTools.WAIT,
+      description: `Wait for a specified duration for page content to load.
+Use this after actions that trigger async content loading.
+After waiting, you receive an updated element list.`,
+      schema: BrowserWaitSchema,
+    })
+  );
+
+  // browser_back
+  tools.push(
+    tool(createToolFunction('back'), {
+      name: EBrowserTools.BACK,
+      description: `Go back to the previous page in browser history.
+After going back, you receive the previous page's element list.`,
+      schema: BrowserBackSchema,
+    })
+  );
+
+  // browser_screenshot
+  tools.push(
+    tool(createToolFunction('screenshot'), {
+      name: EBrowserTools.SCREENSHOT,
+      description: `Capture a screenshot of the current page.
+Returns the page state with a note that screenshot was displayed to the user.
+Use browser_get_page_state to get the element list for automation.`,
+      schema: BrowserScreenshotSchema,
+    })
+  );
+
+  // browser_get_page_state
+  tools.push(
+    tool(createToolFunction('get_page_state'), {
+      name: EBrowserTools.GET_PAGE_STATE,
+      description: `Get the current page state including URL, title, and all interactive elements.
+Use this at the start of a task to see what elements are available.
+Returns a text list of elements with their index numbers for interaction.`,
+      schema: BrowserGetPageStateSchema,
+    })
+  );
+
+  // browser_select_option - Skyvern-inspired for robust dropdown handling
+  tools.push(
+    tool(createToolFunction('select_option'), {
+      name: EBrowserTools.SELECT_OPTION,
+      description: `Select an option from a dropdown or select element.
+For native <select> elements: finds and selects the option by value/label.
+For custom dropdowns: clicks to open, then clicks the matching option.
+Use this instead of click for dropdowns - it handles both native and custom selects.
+After selection, you receive an updated element list.`,
+      schema: BrowserSelectOptionSchema,
+    })
+  );
+
+  // browser_upload_file - Skyvern-inspired for file input handling
+  tools.push(
+    tool(createToolFunction('upload_file'), {
+      name: EBrowserTools.UPLOAD_FILE,
+      description: `Upload a file to a file input element.
+Provide the index of the file input and the URL of the file to upload.
+The system will download the file and attach it to the input.
+After upload, you receive an updated element list.`,
+      schema: BrowserUploadFileSchema,
+    })
+  );
+
+  // browser_keypress - For keyboard shortcuts and special keys
+  tools.push(
+    tool(createToolFunction('keypress'), {
+      name: EBrowserTools.KEYPRESS,
+      description: `Press keyboard key(s) on the page.
+Single keys: "Enter", "Escape", "Tab", "ArrowDown", "ArrowUp", "Backspace", "Delete"
+Key combos: "Control+A" (select all), "Control+C" (copy), "Shift+Enter" (newline)
+Use this for form submission, closing modals, navigating dropdowns.
+After keypress, you receive an updated element list.`,
+      schema: BrowserKeypressSchema,
+    })
+  );
+
+  return tools;
+}