illuma-agents 1.0.35 → 1.0.37

package/dist/cjs/tools/BrowserTools.cjs

@@ -18,6 +18,8 @@ const EBrowserTools = {
     BACK: 'browser_back',
     SCREENSHOT: 'browser_screenshot',
     GET_PAGE_STATE: 'browser_get_page_state',
+    KEYPRESS: 'browser_keypress',
+    SWITCH_TAB: 'browser_switch_tab',
 };
 /**
  * Check if browser capability is available based on request headers or context
@@ -37,17 +39,17 @@ function hasBrowserCapability(req) {
 const BrowserClickSchema = zod.z.object({
     index: zod.z
         .number()
-        .describe('The [index] of the element to click. Use fieldLabel attribute to identify the correct element when multiple similar elements exist (e.g., fieldLabel="To" vs fieldLabel="Cc" for email fields).'),
+        .describe('The [index] of the element to click. Use fieldLabel to identify the correct element. For form fields, target <input> or <textarea> elements, NOT parent <div> containers.'),
 });
 const BrowserTypeSchema = zod.z.object({
     index: zod.z
         .number()
-        .describe('The [index] of the input element to type into. IMPORTANT: Check fieldLabel attribute to ensure you are typing into the correct field (e.g., fieldLabel="To" for recipient, fieldLabel="Subject" for subject line).'),
+        .describe('The [index] of the INPUT element to type into. Target <input> or <textarea> elements. Check fieldLabel to identify the correct field.'),
     text: zod.z.string().describe('The text to type into the element'),
     pressEnter: zod.z
         .boolean()
         .optional()
-        .describe('Whether to press Enter after typing (useful for search forms)'),
+        .describe('Whether to press Enter after typing (useful for search forms and submitting)'),
 });
 const BrowserNavigateSchema = zod.z.object({
     url: zod.z
@@ -81,6 +83,16 @@ const BrowserWaitSchema = zod.z.object({
 const BrowserBackSchema = zod.z.object({});
 const BrowserScreenshotSchema = zod.z.object({});
 const BrowserGetPageStateSchema = zod.z.object({});
+const BrowserKeypressSchema = zod.z.object({
+    keys: zod.z
+        .string()
+        .describe('Keyboard keys to press. Use "+" to combine modifiers (e.g., "Control+Enter", "Control+a", "Escape", "Tab", "Enter"). Common shortcuts: Control+Enter (submit forms/send), Escape (close dialogs), Tab (next field).'),
+});
+const BrowserSwitchTabSchema = zod.z.object({
+    tabId: zod.z
+        .number()
+        .describe('The tab ID to switch to. Use the tab IDs shown in the tabs list from page state.'),
+});
 /**
  * Format browser action result for LLM consumption
  */
@@ -96,8 +108,8 @@ function formatResultForLLM(result, action) {
         parts.push(`**Page Title:** ${result.title}`);
     }
     if (result.elementList != null && result.elementList !== '') {
-        // Add hint about fieldLabel for form interactions
-        parts.push(`\n**Interactive Elements** (use fieldLabel to identify form fields):\n${result.elementList}`);
+        // Add hint about fieldLabel and targeting inputs for form interactions
+        parts.push(`\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\n${result.elementList}`);
     }
     if (result.screenshot != null && result.screenshot !== '') {
         parts.push('\n[Screenshot captured and displayed to user]');
@@ -158,13 +170,13 @@ function createBrowserTools(options) {
     // browser_click
     tools$1.push(tools.tool(createToolFunction('click'), {
         name: EBrowserTools.CLICK,
-        description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. Skip disabled elements.',
+        description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',
         schema: BrowserClickSchema,
     }));
     // browser_type
     tools$1.push(tools.tool(createToolFunction('type'), {
         name: EBrowserTools.TYPE,
-        description: 'Type text into input by [index]. IMPORTANT: Use fieldLabel to identify the correct field (e.g., fieldLabel="To" vs "Cc"). Use pressEnter:true to submit.',
+        description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel="To recipients" for To field).',
         schema: BrowserTypeSchema,
     }));
     // browser_navigate
@@ -215,6 +227,18 @@ function createBrowserTools(options) {
         description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',
         schema: BrowserGetPageStateSchema,
     }));
+    // browser_keypress - for keyboard shortcuts
+    tools$1.push(tools.tool(createToolFunction('keypress'), {
+        name: EBrowserTools.KEYPRESS,
+        description: 'Send keyboard shortcut or key press. Use for: Control+Enter (send email/submit), Escape (close dialog/cancel), Tab (next field), Enter (confirm). The keys are sent to the currently focused element.',
+        schema: BrowserKeypressSchema,
+    }));
+    // browser_switch_tab - for switching between tabs
+    tools$1.push(tools.tool(createToolFunction('switch_tab'), {
+        name: EBrowserTools.SWITCH_TAB,
+        description: 'Switch to a different browser tab by its ID. Tab IDs are shown in the page state. Use this to work with existing open tabs (e.g., use existing Gmail tab instead of opening a new one).',
+        schema: BrowserSwitchTabSchema,
+    }));
     return tools$1;
 }
 

package/dist/cjs/tools/BrowserTools.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"BrowserTools.cjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with ranger-browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the server (Ranger) to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Ranger-Browser-Extension: true\n * - X-Ranger-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-ranger-browser-extension'];\n  const browserCapable = req.headers['x-ranger-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the element to click. Use fieldLabel attribute to identify the correct element when multiple similar elements exist (e.g., fieldLabel=\"To\" vs fieldLabel=\"Cc\" for email fields).'\n    ),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe('The [index] of the input element to type into. IMPORTANT: Check fieldLabel attribute to ensure you are typing into the correct field (e.g., fieldLabel=\"To\" for recipient, fieldLabel=\"Subject\" for subject line).'),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe('Whether to press Enter after typing (useful for search forms)'),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  if (!result.success && result.error) {\n    return `Browser action \"${action}\" failed: ${result.error}`;\n  }\n\n  const parts: string[] = [];\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    // Add hint about fieldLabel for form interactions\n    parts.push(`\\n**Interactive Elements** (use fieldLabel to identify form fields):\\n${result.elementList}`);\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. Skip disabled elements.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description: 'Type text into input by [index]. IMPORTANT: Use fieldLabel to identify the correct field (e.g., fieldLabel=\"To\" vs \"Cc\"). Use pressEnter:true to submit.',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description: 'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description: 'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description: 'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description: 'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description: 'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description: 'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description: 'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":["z","tools","tool"],"mappings":";;;;;AAIA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;;AAkC1C;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,iMAAiM,CAClM;AACJ,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,oNAAoN,CAAC;IACjO,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAEA;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CAAC,+DAA+D,CAAC;AAC7E,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAEA;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAEA;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAEA;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAEA;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAyB9C;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAE,CAAA,CAAC;;AAE9C,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAE,CAAA,CAAC;;AAE/C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;;QAE3D,KAAK,CAAC,IAAI,CAAC,CAAA,sEAAA,EAAyE,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;AAE3G,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;;AAG7D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;;AAG7D,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAMC,OAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAQ,KAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;;AAK/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGDA,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,yGAAyG;AACtH,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,0JAA0J;AACvK,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,oEAAoE;AACjF,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,6DAA6D;AAC1E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,kFAAkF;AAC/F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,+DAA+D;AAC5E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,2EAA2E;AACxF,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,qFAAqF;AAClG,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;AAED,IAAA,OAAOD,OAAK;AACd;;;;;;"}
+{"version":3,"file":"BrowserTools.cjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with ranger-browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n  KEYPRESS: 'browser_keypress',\n  SWITCH_TAB: 'browser_switch_tab',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the server (Ranger) to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Ranger-Browser-Extension: true\n * - X-Ranger-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-ranger-browser-extension'];\n  const browserCapable = req.headers['x-ranger-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the element to click. Use fieldLabel to identify the correct element. For form fields, target <input> or <textarea> elements, NOT parent <div> containers.'\n    ),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe('The [index] of the INPUT element to type into. Target <input> or <textarea> elements. Check fieldLabel to identify the correct field.'),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe('Whether to press Enter after typing (useful for search forms and submitting)'),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\nconst BrowserKeypressSchema = z.object({\n  keys: z\n    .string()\n    .describe('Keyboard keys to press. Use \"+\" to combine modifiers (e.g., \"Control+Enter\", \"Control+a\", \"Escape\", \"Tab\", \"Enter\"). Common shortcuts: Control+Enter (submit forms/send), Escape (close dialogs), Tab (next field).'),\n});\n\nconst BrowserSwitchTabSchema = z.object({\n  tabId: z\n    .number()\n    .describe('The tab ID to switch to. Use the tab IDs shown in the tabs list from page state.'),\n});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  if (!result.success && result.error) {\n    return `Browser action \"${action}\" failed: ${result.error}`;\n  }\n\n  const parts: string[] = [];\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    // Add hint about fieldLabel and targeting inputs for form interactions\n    parts.push(`\\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\\n${result.elementList}`);\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel=\"To recipients\" for To field).',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description: 'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description: 'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description: 'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description: 'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description: 'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description: 'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description: 'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  // browser_keypress - for keyboard shortcuts\n  tools.push(\n    tool(createToolFunction('keypress'), {\n      name: EBrowserTools.KEYPRESS,\n      description: 'Send keyboard shortcut or key press. Use for: Control+Enter (send email/submit), Escape (close dialog/cancel), Tab (next field), Enter (confirm). The keys are sent to the currently focused element.',\n      schema: BrowserKeypressSchema,\n    })\n  );\n\n  // browser_switch_tab - for switching between tabs\n  tools.push(\n    tool(createToolFunction('switch_tab'), {\n      name: EBrowserTools.SWITCH_TAB,\n      description: 'Switch to a different browser tab by its ID. Tab IDs are shown in the page state. Use this to work with existing open tabs (e.g., use existing Gmail tab instead of opening a new one).',\n      schema: BrowserSwitchTabSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":["z","tools","tool"],"mappings":";;;;;AAIA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;AACxC,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,UAAU,EAAE,oBAAoB;;AAkClC;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,2KAA2K,CAC5K;AACJ,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,uIAAuI,CAAC;IACpJ,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAEA;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CAAC,8EAA8E,CAAC;AAC5F,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAEA;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAEA;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAEA;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAEA;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE9C,MAAM,qBAAqB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACrC,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;SACN,QAAQ,CAAC,qNAAqN,CAAC;AACnO,CAAA,CAAC;AAEF,MAAM,sBAAsB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACtC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,kFAAkF,CAAC;AAChG,CAAA,CAAC;AAyBF;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAE,CAAA,CAAC;;AAE9C,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAE,CAAA,CAAC;;AAE/C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;;QAE3D,KAAK,CAAC,IAAI,CAAC,CAAA,gHAAA,EAAmH,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;AAErJ,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;;AAG7D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;;AAG7D,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAMC,OAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAQ,KAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;;AAK/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGDA,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,uJAAuJ;AACpK,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,wNAAwN;AACrO,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,oEAAoE;AACjF,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,6DAA6D;AAC1E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,kFAAkF;AAC/F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,+DAA+D;AAC5E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,2EAA2E;AACxF,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,qFAAqF;AAClG,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,uMAAuM;AACpN,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,yLAAyL;AACtM,QAAA,MAAM,EAAE,sBAAsB;AAC/B,KAAA,CAAC,CACH;AAED,IAAA,OAAOD,OAAK;AACd;;;;;;"}

package/dist/esm/tools/BrowserTools.mjs

@@ -16,6 +16,8 @@ const EBrowserTools = {
     BACK: 'browser_back',
     SCREENSHOT: 'browser_screenshot',
     GET_PAGE_STATE: 'browser_get_page_state',
+    KEYPRESS: 'browser_keypress',
+    SWITCH_TAB: 'browser_switch_tab',
 };
 /**
  * Check if browser capability is available based on request headers or context
@@ -35,17 +37,17 @@ function hasBrowserCapability(req) {
 const BrowserClickSchema = z.object({
     index: z
         .number()
-        .describe('The [index] of the element to click. Use fieldLabel attribute to identify the correct element when multiple similar elements exist (e.g., fieldLabel="To" vs fieldLabel="Cc" for email fields).'),
+        .describe('The [index] of the element to click. Use fieldLabel to identify the correct element. For form fields, target <input> or <textarea> elements, NOT parent <div> containers.'),
 });
 const BrowserTypeSchema = z.object({
     index: z
         .number()
-        .describe('The [index] of the input element to type into. IMPORTANT: Check fieldLabel attribute to ensure you are typing into the correct field (e.g., fieldLabel="To" for recipient, fieldLabel="Subject" for subject line).'),
+        .describe('The [index] of the INPUT element to type into. Target <input> or <textarea> elements. Check fieldLabel to identify the correct field.'),
     text: z.string().describe('The text to type into the element'),
     pressEnter: z
         .boolean()
         .optional()
-        .describe('Whether to press Enter after typing (useful for search forms)'),
+        .describe('Whether to press Enter after typing (useful for search forms and submitting)'),
 });
 const BrowserNavigateSchema = z.object({
     url: z
@@ -79,6 +81,16 @@ const BrowserWaitSchema = z.object({
 const BrowserBackSchema = z.object({});
 const BrowserScreenshotSchema = z.object({});
 const BrowserGetPageStateSchema = z.object({});
+const BrowserKeypressSchema = z.object({
+    keys: z
+        .string()
+        .describe('Keyboard keys to press. Use "+" to combine modifiers (e.g., "Control+Enter", "Control+a", "Escape", "Tab", "Enter"). Common shortcuts: Control+Enter (submit forms/send), Escape (close dialogs), Tab (next field).'),
+});
+const BrowserSwitchTabSchema = z.object({
+    tabId: z
+        .number()
+        .describe('The tab ID to switch to. Use the tab IDs shown in the tabs list from page state.'),
+});
 /**
  * Format browser action result for LLM consumption
  */
@@ -94,8 +106,8 @@ function formatResultForLLM(result, action) {
         parts.push(`**Page Title:** ${result.title}`);
     }
     if (result.elementList != null && result.elementList !== '') {
-        // Add hint about fieldLabel for form interactions
-        parts.push(`\n**Interactive Elements** (use fieldLabel to identify form fields):\n${result.elementList}`);
+        // Add hint about fieldLabel and targeting inputs for form interactions
+        parts.push(`\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\n${result.elementList}`);
     }
     if (result.screenshot != null && result.screenshot !== '') {
         parts.push('\n[Screenshot captured and displayed to user]');
@@ -156,13 +168,13 @@ function createBrowserTools(options) {
     // browser_click
     tools.push(tool(createToolFunction('click'), {
         name: EBrowserTools.CLICK,
-        description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. Skip disabled elements.',
+        description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',
         schema: BrowserClickSchema,
     }));
     // browser_type
     tools.push(tool(createToolFunction('type'), {
         name: EBrowserTools.TYPE,
-        description: 'Type text into input by [index]. IMPORTANT: Use fieldLabel to identify the correct field (e.g., fieldLabel="To" vs "Cc"). Use pressEnter:true to submit.',
+        description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel="To recipients" for To field).',
         schema: BrowserTypeSchema,
     }));
     // browser_navigate
@@ -213,6 +225,18 @@ function createBrowserTools(options) {
         description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',
         schema: BrowserGetPageStateSchema,
     }));
+    // browser_keypress - for keyboard shortcuts
+    tools.push(tool(createToolFunction('keypress'), {
+        name: EBrowserTools.KEYPRESS,
+        description: 'Send keyboard shortcut or key press. Use for: Control+Enter (send email/submit), Escape (close dialog/cancel), Tab (next field), Enter (confirm). The keys are sent to the currently focused element.',
+        schema: BrowserKeypressSchema,
+    }));
+    // browser_switch_tab - for switching between tabs
+    tools.push(tool(createToolFunction('switch_tab'), {
+        name: EBrowserTools.SWITCH_TAB,
+        description: 'Switch to a different browser tab by its ID. Tab IDs are shown in the page state. Use this to work with existing open tabs (e.g., use existing Gmail tab instead of opening a new one).',
+        schema: BrowserSwitchTabSchema,
+    }));
     return tools;
 }
 

package/dist/esm/tools/BrowserTools.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"BrowserTools.mjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with ranger-browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the server (Ranger) to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Ranger-Browser-Extension: true\n * - X-Ranger-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-ranger-browser-extension'];\n  const browserCapable = req.headers['x-ranger-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the element to click. Use fieldLabel attribute to identify the correct element when multiple similar elements exist (e.g., fieldLabel=\"To\" vs fieldLabel=\"Cc\" for email fields).'\n    ),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe('The [index] of the input element to type into. IMPORTANT: Check fieldLabel attribute to ensure you are typing into the correct field (e.g., fieldLabel=\"To\" for recipient, fieldLabel=\"Subject\" for subject line).'),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe('Whether to press Enter after typing (useful for search forms)'),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  if (!result.success && result.error) {\n    return `Browser action \"${action}\" failed: ${result.error}`;\n  }\n\n  const parts: string[] = [];\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    // Add hint about fieldLabel for form interactions\n    parts.push(`\\n**Interactive Elements** (use fieldLabel to identify form fields):\\n${result.elementList}`);\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. Skip disabled elements.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description: 'Type text into input by [index]. IMPORTANT: Use fieldLabel to identify the correct field (e.g., fieldLabel=\"To\" vs \"Cc\"). Use pressEnter:true to submit.',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description: 'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description: 'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description: 'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description: 'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description: 'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description: 'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description: 'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":[],"mappings":";;;AAIA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;;AAkC1C;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,iMAAiM,CAClM;AACJ,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,oNAAoN,CAAC;IACjO,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAE;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CAAC,+DAA+D,CAAC;AAC7E,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAE;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAE;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAE;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAE;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAyB9C;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAE,CAAA,CAAC;;AAE9C,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAE,CAAA,CAAC;;AAE/C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;;QAE3D,KAAK,CAAC,IAAI,CAAC,CAAA,sEAAA,EAAyE,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;AAE3G,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;;AAG7D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;;AAG7D,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAM,KAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAQ,KAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;;AAK/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,yGAAyG;AACtH,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,0JAA0J;AACvK,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,oEAAoE;AACjF,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,6DAA6D;AAC1E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,kFAAkF;AAC/F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,+DAA+D;AAC5E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,2EAA2E;AACxF,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,qFAAqF;AAClG,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;AAED,IAAA,OAAO,KAAK;AACd;;;;"}
+{"version":3,"file":"BrowserTools.mjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with ranger-browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n  KEYPRESS: 'browser_keypress',\n  SWITCH_TAB: 'browser_switch_tab',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the server (Ranger) to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Ranger-Browser-Extension: true\n * - X-Ranger-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-ranger-browser-extension'];\n  const browserCapable = req.headers['x-ranger-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the element to click. Use fieldLabel to identify the correct element. For form fields, target <input> or <textarea> elements, NOT parent <div> containers.'\n    ),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe('The [index] of the INPUT element to type into. Target <input> or <textarea> elements. Check fieldLabel to identify the correct field.'),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe('Whether to press Enter after typing (useful for search forms and submitting)'),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\nconst BrowserKeypressSchema = z.object({\n  keys: z\n    .string()\n    .describe('Keyboard keys to press. Use \"+\" to combine modifiers (e.g., \"Control+Enter\", \"Control+a\", \"Escape\", \"Tab\", \"Enter\"). Common shortcuts: Control+Enter (submit forms/send), Escape (close dialogs), Tab (next field).'),\n});\n\nconst BrowserSwitchTabSchema = z.object({\n  tabId: z\n    .number()\n    .describe('The tab ID to switch to. Use the tab IDs shown in the tabs list from page state.'),\n});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  if (!result.success && result.error) {\n    return `Browser action \"${action}\" failed: ${result.error}`;\n  }\n\n  const parts: string[] = [];\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    // Add hint about fieldLabel and targeting inputs for form interactions\n    parts.push(`\\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\\n${result.elementList}`);\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel=\"To recipients\" for To field).',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description: 'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description: 'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description: 'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description: 'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description: 'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description: 'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description: 'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  // browser_keypress - for keyboard shortcuts\n  tools.push(\n    tool(createToolFunction('keypress'), {\n      name: EBrowserTools.KEYPRESS,\n      description: 'Send keyboard shortcut or key press. Use for: Control+Enter (send email/submit), Escape (close dialog/cancel), Tab (next field), Enter (confirm). The keys are sent to the currently focused element.',\n      schema: BrowserKeypressSchema,\n    })\n  );\n\n  // browser_switch_tab - for switching between tabs\n  tools.push(\n    tool(createToolFunction('switch_tab'), {\n      name: EBrowserTools.SWITCH_TAB,\n      description: 'Switch to a different browser tab by its ID. Tab IDs are shown in the page state. Use this to work with existing open tabs (e.g., use existing Gmail tab instead of opening a new one).',\n      schema: BrowserSwitchTabSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":[],"mappings":";;;AAIA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;AACxC,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,UAAU,EAAE,oBAAoB;;AAkClC;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,2KAA2K,CAC5K;AACJ,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,uIAAuI,CAAC;IACpJ,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAE;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CAAC,8EAA8E,CAAC;AAC5F,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAE;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAE;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAE;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAE;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE9C,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,IAAI,EAAE;AACH,SAAA,MAAM;SACN,QAAQ,CAAC,qNAAqN,CAAC;AACnO,CAAA,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;AACtC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,kFAAkF,CAAC;AAChG,CAAA,CAAC;AAyBF;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAE,CAAA,CAAC;;AAE9C,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAE,CAAA,CAAC;;AAE/C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;;QAE3D,KAAK,CAAC,IAAI,CAAC,CAAA,gHAAA,EAAmH,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;AAErJ,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;;AAG7D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;;AAG7D,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAM,KAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAQ,KAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;;AAK/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,uJAAuJ;AACpK,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,wNAAwN;AACrO,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,oEAAoE;AACjF,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,6DAA6D;AAC1E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,kFAAkF;AAC/F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,+DAA+D;AAC5E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,2EAA2E;AACxF,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,qFAAqF;AAClG,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,uMAAuM;AACpN,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,yLAAyL;AACtM,QAAA,MAAM,EAAE,sBAAsB;AAC/B,KAAA,CAAC,CACH;AAED,IAAA,OAAO,KAAK;AACd;;;;"}

package/dist/types/tools/BrowserTools.d.ts

@@ -14,6 +14,8 @@ export declare const EBrowserTools: {
     readonly BACK: "browser_back";
     readonly SCREENSHOT: "browser_screenshot";
     readonly GET_PAGE_STATE: "browser_get_page_state";
+    readonly KEYPRESS: "browser_keypress";
+    readonly SWITCH_TAB: "browser_switch_tab";
 };
 export type BrowserToolName = (typeof EBrowserTools)[keyof typeof EBrowserTools];
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "illuma-agents",
-  "version": "1.0.35",
+  "version": "1.0.37",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/tools/BrowserTools.ts

@@ -17,6 +17,8 @@ export const EBrowserTools = {
   BACK: 'browser_back',
   SCREENSHOT: 'browser_screenshot',
   GET_PAGE_STATE: 'browser_get_page_state',
+  KEYPRESS: 'browser_keypress',
+  SWITCH_TAB: 'browser_switch_tab',
 } as const;
 
 export type BrowserToolName =
@@ -74,19 +76,19 @@ const BrowserClickSchema = z.object({
   index: z
     .number()
     .describe(
-      'The [index] of the element to click. Use fieldLabel attribute to identify the correct element when multiple similar elements exist (e.g., fieldLabel="To" vs fieldLabel="Cc" for email fields).'
+      'The [index] of the element to click. Use fieldLabel to identify the correct element. For form fields, target <input> or <textarea> elements, NOT parent <div> containers.'
     ),
 });
 
 const BrowserTypeSchema = z.object({
   index: z
     .number()
-    .describe('The [index] of the input element to type into. IMPORTANT: Check fieldLabel attribute to ensure you are typing into the correct field (e.g., fieldLabel="To" for recipient, fieldLabel="Subject" for subject line).'),
+    .describe('The [index] of the INPUT element to type into. Target <input> or <textarea> elements. Check fieldLabel to identify the correct field.'),
   text: z.string().describe('The text to type into the element'),
   pressEnter: z
     .boolean()
     .optional()
-    .describe('Whether to press Enter after typing (useful for search forms)'),
+    .describe('Whether to press Enter after typing (useful for search forms and submitting)'),
 });
 
 const BrowserNavigateSchema = z.object({
@@ -129,6 +131,18 @@ const BrowserScreenshotSchema = z.object({});
 
 const BrowserGetPageStateSchema = z.object({});
 
+const BrowserKeypressSchema = z.object({
+  keys: z
+    .string()
+    .describe('Keyboard keys to press. Use "+" to combine modifiers (e.g., "Control+Enter", "Control+a", "Escape", "Tab", "Enter"). Common shortcuts: Control+Enter (submit forms/send), Escape (close dialogs), Tab (next field).'),
+});
+
+const BrowserSwitchTabSchema = z.object({
+  tabId: z
+    .number()
+    .describe('The tab ID to switch to. Use the tab IDs shown in the tabs list from page state.'),
+});
+
 /**
  * Browser tool response interface
  * This is what the extension returns after executing the action
@@ -172,8 +186,8 @@ function formatResultForLLM(
     parts.push(`**Page Title:** ${result.title}`);
   }
   if (result.elementList != null && result.elementList !== '') {
-    // Add hint about fieldLabel for form interactions
-    parts.push(`\n**Interactive Elements** (use fieldLabel to identify form fields):\n${result.elementList}`);
+    // Add hint about fieldLabel and targeting inputs for form interactions
+    parts.push(`\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\n${result.elementList}`);
   }
   if (result.screenshot != null && result.screenshot !== '') {
     parts.push('\n[Screenshot captured and displayed to user]');
@@ -249,7 +263,7 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('click'), {
       name: EBrowserTools.CLICK,
-      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. Skip disabled elements.',
+      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',
       schema: BrowserClickSchema,
     })
   );
@@ -258,7 +272,7 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('type'), {
       name: EBrowserTools.TYPE,
-      description: 'Type text into input by [index]. IMPORTANT: Use fieldLabel to identify the correct field (e.g., fieldLabel="To" vs "Cc"). Use pressEnter:true to submit.',
+      description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel="To recipients" for To field).',
       schema: BrowserTypeSchema,
     })
   );
@@ -335,5 +349,23 @@ export function createBrowserTools(
     })
   );
 
+  // browser_keypress - for keyboard shortcuts
+  tools.push(
+    tool(createToolFunction('keypress'), {
+      name: EBrowserTools.KEYPRESS,
+      description: 'Send keyboard shortcut or key press. Use for: Control+Enter (send email/submit), Escape (close dialog/cancel), Tab (next field), Enter (confirm). The keys are sent to the currently focused element.',
+      schema: BrowserKeypressSchema,
+    })
+  );
+
+  // browser_switch_tab - for switching between tabs
+  tools.push(
+    tool(createToolFunction('switch_tab'), {
+      name: EBrowserTools.SWITCH_TAB,
+      description: 'Switch to a different browser tab by its ID. Tab IDs are shown in the page state. Use this to work with existing open tabs (e.g., use existing Gmail tab instead of opening a new one).',
+      schema: BrowserSwitchTabSchema,
+    })
+  );
+
   return tools;
 }