illuma-agents 1.0.34 → 1.0.36

package/dist/cjs/tools/BrowserTools.cjs

@@ -37,12 +37,12 @@ function hasBrowserCapability(req) {
 const BrowserClickSchema = zod.z.object({
     index: zod.z
         .number()
-        .describe('The index number [0], [1], etc. of the element to click from the page state element list'),
+        .describe('The [index] of the element to click. CRITICAL: Use fieldLabel to identify the correct element. For form fields, always target the actual <input> or <textarea> element (with type=text or role=combobox), NOT the parent <div> container.'),
 });
 const BrowserTypeSchema = zod.z.object({
     index: zod.z
         .number()
-        .describe('The index number of the input element to type into'),
+        .describe('The [index] of the INPUT element to type into. CRITICAL: Always target <input> or <textarea> elements (look for type=text or role=combobox). Check fieldLabel to identify the correct field (e.g., fieldLabel="To recipients" vs fieldLabel="CC recipients"). Do NOT type into parent container <div> elements.'),
     text: zod.z.string().describe('The text to type into the element'),
     pressEnter: zod.z
         .boolean()
@@ -96,7 +96,8 @@ function formatResultForLLM(result, action) {
         parts.push(`**Page Title:** ${result.title}`);
     }
     if (result.elementList != null && result.elementList !== '') {
-        parts.push(`\n**Interactive Elements:**\n${result.elementList}`);
+        // Add hint about fieldLabel and targeting inputs for form interactions
+        parts.push(`\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\n${result.elementList}`);
     }
     if (result.screenshot != null && result.screenshot !== '') {
         parts.push('\n[Screenshot captured and displayed to user]');
@@ -157,13 +158,13 @@ function createBrowserTools(options) {
     // browser_click
     tools$1.push(tools.tool(createToolFunction('click'), {
         name: EBrowserTools.CLICK,
-        description: 'Click element by [index]. Skip disabled elements. Returns updated element list.',
+        description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',
         schema: BrowserClickSchema,
     }));
     // browser_type
     tools$1.push(tools.tool(createToolFunction('type'), {
         name: EBrowserTools.TYPE,
-        description: 'Type text into input element by [index]. Use pressEnter:true to submit. Returns updated element list.',
+        description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel="To recipients" for To field).',
         schema: BrowserTypeSchema,
     }));
     // browser_navigate

package/dist/cjs/tools/BrowserTools.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"BrowserTools.cjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with ranger-browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the server (Ranger) to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Ranger-Browser-Extension: true\n * - X-Ranger-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-ranger-browser-extension'];\n  const browserCapable = req.headers['x-ranger-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The index number [0], [1], etc. of the element to click from the page state element list'\n    ),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe('The index number of the input element to type into'),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe('Whether to press Enter after typing (useful for search forms)'),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  if (!result.success && result.error) {\n    return `Browser action \"${action}\" failed: ${result.error}`;\n  }\n\n  const parts: string[] = [];\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    parts.push(`\\n**Interactive Elements:**\\n${result.elementList}`);\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description: 'Click element by [index]. Skip disabled elements. Returns updated element list.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description: 'Type text into input element by [index]. Use pressEnter:true to submit. Returns updated element list.',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description: 'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description: 'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description: 'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description: 'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description: 'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description: 'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description: 'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":["z","tools","tool"],"mappings":";;;;;AAIA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;;AAkC1C;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,0FAA0F,CAC3F;AACJ,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,oDAAoD,CAAC;IACjE,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAEA;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CAAC,+DAA+D,CAAC;AAC7E,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAEA;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAEA;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAEA;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAEA;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAyB9C;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAE,CAAA,CAAC;;AAE9C,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAE,CAAA,CAAC;;AAE/C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;QAC3D,KAAK,CAAC,IAAI,CAAC,CAAA,6BAAA,EAAgC,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;AAElE,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;;AAG7D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;;AAG7D,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAMC,OAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAQ,KAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;;AAK/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGDA,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,iFAAiF;AAC9F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,uGAAuG;AACpH,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,oEAAoE;AACjF,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,6DAA6D;AAC1E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,kFAAkF;AAC/F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,+DAA+D;AAC5E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,2EAA2E;AACxF,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,qFAAqF;AAClG,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;AAED,IAAA,OAAOD,OAAK;AACd;;;;;;"}
+{"version":3,"file":"BrowserTools.cjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with ranger-browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the server (Ranger) to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Ranger-Browser-Extension: true\n * - X-Ranger-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-ranger-browser-extension'];\n  const browserCapable = req.headers['x-ranger-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the element to click. CRITICAL: Use fieldLabel to identify the correct element. For form fields, always target the actual <input> or <textarea> element (with type=text or role=combobox), NOT the parent <div> container.'\n    ),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe('The [index] of the INPUT element to type into. CRITICAL: Always target <input> or <textarea> elements (look for type=text or role=combobox). Check fieldLabel to identify the correct field (e.g., fieldLabel=\"To recipients\" vs fieldLabel=\"CC recipients\"). Do NOT type into parent container <div> elements.'),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe('Whether to press Enter after typing (useful for search forms)'),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  if (!result.success && result.error) {\n    return `Browser action \"${action}\" failed: ${result.error}`;\n  }\n\n  const parts: string[] = [];\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    // Add hint about fieldLabel and targeting inputs for form interactions\n    parts.push(`\\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\\n${result.elementList}`);\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel=\"To recipients\" for To field).',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description: 'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description: 'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description: 'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description: 'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description: 'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description: 'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description: 'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":["z","tools","tool"],"mappings":";;;;;AAIA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;;AAkC1C;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,2OAA2O,CAC5O;AACJ,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,iTAAiT,CAAC;IAC9T,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAEA;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CAAC,+DAA+D,CAAC;AAC7E,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAEA;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAEA;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAEA;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAEA;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAyB9C;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAE,CAAA,CAAC;;AAE9C,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAE,CAAA,CAAC;;AAE/C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;;QAE3D,KAAK,CAAC,IAAI,CAAC,CAAA,gHAAA,EAAmH,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;AAErJ,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;;AAG7D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;;AAG7D,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAMC,OAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAQ,KAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;;AAK/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGDA,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,uJAAuJ;AACpK,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,wNAAwN;AACrO,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,oEAAoE;AACjF,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,6DAA6D;AAC1E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,kFAAkF;AAC/F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,+DAA+D;AAC5E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,2EAA2E;AACxF,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,qFAAqF;AAClG,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;AAED,IAAA,OAAOD,OAAK;AACd;;;;;;"}

package/dist/esm/tools/BrowserTools.mjs

@@ -35,12 +35,12 @@ function hasBrowserCapability(req) {
 const BrowserClickSchema = z.object({
     index: z
         .number()
-        .describe('The index number [0], [1], etc. of the element to click from the page state element list'),
+        .describe('The [index] of the element to click. CRITICAL: Use fieldLabel to identify the correct element. For form fields, always target the actual <input> or <textarea> element (with type=text or role=combobox), NOT the parent <div> container.'),
 });
 const BrowserTypeSchema = z.object({
     index: z
         .number()
-        .describe('The index number of the input element to type into'),
+        .describe('The [index] of the INPUT element to type into. CRITICAL: Always target <input> or <textarea> elements (look for type=text or role=combobox). Check fieldLabel to identify the correct field (e.g., fieldLabel="To recipients" vs fieldLabel="CC recipients"). Do NOT type into parent container <div> elements.'),
     text: z.string().describe('The text to type into the element'),
     pressEnter: z
         .boolean()
@@ -94,7 +94,8 @@ function formatResultForLLM(result, action) {
         parts.push(`**Page Title:** ${result.title}`);
     }
     if (result.elementList != null && result.elementList !== '') {
-        parts.push(`\n**Interactive Elements:**\n${result.elementList}`);
+        // Add hint about fieldLabel and targeting inputs for form interactions
+        parts.push(`\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\n${result.elementList}`);
     }
     if (result.screenshot != null && result.screenshot !== '') {
         parts.push('\n[Screenshot captured and displayed to user]');
@@ -155,13 +156,13 @@ function createBrowserTools(options) {
     // browser_click
     tools.push(tool(createToolFunction('click'), {
         name: EBrowserTools.CLICK,
-        description: 'Click element by [index]. Skip disabled elements. Returns updated element list.',
+        description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',
         schema: BrowserClickSchema,
     }));
     // browser_type
     tools.push(tool(createToolFunction('type'), {
         name: EBrowserTools.TYPE,
-        description: 'Type text into input element by [index]. Use pressEnter:true to submit. Returns updated element list.',
+        description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel="To recipients" for To field).',
         schema: BrowserTypeSchema,
     }));
     // browser_navigate

package/dist/esm/tools/BrowserTools.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"BrowserTools.mjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with ranger-browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the server (Ranger) to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Ranger-Browser-Extension: true\n * - X-Ranger-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-ranger-browser-extension'];\n  const browserCapable = req.headers['x-ranger-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The index number [0], [1], etc. of the element to click from the page state element list'\n    ),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe('The index number of the input element to type into'),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe('Whether to press Enter after typing (useful for search forms)'),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  if (!result.success && result.error) {\n    return `Browser action \"${action}\" failed: ${result.error}`;\n  }\n\n  const parts: string[] = [];\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    parts.push(`\\n**Interactive Elements:**\\n${result.elementList}`);\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description: 'Click element by [index]. Skip disabled elements. Returns updated element list.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description: 'Type text into input element by [index]. Use pressEnter:true to submit. Returns updated element list.',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description: 'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description: 'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description: 'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description: 'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description: 'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description: 'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description: 'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":[],"mappings":";;;AAIA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;;AAkC1C;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,0FAA0F,CAC3F;AACJ,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,oDAAoD,CAAC;IACjE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAE;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CAAC,+DAA+D,CAAC;AAC7E,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAE;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAE;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAE;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAE;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAyB9C;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAE,CAAA,CAAC;;AAE9C,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAE,CAAA,CAAC;;AAE/C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;QAC3D,KAAK,CAAC,IAAI,CAAC,CAAA,6BAAA,EAAgC,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;AAElE,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;;AAG7D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;;AAG7D,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAM,KAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAQ,KAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;;AAK/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,iFAAiF;AAC9F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,uGAAuG;AACpH,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,oEAAoE;AACjF,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,6DAA6D;AAC1E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,kFAAkF;AAC/F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,+DAA+D;AAC5E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,2EAA2E;AACxF,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,qFAAqF;AAClG,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;AAED,IAAA,OAAO,KAAK;AACd;;;;"}
+{"version":3,"file":"BrowserTools.mjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with ranger-browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the server (Ranger) to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Ranger-Browser-Extension: true\n * - X-Ranger-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-ranger-browser-extension'];\n  const browserCapable = req.headers['x-ranger-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the element to click. CRITICAL: Use fieldLabel to identify the correct element. For form fields, always target the actual <input> or <textarea> element (with type=text or role=combobox), NOT the parent <div> container.'\n    ),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe('The [index] of the INPUT element to type into. CRITICAL: Always target <input> or <textarea> elements (look for type=text or role=combobox). Check fieldLabel to identify the correct field (e.g., fieldLabel=\"To recipients\" vs fieldLabel=\"CC recipients\"). Do NOT type into parent container <div> elements.'),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe('Whether to press Enter after typing (useful for search forms)'),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  if (!result.success && result.error) {\n    return `Browser action \"${action}\" failed: ${result.error}`;\n  }\n\n  const parts: string[] = [];\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    // Add hint about fieldLabel and targeting inputs for form interactions\n    parts.push(`\\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\\n${result.elementList}`);\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel=\"To recipients\" for To field).',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description: 'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description: 'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description: 'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description: 'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description: 'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description: 'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description: 'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description: 'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":[],"mappings":";;;AAIA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;;AAkC1C;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,2OAA2O,CAC5O;AACJ,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CAAC,iTAAiT,CAAC;IAC9T,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAE;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CAAC,+DAA+D,CAAC;AAC7E,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAE;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAE;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAE;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAE;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAyB9C;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAE,CAAA,CAAC;;AAE9C,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAE,CAAA,CAAC;;AAE/C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;;QAE3D,KAAK,CAAC,IAAI,CAAC,CAAA,gHAAA,EAAmH,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;AAErJ,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;;AAG7D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;;AAG7D,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAM,KAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAQ,KAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;;AAK/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,uJAAuJ;AACpK,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,wNAAwN;AACrO,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,oEAAoE;AACjF,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,6DAA6D;AAC1E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,kFAAkF;AAC/F,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,+DAA+D;AAC5E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,iEAAiE;AAC9E,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,2EAA2E;AACxF,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,qFAAqF;AAClG,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;AAED,IAAA,OAAO,KAAK;AACd;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "illuma-agents",
-  "version": "1.0.34",
+  "version": "1.0.36",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/tools/BrowserTools.ts

@@ -74,14 +74,14 @@ const BrowserClickSchema = z.object({
   index: z
     .number()
     .describe(
-      'The index number [0], [1], etc. of the element to click from the page state element list'
+      'The [index] of the element to click. CRITICAL: Use fieldLabel to identify the correct element. For form fields, always target the actual <input> or <textarea> element (with type=text or role=combobox), NOT the parent <div> container.'
     ),
 });
 
 const BrowserTypeSchema = z.object({
   index: z
     .number()
-    .describe('The index number of the input element to type into'),
+    .describe('The [index] of the INPUT element to type into. CRITICAL: Always target <input> or <textarea> elements (look for type=text or role=combobox). Check fieldLabel to identify the correct field (e.g., fieldLabel="To recipients" vs fieldLabel="CC recipients"). Do NOT type into parent container <div> elements.'),
   text: z.string().describe('The text to type into the element'),
   pressEnter: z
     .boolean()
@@ -172,7 +172,8 @@ function formatResultForLLM(
     parts.push(`**Page Title:** ${result.title}`);
   }
   if (result.elementList != null && result.elementList !== '') {
-    parts.push(`\n**Interactive Elements:**\n${result.elementList}`);
+    // Add hint about fieldLabel and targeting inputs for form interactions
+    parts.push(`\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\n${result.elementList}`);
   }
   if (result.screenshot != null && result.screenshot !== '') {
     parts.push('\n[Screenshot captured and displayed to user]');
@@ -248,7 +249,7 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('click'), {
       name: EBrowserTools.CLICK,
-      description: 'Click element by [index]. Skip disabled elements. Returns updated element list.',
+      description: 'Click element by [index]. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',
       schema: BrowserClickSchema,
     })
   );
@@ -257,7 +258,7 @@ export function createBrowserTools(
   tools.push(
     tool(createToolFunction('type'), {
       name: EBrowserTools.TYPE,
-      description: 'Type text into input element by [index]. Use pressEnter:true to submit. Returns updated element list.',
+      description: 'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel="To recipients" for To field).',
       schema: BrowserTypeSchema,
     })
   );