illuma-agents 1.0.19 → 1.0.21

package/dist/esm/tools/BrowserTools.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"BrowserTools.mjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["/**\r\n * Browser Automation Tools for Ranger Browser Extension\r\n * \r\n * These tools allow the LLM to interact with the browser through the \r\n * ranger-browser extension. They generate structured actions that are\r\n * sent to the extension via SSE streaming for execution.\r\n * \r\n * The extension handles:\r\n * - DOM extraction with element indexing\r\n * - Click, type, hover, scroll actions\r\n * - Navigation and page context\r\n * - Visual element highlighting\r\n */\r\n\r\nimport { z } from 'zod';\r\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\r\n\r\n// ============================================\r\n// Tool Schemas\r\n// ============================================\r\n\r\n/**\r\n * Enhanced click schema that supports both index-based and coordinate-based clicking\r\n */\r\nconst BrowserClickSchema = z.object({\r\n  index: z.number().optional().describe(\r\n    'The index of the element to click, as shown in the page context (e.g., [0], [1], [2]). ' +\r\n    'Use the element index from the interactive elements list provided in the page context. ' +\r\n    'Either index OR coordinates must be provided.'\r\n  ),\r\n  coordinates: z.object({\r\n    x: z.number().describe('X coordinate in viewport pixels'),\r\n    y: z.number().describe('Y coordinate in viewport pixels'),\r\n  }).optional().describe(\r\n    'Coordinates for clicking elements that lack semantic info (marked with ⚠️). ' +\r\n    'The coordinates are provided in the element listing as coords:(x,y). ' +\r\n    'Either index OR coordinates must be provided.'\r\n  ),\r\n  visualDescription: z.string().optional().describe(\r\n    'Description of what the element looks like visually. Used when clicking by appearance ' +\r\n    '(e.g., \"blue button in top right corner\", \"hamburger menu icon\")'\r\n  ),\r\n  reason: z.string().optional().describe(\r\n    'Brief explanation of why you are clicking this element (for user transparency)'\r\n  ),\r\n});\r\n\r\nconst BrowserTypeSchema = z.object({\r\n  index: z.number().describe(\r\n    'The index of the input element to type into, as shown in the page context'\r\n  ),\r\n  text: z.string().describe(\r\n    'The text to type into the input field'\r\n  ),\r\n  clear: z.boolean().optional().describe(\r\n    'Whether to clear the existing content before typing (default: false)'\r\n  ),\r\n  pressEnter: z.boolean().optional().describe(\r\n    'Whether to press Enter after typing (useful for search fields, default: false)'\r\n  ),\r\n});\r\n\r\nconst BrowserNavigateSchema = z.object({\r\n  url: z.string().describe(\r\n    'The URL to navigate to. Can be a full URL or a relative path.'\r\n  ),\r\n  reason: z.string().optional().describe(\r\n    'Brief explanation of why you are navigating to this URL'\r\n  ),\r\n});\r\n\r\nconst BrowserScrollSchema = z.object({\r\n  direction: z.enum(['up', 'down', 'left', 'right']).describe(\r\n    'The direction to scroll'\r\n  ),\r\n  amount: z.number().optional().describe(\r\n    'The amount to scroll in pixels (default: 500)'\r\n  ),\r\n});\r\n\r\nconst BrowserExtractSchema = z.object({\r\n  query: z.string().optional().describe(\r\n    'Optional query to filter extracted content. If provided, only content related to the query will be extracted.'\r\n  ),\r\n  selector: z.string().optional().describe(\r\n    'Optional CSS selector to extract content from a specific element'\r\n  ),\r\n});\r\n\r\nconst BrowserHoverSchema = z.object({\r\n  index: z.number().describe(\r\n    'The index of the element to hover over, as shown in the page context'\r\n  ),\r\n});\r\n\r\nconst BrowserWaitSchema = z.object({\r\n  duration: z.number().optional().describe(\r\n    'Duration to wait in milliseconds (default: 1000)'\r\n  ),\r\n  reason: z.string().optional().describe(\r\n    'Why we are waiting (e.g., \"for page to load\", \"for animation to complete\")'\r\n  ),\r\n});\r\n\r\nconst BrowserGoBackSchema = z.object({\r\n  reason: z.string().optional().describe(\r\n    'Brief explanation of why you are going back'\r\n  ),\r\n});\r\n\r\nconst BrowserScreenshotSchema = z.object({\r\n  fullPage: z.boolean().optional().describe(\r\n    'Whether to capture the full page or just the viewport (default: viewport only)'\r\n  ),\r\n  reason: z.string().optional().describe(\r\n    'Why you need a screenshot (e.g., \"to identify visual elements\", \"to analyze page layout\")'\r\n  ),\r\n});\r\n\r\n// ============================================\r\n// Tool Implementations\r\n// ============================================\r\n\r\n/**\r\n * Browser click tool - clicks an element by index or coordinates\r\n * Supports both semantic (index-based) and vision (coordinate-based) clicking\r\n */\r\nexport function createBrowserClickTool(): DynamicStructuredTool<typeof BrowserClickSchema> {\r\n  return tool<typeof BrowserClickSchema>(\r\n    async ({ index, coordinates, visualDescription, reason }) => {\r\n      // Validate that at least one targeting method is provided\r\n      if (index === undefined && !coordinates) {\r\n        return JSON.stringify({\r\n          type: 'error',\r\n          error: 'Either index or coordinates must be provided to click an element',\r\n        });\r\n      }\r\n\r\n      // Return a structured action for the extension to execute\r\n      // The actual execution happens in the browser extension\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'click',\r\n          ...(index !== undefined && { index }),\r\n          ...(coordinates && { coordinates }),\r\n          ...(visualDescription && { visualDescription }),\r\n          reason,\r\n        },\r\n        // Signal that this requires browser execution\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.CLICK,\r\n      description: `Click an interactive element on the current page.\r\n\r\n**Two ways to target elements:**\r\n\r\n1. **By index (preferred)**: Use the element's index number from the interactive elements list\r\n   - Format: [index] {semantic role} <tag>text</tag>\r\n   - Example: browser_click({ index: 5 }) to click element [5]\r\n\r\n2. **By coordinates (vision fallback)**: For elements marked with ⚠️ that lack semantic info\r\n   - Use the coords:(x,y) shown after the ⚠️ marker\r\n   - Example: browser_click({ coordinates: { x: 150, y: 200 } })\r\n\r\n**When to use coordinates:**\r\n- Elements marked with ⚠️ have poor semantic understanding\r\n- Icon-only buttons without labels\r\n- Custom canvas/SVG elements\r\n- When you identify an element visually in a screenshot\r\n\r\nExample: If element shows \\`[12] {button} <div>⚠️ [left side, small, clickable] coords:(45,120)\\`\r\nUse either: browser_click({ index: 12 }) or browser_click({ coordinates: { x: 45, y: 120 } })`,\r\n      schema: BrowserClickSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser type tool - types text into an input field\r\n */\r\nexport function createBrowserTypeTool(): DynamicStructuredTool<typeof BrowserTypeSchema> {\r\n  return tool<typeof BrowserTypeSchema>(\r\n    async ({ index, text, clear, pressEnter }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'type',\r\n          index,\r\n          text,\r\n          clear: clear ?? false,\r\n          pressEnter: pressEnter ?? false,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.TYPE,\r\n      description: `Type text into an input field on the current page.\r\n\r\nUse this tool when you need to:\r\n- Fill in a text input or textarea\r\n- Enter a search query\r\n- Fill out form fields\r\n\r\nThe element index comes from the page context's interactive elements list.\r\nSet 'clear: true' to clear existing content before typing.\r\nSet 'pressEnter: true' to submit after typing (useful for search fields).\r\n\r\nExample: To type \"hello world\" into a search field shown as \"[2]<input>Search...</input>\",\r\nuse index: 2, text: \"hello world\"`,\r\n      schema: BrowserTypeSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser navigate tool - navigates to a URL\r\n */\r\nexport function createBrowserNavigateTool(): DynamicStructuredTool<typeof BrowserNavigateSchema> {\r\n  return tool<typeof BrowserNavigateSchema>(\r\n    async ({ url, reason }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'navigate',\r\n          url,\r\n          reason,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.NAVIGATE,\r\n      description: `Navigate to a specific URL in the browser.\r\n\r\nUse this tool when you need to:\r\n- Go to a specific website\r\n- Navigate to a different page\r\n- Open a new URL\r\n\r\nProvide the full URL including the protocol (https://).\r\n\r\nExample: browser_navigate({ url: \"https://www.google.com\" })`,\r\n      schema: BrowserNavigateSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser scroll tool - scrolls the page\r\n */\r\nexport function createBrowserScrollTool(): DynamicStructuredTool<typeof BrowserScrollSchema> {\r\n  return tool<typeof BrowserScrollSchema>(\r\n    async ({ direction, amount }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'scroll',\r\n          scroll: {\r\n            direction,\r\n            amount: amount ?? 500,\r\n          },\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.SCROLL,\r\n      description: `Scroll the current page in a specified direction.\r\n\r\nUse this tool when you need to:\r\n- See more content on the page\r\n- Scroll to find elements not currently visible\r\n- Navigate long pages\r\n\r\nDefault scroll amount is 500 pixels. Adjust as needed.\r\n\r\nExample: browser_scroll({ direction: \"down\", amount: 800 })`,\r\n      schema: BrowserScrollSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser extract tool - extracts content from the page\r\n */\r\nexport function createBrowserExtractTool(): DynamicStructuredTool<typeof BrowserExtractSchema> {\r\n  return tool<typeof BrowserExtractSchema>(\r\n    async ({ query, selector }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'extract',\r\n          query,\r\n          selector,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.EXTRACT,\r\n      description: `Extract text content from the current page.\r\n\r\nUse this tool when you need to:\r\n- Get specific information from the page\r\n- Extract text that matches a query\r\n- Read content from a specific element\r\n\r\nIf no query or selector is provided, extracts the main page content.\r\nUse a CSS selector to extract from a specific element.\r\nUse a query to filter for relevant content.\r\n\r\nExample: browser_extract({ query: \"price\" }) - extracts content related to pricing`,\r\n      schema: BrowserExtractSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser hover tool - hovers over an element\r\n */\r\nexport function createBrowserHoverTool(): DynamicStructuredTool<typeof BrowserHoverSchema> {\r\n  return tool<typeof BrowserHoverSchema>(\r\n    async ({ index }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'hover',\r\n          index,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.HOVER,\r\n      description: `Hover over an element to reveal tooltips or dropdown menus.\r\n\r\nUse this tool when you need to:\r\n- Reveal a dropdown menu\r\n- Show a tooltip\r\n- Trigger hover effects\r\n\r\nExample: browser_hover({ index: 3 }) - hovers over element at index 3`,\r\n      schema: BrowserHoverSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser wait tool - waits for a specified duration\r\n */\r\nexport function createBrowserWaitTool(): DynamicStructuredTool<typeof BrowserWaitSchema> {\r\n  return tool<typeof BrowserWaitSchema>(\r\n    async ({ duration, reason }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'wait',\r\n          duration: duration ?? 1000,\r\n          reason,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.WAIT,\r\n      description: `Wait for a specified duration before the next action.\r\n\r\nUse this tool when you need to:\r\n- Wait for a page to load\r\n- Wait for an animation to complete\r\n- Add delay between actions\r\n\r\nDefault wait time is 1000ms (1 second).\r\n\r\nExample: browser_wait({ duration: 2000, reason: \"waiting for page to load\" })`,\r\n      schema: BrowserWaitSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser go back tool - navigates back in history\r\n */\r\nexport function createBrowserGoBackTool(): DynamicStructuredTool<typeof BrowserGoBackSchema> {\r\n  return tool<typeof BrowserGoBackSchema>(\r\n    async ({ reason }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'back',\r\n          reason,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.BACK,\r\n      description: `Navigate back to the previous page in browser history.\r\n\r\nUse this tool when you need to:\r\n- Return to a previous page\r\n- Undo a navigation\r\n\r\nExample: browser_back({ reason: \"returning to search results\" })`,\r\n      schema: BrowserGoBackSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser screenshot tool - captures a screenshot\r\n */\r\nexport function createBrowserScreenshotTool(): DynamicStructuredTool<typeof BrowserScreenshotSchema> {\r\n  return tool<typeof BrowserScreenshotSchema>(\r\n    async ({ fullPage }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'screenshot',\r\n          fullPage: fullPage ?? false,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.SCREENSHOT,\r\n      description: `Capture a screenshot of the current page.\r\n\r\nUse this tool when you need to:\r\n- Capture the current state of a page\r\n- Document visual elements\r\n- Verify page appearance\r\n\r\nSet fullPage: true to capture the entire page (may be large).\r\nDefault captures only the visible viewport.\r\n\r\nExample: browser_screenshot({ fullPage: false })`,\r\n      schema: BrowserScreenshotSchema,\r\n    }\r\n  );\r\n}\r\n\r\n// ============================================\r\n// Tool Collection\r\n// ============================================\r\n\r\nexport type BrowserToolsConfig = {\r\n  /** Enable click tool */\r\n  enableClick?: boolean;\r\n  /** Enable type tool */\r\n  enableType?: boolean;\r\n  /** Enable navigate tool */\r\n  enableNavigate?: boolean;\r\n  /** Enable scroll tool */\r\n  enableScroll?: boolean;\r\n  /** Enable extract tool */\r\n  enableExtract?: boolean;\r\n  /** Enable hover tool */\r\n  enableHover?: boolean;\r\n  /** Enable wait tool */\r\n  enableWait?: boolean;\r\n  /** Enable back tool */\r\n  enableBack?: boolean;\r\n  /** Enable screenshot tool */\r\n  enableScreenshot?: boolean;\r\n};\r\n\r\n/**\r\n * Create all browser automation tools\r\n * \r\n * IMPORTANT: These tools should ONLY be registered when:\r\n * 1. The request comes from a browser extension that can execute them\r\n * 2. The client has indicated browser capability (e.g., via header or parameter)\r\n * \r\n * DO NOT register these for normal web UI users - they cannot execute browser actions.\r\n * \r\n * Detection in Ranger API:\r\n * - Check for `X-Ranger-Browser-Extension: true` header\r\n * - Or check for `browserCapable: true` in request body\r\n * - Or check user agent for extension identifier\r\n * \r\n * @example\r\n * // In Ranger API endpoint:\r\n * const hasBrowserExtension = req.headers['x-ranger-browser-extension'] === 'true';\r\n * const tools = hasBrowserExtension \r\n *   ? [...normalTools, ...createBrowserTools()]\r\n *   : normalTools;\r\n */\r\nexport function createBrowserTools(config: BrowserToolsConfig = {}): DynamicStructuredTool[] {\r\n  const tools: DynamicStructuredTool[] = [];\r\n  \r\n  // Enable all by default\r\n  const {\r\n    enableClick = true,\r\n    enableType = true,\r\n    enableNavigate = true,\r\n    enableScroll = true,\r\n    enableExtract = true,\r\n    enableHover = true,\r\n    enableWait = true,\r\n    enableBack = true,\r\n    enableScreenshot = true,\r\n  } = config;\r\n  \r\n  if (enableClick) tools.push(createBrowserClickTool());\r\n  if (enableType) tools.push(createBrowserTypeTool());\r\n  if (enableNavigate) tools.push(createBrowserNavigateTool());\r\n  if (enableScroll) tools.push(createBrowserScrollTool());\r\n  if (enableExtract) tools.push(createBrowserExtractTool());\r\n  if (enableHover) tools.push(createBrowserHoverTool());\r\n  if (enableWait) tools.push(createBrowserWaitTool());\r\n  if (enableBack) tools.push(createBrowserGoBackTool());\r\n  if (enableScreenshot) tools.push(createBrowserScreenshotTool());\r\n  \r\n  return tools;\r\n}\r\n\r\n/**\r\n * Browser tool name constants\r\n * Use these instead of magic strings\r\n */\r\nexport const EBrowserTools = {\r\n  CLICK: 'browser_click',\r\n  TYPE: 'browser_type',\r\n  NAVIGATE: 'browser_navigate',\r\n  SCROLL: 'browser_scroll',\r\n  EXTRACT: 'browser_extract',\r\n  HOVER: 'browser_hover',\r\n  WAIT: 'browser_wait',\r\n  BACK: 'browser_back',\r\n  SCREENSHOT: 'browser_screenshot',\r\n} as const;\r\n\r\n/**\r\n * Get browser tool names for filtering/identification\r\n */\r\nexport const BROWSER_TOOL_NAMES = [\r\n  EBrowserTools.CLICK,\r\n  EBrowserTools.TYPE,\r\n  EBrowserTools.NAVIGATE,\r\n  EBrowserTools.SCROLL,\r\n  EBrowserTools.EXTRACT,\r\n  EBrowserTools.HOVER,\r\n  EBrowserTools.WAIT,\r\n  EBrowserTools.BACK,\r\n  EBrowserTools.SCREENSHOT,\r\n] as const;\r\n\r\nexport type BrowserToolName = typeof BROWSER_TOOL_NAMES[number];\r\n\r\n/**\r\n * Check if a tool call is a browser action\r\n */\r\nexport function isBrowserToolCall(toolName: string): toolName is BrowserToolName {\r\n  return BROWSER_TOOL_NAMES.includes(toolName as BrowserToolName);\r\n}\r\n\r\n/**\r\n * Check if request indicates browser extension capability\r\n * Use this to conditionally register browser tools\r\n * \r\n * @example\r\n * // In Express middleware or endpoint:\r\n * if (hasBrowserCapability(req.headers)) {\r\n *   tools.push(...createBrowserTools());\r\n * }\r\n */\r\nexport function hasBrowserCapability(headers: Record<string, string | string[] | undefined>): boolean {\r\n  const extensionHeader = headers['x-ranger-browser-extension'];\r\n  const capableHeader = headers['x-ranger-browser-capable'];\r\n  \r\n  return (\r\n    extensionHeader === 'true' || \r\n    capableHeader === 'true' ||\r\n    (Array.isArray(extensionHeader) && extensionHeader.includes('true')) ||\r\n    (Array.isArray(capableHeader) && capableHeader.includes('true'))\r\n  );\r\n}\r\n"],"names":[],"mappings":";;;AAAA;;;;;;;;;;;;AAYG;AAKH;AACA;AACA;AAEA;;AAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnC,yFAAyF;QACzF,yFAAyF;AACzF,QAAA,+CAA+C,CAChD;AACD,IAAA,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;QACzD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;AAC1D,KAAA,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpB,8EAA8E;QAC9E,uEAAuE;AACvE,QAAA,+CAA+C,CAChD;IACD,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/C,wFAAwF;AACxF,QAAA,kEAAkE,CACnE;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,gFAAgF,CACjF;AACF,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACxB,2EAA2E,CAC5E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,uCAAuC,CACxC;AACD,IAAA,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,sEAAsE,CACvE;AACD,IAAA,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzC,gFAAgF,CACjF;AACF,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACtB,+DAA+D,CAChE;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,yDAAyD,CAC1D;AACF,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CACzD,yBAAyB,CAC1B;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,+CAA+C,CAChD;AACF,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnC,+GAA+G,CAChH;AACD,IAAA,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtC,kEAAkE,CACnE;AACF,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACxB,sEAAsE,CACvE;AACF,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtC,kDAAkD,CACnD;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,4EAA4E,CAC7E;AACF,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,6CAA6C,CAC9C;AACF,CAAA,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;AACvC,IAAA,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvC,gFAAgF,CACjF;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,2FAA2F,CAC5F;AACF,CAAA,CAAC;AAEF;AACA;AACA;AAEA;;;AAGG;SACa,sBAAsB,GAAA;AACpC,IAAA,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,EAAE,KAAI;;AAE1D,QAAA,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,WAAW,EAAE;YACvC,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,gBAAA,IAAI,EAAE,OAAO;AACb,gBAAA,KAAK,EAAE,kEAAkE;AAC1E,aAAA,CAAC;;;;QAKJ,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,OAAO;gBACb,IAAI,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;AACrC,gBAAA,IAAI,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;AACnC,gBAAA,IAAI,iBAAiB,IAAI,EAAE,iBAAiB,EAAE,CAAC;gBAC/C,MAAM;AACP,aAAA;;AAED,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;AAmB2E,6FAAA,CAAA;AACxF,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,qBAAqB,GAAA;AACnC,IAAA,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,KAAI;QAC3C,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,MAAM;gBACZ,KAAK;gBACL,IAAI;gBACJ,KAAK,EAAE,KAAK,IAAI,KAAK;gBACrB,UAAU,EAAE,UAAU,IAAI,KAAK;AAChC,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;AAYe,iCAAA,CAAA;AAC5B,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,yBAAyB,GAAA;IACvC,OAAO,IAAI,CACT,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,KAAI;QACxB,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,UAAU;gBAChB,GAAG;gBACH,MAAM;AACP,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;AAS0C,4DAAA,CAAA;AACvD,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,uBAAuB,GAAA;IACrC,OAAO,IAAI,CACT,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAI;QAC9B,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,QAAQ;AACd,gBAAA,MAAM,EAAE;oBACN,SAAS;oBACT,MAAM,EAAE,MAAM,IAAI,GAAG;AACtB,iBAAA;AACF,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;AASyC,2DAAA,CAAA;AACtD,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,wBAAwB,GAAA;IACtC,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAI;QAC5B,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,SAAS;gBACf,KAAK;gBACL,QAAQ;AACT,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;AAWgE,kFAAA,CAAA;AAC7E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,sBAAsB,GAAA;IACpC,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,KAAI;QAClB,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,OAAO;gBACb,KAAK;AACN,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,CAAA;;;;;;;AAOmD,qEAAA,CAAA;AAChE,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,qBAAqB,GAAA;IACnC,OAAO,IAAI,CACT,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAI;QAC7B,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,QAAQ,IAAI,IAAI;gBAC1B,MAAM;AACP,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,CAAA;;;;;;;;;AAS2D,6EAAA,CAAA;AACxE,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,uBAAuB,GAAA;IACrC,OAAO,IAAI,CACT,OAAO,EAAE,MAAM,EAAE,KAAI;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,MAAM;gBACZ,MAAM;AACP,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,CAAA;;;;;;AAM8C,gEAAA,CAAA;AAC3D,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,2BAA2B,GAAA;IACzC,OAAO,IAAI,CACT,OAAO,EAAE,QAAQ,EAAE,KAAI;QACrB,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,QAAQ,IAAI,KAAK;AAC5B,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;AAU8B,gDAAA,CAAA;AAC3C,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CACF;AACH;AA2BA;;;;;;;;;;;;;;;;;;;;AAoBG;AACa,SAAA,kBAAkB,CAAC,MAAA,GAA6B,EAAE,EAAA;IAChE,MAAM,KAAK,GAA4B,EAAE;;AAGzC,IAAA,MAAM,EACJ,WAAW,GAAG,IAAI,EAClB,UAAU,GAAG,IAAI,EACjB,cAAc,GAAG,IAAI,EACrB,YAAY,GAAG,IAAI,EACnB,aAAa,GAAG,IAAI,EACpB,WAAW,GAAG,IAAI,EAClB,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,gBAAgB,GAAG,IAAI,GACxB,GAAG,MAAM;AAEV,IAAA,IAAI,WAAW;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;AACrD,IAAA,IAAI,UAAU;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;AACnD,IAAA,IAAI,cAAc;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC;AAC3D,IAAA,IAAI,YAAY;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC;AACvD,IAAA,IAAI,aAAa;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC;AACzD,IAAA,IAAI,WAAW;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;AACrD,IAAA,IAAI,UAAU;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;AACnD,IAAA,IAAI,UAAU;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC;AACrD,IAAA,IAAI,gBAAgB;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;AAE/D,IAAA,OAAO,KAAK;AACd;AAEA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;;AAGlC;;AAEG;AACU,MAAA,kBAAkB,GAAG;AAChC,IAAA,aAAa,CAAC,KAAK;AACnB,IAAA,aAAa,CAAC,IAAI;AAClB,IAAA,aAAa,CAAC,QAAQ;AACtB,IAAA,aAAa,CAAC,MAAM;AACpB,IAAA,aAAa,CAAC,OAAO;AACrB,IAAA,aAAa,CAAC,KAAK;AACnB,IAAA,aAAa,CAAC,IAAI;AAClB,IAAA,aAAa,CAAC,IAAI;AAClB,IAAA,aAAa,CAAC,UAAU;;AAK1B;;AAEG;AACG,SAAU,iBAAiB,CAAC,QAAgB,EAAA;AAChD,IAAA,OAAO,kBAAkB,CAAC,QAAQ,CAAC,QAA2B,CAAC;AACjE;AAEA;;;;;;;;;AASG;AACG,SAAU,oBAAoB,CAAC,OAAsD,EAAA;AACzF,IAAA,MAAM,eAAe,GAAG,OAAO,CAAC,4BAA4B,CAAC;AAC7D,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAEzD,QACE,eAAe,KAAK,MAAM;AAC1B,QAAA,aAAa,KAAK,MAAM;AACxB,SAAC,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AACpE,SAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAEpE;;;;"}
+{"version":3,"file":"BrowserTools.mjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["/**\r\n * Browser Automation Tools for Ranger Browser Extension\r\n * \r\n * These tools allow the LLM to interact with the browser through the \r\n * ranger-browser extension. They generate structured actions that are\r\n * sent to the extension via SSE streaming for execution.\r\n * \r\n * The extension handles:\r\n * - DOM extraction with element indexing\r\n * - Click, type, hover, scroll actions\r\n * - Navigation and page context\r\n * - Visual element highlighting\r\n */\r\n\r\nimport { z } from 'zod';\r\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\r\n\r\n// ============================================\r\n// Tool Schemas\r\n// ============================================\r\n\r\n/**\r\n * Enhanced click schema that supports both index-based and coordinate-based clicking\r\n */\r\nconst BrowserClickSchema = z.object({\r\n  index: z.number().optional().describe(\r\n    'The index of the element to click, as shown in the page context (e.g., [0], [1], [2]). ' +\r\n    'Use the element index from the interactive elements list provided in the page context. ' +\r\n    'Either index OR coordinates must be provided.'\r\n  ),\r\n  coordinates: z.object({\r\n    x: z.number().describe('X coordinate in viewport pixels'),\r\n    y: z.number().describe('Y coordinate in viewport pixels'),\r\n  }).optional().describe(\r\n    'Coordinates for clicking elements that lack semantic info (marked with ⚠️). ' +\r\n    'The coordinates are provided in the element listing as coords:(x,y). ' +\r\n    'Either index OR coordinates must be provided.'\r\n  ),\r\n  visualDescription: z.string().optional().describe(\r\n    'Description of what the element looks like visually. Used when clicking by appearance ' +\r\n    '(e.g., \"blue button in top right corner\", \"hamburger menu icon\")'\r\n  ),\r\n  reason: z.string().optional().describe(\r\n    'Brief explanation of why you are clicking this element (for user transparency)'\r\n  ),\r\n});\r\n\r\nconst BrowserTypeSchema = z.object({\r\n  index: z.number().describe(\r\n    'The index of the input element to type into, as shown in the page context'\r\n  ),\r\n  text: z.string().describe(\r\n    'The text to type into the input field'\r\n  ),\r\n  clear: z.boolean().optional().describe(\r\n    'Whether to clear the existing content before typing (default: false)'\r\n  ),\r\n  pressEnter: z.boolean().optional().describe(\r\n    'Whether to press Enter after typing (useful for search fields, default: false)'\r\n  ),\r\n});\r\n\r\nconst BrowserNavigateSchema = z.object({\r\n  url: z.string().describe(\r\n    'The URL to navigate to. Can be a full URL or a relative path.'\r\n  ),\r\n  reason: z.string().optional().describe(\r\n    'Brief explanation of why you are navigating to this URL'\r\n  ),\r\n});\r\n\r\nconst BrowserScrollSchema = z.object({\r\n  direction: z.enum(['up', 'down', 'left', 'right']).describe(\r\n    'The direction to scroll'\r\n  ),\r\n  amount: z.number().optional().describe(\r\n    'The amount to scroll in pixels (default: 500)'\r\n  ),\r\n});\r\n\r\nconst BrowserExtractSchema = z.object({\r\n  query: z.string().optional().describe(\r\n    'Optional query to filter extracted content. If provided, only content related to the query will be extracted.'\r\n  ),\r\n  selector: z.string().optional().describe(\r\n    'Optional CSS selector to extract content from a specific element'\r\n  ),\r\n});\r\n\r\nconst BrowserHoverSchema = z.object({\r\n  index: z.number().describe(\r\n    'The index of the element to hover over, as shown in the page context'\r\n  ),\r\n});\r\n\r\nconst BrowserWaitSchema = z.object({\r\n  duration: z.number().optional().describe(\r\n    'Duration to wait in milliseconds (default: 1000)'\r\n  ),\r\n  reason: z.string().optional().describe(\r\n    'Why we are waiting (e.g., \"for page to load\", \"for animation to complete\")'\r\n  ),\r\n});\r\n\r\nconst BrowserGoBackSchema = z.object({\r\n  reason: z.string().optional().describe(\r\n    'Brief explanation of why you are going back'\r\n  ),\r\n});\r\n\r\nconst BrowserScreenshotSchema = z.object({\r\n  fullPage: z.boolean().optional().describe(\r\n    'Whether to capture the full page or just the viewport (default: viewport only)'\r\n  ),\r\n  reason: z.string().optional().describe(\r\n    'Why you need a screenshot (e.g., \"to identify visual elements\", \"to analyze page layout\")'\r\n  ),\r\n});\r\n\r\nconst BrowserGetPageStateSchema = z.object({\r\n  reason: z.string().optional().describe(\r\n    'Why you need fresh page state (e.g., \"after navigation\", \"to see updated elements\")'\r\n  ),\r\n});\r\n\r\n// ============================================\r\n// Tool Implementations\r\n// ============================================\r\n\r\n/**\r\n * Browser click tool - clicks an element by index or coordinates\r\n * Supports both semantic (index-based) and vision (coordinate-based) clicking\r\n */\r\nexport function createBrowserClickTool(): DynamicStructuredTool<typeof BrowserClickSchema> {\r\n  return tool<typeof BrowserClickSchema>(\r\n    async ({ index, coordinates, visualDescription, reason }) => {\r\n      // Validate that at least one targeting method is provided\r\n      if (index === undefined && !coordinates) {\r\n        return JSON.stringify({\r\n          type: 'error',\r\n          error: 'Either index or coordinates must be provided to click an element',\r\n        });\r\n      }\r\n\r\n      // Return a structured action for the extension to execute\r\n      // The actual execution happens in the browser extension\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'click',\r\n          ...(index !== undefined && { index }),\r\n          ...(coordinates && { coordinates }),\r\n          ...(visualDescription && { visualDescription }),\r\n          reason,\r\n        },\r\n        // Signal that this requires browser execution\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.CLICK,\r\n      description: `Click an interactive element on the current page.\r\n\r\n**Two ways to target elements:**\r\n\r\n1. **By index (preferred)**: Use the element's index number from the interactive elements list\r\n   - Format: [index] {semantic role} <tag>text</tag>\r\n   - Example: browser_click({ index: 5 }) to click element [5]\r\n\r\n2. **By coordinates (vision fallback)**: For elements marked with ⚠️ that lack semantic info\r\n   - Use the coords:(x,y) shown after the ⚠️ marker\r\n   - Example: browser_click({ coordinates: { x: 150, y: 200 } })\r\n\r\n**When to use coordinates:**\r\n- Elements marked with ⚠️ have poor semantic understanding\r\n- Icon-only buttons without labels\r\n- Custom canvas/SVG elements\r\n- When you identify an element visually in a screenshot\r\n\r\nExample: If element shows \\`[12] {button} <div>⚠️ [left side, small, clickable] coords:(45,120)\\`\r\nUse either: browser_click({ index: 12 }) or browser_click({ coordinates: { x: 45, y: 120 } })`,\r\n      schema: BrowserClickSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser type tool - types text into an input field\r\n */\r\nexport function createBrowserTypeTool(): DynamicStructuredTool<typeof BrowserTypeSchema> {\r\n  return tool<typeof BrowserTypeSchema>(\r\n    async ({ index, text, clear, pressEnter }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'type',\r\n          index,\r\n          text,\r\n          clear: clear ?? false,\r\n          pressEnter: pressEnter ?? false,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.TYPE,\r\n      description: `Type text into an input field on the current page.\r\n\r\nUse this tool when you need to:\r\n- Fill in a text input or textarea\r\n- Enter a search query\r\n- Fill out form fields\r\n\r\nThe element index comes from the page context's interactive elements list.\r\nSet 'clear: true' to clear existing content before typing.\r\nSet 'pressEnter: true' to submit after typing (useful for search fields).\r\n\r\nExample: To type \"hello world\" into a search field shown as \"[2]<input>Search...</input>\",\r\nuse index: 2, text: \"hello world\"`,\r\n      schema: BrowserTypeSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser navigate tool - navigates to a URL\r\n */\r\nexport function createBrowserNavigateTool(): DynamicStructuredTool<typeof BrowserNavigateSchema> {\r\n  return tool<typeof BrowserNavigateSchema>(\r\n    async ({ url, reason }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'navigate',\r\n          url,\r\n          reason,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.NAVIGATE,\r\n      description: `Navigate to a specific URL in the browser.\r\n\r\nUse this tool when you need to:\r\n- Go to a specific website\r\n- Navigate to a different page\r\n- Open a new URL\r\n\r\n**IMPORTANT**: After calling browser_navigate, you MUST call browser_get_page_state \r\nbefore using browser_click or browser_type. This is because navigation changes the page,\r\nand you need to see the new page's elements before you can interact with them.\r\n\r\nProvide the full URL including the protocol (https://).\r\n\r\n**Correct workflow**:\r\n1. browser_navigate({ url: \"https://www.amazon.com\" })\r\n2. browser_get_page_state({ reason: \"see elements on Amazon\" })\r\n3. Now find the search input's [index] in the returned state\r\n4. browser_type({ index: <search_input_index>, text: \"query\", pressEnter: true })\r\n\r\nExample: browser_navigate({ url: \"https://www.google.com\" })`,\r\n      schema: BrowserNavigateSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser scroll tool - scrolls the page\r\n */\r\nexport function createBrowserScrollTool(): DynamicStructuredTool<typeof BrowserScrollSchema> {\r\n  return tool<typeof BrowserScrollSchema>(\r\n    async ({ direction, amount }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'scroll',\r\n          scroll: {\r\n            direction,\r\n            amount: amount ?? 500,\r\n          },\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.SCROLL,\r\n      description: `Scroll the current page in a specified direction.\r\n\r\nUse this tool when you need to:\r\n- See more content on the page\r\n- Scroll to find elements not currently visible\r\n- Navigate long pages\r\n\r\nDefault scroll amount is 500 pixels. Adjust as needed.\r\n\r\nExample: browser_scroll({ direction: \"down\", amount: 800 })`,\r\n      schema: BrowserScrollSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser extract tool - extracts content from the page\r\n */\r\nexport function createBrowserExtractTool(): DynamicStructuredTool<typeof BrowserExtractSchema> {\r\n  return tool<typeof BrowserExtractSchema>(\r\n    async ({ query, selector }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'extract',\r\n          query,\r\n          selector,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.EXTRACT,\r\n      description: `Extract text content from the current page.\r\n\r\nUse this tool when you need to:\r\n- Get specific information from the page\r\n- Extract text that matches a query\r\n- Read content from a specific element\r\n\r\nIf no query or selector is provided, extracts the main page content.\r\nUse a CSS selector to extract from a specific element.\r\nUse a query to filter for relevant content.\r\n\r\nExample: browser_extract({ query: \"price\" }) - extracts content related to pricing`,\r\n      schema: BrowserExtractSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser hover tool - hovers over an element\r\n */\r\nexport function createBrowserHoverTool(): DynamicStructuredTool<typeof BrowserHoverSchema> {\r\n  return tool<typeof BrowserHoverSchema>(\r\n    async ({ index }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'hover',\r\n          index,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.HOVER,\r\n      description: `Hover over an element to reveal tooltips or dropdown menus.\r\n\r\nUse this tool when you need to:\r\n- Reveal a dropdown menu\r\n- Show a tooltip\r\n- Trigger hover effects\r\n\r\nExample: browser_hover({ index: 3 }) - hovers over element at index 3`,\r\n      schema: BrowserHoverSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser wait tool - waits for a specified duration\r\n */\r\nexport function createBrowserWaitTool(): DynamicStructuredTool<typeof BrowserWaitSchema> {\r\n  return tool<typeof BrowserWaitSchema>(\r\n    async ({ duration, reason }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'wait',\r\n          duration: duration ?? 1000,\r\n          reason,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.WAIT,\r\n      description: `Wait for a specified duration before the next action.\r\n\r\nUse this tool when you need to:\r\n- Wait for a page to load\r\n- Wait for an animation to complete\r\n- Add delay between actions\r\n\r\nDefault wait time is 1000ms (1 second).\r\n\r\nExample: browser_wait({ duration: 2000, reason: \"waiting for page to load\" })`,\r\n      schema: BrowserWaitSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser go back tool - navigates back in history\r\n */\r\nexport function createBrowserGoBackTool(): DynamicStructuredTool<typeof BrowserGoBackSchema> {\r\n  return tool<typeof BrowserGoBackSchema>(\r\n    async ({ reason }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'back',\r\n          reason,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.BACK,\r\n      description: `Navigate back to the previous page in browser history.\r\n\r\nUse this tool when you need to:\r\n- Return to a previous page\r\n- Undo a navigation\r\n\r\nExample: browser_back({ reason: \"returning to search results\" })`,\r\n      schema: BrowserGoBackSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser screenshot tool - captures a screenshot\r\n */\r\nexport function createBrowserScreenshotTool(): DynamicStructuredTool<typeof BrowserScreenshotSchema> {\r\n  return tool<typeof BrowserScreenshotSchema>(\r\n    async ({ fullPage }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'screenshot',\r\n          fullPage: fullPage ?? false,\r\n        },\r\n        requiresBrowserExecution: true,\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.SCREENSHOT,\r\n      description: `Capture a screenshot of the current page.\r\n\r\nUse this tool when you need to:\r\n- Capture the current state of a page\r\n- Document visual elements\r\n- Verify page appearance\r\n\r\nSet fullPage: true to capture the entire page (may be large).\r\nDefault captures only the visible viewport.\r\n\r\nExample: browser_screenshot({ fullPage: false })`,\r\n      schema: BrowserScreenshotSchema,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Browser get page state tool - gets fresh page context after navigation or actions\r\n * CRITICAL: Use this after browser_navigate or any action that changes the page\r\n */\r\nexport function createBrowserGetPageStateTool(): DynamicStructuredTool<typeof BrowserGetPageStateSchema> {\r\n  return tool<typeof BrowserGetPageStateSchema>(\r\n    async ({ reason }) => {\r\n      return JSON.stringify({\r\n        type: 'browser_action',\r\n        action: {\r\n          type: 'get_page_state',\r\n          reason,\r\n        },\r\n        requiresBrowserExecution: true,\r\n        // Special flag: extension should inject fresh context into the conversation\r\n        requiresContextRefresh: true,\r\n        // IMPORTANT: Tell the agent to wait\r\n        message: 'Page state is being captured by the browser extension. The element list will be provided in the next message. DO NOT proceed with click or type actions until you receive the actual element list.',\r\n      });\r\n    },\r\n    {\r\n      name: EBrowserTools.GET_PAGE_STATE,\r\n      description: `Get fresh page state showing current interactive elements.\r\n\r\n**CRITICAL WORKFLOW**: After calling this tool, you MUST STOP and WAIT. The browser extension will capture the page state and return the element list. DO NOT plan any browser_click or browser_type actions in the same response - you don't have the element indices yet!\r\n\r\n**When to use**:\r\n- After browser_navigate (to see elements on the new page)\r\n- After browser_click (if it caused navigation or page changes)\r\n- Any time you need to see what elements are currently on the page\r\n\r\n**IMPORTANT**: This tool captures the page state asynchronously. The actual element list will be provided AFTER this tool completes. You should:\r\n1. Call this tool\r\n2. STOP and wait for the response with the element list\r\n3. In your NEXT response, use the element indices for click/type actions\r\n\r\nExample workflow:\r\n- Turn 1: browser_navigate to amazon.com, then browser_get_page_state\r\n- Turn 2: (After receiving element list) browser_type with the correct search input index\r\n\r\nExample: browser_get_page_state({ reason: \"to see elements after navigation\" })`,\r\n      schema: BrowserGetPageStateSchema,\r\n    }\r\n  );\r\n}\r\n\r\n// ============================================\r\n// Tool Collection\r\n// ============================================\r\n\r\nexport type BrowserToolsConfig = {\r\n  /** Enable click tool */\r\n  enableClick?: boolean;\r\n  /** Enable type tool */\r\n  enableType?: boolean;\r\n  /** Enable navigate tool */\r\n  enableNavigate?: boolean;\r\n  /** Enable scroll tool */\r\n  enableScroll?: boolean;\r\n  /** Enable extract tool */\r\n  enableExtract?: boolean;\r\n  /** Enable hover tool */\r\n  enableHover?: boolean;\r\n  /** Enable wait tool */\r\n  enableWait?: boolean;\r\n  /** Enable back tool */\r\n  enableBack?: boolean;\r\n  /** Enable screenshot tool */\r\n  enableScreenshot?: boolean;\r\n  /** Enable get page state tool */\r\n  enableGetPageState?: boolean;\r\n};\r\n\r\n/**\r\n * Create all browser automation tools\r\n * \r\n * IMPORTANT: These tools should ONLY be registered when:\r\n * 1. The request comes from a browser extension that can execute them\r\n * 2. The client has indicated browser capability (e.g., via header or parameter)\r\n * \r\n * DO NOT register these for normal web UI users - they cannot execute browser actions.\r\n * \r\n * Detection in Ranger API:\r\n * - Check for `X-Ranger-Browser-Extension: true` header\r\n * - Or check for `browserCapable: true` in request body\r\n * - Or check user agent for extension identifier\r\n * \r\n * @example\r\n * // In Ranger API endpoint:\r\n * const hasBrowserExtension = req.headers['x-ranger-browser-extension'] === 'true';\r\n * const tools = hasBrowserExtension \r\n *   ? [...normalTools, ...createBrowserTools()]\r\n *   : normalTools;\r\n */\r\nexport function createBrowserTools(config: BrowserToolsConfig = {}): DynamicStructuredTool[] {\r\n  const tools: DynamicStructuredTool[] = [];\r\n  \r\n  // Enable all by default\r\n  const {\r\n    enableClick = true,\r\n    enableType = true,\r\n    enableNavigate = true,\r\n    enableScroll = true,\r\n    enableExtract = true,\r\n    enableHover = true,\r\n    enableWait = true,\r\n    enableBack = true,\r\n    enableScreenshot = true,\r\n    enableGetPageState = true,\r\n  } = config;\r\n  \r\n  if (enableClick) tools.push(createBrowserClickTool());\r\n  if (enableType) tools.push(createBrowserTypeTool());\r\n  if (enableNavigate) tools.push(createBrowserNavigateTool());\r\n  if (enableScroll) tools.push(createBrowserScrollTool());\r\n  if (enableExtract) tools.push(createBrowserExtractTool());\r\n  if (enableHover) tools.push(createBrowserHoverTool());\r\n  if (enableWait) tools.push(createBrowserWaitTool());\r\n  if (enableBack) tools.push(createBrowserGoBackTool());\r\n  if (enableScreenshot) tools.push(createBrowserScreenshotTool());\r\n  if (enableGetPageState) tools.push(createBrowserGetPageStateTool());\r\n  \r\n  return tools;\r\n}\r\n\r\n/**\r\n * Browser tool name constants\r\n * Use these instead of magic strings\r\n */\r\nexport const EBrowserTools = {\r\n  CLICK: 'browser_click',\r\n  TYPE: 'browser_type',\r\n  NAVIGATE: 'browser_navigate',\r\n  SCROLL: 'browser_scroll',\r\n  EXTRACT: 'browser_extract',\r\n  HOVER: 'browser_hover',\r\n  WAIT: 'browser_wait',\r\n  BACK: 'browser_back',\r\n  SCREENSHOT: 'browser_screenshot',\r\n  GET_PAGE_STATE: 'browser_get_page_state',\r\n} as const;\r\n\r\n/**\r\n * Get browser tool names for filtering/identification\r\n */\r\nexport const BROWSER_TOOL_NAMES = [\r\n  EBrowserTools.CLICK,\r\n  EBrowserTools.TYPE,\r\n  EBrowserTools.NAVIGATE,\r\n  EBrowserTools.SCROLL,\r\n  EBrowserTools.EXTRACT,\r\n  EBrowserTools.HOVER,\r\n  EBrowserTools.WAIT,\r\n  EBrowserTools.BACK,\r\n  EBrowserTools.SCREENSHOT,\r\n  EBrowserTools.GET_PAGE_STATE,\r\n] as const;\r\n\r\nexport type BrowserToolName = typeof BROWSER_TOOL_NAMES[number];\r\n\r\n/**\r\n * Check if a tool call is a browser action\r\n */\r\nexport function isBrowserToolCall(toolName: string): toolName is BrowserToolName {\r\n  return BROWSER_TOOL_NAMES.includes(toolName as BrowserToolName);\r\n}\r\n\r\n/**\r\n * Check if request indicates browser extension capability\r\n * Use this to conditionally register browser tools\r\n * \r\n * @example\r\n * // In Express middleware or endpoint:\r\n * if (hasBrowserCapability(req.headers)) {\r\n *   tools.push(...createBrowserTools());\r\n * }\r\n */\r\nexport function hasBrowserCapability(headers: Record<string, string | string[] | undefined>): boolean {\r\n  const extensionHeader = headers['x-ranger-browser-extension'];\r\n  const capableHeader = headers['x-ranger-browser-capable'];\r\n  \r\n  return (\r\n    extensionHeader === 'true' || \r\n    capableHeader === 'true' ||\r\n    (Array.isArray(extensionHeader) && extensionHeader.includes('true')) ||\r\n    (Array.isArray(capableHeader) && capableHeader.includes('true'))\r\n  );\r\n}\r\n"],"names":[],"mappings":";;;AAAA;;;;;;;;;;;;AAYG;AAKH;AACA;AACA;AAEA;;AAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnC,yFAAyF;QACzF,yFAAyF;AACzF,QAAA,+CAA+C,CAChD;AACD,IAAA,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;QACzD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;AAC1D,KAAA,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpB,8EAA8E;QAC9E,uEAAuE;AACvE,QAAA,+CAA+C,CAChD;IACD,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAC/C,wFAAwF;AACxF,QAAA,kEAAkE,CACnE;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,gFAAgF,CACjF;AACF,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACxB,2EAA2E,CAC5E;IACD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACvB,uCAAuC,CACxC;AACD,IAAA,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,sEAAsE,CACvE;AACD,IAAA,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACzC,gFAAgF,CACjF;AACF,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACtB,+DAA+D,CAChE;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,yDAAyD,CAC1D;AACF,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CACzD,yBAAyB,CAC1B;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,+CAA+C,CAChD;AACF,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACnC,+GAA+G,CAChH;AACD,IAAA,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtC,kEAAkE,CACnE;AACF,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CACxB,sEAAsE,CACvE;AACF,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACtC,kDAAkD,CACnD;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,4EAA4E,CAC7E;AACF,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,6CAA6C,CAC9C;AACF,CAAA,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;AACvC,IAAA,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACvC,gFAAgF,CACjF;AACD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,2FAA2F,CAC5F;AACF,CAAA,CAAC;AAEF,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;AACzC,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CACpC,qFAAqF,CACtF;AACF,CAAA,CAAC;AAEF;AACA;AACA;AAEA;;;AAGG;SACa,sBAAsB,GAAA;AACpC,IAAA,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,EAAE,KAAI;;AAE1D,QAAA,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,WAAW,EAAE;YACvC,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,gBAAA,IAAI,EAAE,OAAO;AACb,gBAAA,KAAK,EAAE,kEAAkE;AAC1E,aAAA,CAAC;;;;QAKJ,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,OAAO;gBACb,IAAI,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;AACrC,gBAAA,IAAI,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;AACnC,gBAAA,IAAI,iBAAiB,IAAI,EAAE,iBAAiB,EAAE,CAAC;gBAC/C,MAAM;AACP,aAAA;;AAED,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;AAmB2E,6FAAA,CAAA;AACxF,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,qBAAqB,GAAA;AACnC,IAAA,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,KAAI;QAC3C,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,MAAM;gBACZ,KAAK;gBACL,IAAI;gBACJ,KAAK,EAAE,KAAK,IAAI,KAAK;gBACrB,UAAU,EAAE,UAAU,IAAI,KAAK;AAChC,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;AAYe,iCAAA,CAAA;AAC5B,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,yBAAyB,GAAA;IACvC,OAAO,IAAI,CACT,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,KAAI;QACxB,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,UAAU;gBAChB,GAAG;gBACH,MAAM;AACP,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;AAmB0C,4DAAA,CAAA;AACvD,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,uBAAuB,GAAA;IACrC,OAAO,IAAI,CACT,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAI;QAC9B,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,QAAQ;AACd,gBAAA,MAAM,EAAE;oBACN,SAAS;oBACT,MAAM,EAAE,MAAM,IAAI,GAAG;AACtB,iBAAA;AACF,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;AASyC,2DAAA,CAAA;AACtD,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,wBAAwB,GAAA;IACtC,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAI;QAC5B,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,SAAS;gBACf,KAAK;gBACL,QAAQ;AACT,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;AAWgE,kFAAA,CAAA;AAC7E,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,sBAAsB,GAAA;IACpC,OAAO,IAAI,CACT,OAAO,EAAE,KAAK,EAAE,KAAI;QAClB,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,OAAO;gBACb,KAAK;AACN,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EAAE,CAAA;;;;;;;AAOmD,qEAAA,CAAA;AAChE,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,qBAAqB,GAAA;IACnC,OAAO,IAAI,CACT,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAI;QAC7B,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,QAAQ,IAAI,IAAI;gBAC1B,MAAM;AACP,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,CAAA;;;;;;;;;AAS2D,6EAAA,CAAA;AACxE,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,uBAAuB,GAAA;IACrC,OAAO,IAAI,CACT,OAAO,EAAE,MAAM,EAAE,KAAI;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,MAAM;gBACZ,MAAM;AACP,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EAAE,CAAA;;;;;;AAM8C,gEAAA,CAAA;AAC3D,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CACF;AACH;AAEA;;AAEG;SACa,2BAA2B,GAAA;IACzC,OAAO,IAAI,CACT,OAAO,EAAE,QAAQ,EAAE,KAAI;QACrB,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,QAAQ,IAAI,KAAK;AAC5B,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;AAC/B,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;AAU8B,gDAAA,CAAA;AAC3C,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CACF;AACH;AAEA;;;AAGG;SACa,6BAA6B,GAAA;IAC3C,OAAO,IAAI,CACT,OAAO,EAAE,MAAM,EAAE,KAAI;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC;AACpB,YAAA,IAAI,EAAE,gBAAgB;AACtB,YAAA,MAAM,EAAE;AACN,gBAAA,IAAI,EAAE,gBAAgB;gBACtB,MAAM;AACP,aAAA;AACD,YAAA,wBAAwB,EAAE,IAAI;;AAE9B,YAAA,sBAAsB,EAAE,IAAI;;AAE5B,YAAA,OAAO,EAAE,oMAAoM;AAC9M,SAAA,CAAC;AACJ,KAAC,EACD;QACE,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;AAkB6D,+EAAA,CAAA;AAC1E,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CACF;AACH;AA6BA;;;;;;;;;;;;;;;;;;;;AAoBG;AACa,SAAA,kBAAkB,CAAC,MAAA,GAA6B,EAAE,EAAA;IAChE,MAAM,KAAK,GAA4B,EAAE;;AAGzC,IAAA,MAAM,EACJ,WAAW,GAAG,IAAI,EAClB,UAAU,GAAG,IAAI,EACjB,cAAc,GAAG,IAAI,EACrB,YAAY,GAAG,IAAI,EACnB,aAAa,GAAG,IAAI,EACpB,WAAW,GAAG,IAAI,EAClB,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,gBAAgB,GAAG,IAAI,EACvB,kBAAkB,GAAG,IAAI,GAC1B,GAAG,MAAM;AAEV,IAAA,IAAI,WAAW;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;AACrD,IAAA,IAAI,UAAU;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;AACnD,IAAA,IAAI,cAAc;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC;AAC3D,IAAA,IAAI,YAAY;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC;AACvD,IAAA,IAAI,aAAa;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC;AACzD,IAAA,IAAI,WAAW;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;AACrD,IAAA,IAAI,UAAU;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;AACnD,IAAA,IAAI,UAAU;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC;AACrD,IAAA,IAAI,gBAAgB;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;AAC/D,IAAA,IAAI,kBAAkB;AAAE,QAAA,KAAK,CAAC,IAAI,CAAC,6BAA6B,EAAE,CAAC;AAEnE,IAAA,OAAO,KAAK;AACd;AAEA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;;AAG1C;;AAEG;AACU,MAAA,kBAAkB,GAAG;AAChC,IAAA,aAAa,CAAC,KAAK;AACnB,IAAA,aAAa,CAAC,IAAI;AAClB,IAAA,aAAa,CAAC,QAAQ;AACtB,IAAA,aAAa,CAAC,MAAM;AACpB,IAAA,aAAa,CAAC,OAAO;AACrB,IAAA,aAAa,CAAC,KAAK;AACnB,IAAA,aAAa,CAAC,IAAI;AAClB,IAAA,aAAa,CAAC,IAAI;AAClB,IAAA,aAAa,CAAC,UAAU;AACxB,IAAA,aAAa,CAAC,cAAc;;AAK9B;;AAEG;AACG,SAAU,iBAAiB,CAAC,QAAgB,EAAA;AAChD,IAAA,OAAO,kBAAkB,CAAC,QAAQ,CAAC,QAA2B,CAAC;AACjE;AAEA;;;;;;;;;AASG;AACG,SAAU,oBAAoB,CAAC,OAAsD,EAAA;AACzF,IAAA,MAAM,eAAe,GAAG,OAAO,CAAC,4BAA4B,CAAC;AAC7D,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAEzD,QACE,eAAe,KAAK,MAAM;AAC1B,QAAA,aAAa,KAAK,MAAM;AACxB,SAAC,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AACpE,SAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAEpE;;;;"}

package/dist/types/index.d.ts

@@ -9,6 +9,7 @@ export * from './tools/CodeExecutor';
 export * from './tools/ProgrammaticToolCalling';
 export * from './tools/ToolSearchRegex';
 export * from './tools/BrowserTools';
+export * from './tools/BrowserInterruptTools';
 export * from './tools/handlers';
 export * from './tools/search';
 export * from './common';

package/dist/types/run.d.ts

@@ -1,15 +1,27 @@
 import './instrumentation';
+import { MemorySaver } from '@langchain/langgraph-checkpoint';
 import type { MessageContentComplex, BaseMessage } from '@langchain/core/messages';
 import type { RunnableConfig } from '@langchain/core/runnables';
 import type * as t from '@/types';
 import { MultiAgentGraph } from '@/graphs/MultiAgentGraph';
 import { StandardGraph } from '@/graphs/Graph';
+import type { BrowserInterrupt, BrowserActionResult } from '@/tools/BrowserInterruptTools';
 export declare const defaultOmitOptions: Set<string>;
+/**
+ * Get or create a checkpointer for browser mode
+ */
+export declare function getBrowserCheckpointer(runId: string): MemorySaver;
+/**
+ * Clean up a browser checkpointer when done
+ */
+export declare function cleanupBrowserCheckpointer(runId: string): void;
 export declare class Run<_T extends t.BaseGraphState> {
     id: string;
     private tokenCounter?;
     private handlerRegistry?;
     private indexTokenCountMap?;
+    /** Whether this run is in browser extension mode */
+    browserMode: boolean;
     graphRunnable?: t.CompiledStateWorkflow;
     Graph: StandardGraph | MultiAgentGraph | undefined;
     returnContent: boolean;
@@ -33,4 +45,39 @@ export declare class Run<_T extends t.BaseGraphState> {
         language?: string;
         title?: string;
     }>;
+    /**
+     * Process stream with browser interrupt support.
+     * Uses regular stream() instead of streamEvents() to properly detect interrupts.
+     * Returns interrupt data when graph is paused waiting for browser action.
+     */
+    processBrowserStream(inputs: t.IState, config: Partial<RunnableConfig> & {
+        version: 'v1' | 'v2';
+        run_id?: string;
+    }, streamOptions?: t.EventStreamOptions): AsyncGenerator<{
+        type: 'event';
+        data: unknown;
+    } | {
+        type: 'interrupt';
+        data: BrowserInterrupt;
+    } | {
+        type: 'done';
+        data?: MessageContentComplex[];
+    }>;
+    /**
+     * Resume a browser stream after interrupt.
+     * Call this with the result from the browser extension.
+     */
+    resumeBrowserStream(result: BrowserActionResult, config: Partial<RunnableConfig> & {
+        version: 'v1' | 'v2';
+        run_id?: string;
+    }): AsyncGenerator<{
+        type: 'event';
+        data: unknown;
+    } | {
+        type: 'interrupt';
+        data: BrowserInterrupt;
+    } | {
+        type: 'done';
+        data?: MessageContentComplex[];
+    }>;
 }

package/dist/types/tools/BrowserInterruptTools.d.ts

@@ -0,0 +1,282 @@
+/**
+ * Browser Automation Tools with LangGraph Interrupt Support
+ *
+ * These tools use LangGraph's interrupt() mechanism to pause execution
+ * and wait for the browser extension to execute actions and return results.
+ *
+ * Flow:
+ * 1. Agent calls browser tool (e.g., browser_navigate)
+ * 2. Tool calls interrupt() with action details
+ * 3. Graph pauses and returns interrupt to client
+ * 4. Extension executes action in browser
+ * 5. Extension sends resume Command with actual result
+ * 6. Graph continues with real browser data
+ *
+ * This enables proper chain-of-thought with browser context because
+ * the agent receives ACTUAL results (page elements, screenshots, etc.)
+ * instead of placeholder acknowledgments.
+ */
+import { z } from 'zod';
+import { DynamicStructuredTool } from '@langchain/core/tools';
+/**
+ * Interrupt payload sent to the extension
+ */
+export interface BrowserInterrupt {
+    /** Type of browser action to execute */
+    type: 'browser_interrupt';
+    /** The specific action to perform */
+    action: BrowserAction;
+    /** Unique ID for this interrupt (for matching resume) */
+    interruptId: string;
+}
+/**
+ * Browser action types
+ */
+export type BrowserAction = {
+    type: 'navigate';
+    url: string;
+    reason?: string;
+} | {
+    type: 'click';
+    index?: number;
+    coordinates?: {
+        x: number;
+        y: number;
+    };
+    reason?: string;
+} | {
+    type: 'type';
+    index: number;
+    text: string;
+    clear?: boolean;
+    pressEnter?: boolean;
+} | {
+    type: 'scroll';
+    direction: 'up' | 'down' | 'left' | 'right';
+    amount?: number;
+} | {
+    type: 'extract';
+    query?: string;
+    selector?: string;
+} | {
+    type: 'hover';
+    index: number;
+} | {
+    type: 'wait';
+    duration?: number;
+    reason?: string;
+} | {
+    type: 'back';
+    reason?: string;
+} | {
+    type: 'screenshot';
+    fullPage?: boolean;
+} | {
+    type: 'get_page_state';
+    reason?: string;
+};
+/**
+ * Result returned from extension after executing action
+ */
+export interface BrowserActionResult {
+    success: boolean;
+    error?: string;
+    /** Page state after action (for navigate, click, get_page_state) */
+    pageState?: {
+        url: string;
+        title: string;
+        /** Formatted element list for LLM */
+        elementList: string;
+        elementCount: number;
+        scrollPosition: number;
+        scrollHeight: number;
+        viewportHeight: number;
+    };
+    /** Screenshot data URL */
+    screenshot?: string;
+    /** Extracted content */
+    extractedContent?: string;
+    /** Any additional data */
+    data?: unknown;
+}
+declare const BrowserClickSchema: z.ZodObject<{
+    index: z.ZodOptional<z.ZodNumber>;
+    coordinates: z.ZodOptional<z.ZodObject<{
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        y: number;
+        x: number;
+    }, {
+        y: number;
+        x: number;
+    }>>;
+    reason: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    index?: number | undefined;
+    coordinates?: {
+        y: number;
+        x: number;
+    } | undefined;
+    reason?: string | undefined;
+}, {
+    index?: number | undefined;
+    coordinates?: {
+        y: number;
+        x: number;
+    } | undefined;
+    reason?: string | undefined;
+}>;
+declare const BrowserTypeSchema: z.ZodObject<{
+    index: z.ZodNumber;
+    text: z.ZodString;
+    clear: z.ZodOptional<z.ZodBoolean>;
+    pressEnter: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    text: string;
+    index: number;
+    clear?: boolean | undefined;
+    pressEnter?: boolean | undefined;
+}, {
+    text: string;
+    index: number;
+    clear?: boolean | undefined;
+    pressEnter?: boolean | undefined;
+}>;
+declare const BrowserNavigateSchema: z.ZodObject<{
+    url: z.ZodString;
+    reason: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    reason?: string | undefined;
+}, {
+    url: string;
+    reason?: string | undefined;
+}>;
+declare const BrowserScrollSchema: z.ZodObject<{
+    direction: z.ZodEnum<["up", "down", "left", "right"]>;
+    amount: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    direction: "up" | "down" | "left" | "right";
+    amount?: number | undefined;
+}, {
+    direction: "up" | "down" | "left" | "right";
+    amount?: number | undefined;
+}>;
+declare const BrowserExtractSchema: z.ZodObject<{
+    query: z.ZodOptional<z.ZodString>;
+    selector: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    query?: string | undefined;
+    selector?: string | undefined;
+}, {
+    query?: string | undefined;
+    selector?: string | undefined;
+}>;
+declare const BrowserHoverSchema: z.ZodObject<{
+    index: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    index: number;
+}, {
+    index: number;
+}>;
+declare const BrowserWaitSchema: z.ZodObject<{
+    duration: z.ZodOptional<z.ZodNumber>;
+    reason: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    duration?: number | undefined;
+    reason?: string | undefined;
+}, {
+    duration?: number | undefined;
+    reason?: string | undefined;
+}>;
+declare const BrowserGoBackSchema: z.ZodObject<{
+    reason: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    reason?: string | undefined;
+}, {
+    reason?: string | undefined;
+}>;
+declare const BrowserScreenshotSchema: z.ZodObject<{
+    fullPage: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    fullPage?: boolean | undefined;
+}, {
+    fullPage?: boolean | undefined;
+}>;
+declare const BrowserGetPageStateSchema: z.ZodObject<{
+    reason: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    reason?: string | undefined;
+}, {
+    reason?: string | undefined;
+}>;
+/**
+ * Navigate tool - navigates to URL and returns page state
+ */
+export declare function createBrowserNavigateInterruptTool(): DynamicStructuredTool<typeof BrowserNavigateSchema>;
+/**
+ * Click tool - clicks element and returns updated state
+ */
+export declare function createBrowserClickInterruptTool(): DynamicStructuredTool<typeof BrowserClickSchema>;
+/**
+ * Type tool - types text into input field
+ */
+export declare function createBrowserTypeInterruptTool(): DynamicStructuredTool<typeof BrowserTypeSchema>;
+/**
+ * Get page state tool - captures current page elements
+ */
+export declare function createBrowserGetPageStateInterruptTool(): DynamicStructuredTool<typeof BrowserGetPageStateSchema>;
+/**
+ * Scroll tool
+ */
+export declare function createBrowserScrollInterruptTool(): DynamicStructuredTool<typeof BrowserScrollSchema>;
+/**
+ * Extract tool
+ */
+export declare function createBrowserExtractInterruptTool(): DynamicStructuredTool<typeof BrowserExtractSchema>;
+/**
+ * Hover tool
+ */
+export declare function createBrowserHoverInterruptTool(): DynamicStructuredTool<typeof BrowserHoverSchema>;
+/**
+ * Wait tool
+ */
+export declare function createBrowserWaitInterruptTool(): DynamicStructuredTool<typeof BrowserWaitSchema>;
+/**
+ * Go back tool
+ */
+export declare function createBrowserGoBackInterruptTool(): DynamicStructuredTool<typeof BrowserGoBackSchema>;
+/**
+ * Screenshot tool
+ */
+export declare function createBrowserScreenshotInterruptTool(): DynamicStructuredTool<typeof BrowserScreenshotSchema>;
+export declare const EBrowserInterruptTools: {
+    readonly CLICK: "browser_click";
+    readonly TYPE: "browser_type";
+    readonly NAVIGATE: "browser_navigate";
+    readonly SCROLL: "browser_scroll";
+    readonly EXTRACT: "browser_extract";
+    readonly HOVER: "browser_hover";
+    readonly WAIT: "browser_wait";
+    readonly BACK: "browser_back";
+    readonly SCREENSHOT: "browser_screenshot";
+    readonly GET_PAGE_STATE: "browser_get_page_state";
+};
+export declare const BROWSER_INTERRUPT_TOOL_NAMES: ("browser_navigate" | "browser_click" | "browser_type" | "browser_get_page_state" | "browser_scroll" | "browser_extract" | "browser_hover" | "browser_wait" | "browser_back" | "browser_screenshot")[];
+export type BrowserInterruptToolName = typeof BROWSER_INTERRUPT_TOOL_NAMES[number];
+export declare function isBrowserInterruptToolCall(toolName: string): toolName is BrowserInterruptToolName;
+/**
+ * Create all interrupt-based browser tools
+ *
+ * Use these when the client is a browser extension that can:
+ * 1. Detect browser_interrupt events in the stream
+ * 2. Execute browser actions locally
+ * 3. Send Command({ resume: result }) to continue the graph
+ */
+export declare function createBrowserInterruptTools(): DynamicStructuredTool[];
+/**
+ * Check if an interrupt is a browser interrupt
+ */
+export declare function isBrowserInterrupt(value: unknown): value is BrowserInterrupt;
+export {};

package/dist/types/tools/BrowserTools.d.ts

@@ -36,16 +36,16 @@ declare const BrowserClickSchema: z.ZodObject<{
         y: number;
         x: number;
     } | undefined;
-    visualDescription?: string | undefined;
     reason?: string | undefined;
+    visualDescription?: string | undefined;
 }, {
     index?: number | undefined;
     coordinates?: {
         y: number;
         x: number;
     } | undefined;
-    visualDescription?: string | undefined;
     reason?: string | undefined;
+    visualDescription?: string | undefined;
 }>;
 declare const BrowserTypeSchema: z.ZodObject<{
     index: z.ZodNumber;
@@ -127,6 +127,13 @@ declare const BrowserScreenshotSchema: z.ZodObject<{
     reason?: string | undefined;
     fullPage?: boolean | undefined;
 }>;
+declare const BrowserGetPageStateSchema: z.ZodObject<{
+    reason: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    reason?: string | undefined;
+}, {
+    reason?: string | undefined;
+}>;
 /**
  * Browser click tool - clicks an element by index or coordinates
  * Supports both semantic (index-based) and vision (coordinate-based) clicking
@@ -164,6 +171,11 @@ export declare function createBrowserGoBackTool(): DynamicStructuredTool<typeof
  * Browser screenshot tool - captures a screenshot
  */
 export declare function createBrowserScreenshotTool(): DynamicStructuredTool<typeof BrowserScreenshotSchema>;
+/**
+ * Browser get page state tool - gets fresh page context after navigation or actions
+ * CRITICAL: Use this after browser_navigate or any action that changes the page
+ */
+export declare function createBrowserGetPageStateTool(): DynamicStructuredTool<typeof BrowserGetPageStateSchema>;
 export type BrowserToolsConfig = {
     /** Enable click tool */
     enableClick?: boolean;
@@ -183,6 +195,8 @@ export type BrowserToolsConfig = {
     enableBack?: boolean;
     /** Enable screenshot tool */
     enableScreenshot?: boolean;
+    /** Enable get page state tool */
+    enableGetPageState?: boolean;
 };
 /**
  * Create all browser automation tools
@@ -220,11 +234,12 @@ export declare const EBrowserTools: {
     readonly WAIT: "browser_wait";
     readonly BACK: "browser_back";
     readonly SCREENSHOT: "browser_screenshot";
+    readonly GET_PAGE_STATE: "browser_get_page_state";
 };
 /**
  * Get browser tool names for filtering/identification
  */
-export declare const BROWSER_TOOL_NAMES: readonly ["browser_click", "browser_type", "browser_navigate", "browser_scroll", "browser_extract", "browser_hover", "browser_wait", "browser_back", "browser_screenshot"];
+export declare const BROWSER_TOOL_NAMES: readonly ["browser_click", "browser_type", "browser_navigate", "browser_scroll", "browser_extract", "browser_hover", "browser_wait", "browser_back", "browser_screenshot", "browser_get_page_state"];
 export type BrowserToolName = typeof BROWSER_TOOL_NAMES[number];
 /**
  * Check if a tool call is a browser action

package/dist/types/types/run.d.ts

@@ -103,6 +103,14 @@ export type RunConfig = {
     returnContent?: boolean;
     tokenCounter?: TokenCounter;
     indexTokenCountMap?: Record<string, number>;
+    /**
+     * Enable browser extension mode with interrupt-based tool execution.
+     * When true:
+     * - Uses MemorySaver checkpointer for pause/resume
+     * - Browser tools will interrupt execution and wait for extension results
+     * - Extension must call resume endpoint with Command to continue
+     */
+    browserMode?: boolean;
 };
 export type ProvidedCallbacks = (BaseCallbackHandler | CallbackHandlerMethods)[] | undefined;
 export type TokenCounter = (message: BaseMessage) => number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "illuma-agents",
-  "version": "1.0.19",
+  "version": "1.0.21",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/graphs/Graph.ts

@@ -1240,10 +1240,10 @@ If I seem to be missing something we discussed earlier, just give me a quick rem
     });
     const workflow = new StateGraph(StateAnnotation)
       .addNode(this.defaultAgentId, agentNode, { ends: [END] })
-      .addEdge(START, this.defaultAgentId)
-      .compile();
+      .addEdge(START, this.defaultAgentId);
 
-    return workflow;
+    // Pass compileOptions (e.g., checkpointer for browser interrupt support)
+    return workflow.compile(this.compileOptions as unknown as never);
   }
 
   /* Dispatchers */

package/src/index.ts

@@ -14,6 +14,7 @@ export * from './tools/CodeExecutor';
 export * from './tools/ProgrammaticToolCalling';
 export * from './tools/ToolSearchRegex';
 export * from './tools/BrowserTools';
+export * from './tools/BrowserInterruptTools';
 export * from './tools/handlers';
 export * from './tools/search';