@illuma-ai/agents 1.1.25 → 1.3.0

package/dist/esm/tools/BrowserTools.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"BrowserTools.mjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with ranger-browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n  KEYPRESS: 'browser_keypress',\n  SWITCH_TAB: 'browser_switch_tab',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the server (Ranger) to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Illuma-Browser-Extension: true\n * - X-Illuma-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-illuma-browser-extension'];\n  const browserCapable = req.headers['x-illuma-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the element to click. Use fieldLabel to identify the correct element. For form fields, target <input> or <textarea> elements, NOT parent <div> containers.'\n    ),\n  text: z\n    .string()\n    .optional()\n    .describe(\n      'The visible text of the element being clicked (e.g., \"Send\", \"Delete\", \"Buy Now\"). Always include this for buttons and links.'\n    ),\n  label: z\n    .string()\n    .optional()\n    .describe(\n      'The fieldLabel or ariaLabel of the element, if available.'\n    ),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the INPUT element to type into. Target <input> or <textarea> elements. Check fieldLabel to identify the correct field.'\n    ),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe(\n      'Whether to press Enter after typing (useful for search forms and submitting)'\n    ),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\nconst BrowserKeypressSchema = z.object({\n  keys: z\n    .string()\n    .describe(\n      'Keyboard keys to press. Use \"+\" to combine modifiers (e.g., \"Control+Enter\", \"Control+a\", \"Escape\", \"Tab\", \"Enter\"). Common shortcuts: Control+Enter (submit forms/send), Escape (close dialogs), Tab (next field).'\n    ),\n});\n\nconst BrowserSwitchTabSchema = z.object({\n  tabId: z\n    .number()\n    .describe(\n      'The tab ID to switch to. Use the tab IDs shown in the tabs list from page state.'\n    ),\n});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  const parts: string[] = [];\n\n  // Include error info but DON'T discard page state — the extension often sends\n  // success=false with valid page state (URL, elementList) that the LLM needs\n  if (!result.success && result.error) {\n    parts.push(`**Note:** Action \"${action}\" reported: ${result.error}`);\n  }\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    parts.push(\n      `\\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\\n${result.elementList}`\n    );\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    if (!result.success) {\n      return `Browser action \"${action}\" failed: ${result.error || 'Unknown error'}`;\n    }\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description:\n        'Click element by [index]. ALWAYS include the element text and label in your call for safety review. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description:\n        'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel=\"To recipients\" for To field).',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description:\n        'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description:\n        'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description:\n        'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description:\n        'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description:\n        'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description:\n        'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description:\n        'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description:\n        'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  // browser_keypress - for keyboard shortcuts\n  tools.push(\n    tool(createToolFunction('keypress'), {\n      name: EBrowserTools.KEYPRESS,\n      description:\n        'Send keyboard shortcut or key press. Use for: Control+Enter (send email/submit), Escape (close dialog/cancel), Tab (next field), Enter (confirm). The keys are sent to the currently focused element.',\n      schema: BrowserKeypressSchema,\n    })\n  );\n\n  // browser_switch_tab - for switching between tabs\n  tools.push(\n    tool(createToolFunction('switch_tab'), {\n      name: EBrowserTools.SWITCH_TAB,\n      description:\n        'Switch to a different browser tab by its ID. Tab IDs are shown in the page state. Use this to work with existing open tabs (e.g., use existing Gmail tab instead of opening a new one).',\n      schema: BrowserSwitchTabSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":[],"mappings":";;;AAIA;;;AAGG;AACI,MAAM,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;AACxC,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,UAAU,EAAE,oBAAoB;;AAkClC;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;IACd;IAEA,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,2KAA2K,CAC5K;AACH,IAAA,IAAI,EAAE;AACH,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CACP,+HAA+H,CAChI;AACH,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CACP,2DAA2D,CAC5D;AACJ,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,uIAAuI,CACxI;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAE;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CACP,8EAA8E,CAC/E;AACJ,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAE;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAE;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAE;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAE;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE9C,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,IAAI,EAAE;AACH,SAAA,MAAM;SACN,QAAQ,CACP,qNAAqN,CACtN;AACJ,CAAA,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;AACtC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,kFAAkF,CACnF;AACJ,CAAA,CAAC;AAyBF;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,MAAM,KAAK,GAAa,EAAE;;;IAI1B,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;QACnC,KAAK,CAAC,IAAI,CAAC,CAAA,kBAAA,EAAqB,MAAM,CAAA,YAAA,EAAe,MAAM,CAAC,KAAK,CAAA,CAAE,CAAC;IACtE;AAEA,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAA,CAAE,CAAC;IAC9C;AACA,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAA,CAAE,CAAC;IAC/C;AACA,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;QAC3D,KAAK,CAAC,IAAI,CACR,CAAA,gHAAA,EAAmH,MAAM,CAAC,WAAW,CAAA,CAAE,CACxI;IACH;AACA,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;IAC7D;AAEA,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;AACtB,QAAA,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;YACnB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE;QAChF;QACA,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;IAC7D;AAEA,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAM,KAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAA,KAAA,EAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAC/B;;;AAIA,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;YAC3C;YAAE,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,UAAA,EAAa,YAAY,EAAE;YAC7D;AACF,QAAA,CAAC;AACH,IAAA,CAAC;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EACT,iOAAiO;AACnO,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,wNAAwN;AAC1N,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EACT,oEAAoE;AACtE,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EACT,iEAAiE;AACnE,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EACT,6DAA6D;AAC/D,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EACT,kFAAkF;AACpF,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,+DAA+D;AACjE,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,iEAAiE;AACnE,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EACT,2EAA2E;AAC7E,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EACT,qFAAqF;AACvF,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EACT,uMAAuM;AACzM,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EACT,yLAAyL;AAC3L,QAAA,MAAM,EAAE,sBAAsB;AAC/B,KAAA,CAAC,CACH;AAED,IAAA,OAAO,KAAK;AACd;;;;"}
+{"version":3,"file":"BrowserTools.mjs","sources":["../../../src/tools/BrowserTools.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as _t from '@/types';\n\n/**\n * Browser tool names - keep in sync with the browser extension\n * These tools execute locally in the browser extension, NOT on the server\n */\nexport const EBrowserTools = {\n  CLICK: 'browser_click',\n  TYPE: 'browser_type',\n  NAVIGATE: 'browser_navigate',\n  SCROLL: 'browser_scroll',\n  EXTRACT: 'browser_extract',\n  HOVER: 'browser_hover',\n  WAIT: 'browser_wait',\n  BACK: 'browser_back',\n  SCREENSHOT: 'browser_screenshot',\n  GET_PAGE_STATE: 'browser_get_page_state',\n  KEYPRESS: 'browser_keypress',\n  SWITCH_TAB: 'browser_switch_tab',\n} as const;\n\nexport type BrowserToolName =\n  (typeof EBrowserTools)[keyof typeof EBrowserTools];\n\n/**\n * Callback function type for waiting on browser action results\n * This allows the host server to provide a callback that waits for the extension\n * to POST results back to the server before returning to the LLM.\n *\n * @param action - The browser action (click, type, navigate, etc.)\n * @param args - Arguments for the action\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\n * @returns Promise that resolves with the actual browser result (page state, etc.)\n */\nexport type BrowserToolCallback = (\n  action: string,\n  args: Record<string, unknown>,\n  toolCallId: string\n) => Promise<BrowserActionResult>;\n\n/**\n * Result returned from browser action execution\n */\nexport interface BrowserActionResult {\n  success: boolean;\n  url?: string;\n  title?: string;\n  elementList?: string; // Text-based element list\n  error?: string;\n  screenshot?: string; // Base64 screenshot (if requested)\n}\n\n/**\n * Check if browser capability is available based on request headers or context\n * The browser extension sets these headers when connected:\n * - X-Illuma-Browser-Extension: true\n * - X-Illuma-Browser-Capable: true\n */\nexport function hasBrowserCapability(req?: {\n  headers?: Record<string, string | string[] | undefined>;\n}): boolean {\n  if (!req?.headers) {\n    return false;\n  }\n\n  const browserExtension = req.headers['x-illuma-browser-extension'];\n  const browserCapable = req.headers['x-illuma-browser-capable'];\n\n  return browserExtension === 'true' || browserCapable === 'true';\n}\n\n// Tool schemas\nconst BrowserClickSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the element to click. Use fieldLabel to identify the correct element. For form fields, target <input> or <textarea> elements, NOT parent <div> containers.'\n    ),\n  text: z\n    .string()\n    .optional()\n    .describe(\n      'The visible text of the element being clicked (e.g., \"Send\", \"Delete\", \"Buy Now\"). Always include this for buttons and links.'\n    ),\n  label: z\n    .string()\n    .optional()\n    .describe('The fieldLabel or ariaLabel of the element, if available.'),\n});\n\nconst BrowserTypeSchema = z.object({\n  index: z\n    .number()\n    .describe(\n      'The [index] of the INPUT element to type into. Target <input> or <textarea> elements. Check fieldLabel to identify the correct field.'\n    ),\n  text: z.string().describe('The text to type into the element'),\n  pressEnter: z\n    .boolean()\n    .optional()\n    .describe(\n      'Whether to press Enter after typing (useful for search forms and submitting)'\n    ),\n});\n\nconst BrowserNavigateSchema = z.object({\n  url: z\n    .string()\n    .describe('The full URL to navigate to (must include https://)'),\n});\n\nconst BrowserScrollSchema = z.object({\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .describe('Direction to scroll'),\n  amount: z\n    .number()\n    .optional()\n    .describe('Pixels to scroll (default: one viewport height)'),\n});\n\nconst BrowserExtractSchema = z.object({\n  query: z\n    .string()\n    .optional()\n    .describe('Optional: specific content to extract from the page'),\n});\n\nconst BrowserHoverSchema = z.object({\n  index: z.number().describe('The index number of the element to hover over'),\n});\n\nconst BrowserWaitSchema = z.object({\n  duration: z\n    .number()\n    .optional()\n    .describe('Milliseconds to wait (default: 1000)'),\n});\n\nconst BrowserBackSchema = z.object({});\n\nconst BrowserScreenshotSchema = z.object({});\n\nconst BrowserGetPageStateSchema = z.object({});\n\nconst BrowserKeypressSchema = z.object({\n  keys: z\n    .string()\n    .describe(\n      'Keyboard keys to press. Use \"+\" to combine modifiers (e.g., \"Control+Enter\", \"Control+a\", \"Escape\", \"Tab\", \"Enter\"). Common shortcuts: Control+Enter (submit forms/send), Escape (close dialogs), Tab (next field).'\n    ),\n});\n\nconst BrowserSwitchTabSchema = z.object({\n  tabId: z\n    .number()\n    .describe(\n      'The tab ID to switch to. Use the tab IDs shown in the tabs list from page state.'\n    ),\n});\n\n/**\n * Browser tool response interface\n * This is what the extension returns after executing the action\n */\nexport interface BrowserToolResponse {\n  requiresBrowserExecution: true;\n  action: string;\n  args: Record<string, unknown>;\n  toolCallId?: string; // Added to help extension correlate with callback\n}\n\n/**\n * Options for creating browser tools\n */\nexport interface CreateBrowserToolsOptions {\n  /**\n   * Optional callback that waits for browser action results.\n   * When provided, tools will await this callback to get actual results from the extension.\n   * When not provided, tools return markers immediately (for non-server contexts).\n   */\n  waitForResult?: BrowserToolCallback;\n}\n\n/**\n * Format browser action result for LLM consumption\n */\nfunction formatResultForLLM(\n  result: BrowserActionResult,\n  action: string\n): string {\n  const parts: string[] = [];\n\n  // Include error info but DON'T discard page state — the extension often sends\n  // success=false with valid page state (URL, elementList) that the LLM needs\n  if (!result.success && result.error) {\n    parts.push(`**Note:** Action \"${action}\" reported: ${result.error}`);\n  }\n\n  if (result.url != null && result.url !== '') {\n    parts.push(`**Current URL:** ${result.url}`);\n  }\n  if (result.title != null && result.title !== '') {\n    parts.push(`**Page Title:** ${result.title}`);\n  }\n  if (result.elementList != null && result.elementList !== '') {\n    parts.push(\n      `\\n**Interactive Elements** (for typing: target <input> elements with fieldLabel, NOT parent <div> containers):\\n${result.elementList}`\n    );\n  }\n  if (result.screenshot != null && result.screenshot !== '') {\n    parts.push('\\n[Screenshot captured and displayed to user]');\n  }\n\n  if (parts.length === 0) {\n    if (!result.success) {\n      return `Browser action \"${action}\" failed: ${result.error || 'Unknown error'}`;\n    }\n    return `Browser action \"${action}\" completed successfully.`;\n  }\n\n  return parts.join('\\n');\n}\n\n/**\n * Create browser tools with optional callback for waiting on results\n *\n * When waitForResult callback is provided:\n * 1. Tool returns marker that triggers extension\n * 2. Tool then awaits callback to get actual results\n * 3. Returns real page state to LLM\n *\n * When no callback:\n * 1. Tool returns marker only (for non-server contexts)\n *\n * NOTE: These tools use TEXT-BASED element lists, NOT screenshots\n * Screenshots would be 100K+ tokens each - element lists are ~100 tokens\n */\nexport function createBrowserTools(\n  options?: CreateBrowserToolsOptions\n): DynamicStructuredTool[] {\n  const { waitForResult } = options || {};\n  const tools: DynamicStructuredTool[] = [];\n\n  /**\n   * Helper to create tool function that optionally waits for results\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\n   */\n  const createToolFunction = (action: string) => {\n    return async (\n      args: Record<string, unknown>,\n      config?: { toolCall?: { id?: string } }\n    ): Promise<string> => {\n      const toolCallId =\n        config?.toolCall?.id ??\n        `tool_${Date.now()}_${Math.random().toString(36).slice(2)}`;\n\n      // Create marker for extension\n      const marker: BrowserToolResponse = {\n        requiresBrowserExecution: true,\n        action,\n        args,\n        toolCallId,\n      };\n\n      // If no callback, return marker immediately (extension handles via SSE interception)\n      if (!waitForResult) {\n        return JSON.stringify(marker);\n      }\n\n      // With callback: wait for actual results from extension\n      // The marker is still returned initially via SSE, but we wait for the callback\n      try {\n        const result = await waitForResult(action, args, toolCallId);\n        return formatResultForLLM(result, action);\n      } catch (error) {\n        const errorMessage =\n          error instanceof Error ? error.message : String(error);\n        return `Browser action \"${action}\" failed: ${errorMessage}`;\n      }\n    };\n  };\n\n  // browser_click\n  tools.push(\n    tool(createToolFunction('click'), {\n      name: EBrowserTools.CLICK,\n      description:\n        'Click element by [index]. ALWAYS include the element text and label in your call for safety review. Use fieldLabel attribute to identify correct element. For form fields, target <input> elements NOT parent <div> containers.',\n      schema: BrowserClickSchema,\n    })\n  );\n\n  // browser_type\n  tools.push(\n    tool(createToolFunction('type'), {\n      name: EBrowserTools.TYPE,\n      description:\n        'Type text into <input> element by [index]. CRITICAL: Always target <input> or <textarea> tags (NOT parent <div> containers). Use fieldLabel to identify correct field (e.g., fieldLabel=\"To recipients\" for To field).',\n      schema: BrowserTypeSchema,\n    })\n  );\n\n  // browser_navigate\n  tools.push(\n    tool(createToolFunction('navigate'), {\n      name: EBrowserTools.NAVIGATE,\n      description:\n        'Navigate to URL (include https://). Returns new page element list.',\n      schema: BrowserNavigateSchema,\n    })\n  );\n\n  // browser_scroll\n  tools.push(\n    tool(createToolFunction('scroll'), {\n      name: EBrowserTools.SCROLL,\n      description:\n        'Scroll page (up/down/left/right). Returns updated element list.',\n      schema: BrowserScrollSchema,\n    })\n  );\n\n  // browser_extract\n  tools.push(\n    tool(createToolFunction('extract'), {\n      name: EBrowserTools.EXTRACT,\n      description:\n        'Extract page content. Returns URL, title, and element list.',\n      schema: BrowserExtractSchema,\n    })\n  );\n\n  // browser_hover\n  tools.push(\n    tool(createToolFunction('hover'), {\n      name: EBrowserTools.HOVER,\n      description:\n        'Hover element by [index] to reveal menus/tooltips. Returns updated element list.',\n      schema: BrowserHoverSchema,\n    })\n  );\n\n  // browser_wait\n  tools.push(\n    tool(createToolFunction('wait'), {\n      name: EBrowserTools.WAIT,\n      description:\n        'Wait for async content to load. Returns updated element list.',\n      schema: BrowserWaitSchema,\n    })\n  );\n\n  // browser_back\n  tools.push(\n    tool(createToolFunction('back'), {\n      name: EBrowserTools.BACK,\n      description:\n        'Go back in browser history. Returns previous page element list.',\n      schema: BrowserBackSchema,\n    })\n  );\n\n  // browser_screenshot\n  tools.push(\n    tool(createToolFunction('screenshot'), {\n      name: EBrowserTools.SCREENSHOT,\n      description:\n        'Capture screenshot. Displayed to user. Use get_page_state for automation.',\n      schema: BrowserScreenshotSchema,\n    })\n  );\n\n  // browser_get_page_state\n  tools.push(\n    tool(createToolFunction('get_page_state'), {\n      name: EBrowserTools.GET_PAGE_STATE,\n      description:\n        'Get page URL, title, and interactive elements with [index] for actions. Start here.',\n      schema: BrowserGetPageStateSchema,\n    })\n  );\n\n  // browser_keypress - for keyboard shortcuts\n  tools.push(\n    tool(createToolFunction('keypress'), {\n      name: EBrowserTools.KEYPRESS,\n      description:\n        'Send keyboard shortcut or key press. Use for: Control+Enter (send email/submit), Escape (close dialog/cancel), Tab (next field), Enter (confirm). The keys are sent to the currently focused element.',\n      schema: BrowserKeypressSchema,\n    })\n  );\n\n  // browser_switch_tab - for switching between tabs\n  tools.push(\n    tool(createToolFunction('switch_tab'), {\n      name: EBrowserTools.SWITCH_TAB,\n      description:\n        'Switch to a different browser tab by its ID. Tab IDs are shown in the page state. Use this to work with existing open tabs (e.g., use existing Gmail tab instead of opening a new one).',\n      schema: BrowserSwitchTabSchema,\n    })\n  );\n\n  return tools;\n}\n"],"names":[],"mappings":";;;AAIA;;;AAGG;AACI,MAAM,aAAa,GAAG;AAC3B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,MAAM,EAAE,gBAAgB;AACxB,IAAA,OAAO,EAAE,iBAAiB;AAC1B,IAAA,KAAK,EAAE,eAAe;AACtB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,IAAI,EAAE,cAAc;AACpB,IAAA,UAAU,EAAE,oBAAoB;AAChC,IAAA,cAAc,EAAE,wBAAwB;AACxC,IAAA,QAAQ,EAAE,kBAAkB;AAC5B,IAAA,UAAU,EAAE,oBAAoB;;AAkClC;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;IACd;IAEA,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC;IAClE,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,gBAAgB,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AACjE;AAEA;AACA,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;AAClC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,2KAA2K,CAC5K;AACH,IAAA,IAAI,EAAE;AACH,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CACP,+HAA+H,CAChI;AACH,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,2DAA2D,CAAC;AACzE,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,uIAAuI,CACxI;IACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AAC9D,IAAA,UAAU,EAAE;AACT,SAAA,OAAO;AACP,SAAA,QAAQ;SACR,QAAQ,CACP,8EAA8E,CAC/E;AACJ,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,GAAG,EAAE;AACF,SAAA,MAAM;SACN,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,SAAS,EAAE;SACR,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;SACpC,QAAQ,CAAC,qBAAqB,CAAC;AAClC,IAAA,MAAM,EAAE;AACL,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,iDAAiD,CAAC;AAC/D,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,qDAAqD,CAAC;AACnE,CAAA,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;AAC5E,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;AACjC,IAAA,QAAQ,EAAE;AACP,SAAA,MAAM;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,sCAAsC,CAAC;AACpD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEtC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE5C,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE9C,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,IAAI,EAAE;AACH,SAAA,MAAM;SACN,QAAQ,CACP,qNAAqN,CACtN;AACJ,CAAA,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;AACtC,IAAA,KAAK,EAAE;AACJ,SAAA,MAAM;SACN,QAAQ,CACP,kFAAkF,CACnF;AACJ,CAAA,CAAC;AAyBF;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,MAAM,KAAK,GAAa,EAAE;;;IAI1B,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;QACnC,KAAK,CAAC,IAAI,CAAC,CAAA,kBAAA,EAAqB,MAAM,CAAA,YAAA,EAAe,MAAM,CAAC,KAAK,CAAA,CAAE,CAAC;IACtE;AAEA,IAAA,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE;QAC3C,KAAK,CAAC,IAAI,CAAC,CAAA,iBAAA,EAAoB,MAAM,CAAC,GAAG,CAAA,CAAE,CAAC;IAC9C;AACA,IAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,EAAE,EAAE;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAA,gBAAA,EAAmB,MAAM,CAAC,KAAK,CAAA,CAAE,CAAC;IAC/C;AACA,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;QAC3D,KAAK,CAAC,IAAI,CACR,CAAA,gHAAA,EAAmH,MAAM,CAAC,WAAW,CAAA,CAAE,CACxI;IACH;AACA,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,EAAE,EAAE;AACzD,QAAA,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;IAC7D;AAEA,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;AACtB,QAAA,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;YACnB,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE;QAChF;QACA,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,yBAAA,CAA2B;IAC7D;AAEA,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;;;;;;;;;;;AAaG;AACG,SAAU,kBAAkB,CAChC,OAAmC,EAAA;AAEnC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE;IACvC,MAAM,KAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAA,KAAA,EAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAG7D,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAC/B;;;AAIA,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;YAC3C;YAAE,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAA,gBAAA,EAAmB,MAAM,CAAA,UAAA,EAAa,YAAY,EAAE;YAC7D;AACF,QAAA,CAAC;AACH,IAAA,CAAC;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EACT,iOAAiO;AACnO,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,wNAAwN;AAC1N,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EACT,oEAAoE;AACtE,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE;QACjC,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EACT,iEAAiE;AACnE,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClC,IAAI,EAAE,aAAa,CAAC,OAAO;AAC3B,QAAA,WAAW,EACT,6DAA6D;AAC/D,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAChC,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EACT,kFAAkF;AACpF,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,+DAA+D;AACjE,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE;QAC/B,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,iEAAiE;AACnE,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EACT,2EAA2E;AAC7E,QAAA,MAAM,EAAE,uBAAuB;AAChC,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE;QACzC,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EACT,qFAAqF;AACvF,QAAA,MAAM,EAAE,yBAAyB;AAClC,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE;QACnC,IAAI,EAAE,aAAa,CAAC,QAAQ;AAC5B,QAAA,WAAW,EACT,uMAAuM;AACzM,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,EAAE;QACrC,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EACT,yLAAyL;AAC3L,QAAA,MAAM,EAAE,sBAAsB;AAC/B,KAAA,CAAC,CACH;AAED,IAAA,OAAO,KAAK;AACd;;;;"}

package/dist/esm/tools/ToolNode.mjs

@@ -1,7 +1,7 @@
 import { ToolMessage, isBaseMessage, isAIMessage } from '@langchain/core/messages';
 import { interrupt, isCommand, isGraphInterrupt, Command, Send, END } from '@langchain/langgraph';
 import { ExecutionContext } from './approval/constants.mjs';
-import { GraphEvents, Constants } from '../common/enum.mjs';
+import { Constants, GraphEvents } from '../common/enum.mjs';
 import 'nanoid';
 import '../messages/core.mjs';
 import { processToolOutput } from '../utils/toonFormat.mjs';
@@ -137,7 +137,8 @@ class ToolNode extends RunnableCallable {
         // Resolve the effective execution context for this agent.
         // Per-agent overrides take precedence — allows handoff agents to bypass HITL
         // while the primary agent retains interactive approval.
-        const effectiveContext = (this.agentId && agentExecutionContextOverrides?.[this.agentId]) ?? executionContext;
+        const effectiveContext = (this.agentId && agentExecutionContextOverrides?.[this.agentId]) ??
+            executionContext;
         // Scheduled executions bypass all approval checks — no user is present
         if (effectiveContext === ExecutionContext.SCHEDULED) {
             return false;
@@ -149,9 +150,19 @@ class ToolNode extends RunnableCallable {
         if (effectiveContext === ExecutionContext.HANDOFF) {
             return false;
         }
-        // Graph-managed tools (handoff/transfer tools) bypass HITL approval —
-        // these are internal routing mechanisms, not user-facing tool executions
-        if (this.directToolNames?.has(toolName)) {
+        // Graph-managed routing tools (handoff/transfer) bypass HITL approval —
+        // these are internal routing mechanisms, not user-facing tool executions.
+        //
+        // NOTE: `directToolNames` is used for two purposes — (1) marking tools that
+        // are loaded as full instances and don't need on-demand ON_TOOL_EXECUTE loading,
+        // and (2) bypassing HITL. In event-driven mode ALL built-in tools (including
+        // `ask_user`) end up in directToolNames for reason (1), so we cannot use
+        // `directToolNames.has(toolName)` as the HITL-bypass test — it would let
+        // `ask_user` execute without ever firing interrupt(), defeating the whole tool.
+        // Instead, gate the bypass on the actual routing-tool name prefix.
+        if (this.directToolNames?.has(toolName) &&
+            (toolName.startsWith(Constants.LC_TRANSFER_TO_) ||
+                toolName.startsWith(Constants.LC_HANDOFF_TO_))) {
             return false;
         }
         // Determine the effective policy for this tool
@@ -183,7 +194,7 @@ class ToolNode extends RunnableCallable {
      * @param config - The runnable config for event dispatch
      * @returns The approval response from the human
      */
-    requestApproval(call, config) {
+    async requestApproval(call, config) {
         const approvalRequest = {
             type: 'tool_approval_required',
             toolCallId: call.id ?? '',
@@ -192,9 +203,13 @@ class ToolNode extends RunnableCallable {
             agentId: this.agentId,
             description: `Tool "${call.name}" wants to execute with the provided arguments.`,
         };
-        // Notify host before interrupting — allows SSE event emission and DB persistence.
-        // This is a fire-and-forget notification, NOT the approval mechanism.
-        safeDispatchCustomEvent(GraphEvents.ON_TOOL_APPROVAL_REQUIRED, approvalRequest, config);
+        // MUST await — interrupt() throws GraphInterrupt synchronously which unwinds
+        // the call stack. Any un-awaited dispatch Promise is abandoned before the
+        // host's ON_TOOL_APPROVAL_REQUIRED handler runs, so the MongoDB row never
+        // gets written and the subsequent approve-tool POST 404s with
+        // "No pending approval found". Awaiting guarantees the handler has persisted
+        // the request before we suspend the graph.
+        await safeDispatchCustomEvent(GraphEvents.ON_TOOL_APPROVAL_REQUIRED, approvalRequest, config);
         // interrupt() throws GraphInterrupt on first call (checkpoints state),
         // returns the resume value on re-execution after Command({ resume }).
         const response = interrupt(approvalRequest);
@@ -292,7 +307,7 @@ class ToolNode extends RunnableCallable {
             // Resumes when host sends Command({ resume: ToolApprovalResponse }).
             // ========================================================================
             if (this.requiresApproval(call.name, call.args)) {
-                const approvalResponse = this.requestApproval(call, config);
+                const approvalResponse = await this.requestApproval(call, config);
                 if (!approvalResponse.approved) {
                     // Human denied the tool call - return a denial message
                     return new ToolMessage({
@@ -657,7 +672,7 @@ class ToolNode extends RunnableCallable {
         const denialMessages = [];
         for (const call of toolCalls) {
             if (this.requiresApproval(call.name, call.args)) {
-                const approvalResponse = this.requestApproval(call, config);
+                const approvalResponse = await this.requestApproval(call, config);
                 if (!approvalResponse.approved) {
                     denialMessages.push(new ToolMessage({
                         status: 'error',
@@ -911,6 +926,108 @@ class ToolNode extends RunnableCallable {
                 combinedOutputs.push(Array.isArray(input) ? [output] : { messages: [output] });
             }
         }
+        /**
+         * Dedupe handoffs targeting the same destination.
+         *
+         * Parent LLMs sometimes emit multiple parallel transfer_to_<agent> tool
+         * calls for the same child agent in one model tick (e.g. "send email 1"
+         * and "send email 2" both routed to the Productivity Assistant). LangGraph
+         * would otherwise create two parallel Sends into the same subgraph, which
+         * produces duplicate handoff entries in the UI and leaves extra
+         * tool_call_ids without a clean result when the single child run finishes.
+         *
+         * Merge same-destination handoffs into one Command whose update.messages
+         * carries every original ToolMessage — the primary one has its
+         * Instructions field rebuilt from the combined set, and the rest are kept
+         * verbatim so the parent message history has a ToolMessage for every
+         * tool_call_id (LangChain requires every AI tool_call to be resolved).
+         * The child's _extractTransferContext filters all transfer ToolMessages
+         * out of the child's view, so it only ever sees the merged instructions.
+         */
+        if (handoffCommands.length > 1) {
+            const byDestination = new Map();
+            for (const cmd of handoffCommands) {
+                const goto = cmd.goto;
+                const dest = typeof goto === 'string' ? goto : goto[0];
+                const arr = byDestination.get(dest) ?? [];
+                arr.push(cmd);
+                byDestination.set(dest, arr);
+            }
+            const hasDuplicates = Array.from(byDestination.values()).some((arr) => arr.length > 1);
+            if (hasDuplicates) {
+                const TRANSFER_INSTRUCTIONS_PATTERN = /(?:Instructions?|Context):\s*(.+)/is;
+                const mergedHandoffs = [];
+                for (const [dest, cmds] of byDestination) {
+                    if (cmds.length === 1) {
+                        mergedHandoffs.push(cmds[0]);
+                        continue;
+                    }
+                    const allToolMessages = [];
+                    const allOtherMessages = [];
+                    for (const cmd of cmds) {
+                        const upd = cmd.update;
+                        for (const m of upd?.messages ?? []) {
+                            if (m.getType() === 'tool') {
+                                allToolMessages.push(m);
+                            }
+                            else {
+                                allOtherMessages.push(m);
+                            }
+                        }
+                    }
+                    if (allToolMessages.length === 0) {
+                        mergedHandoffs.push(cmds[0]);
+                        continue;
+                    }
+                    const primary = allToolMessages[0];
+                    const primaryContent = typeof primary.content === 'string'
+                        ? primary.content
+                        : JSON.stringify(primary.content);
+                    const mergedInstructions = [];
+                    for (const tm of allToolMessages) {
+                        const c = typeof tm.content === 'string'
+                            ? tm.content
+                            : JSON.stringify(tm.content);
+                        const match = c.match(TRANSFER_INSTRUCTIONS_PATTERN);
+                        if (match?.[1]) {
+                            const instr = match[1].trim();
+                            if (instr && !mergedInstructions.includes(instr)) {
+                                mergedInstructions.push(instr);
+                            }
+                        }
+                    }
+                    let mergedPrimaryContent = primaryContent;
+                    if (mergedInstructions.length > 1) {
+                        const header = primaryContent
+                            .replace(TRANSFER_INSTRUCTIONS_PATTERN, '')
+                            .trimEnd();
+                        const combinedInstr = mergedInstructions
+                            .map((s, i) => `${i + 1}. ${s}`)
+                            .join('\n');
+                        mergedPrimaryContent =
+                            `${header}\nInstructions: ${combinedInstr}`.trim();
+                    }
+                    const mergedPrimary = new ToolMessage({
+                        content: mergedPrimaryContent,
+                        tool_call_id: primary.tool_call_id,
+                        name: primary.name,
+                        additional_kwargs: { ...primary.additional_kwargs },
+                    });
+                    const mergedMessages = [
+                        mergedPrimary,
+                        ...allToolMessages.slice(1),
+                        ...allOtherMessages,
+                    ];
+                    mergedHandoffs.push(new Command({
+                        graph: Command.PARENT,
+                        goto: dest,
+                        update: { messages: mergedMessages },
+                    }));
+                }
+                handoffCommands.length = 0;
+                handoffCommands.push(...mergedHandoffs);
+            }
+        }
         /**
          * Handle handoff commands - convert to Send objects for parallel execution
          * when multiple handoffs are requested