@browserbasehq/orca 3.1.0-patch.3 → 3.2.0-middleware.1

package/dist/esm/lib/v3/agent/tools/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AA8BzC;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAE3C,MAAM,QAAQ,GAAY;QACxB,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,CAAC;QAChC,QAAQ,EAAE,YAAY,CAAC,EAAE,CAAC;QAC1B,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,CAAC;QACxC,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,CAAC;QAC1C,cAAc,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAChD,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,SAAS,EAAE;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5B,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,0DAA0D;IAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QAC9B,QAAQ,CAAC,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool } from \"./search.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n  AgentToolMode,\n  AgentModelConfig,\n} from \"../../types/public/agent.js\";\n\nexport interface V3AgentToolOptions {\n  executionModel?: string | AgentModelConfig;\n  logger?: (message: LogLine) => void;\n  /**\n   * Tool mode determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n   */\n  mode?: AgentToolMode;\n  /**\n   * The model provider. Used for model-specific coordinate handling\n   */\n  provider?: string;\n  /**\n   * Tools to exclude from the available toolset.\n   * These tools will be filtered out after mode-based filtering.\n   */\n  excludeTools?: string[];\n}\n\n/**\n * Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n */\nfunction filterTools(\n  tools: ToolSet,\n  mode: AgentToolMode,\n  excludeTools?: string[],\n): ToolSet {\n  const filtered: ToolSet = { ...tools };\n\n  // Mode-based filtering\n  if (mode === \"hybrid\") {\n    delete filtered.fillForm;\n  } else {\n    // DOM mode (default)\n    delete filtered.click;\n    delete filtered.type;\n    delete filtered.dragAndDrop;\n    delete filtered.clickAndHold;\n    delete filtered.fillFormVision;\n  }\n\n  if (excludeTools) {\n    for (const toolName of excludeTools) {\n      delete filtered[toolName];\n    }\n  }\n\n  return filtered;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n  const executionModel = options?.executionModel;\n  const mode = options?.mode ?? \"dom\";\n  const provider = options?.provider;\n  const excludeTools = options?.excludeTools;\n\n  const allTools: ToolSet = {\n    act: actTool(v3, executionModel),\n    ariaTree: ariaTreeTool(v3),\n    click: clickTool(v3, provider),\n    clickAndHold: clickAndHoldTool(v3, provider),\n    dragAndDrop: dragAndDropTool(v3, provider),\n    extract: extractTool(v3, executionModel),\n    fillForm: fillFormTool(v3, executionModel),\n    fillFormVision: fillFormVisionTool(v3, provider),\n    goto: gotoTool(v3),\n    keys: keysTool(v3),\n    navback: navBackTool(v3),\n    screenshot: screenshotTool(v3),\n    scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n    think: thinkTool(),\n    type: typeTool(v3, provider),\n    wait: waitTool(v3, mode),\n  };\n\n  // Only include search tool if BRAVE_API_KEY is configured\n  if (process.env.BRAVE_API_KEY) {\n    allTools.search = searchTool(v3);\n  }\n\n  return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/**\n * Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional as it's only available when BRAVE_API_KEY is configured.\n */\nexport type AgentToolTypesMap = {\n  act: ReturnType<typeof actTool>;\n  ariaTree: ReturnType<typeof ariaTreeTool>;\n  click: ReturnType<typeof clickTool>;\n  clickAndHold: ReturnType<typeof clickAndHoldTool>;\n  dragAndDrop: ReturnType<typeof dragAndDropTool>;\n  extract: ReturnType<typeof extractTool>;\n  fillForm: ReturnType<typeof fillFormTool>;\n  fillFormVision: ReturnType<typeof fillFormVisionTool>;\n  goto: ReturnType<typeof gotoTool>;\n  keys: ReturnType<typeof keysTool>;\n  navback: ReturnType<typeof navBackTool>;\n  screenshot: ReturnType<typeof screenshotTool>;\n  scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;\n  search?: ReturnType<typeof searchTool>;\n  think: ReturnType<typeof thinkTool>;\n  type: ReturnType<typeof typeTool>;\n  wait: ReturnType<typeof waitTool>;\n};\n\n/**\n * Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n */\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/**\n * Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n */\nexport type AgentToolCall = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    args: AgentUITools[K][\"input\"];\n  };\n}[keyof AgentToolTypesMap];\n\n/**\n * Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    result: AgentUITools[K][\"output\"];\n  };\n}[keyof AgentToolTypesMap];\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,IAAI,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,EAAE,UAAU,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAUjE,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AA0C/D;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,8DAA8D;AAC9D,SAAS,mBAAmB,CAC1B,SAAY,EACZ,QAAgB,EAChB,EAAM,EACN,SAAkB,EAClB,WAAoB;IAEpB,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAEvD,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,CAAC;IAC1C,OAAO;QACL,GAAG,SAAS;QACZ,OAAO,EAAE,KAAK,EAAE,GAAG,IAAe,EAAE,EAAE;YACpC,IAAI,CAAC;gBACH,OAAO,MAAM,WAAW,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC1E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,OAAO,GAAG,iBAAiB,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;oBACxF,EAAE,CAAC,MAAM,CAAC;wBACR,QAAQ,EAAE,OAAO;wBACjB,OAAO;wBACP,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,OAAO;qBACf,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;KACG,CAAC;AACT,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACrC,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,CAAC;IAEzC,MAAM,YAAY,GAA2B;QAC3C,GAAG,EAAE,kGAAkG;QACvG,QAAQ,EAAE,6BAA6B;QACvC,OAAO,EAAE,yCAAyC;QAClD,QAAQ,EACN,0GAA0G;KAC7G,CAAC;IAEF,MAAM,cAAc,GAAY;QAC9B,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QACxD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,WAAW,CAAC;QACvC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC;QACrD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QAClE,cAAc,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QAC3D,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;KACxC,CAAC;IAEF,IAAI,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACpD,cAAc,CAAC,MAAM,GAAG,qBAAqB,CAC3C,EAAE,EACF,OAAO,CAAC,iBAAiB,CAC1B,CAAC;IACJ,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QACrC,cAAc,CAAC,MAAM,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,QAAQ,GAAY;QACxB,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAChD,IAAI;YACJ,mBAAmB,CACjB,CAAC,EACD,GAAG,IAAI,IAAI,EACX,EAAE,EACF,WAAW,EACX,YAAY,CAAC,IAAI,CAAC,CACnB;SACF,CAAC,CACH;QACD,KAAK,EAAE,SAAS,EAAE;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool as browserbaseSearchTool } from \"./browserbaseSearch.js\";\nimport { searchTool as braveSearchTool } from \"./braveSearch.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n  AgentToolMode,\n  AgentModelConfig,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { withTimeout } from \"../../timeoutConfig.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport interface V3AgentToolOptions {\n  executionModel?: string | AgentModelConfig;\n  logger?: (message: LogLine) => void;\n  /**\n   * Tool mode determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n   */\n  mode?: AgentToolMode;\n  /**\n   * The model provider. Used for model-specific coordinate handling\n   */\n  provider?: string;\n  /**\n   * Tools to exclude from the available toolset.\n   * These tools will be filtered out after mode-based filtering.\n   */\n  excludeTools?: string[];\n  /**\n   * Variables available to the agent for use in act/type tools.\n   * When provided, these tools will have an optional useVariable field.\n   */\n  variables?: Variables;\n  /**\n   * Timeout in milliseconds for async tool calls.\n   * Applied to all tools that perform I/O (except wait and think).\n   */\n  toolTimeout?: number;\n  /**\n   * Whether to enable the Browserbase-powered web search tool.\n   * Requires a valid Browserbase API key.\n   */\n  useSearch?: boolean;\n  /**\n   * The Browserbase API key used for the search tool.\n   * Resolved from BROWSERBASE_API_KEY env var or the Stagehand constructor.\n   */\n  browserbaseApiKey?: string;\n}\n\n/**\n * Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n */\nfunction filterTools(\n  tools: ToolSet,\n  mode: AgentToolMode,\n  excludeTools?: string[],\n): ToolSet {\n  const filtered: ToolSet = { ...tools };\n\n  // Mode-based filtering\n  if (mode === \"hybrid\") {\n    delete filtered.fillForm;\n  } else {\n    // DOM mode (default)\n    delete filtered.click;\n    delete filtered.type;\n    delete filtered.dragAndDrop;\n    delete filtered.clickAndHold;\n    delete filtered.fillFormVision;\n  }\n\n  if (excludeTools) {\n    for (const toolName of excludeTools) {\n      delete filtered[toolName];\n    }\n  }\n\n  return filtered;\n}\n\n/**\n * Wraps an AI SDK tool's execute function with a timeout guard.\n * On timeout, returns `{ success: false, error: \"TimeoutError: ...\" }` to the LLM\n * and logs the error. Also acts as a safety net for any uncaught errors.\n */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction wrapToolWithTimeout<T extends Record<string, any>>(\n  agentTool: T,\n  toolName: string,\n  v3: V3,\n  timeoutMs?: number,\n  timeoutHint?: string,\n): T {\n  if (!timeoutMs || !agentTool.execute) return agentTool;\n\n  const originalExecute = agentTool.execute;\n  return {\n    ...agentTool,\n    execute: async (...args: unknown[]) => {\n      try {\n        return await withTimeout(originalExecute(...args), timeoutMs, toolName);\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const message = `TimeoutError: ${error.message}${timeoutHint ? ` ${timeoutHint}` : \"\"}`;\n          v3.logger({\n            category: \"agent\",\n            message,\n            level: 0,\n          });\n          return {\n            success: false,\n            error: message,\n          };\n        }\n        throw error;\n      }\n    },\n  } as T;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n  const executionModel = options?.executionModel;\n  const mode = options?.mode ?? \"dom\";\n  const provider = options?.provider;\n  const excludeTools = options?.excludeTools;\n  const variables = options?.variables;\n  const toolTimeout = options?.toolTimeout;\n\n  const timeoutHints: Record<string, string> = {\n    act: \"(it may continue executing in the background) — try using a different description for the action\",\n    ariaTree: \"— the page may be too large\",\n    extract: \"— try using a smaller or simpler schema\",\n    fillForm:\n      \"(it may continue executing in the background) — try filling fewer fields at once or use a different tool\",\n  };\n\n  const unwrappedTools: ToolSet = {\n    act: actTool(v3, executionModel, variables, toolTimeout),\n    ariaTree: ariaTreeTool(v3, toolTimeout),\n    click: clickTool(v3, provider),\n    clickAndHold: clickAndHoldTool(v3, provider),\n    dragAndDrop: dragAndDropTool(v3, provider),\n    extract: extractTool(v3, executionModel, toolTimeout),\n    fillForm: fillFormTool(v3, executionModel, variables, toolTimeout),\n    fillFormVision: fillFormVisionTool(v3, provider, variables),\n    goto: gotoTool(v3),\n    keys: keysTool(v3),\n    navback: navBackTool(v3),\n    screenshot: screenshotTool(v3),\n    scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n    type: typeTool(v3, provider, variables),\n  };\n\n  if (options?.useSearch && options.browserbaseApiKey) {\n    unwrappedTools.search = browserbaseSearchTool(\n      v3,\n      options.browserbaseApiKey,\n    );\n  } else if (process.env.BRAVE_API_KEY) {\n    unwrappedTools.search = braveSearchTool(v3);\n  }\n\n  const allTools: ToolSet = {\n    ...Object.fromEntries(\n      Object.entries(unwrappedTools).map(([name, t]) => [\n        name,\n        wrapToolWithTimeout(\n          t,\n          `${name}()`,\n          v3,\n          toolTimeout,\n          timeoutHints[name],\n        ),\n      ]),\n    ),\n    think: thinkTool(),\n    wait: waitTool(v3, mode),\n  };\n\n  return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/**\n * Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional — enabled via useSearch: true (Browserbase) or BRAVE_API_KEY env var (legacy).\n */\nexport type AgentToolTypesMap = {\n  act: ReturnType<typeof actTool>;\n  ariaTree: ReturnType<typeof ariaTreeTool>;\n  click: ReturnType<typeof clickTool>;\n  clickAndHold: ReturnType<typeof clickAndHoldTool>;\n  dragAndDrop: ReturnType<typeof dragAndDropTool>;\n  extract: ReturnType<typeof extractTool>;\n  fillForm: ReturnType<typeof fillFormTool>;\n  fillFormVision: ReturnType<typeof fillFormVisionTool>;\n  goto: ReturnType<typeof gotoTool>;\n  keys: ReturnType<typeof keysTool>;\n  navback: ReturnType<typeof navBackTool>;\n  screenshot: ReturnType<typeof screenshotTool>;\n  scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;\n  search?:\n    | ReturnType<typeof browserbaseSearchTool>\n    | ReturnType<typeof braveSearchTool>;\n  think: ReturnType<typeof thinkTool>;\n  type: ReturnType<typeof typeTool>;\n  wait: ReturnType<typeof waitTool>;\n};\n\n/**\n * Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n */\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/**\n * Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n */\nexport type AgentToolCall = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    args: AgentUITools[K][\"input\"];\n  };\n}[keyof AgentToolTypesMap];\n\n/**\n * Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    result: AgentUITools[K][\"output\"];\n  };\n}[keyof AgentToolTypesMap];\n"]}

package/dist/esm/lib/v3/agent/tools/keys.d.ts

@@ -17,7 +17,7 @@ export declare const keysTool: (v3: V3) => import("ai").Tool<{
     error?: undefined;
 } | {
     success: boolean;
-    error: string;
+    error: any;
     method?: undefined;
     value?: undefined;
     times?: undefined;

package/dist/esm/lib/v3/agent/tools/keys.js.map

@@ -1 +1 @@
-{"version":3,"file":"keys.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/keys.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,EAAE,CACjC,IAAI,CAAC;IACH,WAAW,EAAE;;;;kIAIiH;IAC9H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjC,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,QAAQ,CACP,iEAAiE,CAClE;QACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC9B,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;wBAChD,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAEvC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,GAAG;oBAC9B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,EAAE;oBAC7B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,MAAM,EAAE,EAAE,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC;QAC7D,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const keysTool = (v3: V3) =>\n  tool({\n    description: `Send keyboard input to the page without targeting a specific element. Unlike the type tool which clicks then types into coordinates, this sends keystrokes directly to wherever focus currently is.\n\nUse method=\"type\" to enter text into the currently focused element. Preferred when: input is already focused, text needs to flow across multiple fields (e.g., verification codes)\n\nUse method=\"press\" for navigation keys (Enter, Tab, Escape, Backspace, arrows) and keyboard shortcuts (Cmd+A, Ctrl+C, Shift+Tab).`,\n    inputSchema: z.object({\n      method: z.enum([\"press\", \"type\"]),\n      value: z\n        .string()\n        .describe(\n          \"The text to type, or the key/combo to press (Enter, Tab, Cmd+A)\",\n        ),\n      repeat: z.number().optional(),\n    }),\n    execute: async ({ method, value, repeat }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: keys`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ method, value, repeat }),\n              type: \"object\",\n            },\n          },\n        });\n\n        const times = Math.max(1, repeat ?? 1);\n\n        if (method === \"type\") {\n          for (let i = 0; i < times; i++) {\n            await page.type(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `type \"${value}\"`,\n            playwrightArguments: { method, text: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        if (method === \"press\") {\n          for (let i = 0; i < times; i++) {\n            await page.keyPress(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `press ${value}`,\n            playwrightArguments: { method, keys: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        return { success: false, error: `Unsupported method: ${method}` };\n      } catch (error) {\n        return { success: false, error: (error as Error).message };\n      }\n    },\n  });\n"]}
+{"version":3,"file":"keys.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/keys.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,EAAE,CACjC,IAAI,CAAC;IACH,WAAW,EAAE;;;;kIAIiH;IAC9H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjC,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,QAAQ,CACP,iEAAiE,CAClE;QACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC9B,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;wBAChD,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAEvC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,GAAG;oBAC9B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,EAAE;oBAC7B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,MAAM,EAAE,EAAE,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QAClD,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const keysTool = (v3: V3) =>\n  tool({\n    description: `Send keyboard input to the page without targeting a specific element. Unlike the type tool which clicks then types into coordinates, this sends keystrokes directly to wherever focus currently is.\n\nUse method=\"type\" to enter text into the currently focused element. Preferred when: input is already focused, text needs to flow across multiple fields (e.g., verification codes)\n\nUse method=\"press\" for navigation keys (Enter, Tab, Escape, Backspace, arrows) and keyboard shortcuts (Cmd+A, Ctrl+C, Shift+Tab).`,\n    inputSchema: z.object({\n      method: z.enum([\"press\", \"type\"]),\n      value: z\n        .string()\n        .describe(\n          \"The text to type, or the key/combo to press (Enter, Tab, Cmd+A)\",\n        ),\n      repeat: z.number().optional(),\n    }),\n    execute: async ({ method, value, repeat }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: keys`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ method, value, repeat }),\n              type: \"object\",\n            },\n          },\n        });\n\n        const times = Math.max(1, repeat ?? 1);\n\n        if (method === \"type\") {\n          for (let i = 0; i < times; i++) {\n            await page.type(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `type \"${value}\"`,\n            playwrightArguments: { method, text: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        if (method === \"press\") {\n          for (let i = 0; i < times; i++) {\n            await page.keyPress(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `press ${value}`,\n            playwrightArguments: { method, keys: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        return { success: false, error: `Unsupported method: ${method}` };\n      } catch (error) {\n        return { success: false, error: error.message };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/screenshot.d.ts

@@ -1,6 +1,14 @@
 import type { V3 } from "../../v3.js";
 export declare const screenshotTool: (v3: V3) => import("ai").Tool<Record<string, never>, {
+    success: boolean;
     base64: string;
     timestamp: number;
     pageUrl: string;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: string;
+    base64?: undefined;
+    timestamp?: undefined;
+    pageUrl?: undefined;
 }>;

package/dist/esm/lib/v3/agent/tools/screenshot.js

@@ -4,23 +4,40 @@ export const screenshotTool = (v3) => tool({
     description: "Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.",
     inputSchema: z.object({}),
     execute: async () => {
-        v3.logger({
-            category: "agent",
-            message: `Agent calling tool: screenshot`,
-            level: 1,
-        });
-        const page = await v3.context.awaitActivePage();
-        const buffer = await page.screenshot({ fullPage: false });
-        const pageUrl = page.url();
+        try {
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: screenshot`,
+                level: 1,
+            });
+            const page = await v3.context.awaitActivePage();
+            const buffer = await page.screenshot({ fullPage: false });
+            const pageUrl = page.url();
+            return {
+                success: true,
+                base64: buffer.toString("base64"),
+                timestamp: Date.now(),
+                pageUrl,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error taking screenshot: ${error.message}`,
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         return {
-            base64: buffer.toString("base64"),
-            timestamp: Date.now(),
-            pageUrl,
+            type: "content",
+            value: [{ type: "media", mediaType: "image/png", data: result.base64 }],
         };
     },
-    toModelOutput: (result) => ({
-        type: "content",
-        value: [{ type: "media", mediaType: "image/png", data: result.base64 }],
-    }),
 });
 //# sourceMappingURL=screenshot.js.map

package/dist/esm/lib/v3/agent/tools/screenshot.js.map

@@ -1 +1 @@
-{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAM,EAAE,EAAE,CACvC,IAAI,CAAC;IACH,WAAW,EACT,sFAAsF;IACxF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,gCAAgC;YACzC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,OAAO;SACR,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;KACxE,CAAC;CACH,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const screenshotTool = (v3: V3) =>\n  tool({\n    description:\n      \"Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: screenshot`,\n        level: 1,\n      });\n      const page = await v3.context.awaitActivePage();\n      const buffer = await page.screenshot({ fullPage: false });\n      const pageUrl = page.url();\n      return {\n        base64: buffer.toString(\"base64\"),\n        timestamp: Date.now(),\n        pageUrl,\n      };\n    },\n    toModelOutput: (result) => ({\n      type: \"content\",\n      value: [{ type: \"media\", mediaType: \"image/png\", data: result.base64 }],\n    }),\n  });\n"]}
+{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAM,EAAE,EAAE,CACvC,IAAI,CAAC;IACH,WAAW,EACT,sFAAsF;IACxF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,gCAAgC;gBACzC,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,OAAO;aACR,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,4BAA6B,KAAe,CAAC,OAAO,EAAE;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;SACxE,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const screenshotTool = (v3: V3) =>\n  tool({\n    description:\n      \"Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: screenshot`,\n          level: 1,\n        });\n        const page = await v3.context.awaitActivePage();\n        const buffer = await page.screenshot({ fullPage: false });\n        const pageUrl = page.url();\n        return {\n          success: true,\n          base64: buffer.toString(\"base64\"),\n          timestamp: Date.now(),\n          pageUrl,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error taking screenshot: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"content\",\n        value: [{ type: \"media\", mediaType: \"image/png\", data: result.base64 }],\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/scroll.js

@@ -44,6 +44,12 @@ export const scrollTool = (v3) => tool({
         };
     },
     toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         return {
             type: "json",
             value: {
@@ -119,6 +125,12 @@ export const scrollVisionTool = (v3, provider) => tool({
         };
     },
     toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         const content = [
             {
                 type: "text",

package/dist/esm/lib/v3/agent/tools/scroll.js.map

@@ -1 +1 @@
-{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  ScrollToolResult,\n  ScrollVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n  tool({\n    description:\n      \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      percentage = 80,\n    }): Promise<ScrollToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ direction, percentage }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const cx = Math.floor(w / 2);\n      const cy = Math.floor(h / 2);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n        scrolledPixels: scrollDistance,\n      };\n    },\n    toModelOutput: (result) => {\n      return {\n        type: \"json\",\n        value: {\n          success: result.success,\n          message: result.message,\n          scrolledPixels: result.scrolledPixels,\n        },\n      };\n    },\n  });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      coordinates: z\n        .array(z.number())\n        .optional()\n        .describe(\n          \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n        ),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      coordinates,\n      percentage = 80,\n    }): Promise<ScrollVisionToolResult> => {\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      // Process coordinates if provided, otherwise use viewport center\n      let cx: number;\n      let cy: number;\n      if (coordinates) {\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n        cx = processed.x;\n        cy = processed.y;\n      } else {\n        cx = Math.floor(w / 2);\n        cy = Math.floor(h / 2);\n      }\n\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({\n              direction,\n              coordinates,\n              percentage,\n              processed: { cx, cy },\n            }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: coordinates\n          ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n          : `Scrolled ${percentage}% ${direction}`,\n        scrolledPixels: scrollDistance,\n        screenshotBase64,\n      };\n    },\n    toModelOutput: (result) => {\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            message: result.message,\n            scrolledPixels: result.scrolledPixels,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}
+{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  ScrollToolResult,\n  ScrollVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n  tool({\n    description:\n      \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      percentage = 80,\n    }): Promise<ScrollToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ direction, percentage }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const cx = Math.floor(w / 2);\n      const cy = Math.floor(h / 2);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n        scrolledPixels: scrollDistance,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"json\",\n        value: {\n          success: result.success,\n          message: result.message,\n          scrolledPixels: result.scrolledPixels,\n        },\n      };\n    },\n  });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      coordinates: z\n        .array(z.number())\n        .optional()\n        .describe(\n          \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n        ),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      coordinates,\n      percentage = 80,\n    }): Promise<ScrollVisionToolResult> => {\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      // Process coordinates if provided, otherwise use viewport center\n      let cx: number;\n      let cy: number;\n      if (coordinates) {\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n        cx = processed.x;\n        cy = processed.y;\n      } else {\n        cx = Math.floor(w / 2);\n        cy = Math.floor(h / 2);\n      }\n\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({\n              direction,\n              coordinates,\n              percentage,\n              processed: { cx, cy },\n            }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: coordinates\n          ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n          : `Scrolled ${percentage}% ${direction}`,\n        scrolledPixels: scrollDistance,\n        screenshotBase64,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            message: result.message,\n            scrolledPixels: result.scrolledPixels,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/type.d.ts

@@ -1,6 +1,6 @@
 import type { V3 } from "../../v3.js";
-import type { TypeToolResult } from "../../types/public/agent.js";
-export declare const typeTool: (v3: V3, provider?: string) => import("ai").Tool<{
+import type { TypeToolResult, Variables } from "../../types/public/agent.js";
+export declare const typeTool: (v3: V3, provider?: string, variables?: Variables) => import("ai").Tool<{
     describe: string;
     text: string;
     coordinates: number[];

package/dist/esm/lib/v3/agent/tools/type.js

@@ -3,73 +3,86 @@ import { z } from "zod";
 import { processCoordinates } from "../utils/coordinateNormalization.js";
 import { ensureXPath } from "../utils/xpath.js";
 import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
-export const typeTool = (v3, provider) => tool({
-    description: "Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
-    inputSchema: z.object({
-        describe: z
-            .string()
-            .describe("Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description"),
-        text: z.string().describe("The text to type into the element"),
-        coordinates: z
-            .array(z.number())
-            .describe("The (x, y) coordinates to type into the element"),
-    }),
-    execute: async ({ describe, coordinates, text, }) => {
-        try {
-            const page = await v3.context.awaitActivePage();
-            const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
-            v3.logger({
-                category: "agent",
-                message: `Agent calling tool: type`,
-                level: 1,
-                auxiliary: {
-                    arguments: {
-                        value: JSON.stringify({ describe, text }),
-                        type: "object",
+import { substituteVariables } from "../utils/variables.js";
+export const typeTool = (v3, provider, variables) => {
+    const hasVariables = variables && Object.keys(variables).length > 0;
+    const textDescription = hasVariables
+        ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
+        : "The text to type into the element";
+    return tool({
+        description: "Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
+        inputSchema: z.object({
+            describe: z
+                .string()
+                .describe("Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description"),
+            text: z.string().describe(textDescription),
+            coordinates: z
+                .array(z.number())
+                .describe("The (x, y) coordinates to type into the element"),
+        }),
+        execute: async ({ describe, coordinates, text, }) => {
+            try {
+                const page = await v3.context.awaitActivePage();
+                const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+                // Substitute any %variableName% tokens in the text
+                const actualText = substituteVariables(text, variables);
+                v3.logger({
+                    category: "agent",
+                    message: `Agent calling tool: type`,
+                    level: 1,
+                    auxiliary: {
+                        arguments: {
+                            value: JSON.stringify({ describe, text }),
+                            type: "object",
+                        },
                     },
-                },
-            });
-            // Only request XPath when caching is enabled to avoid unnecessary computation
-            const shouldCollectXpath = v3.isAgentReplayActive();
-            const xpath = await page.click(processed.x, processed.y, {
-                returnXpath: shouldCollectXpath,
-            });
-            await page.type(text);
-            const screenshotBase64 = await waitAndCaptureScreenshot(page);
-            // Record as an "act" step with proper Action for deterministic replay (only when caching)
-            if (shouldCollectXpath) {
-                const normalizedXpath = ensureXPath(xpath);
-                if (normalizedXpath) {
-                    const action = {
-                        selector: normalizedXpath,
-                        description: describe,
-                        method: "type",
-                        arguments: [text],
-                    };
-                    v3.recordAgentReplayStep({
-                        type: "act",
-                        instruction: describe,
-                        actions: [action],
-                        actionDescription: describe,
-                    });
+                });
+                // Only request XPath when caching is enabled to avoid unnecessary computation
+                const shouldCollectXpath = v3.isAgentReplayActive();
+                const xpath = await page.click(processed.x, processed.y, {
+                    returnXpath: shouldCollectXpath,
+                });
+                await page.type(actualText);
+                const screenshotBase64 = await waitAndCaptureScreenshot(page);
+                // Record as an "act" step with proper Action for deterministic replay (only when caching)
+                if (shouldCollectXpath) {
+                    const normalizedXpath = ensureXPath(xpath);
+                    if (normalizedXpath) {
+                        const action = {
+                            selector: normalizedXpath,
+                            description: describe,
+                            method: "type",
+                            arguments: [text],
+                        };
+                        v3.recordAgentReplayStep({
+                            type: "act",
+                            instruction: describe,
+                            actions: [action],
+                            actionDescription: describe,
+                        });
+                    }
                 }
+                return {
+                    success: true,
+                    describe,
+                    text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM
+                    screenshotBase64,
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    error: `Error typing: ${error.message}`,
+                };
+            }
+        },
+        toModelOutput: (result) => {
+            if (result.success === false || result.error !== undefined) {
+                return {
+                    type: "content",
+                    value: [{ type: "text", text: JSON.stringify(result) }],
+                };
             }
-            return {
-                success: true,
-                describe,
-                text,
-                screenshotBase64,
-            };
-        }
-        catch (error) {
-            return {
-                success: false,
-                error: `Error typing: ${error.message}`,
-            };
-        }
-    },
-    toModelOutput: (result) => {
-        if (result.success) {
             const content = [
                 {
                     type: "text",
@@ -88,19 +101,7 @@ export const typeTool = (v3, provider) => tool({
                 });
             }
             return { type: "content", value: content };
-        }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
-    },
-});
+        },
+    });
+};
 //# sourceMappingURL=type.js.map

package/dist/esm/lib/v3/agent/tools/type.js.map

@@ -1 +1 @@
-{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACpD,IAAI,CAAC;IACH,WAAW,EACT,2QAA2Q;IAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,4HAA4H,CAC7H;QACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;QAC9D,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,iDAAiD,CAAC;KAC/D,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;QAC5B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;wBACzC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEtB,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,MAAM;wBACd,SAAS,EAAE,CAAC,IAAI,CAAC;qBAClB,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,IAAI;gBACJ,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,iBAAkB,KAAe,CAAC,OAAO,EAAE;aACnD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const typeTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(\"The text to type into the element\"),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        await page.type(text);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n              text: result.text,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}
+{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvD,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;gBAEH,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const textDescription = hasVariables\n    ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"The text to type into the element\";\n\n  return tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(textDescription),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        // Substitute any %variableName% tokens in the text\n        const actualText = substituteVariables(text, variables);\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        await page.type(actualText);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            text: result.text,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/wait.js

@@ -31,6 +31,12 @@ export const waitTool = (v3, mode) => tool({
         return { success: true, waited: timeMs };
     },
     toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         const content = [
             {
                 type: "text",

package/dist/esm/lib/v3/agent/tools/wait.js.map

@@ -1 +1 @@
-{"version":3,"file":"wait.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/wait.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,IAAoB,EAAE,EAAE,CACvD,IAAI,CAAC;IACH,WAAW,EAAE,2BAA2B;IACxC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;KACpD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAA2B,EAAE;QACrD,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,0BAA0B;YACnC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,eAAe,MAAM,eAAe;oBAC3C,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QACH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,EAAE,CAAC,qBAAqB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,gEAAgE;QAChE,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAC7D,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC3C,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  AgentToolMode,\n  WaitToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const waitTool = (v3: V3, mode?: AgentToolMode) =>\n  tool({\n    description: \"Wait for a specified time\",\n    inputSchema: z.object({\n      timeMs: z.number().describe(\"Time in milliseconds\"),\n    }),\n    execute: async ({ timeMs }): Promise<WaitToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: wait`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: `Waiting for ${timeMs} milliseconds`,\n            type: \"string\",\n          },\n        },\n      });\n      await new Promise((resolve) => setTimeout(resolve, timeMs));\n      if (timeMs > 0) {\n        v3.recordAgentReplayStep({ type: \"wait\", timeMs });\n      }\n\n      // Take screenshot after wait in hybrid mode for visual feedback\n      if (mode === \"hybrid\") {\n        const page = await v3.context.awaitActivePage();\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 0);\n        return { success: true, waited: timeMs, screenshotBase64 };\n      }\n\n      return { success: true, waited: timeMs };\n    },\n    toModelOutput: (result) => {\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            waited: result.waited,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}
+{"version":3,"file":"wait.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/wait.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,IAAoB,EAAE,EAAE,CACvD,IAAI,CAAC;IACH,WAAW,EAAE,2BAA2B;IACxC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;KACpD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAA2B,EAAE;QACrD,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,0BAA0B;YACnC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,eAAe,MAAM,eAAe;oBAC3C,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QACH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,EAAE,CAAC,qBAAqB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,gEAAgE;QAChE,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAC7D,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC3C,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  AgentToolMode,\n  WaitToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const waitTool = (v3: V3, mode?: AgentToolMode) =>\n  tool({\n    description: \"Wait for a specified time\",\n    inputSchema: z.object({\n      timeMs: z.number().describe(\"Time in milliseconds\"),\n    }),\n    execute: async ({ timeMs }): Promise<WaitToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: wait`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: `Waiting for ${timeMs} milliseconds`,\n            type: \"string\",\n          },\n        },\n      });\n      await new Promise((resolve) => setTimeout(resolve, timeMs));\n      if (timeMs > 0) {\n        v3.recordAgentReplayStep({ type: \"wait\", timeMs });\n      }\n\n      // Take screenshot after wait in hybrid mode for visual feedback\n      if (mode === \"hybrid\") {\n        const page = await v3.context.awaitActivePage();\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 0);\n        return { success: true, waited: timeMs, screenshotBase64 };\n      }\n\n      return { success: true, waited: timeMs };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            waited: result.waited,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/utils/captchaSolver.d.ts

@@ -0,0 +1,76 @@
+import type { Page } from "../../understudy/page.js";
+/** Injected into the agent message stream after a successful captcha solve. */
+export declare const CAPTCHA_SOLVED_MSG = "A captcha was automatically detected and solved \u2014 no further interaction with the captcha is needed, even if it does not visually appear solved. Do not click the captcha checkbox, widget, or challenge again. Continue with your task.";
+/** Injected into the agent message stream when the captcha solver fails. */
+export declare const CAPTCHA_ERRORED_MSG = "A captcha was detected but the automatic captcha solver failed to solve it. You may need to try a different approach or navigate around the captcha.";
+/** Appended to the system prompt (DOM/hybrid agents) when captchas auto-solve. */
+export declare const CAPTCHA_SYSTEM_PROMPT_NOTE = "Captchas on this page are automatically detected and solved by the browser environment. Do not interact with or attempt to solve any captchas yourself \u2014 they will be handled for you. Do not click the captcha checkbox, widget, or challenge again after it has been solved, even if it still looks unresolved. Continue with your task as if the captcha does not exist.";
+/** Appended to the CUA system prompt when captchas auto-solve. */
+export declare const CAPTCHA_CUA_SYSTEM_PROMPT_NOTE = "\n\nCaptchas on this page are automatically detected and solved by the browser environment. Do not interact with or attempt to solve any captchas yourself \u2014 they will be handled for you. Continue with your task as if the captcha does not exist.";
+/**
+ * Tracks Browserbase captcha solver state via console messages and provides
+ * a blocking `waitIfSolving()` that agents call before each step/action.
+ *
+ * Accepts a page-provider callback so the listener is automatically
+ * re-attached when the active page changes (e.g. popup / new tab).
+ *
+ * All concurrent callers of `waitIfSolving()` share the same underlying
+ * promise, so multiple waiters are safely resolved together.
+ */
+export declare class CaptchaSolver {
+    private solving;
+    private _solvedSinceLastConsume;
+    private _erroredSinceLastConsume;
+    private listener;
+    private attachedPage;
+    private pageProvider;
+    /** Shared promise that all concurrent waitIfSolving() callers await. */
+    private waitPromise;
+    /** Resolves the shared waitPromise. */
+    private resolveWait;
+    /** Timeout handle for the 90s deadline. */
+    private waitTimer;
+    /**
+     * Initialise with a callback that returns the current active page.
+     * The listener is lazily (re-)attached whenever the active page changes.
+     */
+    init(pageProvider: () => Promise<Page>): void;
+    /** Whether a captcha solve is currently in progress. */
+    isSolving(): boolean;
+    /**
+     * Ensure the console listener is attached to the current active page.
+     * If the active page has changed since the last call, the old listener
+     * is removed and a new one is installed.
+     */
+    ensureAttached(): Promise<void>;
+    /**
+     * Returns a promise that resolves immediately if no captcha is being
+     * solved, or blocks until the solver finishes, errors, or the 90s
+     * timeout is reached.
+     *
+     * Also re-attaches the listener to the current active page if it has
+     * changed since the last call.
+     *
+     * All concurrent callers share the same promise, so no waiter is
+     * orphaned.
+     */
+    waitIfSolving(): Promise<void>;
+    /**
+     * Returns and resets the solve event flags.
+     * Call after `waitIfSolving()` to check whether a captcha was solved
+     * (or errored) since the last consume.  This captures events even if
+     * the solve completed between two `waitIfSolving()` calls.
+     */
+    consumeSolveResult(): {
+        solved: boolean;
+        errored: boolean;
+    };
+    /**
+     * Remove the console listener and reset all state.
+     */
+    dispose(): void;
+    /** Remove the console listener from the currently attached page. */
+    private detachListener;
+    /** Resolve the shared wait promise and clear the timeout. */
+    private settle;
+}