@browserbasehq/orca 3.1.0-patch.4 → 3.2.0-middleware.1

package/dist/esm/lib/v3/agent/tools/fillform.js

@@ -1,62 +1,73 @@
 import { tool } from "ai";
 import { z } from "zod";
-export const fillFormTool = (v3, executionModel, variables) => {
+import { TimeoutError } from "../../types/public/sdkErrors.js";
+export const fillFormTool = (v3, executionModel, variables, toolTimeout) => {
     const hasVariables = variables && Object.keys(variables).length > 0;
-    const valueDescription = hasVariables
-        ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
-        : "Text to type into the target";
+    const actionDescription = hasVariables
+        ? `Must follow the pattern: "type <exact value> into the <field name> <fieldType>". Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}. Examples: "type %email% into the email input", "type %password% into the password input"`
+        : 'Must follow the pattern: "type <exact value> into the <field name> <fieldType>". Examples: "type john@example.com into the email input", "type John into the first name input"';
     return tool({
-        description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,
+        description: 'FORM FILL - MULTI-FIELD INPUT TOOL\nFill 2+ form inputs/textareas at once. Each action MUST include the exact text to type and the target field, e.g. "type john@example.com into the email field".',
         inputSchema: z.object({
             fields: z
                 .array(z.object({
-                action: z
-                    .string()
-                    .describe('Description of typing action, e.g. "type foo into the email field"'),
-                value: z.string().describe(valueDescription),
+                action: z.string().describe(actionDescription),
             }))
                 .min(1, "Provide at least one field to fill"),
         }),
         execute: async ({ fields }) => {
-            v3.logger({
-                category: "agent",
-                message: `Agent calling tool: fillForm`,
-                level: 1,
-                auxiliary: {
-                    arguments: {
-                        value: JSON.stringify(fields),
-                        type: "object",
+            try {
+                v3.logger({
+                    category: "agent",
+                    message: `Agent calling tool: fillForm`,
+                    level: 1,
+                    auxiliary: {
+                        arguments: {
+                            value: JSON.stringify(fields),
+                            type: "object",
+                        },
                     },
-                },
-            });
-            const instruction = `Return observation results for the following actions: ${fields
-                .map((f) => f.action)
-                .join(", ")}`;
-            const observeOptions = executionModel
-                ? { model: executionModel }
-                : undefined;
-            const observeResults = await v3.observe(instruction, observeOptions);
-            const completed = [];
-            const replayableActions = [];
-            for (const res of observeResults) {
-                const actOptions = variables ? { variables } : undefined;
-                const actResult = await v3.act(res, actOptions);
-                completed.push(actResult);
-                if (Array.isArray(actResult.actions)) {
-                    replayableActions.push(...actResult.actions);
+                });
+                const instruction = `Return observation results for the following actions: ${fields
+                    .map((f) => f.action)
+                    .join(", ")}`;
+                const observeOptions = executionModel
+                    ? { model: executionModel, timeout: toolTimeout }
+                    : { timeout: toolTimeout };
+                const observeResults = await v3.observe(instruction, observeOptions);
+                const completed = [];
+                const replayableActions = [];
+                for (const res of observeResults) {
+                    const actOptions = variables
+                        ? { variables, timeout: toolTimeout }
+                        : { timeout: toolTimeout };
+                    const actResult = await v3.act(res, actOptions);
+                    completed.push(actResult);
+                    if (Array.isArray(actResult.actions)) {
+                        replayableActions.push(...actResult.actions);
+                    }
                 }
+                v3.recordAgentReplayStep({
+                    type: "fillForm",
+                    fields,
+                    observeResults,
+                    actions: replayableActions,
+                });
+                return {
+                    success: true,
+                    actions: completed,
+                    playwrightArguments: replayableActions,
+                };
+            }
+            catch (error) {
+                if (error instanceof TimeoutError) {
+                    throw error;
+                }
+                return {
+                    success: false,
+                    error: error?.message ?? String(error),
+                };
             }
-            v3.recordAgentReplayStep({
-                type: "fillForm",
-                fields,
-                observeResults,
-                actions: replayableActions,
-            });
-            return {
-                success: true,
-                actions: completed,
-                playwrightArguments: replayableActions,
-            };
         },
     });
 };

package/dist/esm/lib/v3/agent/tools/fillform.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,+FAA+F,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpI,CAAC,CAAC,8BAA8B,CAAC;IAEnC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE,8GAA8G;QAC3H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,oEAAoE,CACrE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;aAC7C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,8BAA8B;gBACvC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;wBAC7B,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;iBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;iBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAEhB,MAAM,cAAc,GAAG,cAAc;gBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE;gBAC3B,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;YAErE,MAAM,SAAS,GAAG,EAAe,CAAC;YAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;gBACjC,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;oBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;YACD,EAAE,CAAC,qBAAqB,CAAC;gBACvB,IAAI,EAAE,UAAU;gBAChB,MAAM;gBACN,cAAc;gBACd,OAAO,EAAE,iBAAiB;aAC3B,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,SAAS;gBAClB,mBAAmB,EAAE,iBAAiB;aACvC,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\n\nexport const fillFormTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target\";\n\n  return tool({\n    description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                'Description of typing action, e.g. \"type foo into the email field\"',\n              ),\n            value: z.string().describe(valueDescription),\n          }),\n        )\n        .min(1, \"Provide at least one field to fill\"),\n    }),\n    execute: async ({ fields }) => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: fillForm`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify(fields),\n            type: \"object\",\n          },\n        },\n      });\n      const instruction = `Return observation results for the following actions: ${fields\n        .map((f) => f.action)\n        .join(\", \")}`;\n\n      const observeOptions = executionModel\n        ? { model: executionModel }\n        : undefined;\n      const observeResults = await v3.observe(instruction, observeOptions);\n\n      const completed = [] as unknown[];\n      const replayableActions: Action[] = [];\n      for (const res of observeResults) {\n        const actOptions = variables ? { variables } : undefined;\n        const actResult = await v3.act(res, actOptions);\n        completed.push(actResult);\n        if (Array.isArray(actResult.actions)) {\n          replayableActions.push(...(actResult.actions as Action[]));\n        }\n      }\n      v3.recordAgentReplayStep({\n        type: \"fillForm\",\n        fields,\n        observeResults,\n        actions: replayableActions,\n      });\n      return {\n        success: true,\n        actions: completed,\n        playwrightArguments: replayableActions,\n      };\n    },\n  });\n};\n"]}
+{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,kJAAkJ,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,4FAA4F;QACjR,CAAC,CAAC,gLAAgL,CAAC;IAErL,OAAO,IAAI,CAAC;QACV,WAAW,EACT,qMAAqM;QACvM,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,8BAA8B;oBACvC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;4BAC7B,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;qBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;qBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEhB,MAAM,cAAc,GAAG,cAAc;oBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE;oBACjD,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBAErE,MAAM,SAAS,GAAG,EAAe,CAAC;gBAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;gBACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,SAAS;wBAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;wBACrC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;wBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,UAAU;oBAChB,MAAM;oBACN,cAAc;oBACd,OAAO,EAAE,iBAAiB;iBAC3B,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,SAAS;oBAClB,mBAAmB,EAAE,iBAAiB;iBACvC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const fillFormTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  variables?: Variables,\n  toolTimeout?: number,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const actionDescription = hasVariables\n    ? `Must follow the pattern: \"type <exact value> into the <field name> <fieldType>\". Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}. Examples: \"type %email% into the email input\", \"type %password% into the password input\"`\n    : 'Must follow the pattern: \"type <exact value> into the <field name> <fieldType>\". Examples: \"type john@example.com into the email input\", \"type John into the first name input\"';\n\n  return tool({\n    description:\n      'FORM FILL - MULTI-FIELD INPUT TOOL\\nFill 2+ form inputs/textareas at once. Each action MUST include the exact text to type and the target field, e.g. \"type john@example.com into the email field\".',\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z.string().describe(actionDescription),\n          }),\n        )\n        .min(1, \"Provide at least one field to fill\"),\n    }),\n    execute: async ({ fields }) => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillForm`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify(fields),\n              type: \"object\",\n            },\n          },\n        });\n        const instruction = `Return observation results for the following actions: ${fields\n          .map((f) => f.action)\n          .join(\", \")}`;\n\n        const observeOptions = executionModel\n          ? { model: executionModel, timeout: toolTimeout }\n          : { timeout: toolTimeout };\n        const observeResults = await v3.observe(instruction, observeOptions);\n\n        const completed = [] as unknown[];\n        const replayableActions: Action[] = [];\n        for (const res of observeResults) {\n          const actOptions = variables\n            ? { variables, timeout: toolTimeout }\n            : { timeout: toolTimeout };\n          const actResult = await v3.act(res, actOptions);\n          completed.push(actResult);\n          if (Array.isArray(actResult.actions)) {\n            replayableActions.push(...(actResult.actions as Action[]));\n          }\n        }\n        v3.recordAgentReplayStep({\n          type: \"fillForm\",\n          fields,\n          observeResults,\n          actions: replayableActions,\n        });\n        return {\n          success: true,\n          actions: completed,\n          playwrightArguments: replayableActions,\n        };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          throw error;\n        }\n        return {\n          success: false,\n          error: error?.message ?? String(error),\n        };\n      }\n    },\n  });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/index.d.ts

@@ -14,7 +14,8 @@ import { clickAndHoldTool } from "./clickAndHold.js";
 import { keysTool } from "./keys.js";
 import { fillFormVisionTool } from "./fillFormVision.js";
 import { thinkTool } from "./think.js";
-import { searchTool } from "./search.js";
+import { searchTool as browserbaseSearchTool } from "./browserbaseSearch.js";
+import { searchTool as braveSearchTool } from "./braveSearch.js";
 import type { ToolSet, InferUITools } from "ai";
 import type { V3 } from "../../v3.js";
 import type { LogLine } from "../../types/public/logs.js";
@@ -42,12 +43,27 @@ export interface V3AgentToolOptions {
      * When provided, these tools will have an optional useVariable field.
      */
     variables?: Variables;
+    /**
+     * Timeout in milliseconds for async tool calls.
+     * Applied to all tools that perform I/O (except wait and think).
+     */
+    toolTimeout?: number;
+    /**
+     * Whether to enable the Browserbase-powered web search tool.
+     * Requires a valid Browserbase API key.
+     */
+    useSearch?: boolean;
+    /**
+     * The Browserbase API key used for the search tool.
+     * Resolved from BROWSERBASE_API_KEY env var or the Stagehand constructor.
+     */
+    browserbaseApiKey?: string;
 }
 export declare function createAgentTools(v3: V3, options?: V3AgentToolOptions): ToolSet;
 export type AgentTools = ReturnType<typeof createAgentTools>;
 /**
  * Type map of all agent tools for strong typing of tool calls and results.
- * Note: `search` is optional as it's only available when BRAVE_API_KEY is configured.
+ * Note: `search` is optional — enabled via useSearch: true (Browserbase) or BRAVE_API_KEY env var (legacy).
  */
 export type AgentToolTypesMap = {
     act: ReturnType<typeof actTool>;
@@ -63,7 +79,7 @@ export type AgentToolTypesMap = {
     navback: ReturnType<typeof navBackTool>;
     screenshot: ReturnType<typeof screenshotTool>;
     scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;
-    search?: ReturnType<typeof searchTool>;
+    search?: ReturnType<typeof browserbaseSearchTool> | ReturnType<typeof braveSearchTool>;
     think: ReturnType<typeof thinkTool>;
     type: ReturnType<typeof typeTool>;
     wait: ReturnType<typeof waitTool>;

package/dist/esm/lib/v3/agent/tools/index.js

@@ -14,7 +14,10 @@ import { clickAndHoldTool } from "./clickAndHold.js";
 import { keysTool } from "./keys.js";
 import { fillFormVisionTool } from "./fillFormVision.js";
 import { thinkTool } from "./think.js";
-import { searchTool } from "./search.js";
+import { searchTool as browserbaseSearchTool } from "./browserbaseSearch.js";
+import { searchTool as braveSearchTool } from "./braveSearch.js";
+import { withTimeout } from "../../timeoutConfig.js";
+import { TimeoutError } from "../../types/public/sdkErrors.js";
 /**
  * Filters tools based on mode and explicit exclusions.
  * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)
@@ -42,34 +45,83 @@ function filterTools(tools, mode, excludeTools) {
     }
     return filtered;
 }
+/**
+ * Wraps an AI SDK tool's execute function with a timeout guard.
+ * On timeout, returns `{ success: false, error: "TimeoutError: ..." }` to the LLM
+ * and logs the error. Also acts as a safety net for any uncaught errors.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function wrapToolWithTimeout(agentTool, toolName, v3, timeoutMs, timeoutHint) {
+    if (!timeoutMs || !agentTool.execute)
+        return agentTool;
+    const originalExecute = agentTool.execute;
+    return {
+        ...agentTool,
+        execute: async (...args) => {
+            try {
+                return await withTimeout(originalExecute(...args), timeoutMs, toolName);
+            }
+            catch (error) {
+                if (error instanceof TimeoutError) {
+                    const message = `TimeoutError: ${error.message}${timeoutHint ? ` ${timeoutHint}` : ""}`;
+                    v3.logger({
+                        category: "agent",
+                        message,
+                        level: 0,
+                    });
+                    return {
+                        success: false,
+                        error: message,
+                    };
+                }
+                throw error;
+            }
+        },
+    };
+}
 export function createAgentTools(v3, options) {
     const executionModel = options?.executionModel;
     const mode = options?.mode ?? "dom";
     const provider = options?.provider;
     const excludeTools = options?.excludeTools;
     const variables = options?.variables;
-    const allTools = {
-        act: actTool(v3, executionModel, variables),
-        ariaTree: ariaTreeTool(v3),
+    const toolTimeout = options?.toolTimeout;
+    const timeoutHints = {
+        act: "(it may continue executing in the background) — try using a different description for the action",
+        ariaTree: "— the page may be too large",
+        extract: "— try using a smaller or simpler schema",
+        fillForm: "(it may continue executing in the background) — try filling fewer fields at once or use a different tool",
+    };
+    const unwrappedTools = {
+        act: actTool(v3, executionModel, variables, toolTimeout),
+        ariaTree: ariaTreeTool(v3, toolTimeout),
         click: clickTool(v3, provider),
         clickAndHold: clickAndHoldTool(v3, provider),
         dragAndDrop: dragAndDropTool(v3, provider),
-        extract: extractTool(v3, executionModel),
-        fillForm: fillFormTool(v3, executionModel, variables),
+        extract: extractTool(v3, executionModel, toolTimeout),
+        fillForm: fillFormTool(v3, executionModel, variables, toolTimeout),
         fillFormVision: fillFormVisionTool(v3, provider, variables),
         goto: gotoTool(v3),
         keys: keysTool(v3),
         navback: navBackTool(v3),
         screenshot: screenshotTool(v3),
         scroll: mode === "hybrid" ? scrollVisionTool(v3, provider) : scrollTool(v3),
-        think: thinkTool(),
         type: typeTool(v3, provider, variables),
-        wait: waitTool(v3, mode),
     };
-    // Only include search tool if BRAVE_API_KEY is configured
-    if (process.env.BRAVE_API_KEY) {
-        allTools.search = searchTool(v3);
+    if (options?.useSearch && options.browserbaseApiKey) {
+        unwrappedTools.search = browserbaseSearchTool(v3, options.browserbaseApiKey);
+    }
+    else if (process.env.BRAVE_API_KEY) {
+        unwrappedTools.search = braveSearchTool(v3);
     }
+    const allTools = {
+        ...Object.fromEntries(Object.entries(unwrappedTools).map(([name, t]) => [
+            name,
+            wrapToolWithTimeout(t, `${name}()`, v3, toolTimeout, timeoutHints[name]),
+        ])),
+        think: thinkTool(),
+        wait: waitTool(v3, mode),
+    };
     return filterTools(allTools, mode, excludeTools);
 }
 //# sourceMappingURL=index.js.map

package/dist/esm/lib/v3/agent/tools/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAoCzC;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IAErC,MAAM,QAAQ,GAAY;QACxB,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,CAAC;QAC3C,QAAQ,EAAE,YAAY,CAAC,EAAE,CAAC;QAC1B,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,CAAC;QACxC,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,CAAC;QACrD,cAAc,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QAC3D,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,SAAS,EAAE;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QACvC,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,0DAA0D;IAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QAC9B,QAAQ,CAAC,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool } from \"./search.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n  AgentToolMode,\n  AgentModelConfig,\n  Variables,\n} from \"../../types/public/agent.js\";\n\nexport interface V3AgentToolOptions {\n  executionModel?: string | AgentModelConfig;\n  logger?: (message: LogLine) => void;\n  /**\n   * Tool mode determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n   */\n  mode?: AgentToolMode;\n  /**\n   * The model provider. Used for model-specific coordinate handling\n   */\n  provider?: string;\n  /**\n   * Tools to exclude from the available toolset.\n   * These tools will be filtered out after mode-based filtering.\n   */\n  excludeTools?: string[];\n  /**\n   * Variables available to the agent for use in act/type tools.\n   * When provided, these tools will have an optional useVariable field.\n   */\n  variables?: Variables;\n}\n\n/**\n * Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n */\nfunction filterTools(\n  tools: ToolSet,\n  mode: AgentToolMode,\n  excludeTools?: string[],\n): ToolSet {\n  const filtered: ToolSet = { ...tools };\n\n  // Mode-based filtering\n  if (mode === \"hybrid\") {\n    delete filtered.fillForm;\n  } else {\n    // DOM mode (default)\n    delete filtered.click;\n    delete filtered.type;\n    delete filtered.dragAndDrop;\n    delete filtered.clickAndHold;\n    delete filtered.fillFormVision;\n  }\n\n  if (excludeTools) {\n    for (const toolName of excludeTools) {\n      delete filtered[toolName];\n    }\n  }\n\n  return filtered;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n  const executionModel = options?.executionModel;\n  const mode = options?.mode ?? \"dom\";\n  const provider = options?.provider;\n  const excludeTools = options?.excludeTools;\n  const variables = options?.variables;\n\n  const allTools: ToolSet = {\n    act: actTool(v3, executionModel, variables),\n    ariaTree: ariaTreeTool(v3),\n    click: clickTool(v3, provider),\n    clickAndHold: clickAndHoldTool(v3, provider),\n    dragAndDrop: dragAndDropTool(v3, provider),\n    extract: extractTool(v3, executionModel),\n    fillForm: fillFormTool(v3, executionModel, variables),\n    fillFormVision: fillFormVisionTool(v3, provider, variables),\n    goto: gotoTool(v3),\n    keys: keysTool(v3),\n    navback: navBackTool(v3),\n    screenshot: screenshotTool(v3),\n    scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n    think: thinkTool(),\n    type: typeTool(v3, provider, variables),\n    wait: waitTool(v3, mode),\n  };\n\n  // Only include search tool if BRAVE_API_KEY is configured\n  if (process.env.BRAVE_API_KEY) {\n    allTools.search = searchTool(v3);\n  }\n\n  return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/**\n * Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional as it's only available when BRAVE_API_KEY is configured.\n */\nexport type AgentToolTypesMap = {\n  act: ReturnType<typeof actTool>;\n  ariaTree: ReturnType<typeof ariaTreeTool>;\n  click: ReturnType<typeof clickTool>;\n  clickAndHold: ReturnType<typeof clickAndHoldTool>;\n  dragAndDrop: ReturnType<typeof dragAndDropTool>;\n  extract: ReturnType<typeof extractTool>;\n  fillForm: ReturnType<typeof fillFormTool>;\n  fillFormVision: ReturnType<typeof fillFormVisionTool>;\n  goto: ReturnType<typeof gotoTool>;\n  keys: ReturnType<typeof keysTool>;\n  navback: ReturnType<typeof navBackTool>;\n  screenshot: ReturnType<typeof screenshotTool>;\n  scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;\n  search?: ReturnType<typeof searchTool>;\n  think: ReturnType<typeof thinkTool>;\n  type: ReturnType<typeof typeTool>;\n  wait: ReturnType<typeof waitTool>;\n};\n\n/**\n * Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n */\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/**\n * Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n */\nexport type AgentToolCall = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    args: AgentUITools[K][\"input\"];\n  };\n}[keyof AgentToolTypesMap];\n\n/**\n * Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    result: AgentUITools[K][\"output\"];\n  };\n}[keyof AgentToolTypesMap];\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,IAAI,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,EAAE,UAAU,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAUjE,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AA0C/D;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,8DAA8D;AAC9D,SAAS,mBAAmB,CAC1B,SAAY,EACZ,QAAgB,EAChB,EAAM,EACN,SAAkB,EAClB,WAAoB;IAEpB,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAEvD,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,CAAC;IAC1C,OAAO;QACL,GAAG,SAAS;QACZ,OAAO,EAAE,KAAK,EAAE,GAAG,IAAe,EAAE,EAAE;YACpC,IAAI,CAAC;gBACH,OAAO,MAAM,WAAW,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC1E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,OAAO,GAAG,iBAAiB,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;oBACxF,EAAE,CAAC,MAAM,CAAC;wBACR,QAAQ,EAAE,OAAO;wBACjB,OAAO;wBACP,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,OAAO;qBACf,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;KACG,CAAC;AACT,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACrC,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,CAAC;IAEzC,MAAM,YAAY,GAA2B;QAC3C,GAAG,EAAE,kGAAkG;QACvG,QAAQ,EAAE,6BAA6B;QACvC,OAAO,EAAE,yCAAyC;QAClD,QAAQ,EACN,0GAA0G;KAC7G,CAAC;IAEF,MAAM,cAAc,GAAY;QAC9B,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QACxD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,WAAW,CAAC;QACvC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC;QACrD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QAClE,cAAc,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QAC3D,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;KACxC,CAAC;IAEF,IAAI,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACpD,cAAc,CAAC,MAAM,GAAG,qBAAqB,CAC3C,EAAE,EACF,OAAO,CAAC,iBAAiB,CAC1B,CAAC;IACJ,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QACrC,cAAc,CAAC,MAAM,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,QAAQ,GAAY;QACxB,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAChD,IAAI;YACJ,mBAAmB,CACjB,CAAC,EACD,GAAG,IAAI,IAAI,EACX,EAAE,EACF,WAAW,EACX,YAAY,CAAC,IAAI,CAAC,CACnB;SACF,CAAC,CACH;QACD,KAAK,EAAE,SAAS,EAAE;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool as browserbaseSearchTool } from \"./browserbaseSearch.js\";\nimport { searchTool as braveSearchTool } from \"./braveSearch.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n  AgentToolMode,\n  AgentModelConfig,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { withTimeout } from \"../../timeoutConfig.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport interface V3AgentToolOptions {\n  executionModel?: string | AgentModelConfig;\n  logger?: (message: LogLine) => void;\n  /**\n   * Tool mode determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n   */\n  mode?: AgentToolMode;\n  /**\n   * The model provider. Used for model-specific coordinate handling\n   */\n  provider?: string;\n  /**\n   * Tools to exclude from the available toolset.\n   * These tools will be filtered out after mode-based filtering.\n   */\n  excludeTools?: string[];\n  /**\n   * Variables available to the agent for use in act/type tools.\n   * When provided, these tools will have an optional useVariable field.\n   */\n  variables?: Variables;\n  /**\n   * Timeout in milliseconds for async tool calls.\n   * Applied to all tools that perform I/O (except wait and think).\n   */\n  toolTimeout?: number;\n  /**\n   * Whether to enable the Browserbase-powered web search tool.\n   * Requires a valid Browserbase API key.\n   */\n  useSearch?: boolean;\n  /**\n   * The Browserbase API key used for the search tool.\n   * Resolved from BROWSERBASE_API_KEY env var or the Stagehand constructor.\n   */\n  browserbaseApiKey?: string;\n}\n\n/**\n * Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n */\nfunction filterTools(\n  tools: ToolSet,\n  mode: AgentToolMode,\n  excludeTools?: string[],\n): ToolSet {\n  const filtered: ToolSet = { ...tools };\n\n  // Mode-based filtering\n  if (mode === \"hybrid\") {\n    delete filtered.fillForm;\n  } else {\n    // DOM mode (default)\n    delete filtered.click;\n    delete filtered.type;\n    delete filtered.dragAndDrop;\n    delete filtered.clickAndHold;\n    delete filtered.fillFormVision;\n  }\n\n  if (excludeTools) {\n    for (const toolName of excludeTools) {\n      delete filtered[toolName];\n    }\n  }\n\n  return filtered;\n}\n\n/**\n * Wraps an AI SDK tool's execute function with a timeout guard.\n * On timeout, returns `{ success: false, error: \"TimeoutError: ...\" }` to the LLM\n * and logs the error. Also acts as a safety net for any uncaught errors.\n */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction wrapToolWithTimeout<T extends Record<string, any>>(\n  agentTool: T,\n  toolName: string,\n  v3: V3,\n  timeoutMs?: number,\n  timeoutHint?: string,\n): T {\n  if (!timeoutMs || !agentTool.execute) return agentTool;\n\n  const originalExecute = agentTool.execute;\n  return {\n    ...agentTool,\n    execute: async (...args: unknown[]) => {\n      try {\n        return await withTimeout(originalExecute(...args), timeoutMs, toolName);\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const message = `TimeoutError: ${error.message}${timeoutHint ? ` ${timeoutHint}` : \"\"}`;\n          v3.logger({\n            category: \"agent\",\n            message,\n            level: 0,\n          });\n          return {\n            success: false,\n            error: message,\n          };\n        }\n        throw error;\n      }\n    },\n  } as T;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n  const executionModel = options?.executionModel;\n  const mode = options?.mode ?? \"dom\";\n  const provider = options?.provider;\n  const excludeTools = options?.excludeTools;\n  const variables = options?.variables;\n  const toolTimeout = options?.toolTimeout;\n\n  const timeoutHints: Record<string, string> = {\n    act: \"(it may continue executing in the background) — try using a different description for the action\",\n    ariaTree: \"— the page may be too large\",\n    extract: \"— try using a smaller or simpler schema\",\n    fillForm:\n      \"(it may continue executing in the background) — try filling fewer fields at once or use a different tool\",\n  };\n\n  const unwrappedTools: ToolSet = {\n    act: actTool(v3, executionModel, variables, toolTimeout),\n    ariaTree: ariaTreeTool(v3, toolTimeout),\n    click: clickTool(v3, provider),\n    clickAndHold: clickAndHoldTool(v3, provider),\n    dragAndDrop: dragAndDropTool(v3, provider),\n    extract: extractTool(v3, executionModel, toolTimeout),\n    fillForm: fillFormTool(v3, executionModel, variables, toolTimeout),\n    fillFormVision: fillFormVisionTool(v3, provider, variables),\n    goto: gotoTool(v3),\n    keys: keysTool(v3),\n    navback: navBackTool(v3),\n    screenshot: screenshotTool(v3),\n    scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n    type: typeTool(v3, provider, variables),\n  };\n\n  if (options?.useSearch && options.browserbaseApiKey) {\n    unwrappedTools.search = browserbaseSearchTool(\n      v3,\n      options.browserbaseApiKey,\n    );\n  } else if (process.env.BRAVE_API_KEY) {\n    unwrappedTools.search = braveSearchTool(v3);\n  }\n\n  const allTools: ToolSet = {\n    ...Object.fromEntries(\n      Object.entries(unwrappedTools).map(([name, t]) => [\n        name,\n        wrapToolWithTimeout(\n          t,\n          `${name}()`,\n          v3,\n          toolTimeout,\n          timeoutHints[name],\n        ),\n      ]),\n    ),\n    think: thinkTool(),\n    wait: waitTool(v3, mode),\n  };\n\n  return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/**\n * Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional — enabled via useSearch: true (Browserbase) or BRAVE_API_KEY env var (legacy).\n */\nexport type AgentToolTypesMap = {\n  act: ReturnType<typeof actTool>;\n  ariaTree: ReturnType<typeof ariaTreeTool>;\n  click: ReturnType<typeof clickTool>;\n  clickAndHold: ReturnType<typeof clickAndHoldTool>;\n  dragAndDrop: ReturnType<typeof dragAndDropTool>;\n  extract: ReturnType<typeof extractTool>;\n  fillForm: ReturnType<typeof fillFormTool>;\n  fillFormVision: ReturnType<typeof fillFormVisionTool>;\n  goto: ReturnType<typeof gotoTool>;\n  keys: ReturnType<typeof keysTool>;\n  navback: ReturnType<typeof navBackTool>;\n  screenshot: ReturnType<typeof screenshotTool>;\n  scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;\n  search?:\n    | ReturnType<typeof browserbaseSearchTool>\n    | ReturnType<typeof braveSearchTool>;\n  think: ReturnType<typeof thinkTool>;\n  type: ReturnType<typeof typeTool>;\n  wait: ReturnType<typeof waitTool>;\n};\n\n/**\n * Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n */\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/**\n * Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n */\nexport type AgentToolCall = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    args: AgentUITools[K][\"input\"];\n  };\n}[keyof AgentToolTypesMap];\n\n/**\n * Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    result: AgentUITools[K][\"output\"];\n  };\n}[keyof AgentToolTypesMap];\n"]}

package/dist/esm/lib/v3/agent/tools/keys.d.ts

@@ -17,7 +17,7 @@ export declare const keysTool: (v3: V3) => import("ai").Tool<{
     error?: undefined;
 } | {
     success: boolean;
-    error: string;
+    error: any;
     method?: undefined;
     value?: undefined;
     times?: undefined;

package/dist/esm/lib/v3/agent/tools/keys.js.map

@@ -1 +1 @@
-{"version":3,"file":"keys.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/keys.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,EAAE,CACjC,IAAI,CAAC;IACH,WAAW,EAAE;;;;kIAIiH;IAC9H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjC,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,QAAQ,CACP,iEAAiE,CAClE;QACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC9B,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;wBAChD,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAEvC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,GAAG;oBAC9B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,EAAE;oBAC7B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,MAAM,EAAE,EAAE,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC;QAC7D,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const keysTool = (v3: V3) =>\n  tool({\n    description: `Send keyboard input to the page without targeting a specific element. Unlike the type tool which clicks then types into coordinates, this sends keystrokes directly to wherever focus currently is.\n\nUse method=\"type\" to enter text into the currently focused element. Preferred when: input is already focused, text needs to flow across multiple fields (e.g., verification codes)\n\nUse method=\"press\" for navigation keys (Enter, Tab, Escape, Backspace, arrows) and keyboard shortcuts (Cmd+A, Ctrl+C, Shift+Tab).`,\n    inputSchema: z.object({\n      method: z.enum([\"press\", \"type\"]),\n      value: z\n        .string()\n        .describe(\n          \"The text to type, or the key/combo to press (Enter, Tab, Cmd+A)\",\n        ),\n      repeat: z.number().optional(),\n    }),\n    execute: async ({ method, value, repeat }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: keys`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ method, value, repeat }),\n              type: \"object\",\n            },\n          },\n        });\n\n        const times = Math.max(1, repeat ?? 1);\n\n        if (method === \"type\") {\n          for (let i = 0; i < times; i++) {\n            await page.type(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `type \"${value}\"`,\n            playwrightArguments: { method, text: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        if (method === \"press\") {\n          for (let i = 0; i < times; i++) {\n            await page.keyPress(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `press ${value}`,\n            playwrightArguments: { method, keys: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        return { success: false, error: `Unsupported method: ${method}` };\n      } catch (error) {\n        return { success: false, error: (error as Error).message };\n      }\n    },\n  });\n"]}
+{"version":3,"file":"keys.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/keys.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,EAAE,CACjC,IAAI,CAAC;IACH,WAAW,EAAE;;;;kIAIiH;IAC9H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjC,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,QAAQ,CACP,iEAAiE,CAClE;QACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC9B,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;wBAChD,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAEvC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,GAAG;oBAC9B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,EAAE;oBAC7B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,MAAM,EAAE,EAAE,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QAClD,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const keysTool = (v3: V3) =>\n  tool({\n    description: `Send keyboard input to the page without targeting a specific element. Unlike the type tool which clicks then types into coordinates, this sends keystrokes directly to wherever focus currently is.\n\nUse method=\"type\" to enter text into the currently focused element. Preferred when: input is already focused, text needs to flow across multiple fields (e.g., verification codes)\n\nUse method=\"press\" for navigation keys (Enter, Tab, Escape, Backspace, arrows) and keyboard shortcuts (Cmd+A, Ctrl+C, Shift+Tab).`,\n    inputSchema: z.object({\n      method: z.enum([\"press\", \"type\"]),\n      value: z\n        .string()\n        .describe(\n          \"The text to type, or the key/combo to press (Enter, Tab, Cmd+A)\",\n        ),\n      repeat: z.number().optional(),\n    }),\n    execute: async ({ method, value, repeat }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: keys`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ method, value, repeat }),\n              type: \"object\",\n            },\n          },\n        });\n\n        const times = Math.max(1, repeat ?? 1);\n\n        if (method === \"type\") {\n          for (let i = 0; i < times; i++) {\n            await page.type(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `type \"${value}\"`,\n            playwrightArguments: { method, text: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        if (method === \"press\") {\n          for (let i = 0; i < times; i++) {\n            await page.keyPress(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `press ${value}`,\n            playwrightArguments: { method, keys: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        return { success: false, error: `Unsupported method: ${method}` };\n      } catch (error) {\n        return { success: false, error: error.message };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/screenshot.d.ts

@@ -1,6 +1,14 @@
 import type { V3 } from "../../v3.js";
 export declare const screenshotTool: (v3: V3) => import("ai").Tool<Record<string, never>, {
+    success: boolean;
     base64: string;
     timestamp: number;
     pageUrl: string;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: string;
+    base64?: undefined;
+    timestamp?: undefined;
+    pageUrl?: undefined;
 }>;

package/dist/esm/lib/v3/agent/tools/screenshot.js

@@ -4,23 +4,40 @@ export const screenshotTool = (v3) => tool({
     description: "Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.",
     inputSchema: z.object({}),
     execute: async () => {
-        v3.logger({
-            category: "agent",
-            message: `Agent calling tool: screenshot`,
-            level: 1,
-        });
-        const page = await v3.context.awaitActivePage();
-        const buffer = await page.screenshot({ fullPage: false });
-        const pageUrl = page.url();
+        try {
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: screenshot`,
+                level: 1,
+            });
+            const page = await v3.context.awaitActivePage();
+            const buffer = await page.screenshot({ fullPage: false });
+            const pageUrl = page.url();
+            return {
+                success: true,
+                base64: buffer.toString("base64"),
+                timestamp: Date.now(),
+                pageUrl,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error taking screenshot: ${error.message}`,
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         return {
-            base64: buffer.toString("base64"),
-            timestamp: Date.now(),
-            pageUrl,
+            type: "content",
+            value: [{ type: "media", mediaType: "image/png", data: result.base64 }],
         };
     },
-    toModelOutput: (result) => ({
-        type: "content",
-        value: [{ type: "media", mediaType: "image/png", data: result.base64 }],
-    }),
 });
 //# sourceMappingURL=screenshot.js.map

package/dist/esm/lib/v3/agent/tools/screenshot.js.map

@@ -1 +1 @@
-{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAM,EAAE,EAAE,CACvC,IAAI,CAAC;IACH,WAAW,EACT,sFAAsF;IACxF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,gCAAgC;YACzC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,OAAO;SACR,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;KACxE,CAAC;CACH,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const screenshotTool = (v3: V3) =>\n  tool({\n    description:\n      \"Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: screenshot`,\n        level: 1,\n      });\n      const page = await v3.context.awaitActivePage();\n      const buffer = await page.screenshot({ fullPage: false });\n      const pageUrl = page.url();\n      return {\n        base64: buffer.toString(\"base64\"),\n        timestamp: Date.now(),\n        pageUrl,\n      };\n    },\n    toModelOutput: (result) => ({\n      type: \"content\",\n      value: [{ type: \"media\", mediaType: \"image/png\", data: result.base64 }],\n    }),\n  });\n"]}
+{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAM,EAAE,EAAE,CACvC,IAAI,CAAC;IACH,WAAW,EACT,sFAAsF;IACxF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,gCAAgC;gBACzC,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,OAAO;aACR,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,4BAA6B,KAAe,CAAC,OAAO,EAAE;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;SACxE,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const screenshotTool = (v3: V3) =>\n  tool({\n    description:\n      \"Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: screenshot`,\n          level: 1,\n        });\n        const page = await v3.context.awaitActivePage();\n        const buffer = await page.screenshot({ fullPage: false });\n        const pageUrl = page.url();\n        return {\n          success: true,\n          base64: buffer.toString(\"base64\"),\n          timestamp: Date.now(),\n          pageUrl,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error taking screenshot: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"content\",\n        value: [{ type: \"media\", mediaType: \"image/png\", data: result.base64 }],\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/scroll.js

@@ -44,6 +44,12 @@ export const scrollTool = (v3) => tool({
         };
     },
     toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         return {
             type: "json",
             value: {
@@ -119,6 +125,12 @@ export const scrollVisionTool = (v3, provider) => tool({
         };
     },
     toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         const content = [
             {
                 type: "text",

package/dist/esm/lib/v3/agent/tools/scroll.js.map

@@ -1 +1 @@
-{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  ScrollToolResult,\n  ScrollVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n  tool({\n    description:\n      \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      percentage = 80,\n    }): Promise<ScrollToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ direction, percentage }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const cx = Math.floor(w / 2);\n      const cy = Math.floor(h / 2);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n        scrolledPixels: scrollDistance,\n      };\n    },\n    toModelOutput: (result) => {\n      return {\n        type: \"json\",\n        value: {\n          success: result.success,\n          message: result.message,\n          scrolledPixels: result.scrolledPixels,\n        },\n      };\n    },\n  });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      coordinates: z\n        .array(z.number())\n        .optional()\n        .describe(\n          \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n        ),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      coordinates,\n      percentage = 80,\n    }): Promise<ScrollVisionToolResult> => {\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      // Process coordinates if provided, otherwise use viewport center\n      let cx: number;\n      let cy: number;\n      if (coordinates) {\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n        cx = processed.x;\n        cy = processed.y;\n      } else {\n        cx = Math.floor(w / 2);\n        cy = Math.floor(h / 2);\n      }\n\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({\n              direction,\n              coordinates,\n              percentage,\n              processed: { cx, cy },\n            }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: coordinates\n          ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n          : `Scrolled ${percentage}% ${direction}`,\n        scrolledPixels: scrollDistance,\n        screenshotBase64,\n      };\n    },\n    toModelOutput: (result) => {\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            message: result.message,\n            scrolledPixels: result.scrolledPixels,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}
+{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  ScrollToolResult,\n  ScrollVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n  tool({\n    description:\n      \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      percentage = 80,\n    }): Promise<ScrollToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ direction, percentage }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const cx = Math.floor(w / 2);\n      const cy = Math.floor(h / 2);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n        scrolledPixels: scrollDistance,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"json\",\n        value: {\n          success: result.success,\n          message: result.message,\n          scrolledPixels: result.scrolledPixels,\n        },\n      };\n    },\n  });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      coordinates: z\n        .array(z.number())\n        .optional()\n        .describe(\n          \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n        ),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      coordinates,\n      percentage = 80,\n    }): Promise<ScrollVisionToolResult> => {\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      // Process coordinates if provided, otherwise use viewport center\n      let cx: number;\n      let cy: number;\n      if (coordinates) {\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n        cx = processed.x;\n        cy = processed.y;\n      } else {\n        cx = Math.floor(w / 2);\n        cy = Math.floor(h / 2);\n      }\n\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({\n              direction,\n              coordinates,\n              percentage,\n              processed: { cx, cy },\n            }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: coordinates\n          ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n          : `Scrolled ${percentage}% ${direction}`,\n        scrolledPixels: scrollDistance,\n        screenshotBase64,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            message: result.message,\n            scrolledPixels: result.scrolledPixels,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}